From e44bc9394c8c784dcf80940e240da1a46c3a8290 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Fri, 31 Aug 2018 16:50:21 +0530 Subject: [PATCH 01/22] unified architecture changes --- app/com/linkedin/drelephant/AutoTuner.java | 23 +- .../tuning/AbstractBaselineManager.java | 165 ++++ ...eUtil.java => AbstractFitnessManager.java} | 720 +++++++++--------- .../tuning/AbstractJobStatusManager.java | 90 +++ .../tuning/AbstractTuningTypeManager.java | 174 +++++ .../tuning/AutoTuningAPIHelper.java | 326 +++++--- .../tuning/AzkabanJobCompleteDetector.java | 120 --- .../tuning/BaselineComputeUtil.java | 154 ---- .../linkedin/drelephant/tuning/Constant.java | 13 + .../drelephant/tuning/ExecutionEngine.java | 41 + app/com/linkedin/drelephant/tuning/Flow.java | 107 +++ .../tuning/JobCompleteDetector.java | 88 --- .../drelephant/tuning/JobTuningInfo.java | 2 +- .../linkedin/drelephant/tuning/Manager.java | 13 + .../drelephant/tuning/ManagerFactory.java | 109 +++ .../drelephant/tuning/PSOParamGenerator.java | 104 --- .../drelephant/tuning/ParamGenerator.java | 522 ------------- .../Schduler/AzkabanJobStatusManager.java | 111 +++ .../drelephant/tuning/TuningHelper.java | 53 ++ .../tuning/engine/MRExecutionEngine.java | 425 +++++++++++ .../tuning/engine/SparkExecutionEngine.java | 74 ++ .../tuning/hbt/BaselineManagerHBT.java | 39 + .../tuning/hbt/FitnessManagerHBT.java | 136 ++++ .../tuning/hbt/TuningTypeManagerHBT.java | 288 +++++++ .../tuning/obt/BaselineManagerOBT.java | 43 ++ .../tuning/obt/FitnessManagerOBT.java | 75 ++ .../tuning/obt/FitnessManagerOBTAlgoIPSO.java | 91 +++ .../tuning/obt/FitnessManagerOBTAlgoPSO.java | 78 ++ .../tuning/obt/OptimizationAlgoFactory.java | 36 + .../tuning/obt/TuningTypeManagerOBT.java | 403 ++++++++++ .../obt/TuningTypeManagerOBTAlgoIPSO.java | 135 ++++ .../obt/TuningTypeManagerOBTAlgoPSO.java | 71 ++ app/controllers/Application.java | 5 +- app/models/JobSuggestedParamSet.java | 4 + app/models/TuningAlgorithm.java | 2 +- app/models/TuningJobDefinition.java | 9 + app/models/TuningParameterConstraint.java | 105 +++ conf/evolutions/default/5.sql | 3 + conf/evolutions/default/6.sql | 60 ++ .../tuning/BaselineManagerTestRunner.java | 37 + .../drelephant/tuning/IPSOManagerTest.java | 56 ++ .../tuning/IPSOManagerTestRunner.java | 211 +++++ .../tuning/PSOParamGeneratorTest.java | 22 +- test/common/DBTestUtil.java | 11 +- test/common/TestConstants.java | 2 + test/resources/test-init-baseline.sql | 163 ++++ test/resources/test-init-ipso.sql | 167 ++++ test/rest/RestAPITest.java | 21 +- 48 files changed, 4217 insertions(+), 1490 deletions(-) create mode 100644 app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java rename app/com/linkedin/drelephant/tuning/{FitnessComputeUtil.java => AbstractFitnessManager.java} (53%) create mode 100644 app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java create mode 100644 app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java delete mode 100644 app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java delete mode 100644 app/com/linkedin/drelephant/tuning/BaselineComputeUtil.java create mode 100644 app/com/linkedin/drelephant/tuning/Constant.java create mode 100644 app/com/linkedin/drelephant/tuning/ExecutionEngine.java create mode 100644 app/com/linkedin/drelephant/tuning/Flow.java delete mode 100644 app/com/linkedin/drelephant/tuning/JobCompleteDetector.java create mode 100644 app/com/linkedin/drelephant/tuning/Manager.java create mode 100644 app/com/linkedin/drelephant/tuning/ManagerFactory.java delete mode 100644 app/com/linkedin/drelephant/tuning/PSOParamGenerator.java delete mode 100644 app/com/linkedin/drelephant/tuning/ParamGenerator.java create mode 100644 app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java create mode 100644 app/com/linkedin/drelephant/tuning/TuningHelper.java create mode 100644 app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java create mode 100644 app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java create mode 100644 app/com/linkedin/drelephant/tuning/hbt/BaselineManagerHBT.java create mode 100644 app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java create mode 100644 app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java create mode 100644 app/com/linkedin/drelephant/tuning/obt/BaselineManagerOBT.java create mode 100644 app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java create mode 100644 app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java create mode 100644 app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoPSO.java create mode 100644 app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java create mode 100644 app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java create mode 100644 app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoIPSO.java create mode 100644 app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoPSO.java create mode 100644 app/models/TuningParameterConstraint.java create mode 100644 conf/evolutions/default/6.sql create mode 100644 test/com/linkedin/drelephant/tuning/BaselineManagerTestRunner.java create mode 100644 test/com/linkedin/drelephant/tuning/IPSOManagerTest.java create mode 100644 test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java create mode 100644 test/resources/test-init-baseline.sql create mode 100644 test/resources/test-init-ipso.sql diff --git a/app/com/linkedin/drelephant/AutoTuner.java b/app/com/linkedin/drelephant/AutoTuner.java index c4180e220..4a1da8cbd 100644 --- a/app/com/linkedin/drelephant/AutoTuner.java +++ b/app/com/linkedin/drelephant/AutoTuner.java @@ -16,16 +16,11 @@ package com.linkedin.drelephant; +import com.linkedin.drelephant.tuning.Flow; import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; import com.linkedin.drelephant.analysis.HDFSContext; -import com.linkedin.drelephant.tuning.AzkabanJobCompleteDetector; -import com.linkedin.drelephant.tuning.BaselineComputeUtil; -import com.linkedin.drelephant.tuning.FitnessComputeUtil; -import com.linkedin.drelephant.tuning.JobCompleteDetector; -import com.linkedin.drelephant.tuning.PSOParamGenerator; -import com.linkedin.drelephant.tuning.ParamGenerator; import com.linkedin.drelephant.util.Utils; import controllers.AutoTuningMetricsController; @@ -43,30 +38,22 @@ public class AutoTuner implements Runnable { public static final long ONE_MIN = 60 * 1000; private static final Logger logger = Logger.getLogger(AutoTuner.class); private static final long DEFAULT_METRICS_COMPUTATION_INTERVAL = ONE_MIN / 5; - public static final String AUTO_TUNING_DAEMON_WAIT_INTERVAL = "autotuning.daemon.wait.interval.ms"; + private static final String tuningTypes[] = {"OBT"}; public void run() { logger.info("Starting Auto Tuning thread"); HDFSContext.load(); Configuration configuration = ElephantContext.instance().getAutoTuningConf(); - Long interval = Utils.getNonNegativeLong(configuration, AUTO_TUNING_DAEMON_WAIT_INTERVAL, DEFAULT_METRICS_COMPUTATION_INTERVAL); - try { AutoTuningMetricsController.init(); - BaselineComputeUtil baselineComputeUtil = new BaselineComputeUtil(); - FitnessComputeUtil fitnessComputeUtil = new FitnessComputeUtil(); - ParamGenerator paramGenerator = new PSOParamGenerator(); - JobCompleteDetector jobCompleteDetector = new AzkabanJobCompleteDetector(); + Flow autoTuningFlow= new Flow(); while (!Thread.currentThread().isInterrupted()) { try { - baselineComputeUtil.computeBaseline(); - jobCompleteDetector.updateCompletedExecutions(); - fitnessComputeUtil.updateFitness(); - paramGenerator.getParams(); + autoTuningFlow.executeFlow(); } catch (Exception e) { logger.error("Error in auto tuner thread ", e); } @@ -77,4 +64,6 @@ public void run() { } logger.info("Auto tuning thread shutting down"); } + + } diff --git a/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java b/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java new file mode 100644 index 000000000..7336efb4c --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java @@ -0,0 +1,165 @@ +package com.linkedin.drelephant.tuning; + +import com.avaje.ebean.Ebean; +import com.avaje.ebean.SqlRow; +import com.linkedin.drelephant.ElephantContext; +import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; +import java.util.List; +import models.JobExecution; +import models.TuningJobDefinition; +import models.TuningJobExecutionParamSet; +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; +import controllers.AutoTuningMetricsController; +import org.apache.hadoop.conf.Configuration; + +public abstract class AbstractBaselineManager implements Manager { + protected final String BASELINE_EXECUTION_COUNT = "baseline.execution.count"; + protected Integer NUM_JOBS_FOR_BASELINE_DEFAULT = 30; + protected String baseLineCalculationSQL = + "SELECT AVG(resource_used) AS resource_used, AVG(execution_time) AS execution_time FROM " + + "(SELECT job_exec_id, SUM(resource_used/(1024 * 3600)) AS resource_used, " + + "SUM((finish_time - start_time - total_delay)/(1000 * 60)) AS execution_time, " + + "MAX(start_time) AS start_time " + "FROM yarn_app_result WHERE job_def_id=:jobDefId " + + "GROUP BY job_exec_id " + "ORDER BY start_time DESC " + "LIMIT :num) temp"; + protected String avgInputSizeSQL = "SELECT AVG(inputSizeInBytes) as avgInputSizeInMB FROM " + + "(SELECT job_exec_id, SUM(cast(value as decimal)) inputSizeInBytes, MAX(start_time) AS start_time " + + "FROM yarn_app_result yar INNER JOIN yarn_app_heuristic_result yahr " + "ON yar.id=yahr.yarn_app_result_id " + + "INNER JOIN yarn_app_heuristic_result_details yahrd " + "ON yahr.id=yahrd.yarn_app_heuristic_result_id " + + "WHERE job_def_id=:jobDefId AND yahr.heuristic_name='" + CommonConstantsHeuristic.MAPPER_SPEED + "' " + + "AND yahrd.name='" + CommonConstantsHeuristic.TOTAL_INPUT_SIZE_IN_MB + "' " + + "GROUP BY job_exec_id ORDER BY start_time DESC LIMIT :num ) temp"; + private final Logger logger = Logger.getLogger(getClass()); + protected Integer _numJobsForBaseline = null; + protected Configuration configuration = null; + + public AbstractBaselineManager(){ + configuration = ElephantContext.instance().getAutoTuningConf(); + } + + /* + Execute the whole logic for Base line Computing + 1) Get the jobs for which Baseline have to be calculated + 2) Calculate the baseline + 3) Update the base line + 4) Update the metrics . + */ + @Override + public final Boolean execute() { + logger.info("Executing BaseLine"); + Boolean baseLineComputationDone = false, databaseUpdateDone = false, updateMetricsDone = false; + List tuningJobDefinitions = detectJobsForBaseLineComputation(); + if (tuningJobDefinitions != null && tuningJobDefinitions.size() >= 1) { + logger.info("Computing BaseLine"); + baseLineComputationDone = calculateBaseLine(tuningJobDefinitions); + } + if (baseLineComputationDone) { + logger.info("Updating Database"); + databaseUpdateDone = updateDataBase(tuningJobDefinitions); + } + if (databaseUpdateDone) { + logger.info("Updating Metrics"); + updateMetricsDone = updateMetrics(tuningJobDefinitions); + } + logger.info("Baseline Done "); + return updateMetricsDone; + } + + /** + * Fetches the jobs whose baseline is to be computed. + * This is done by returning the jobs with null average resource usage + * @return List of jobs whose baseline needs to be added + */ + + protected abstract List detectJobsForBaseLineComputation(); + + /** + * Adds baseline metric values for a job + * @param tuningJobDefinitions Job for which baseline is to be computed + */ + + protected Boolean calculateBaseLine(List tuningJobDefinitions) { + for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitions) { + try { + logger.info("Computing and updating baseline metric values for job: " + tuningJobDefinition.job.jobName); + logger.debug("Running query for baseline computation " + baseLineCalculationSQL); + SqlRow baseline = Ebean.createSqlQuery(baseLineCalculationSQL) + .setParameter("jobDefId", tuningJobDefinition.job.jobDefId) + .setParameter("num", _numJobsForBaseline) + .findUnique(); + Double avgResourceUsage = 0D; + Double avgExecutionTime = 0D; + avgResourceUsage = baseline.getDouble("resource_used"); + avgExecutionTime = baseline.getDouble("execution_time"); + tuningJobDefinition.averageExecutionTime = avgExecutionTime; + tuningJobDefinition.averageResourceUsage = avgResourceUsage; + tuningJobDefinition.averageInputSizeInBytes = getAvgInputSizeInBytes(tuningJobDefinition.job.jobDefId); + logger.debug( + "Baseline metric values: Average resource usage=" + avgResourceUsage + " and Average execution time=" + + avgExecutionTime); + } catch (Exception e) { + logger.error("Error in computing baseline for job: " + tuningJobDefinition.job.jobName, e); + return false; + } + } + return true; + } + + /** + * Returns the average input size in bytes of a job (over last _numJobsForBaseline executions) + * @param jobDefId job definition id of the job + * @return average input size in bytes as long + */ + private Long getAvgInputSizeInBytes(String jobDefId) { + logger.debug("Running query for average input size computation " + avgInputSizeSQL + " Job definintion ID "+jobDefId); + + SqlRow baseline = Ebean.createSqlQuery(avgInputSizeSQL) + .setParameter("jobDefId", jobDefId) + .setParameter("num", _numJobsForBaseline) + .findUnique(); + Double avgInputSizeInBytes = baseline.getDouble("avgInputSizeInMB") * FileUtils.ONE_MB; + return avgInputSizeInBytes.longValue(); + } + + /** + * This method update database for auto tuning monitoring for baseline computation + * @param tuningJobDefinitions + */ + protected Boolean updateDataBase(List tuningJobDefinitions) { + try { + for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitions) { + tuningJobDefinition.update(); + logger.info("Updated baseline metric value for job: " + tuningJobDefinition.job.jobName); + } + } catch (Exception e) { + logger.error(" Error Updating Database " + e.getMessage()); + return false; + } + return true; + } + + /** + * This method update metrics for auto tuning monitoring for baseline computation + * @param tuningJobDefinitions + */ + + protected Boolean updateMetrics(List tuningJobDefinitions) { + try { + int baselineComputeWaitJobs = 0; + for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitions) { + if (tuningJobDefinition.averageResourceUsage == null) { + baselineComputeWaitJobs++; + } else { + AutoTuningMetricsController.markBaselineComputed(); + } + } + AutoTuningMetricsController.setBaselineComputeWaitJobs(baselineComputeWaitJobs); + } catch (Exception e) { + logger.error(" Error Updating Metrics in Ingraph" + e.getMessage()); + return false; + } + return true; + } + + +} diff --git a/app/com/linkedin/drelephant/tuning/FitnessComputeUtil.java b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java similarity index 53% rename from app/com/linkedin/drelephant/tuning/FitnessComputeUtil.java rename to app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java index bf6f64d66..98295b8c1 100644 --- a/app/com/linkedin/drelephant/tuning/FitnessComputeUtil.java +++ b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java @@ -1,28 +1,8 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - package com.linkedin.drelephant.tuning; import com.avaje.ebean.Expr; -import com.linkedin.drelephant.AutoTuner; -import com.linkedin.drelephant.ElephantContext; import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; -import com.linkedin.drelephant.util.Utils; import controllers.AutoTuningMetricsController; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; @@ -35,14 +15,12 @@ import models.JobDefinition; import models.JobExecution; import models.JobSuggestedParamSet; -import models.JobSuggestedParamSet.ParamSetStatus; import models.JobSuggestedParamValue; import models.TuningAlgorithm; import models.TuningJobDefinition; import models.TuningJobExecutionParamSet; import models.TuningParameter; import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; @@ -53,137 +31,235 @@ * In case there is failure or resource usage/execution time goes beyond configured limit, fitness is computed by * adding a penalty. */ -public class FitnessComputeUtil { - private static final Logger logger = Logger.getLogger(FitnessComputeUtil.class); - private static final String FITNESS_COMPUTE_WAIT_INTERVAL = "fitness.compute.wait_interval.ms"; - private static final String IGNORE_EXECUTION_WAIT_INTERVAL = "ignore.execution.wait.interval.ms"; - private static final String MAX_TUNING_EXECUTIONS = "max.tuning.executions"; - private static final String MIN_TUNING_EXECUTIONS = "min.tuning.executions"; - private int maxTuningExecutions; - private int minTuningExecutions; - private Long fitnessComputeWaitInterval; - private Long ignoreExecutionWaitInterval; - - public FitnessComputeUtil() { - Configuration configuration = ElephantContext.instance().getAutoTuningConf(); - - // Time duration to wait for computing the fitness of a param set once the corresponding execution is completed - fitnessComputeWaitInterval = - Utils.getNonNegativeLong(configuration, FITNESS_COMPUTE_WAIT_INTERVAL, 5 * AutoTuner.ONE_MIN); - - // Time duration to wait for metrics (resource usage, execution time) of an execution to be computed before - // discarding it for fitness computation - ignoreExecutionWaitInterval = - Utils.getNonNegativeLong(configuration, IGNORE_EXECUTION_WAIT_INTERVAL, 2 * 60 * AutoTuner.ONE_MIN); - - // #executions after which tuning will stop even if parameters don't converge - maxTuningExecutions = - Utils.getNonNegativeInt(configuration, MAX_TUNING_EXECUTIONS, 39); - - // #executions before which tuning cannot stop even if parameters converge - minTuningExecutions = - Utils.getNonNegativeInt(configuration, MIN_TUNING_EXECUTIONS, 18); + +public abstract class AbstractFitnessManager implements Manager { + private final Logger logger = Logger.getLogger(getClass()); + protected final String FITNESS_COMPUTE_WAIT_INTERVAL = "fitness.compute.wait_interval.ms"; + protected final String IGNORE_EXECUTION_WAIT_INTERVAL = "ignore.execution.wait.interval.ms"; + protected final String MAX_TUNING_EXECUTIONS = "max.tuning.executions"; + protected final String MIN_TUNING_EXECUTIONS = "min.tuning.executions"; + protected int maxTuningExecutions; + protected int minTuningExecutions; + protected Long fitnessComputeWaitInterval; + protected Long ignoreExecutionWaitInterval; + + protected abstract List detectJobsForFitnessComputation(); + + protected Boolean calculateFitness(List completedJobExecutionParamSets) { + for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { + JobExecution jobExecution = completedJobExecutionParamSet.jobExecution; + JobSuggestedParamSet jobSuggestedParamSet = completedJobExecutionParamSet.jobSuggestedParamSet; + JobDefinition job = jobExecution.job; + + logger.info("Updating execution metrics and fitness for execution: " + jobExecution.jobExecId); + try { + TuningJobDefinition tuningJobDefinition = getTuningJobDefinition(job); + List results = getAppResults(jobExecution); + handleFitnessCalculation(jobExecution, results, tuningJobDefinition, jobSuggestedParamSet); + } catch (Exception e) { + logger.error("Error updating fitness of execution: " + jobExecution.id + "\n Stacktrace: ", e); + } + } + logger.info("Execution metrics updated"); + return false; } - private boolean isTuningEnabled(Integer jobDefinitionId) { - TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() - .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinitionId) + private TuningJobDefinition getTuningJobDefinition(JobDefinition job) { + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") + .fetch(TuningJobDefinition.TABLE.job, "*") + .where() + .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, job.id) .order() - // There can be multiple entries in tuningJobDefinition if the job is switch on/off multiple times. - // The latest entry gives the information regarding whether tuning is enabled or not .desc(TuningJobDefinition.TABLE.createdTs) - .setMaxRows(1) .findUnique(); + return tuningJobDefinition; + } - return tuningJobDefinition != null && tuningJobDefinition.tuningEnabled; + private List getAppResults(JobExecution jobExecution) { + List results = AppResult.find.select("*") + .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*") + .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, "*") + .where() + .eq(AppResult.TABLE.FLOW_EXEC_ID, jobExecution.flowExecution.flowExecId) + .eq(AppResult.TABLE.JOB_EXEC_ID, jobExecution.jobExecId) + .findList(); + return results; + } + + private void handleFitnessCalculation(JobExecution jobExecution, List results, + TuningJobDefinition tuningJobDefinition, JobSuggestedParamSet jobSuggestedParamSet) { + if (results != null && results.size() > 0) { + calculateAndUpdateFitness(jobExecution, results, tuningJobDefinition, jobSuggestedParamSet); + } else { + handleEmptyResultScenario(jobExecution, jobSuggestedParamSet); + } + } + + protected abstract void calculateAndUpdateFitness(JobExecution jobExecution, List results, + TuningJobDefinition tuningJobDefinition, JobSuggestedParamSet jobSuggestedParamSet); + + protected void updateJobExecution(JobExecution jobExecution, Double totalResourceUsed, Double totalInputBytesInBytes, + Long totalExecutionTime) { + jobExecution.executionTime = totalExecutionTime * 1.0 / (1000 * 60); + jobExecution.resourceUsage = totalResourceUsed * 1.0 / (1024 * 3600); + jobExecution.inputSizeInBytes = totalInputBytesInBytes; + jobExecution.update(); + logger.info("Metric Values for execution " + jobExecution.jobExecId + ": Execution time = " + totalExecutionTime + + ", Resource usage = " + totalResourceUsed + " and total input size = " + totalInputBytesInBytes); + } + + protected void updateTuningJobDefinition(TuningJobDefinition tuningJobDefinition, JobExecution jobExecution) { + tuningJobDefinition.averageResourceUsage = jobExecution.resourceUsage; + tuningJobDefinition.averageExecutionTime = jobExecution.executionTime; + tuningJobDefinition.averageInputSizeInBytes = jobExecution.inputSizeInBytes.longValue(); + tuningJobDefinition.update(); + } + + private void handleEmptyResultScenario(JobExecution jobExecution, JobSuggestedParamSet jobSuggestedParamSet) { + long diff = System.currentTimeMillis() - jobExecution.updatedTs.getTime(); + logger.debug("Current Time in millis: " + System.currentTimeMillis() + ", job execution last updated time " + + jobExecution.updatedTs.getTime()); + if (diff > ignoreExecutionWaitInterval) { + logger.info( + "Fitness of param set " + jobSuggestedParamSet.id + " corresponding to execution id: " + jobExecution.id + + " not computed for more than the maximum duration specified to compute fitness. " + + "Resetting the param set to CREATED state"); + resetParamSetToCreated(jobSuggestedParamSet); + } } /** - * Updates the metrics (execution time, resource usage, cost function) of the completed executions whose metrics are - * not computed. + * Returns the total input size + * @param appResult appResult + * @return total input size */ - public void updateFitness() { - logger.info("Computing and updating fitness for completed executions"); - List completedJobExecutionParamSets = getCompletedJobExecutionParamSets(); - updateExecutionMetrics(completedJobExecutionParamSets); - updateMetrics(completedJobExecutionParamSets); - - Set jobDefinitionSet = new HashSet(); - for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { - JobDefinition jobDefinition = completedJobExecutionParamSet.jobSuggestedParamSet.jobDefinition; - if (isTuningEnabled(jobDefinition.id)) { - jobDefinitionSet.add(jobDefinition); + protected Long getTotalInputBytes(AppResult appResult) { + Long totalInputBytes = 0L; + if (appResult.yarnAppHeuristicResults != null) { + for (AppHeuristicResult appHeuristicResult : appResult.yarnAppHeuristicResults) { + if (appHeuristicResult.heuristicName.equals(CommonConstantsHeuristic.MAPPER_SPEED)) { + if (appHeuristicResult.yarnAppHeuristicResultDetails != null) { + for (AppHeuristicResultDetails appHeuristicResultDetails : appHeuristicResult.yarnAppHeuristicResultDetails) { + if (appHeuristicResultDetails.name.equals(CommonConstantsHeuristic.TOTAL_INPUT_SIZE_IN_MB)) { + totalInputBytes += Math.round(Double.parseDouble(appHeuristicResultDetails.value) * FileUtils.ONE_MB); + } + } + } + } } } - checkToDisableTuning(jobDefinitionSet); + return totalInputBytes; } /** - * Checks if the tuning parameters converge - * @param tuningJobExecutionParamSets List of previous executions and corresponding param sets - * @return true if the parameters converge, else false + * Resets the param set to CREATED state if its fitness is not already computed + * @param jobSuggestedParamSet Param set which is to be reset */ - private boolean didParameterSetConverge(List tuningJobExecutionParamSets) { - boolean result = false; - int numParamSetForConvergence = 3; - - if (tuningJobExecutionParamSets.size() < numParamSetForConvergence) { - return false; + protected void resetParamSetToCreated(JobSuggestedParamSet jobSuggestedParamSet) { + if (!jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED)) { + logger.info("Resetting parameter set to created: " + jobSuggestedParamSet.id); + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; + jobSuggestedParamSet.save(); } + } - TuningAlgorithm.JobType jobType = tuningJobExecutionParamSets.get(0).jobSuggestedParamSet.tuningAlgorithm.jobType; - - if (jobType == TuningAlgorithm.JobType.PIG) { - - Map> paramValueSet = new HashMap>(); - - for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { - - JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; - - List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, - jobSuggestedParamSet.id) - .or(Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, - "mapreduce.map.memory.mb"), - Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, - "mapreduce.reduce.memory.mb")) - .findList(); + /** + * Updates the job suggested param set when the corresponding execution was succeeded + * @param jobExecution JobExecution: succeeded job execution corresponding to the param set which is to be updated + * @param jobSuggestedParamSet param set which is to be updated + * @param tuningJobDefinition TuningJobDefinition of the job to which param set corresponds + */ + protected void updateJobSuggestedParamSetSucceededExecution(JobExecution jobExecution, + JobSuggestedParamSet jobSuggestedParamSet, TuningJobDefinition tuningJobDefinition) { + int penaltyConstant = 3; + Double averageResourceUsagePerGBInput = + tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; + Double maxDesiredResourceUsagePerGBInput = + averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; + Double averageExecutionTimePerGBInput = + tuningJobDefinition.averageExecutionTime * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; + Double maxDesiredExecutionTimePerGBInput = + averageExecutionTimePerGBInput * tuningJobDefinition.allowedMaxExecutionTimePercent / 100.0; + Double resourceUsagePerGBInput = jobExecution.resourceUsage * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; + Double executionTimePerGBInput = jobExecution.executionTime * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; - // if jobSuggestedParamValueList contains both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb - // ie, if the size of jobSuggestedParamValueList is 2 - if (jobSuggestedParamValueList != null && jobSuggestedParamValueList.size() == 2) { - numParamSetForConvergence -= 1; - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - Set tmp; - if (paramValueSet.containsKey(jobSuggestedParamValue.id)) { - tmp = paramValueSet.get(jobSuggestedParamValue.id); - } else { - tmp = new HashSet(); - } - tmp.add(jobSuggestedParamValue.paramValue); - paramValueSet.put(jobSuggestedParamValue.id, tmp); - } - } + if (resourceUsagePerGBInput > maxDesiredResourceUsagePerGBInput + || executionTimePerGBInput > maxDesiredExecutionTimePerGBInput) { + logger.info("Execution " + jobExecution.jobExecId + " violates constraint on resource usage per GB input"); + jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; + } else { + jobSuggestedParamSet.fitness = resourceUsagePerGBInput; + } + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED; + jobSuggestedParamSet.fitnessJobExecution = jobExecution; + jobSuggestedParamSet = updateBestJobSuggestedParamSet(jobSuggestedParamSet); + jobSuggestedParamSet.update(); + } - if (numParamSetForConvergence == 0) { - break; - } + /** + * Updates the given job suggested param set to be the best param set if its fitness is less than the current best param set + * (since the objective is to minimize the fitness, the param set with the lowest fitness is the best) + * @param jobSuggestedParamSet JobSuggestedParamSet + */ + private JobSuggestedParamSet updateBestJobSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { + logger.info("Checking if a new best param set is found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId); + JobSuggestedParamSet currentBestJobSuggestedParamSet = JobSuggestedParamSet.find.where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, + jobSuggestedParamSet.jobDefinition.id) + .eq(JobSuggestedParamSet.TABLE.isParamSetBest, 1) + .findUnique(); + if (currentBestJobSuggestedParamSet != null) { + if (currentBestJobSuggestedParamSet.fitness > jobSuggestedParamSet.fitness) { + logger.info("Param set: " + jobSuggestedParamSet.id + " is the new best param set for job: " + + jobSuggestedParamSet.jobDefinition.jobDefId); + currentBestJobSuggestedParamSet.isParamSetBest = false; + jobSuggestedParamSet.isParamSetBest = true; + currentBestJobSuggestedParamSet.save(); } + } else { + logger.info("No best param set found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId + + ". Marking current param set " + jobSuggestedParamSet.id + " as best"); + jobSuggestedParamSet.isParamSetBest = true; + } + return jobSuggestedParamSet; + } - result = true; - for (Integer paramId : paramValueSet.keySet()) { - if (paramValueSet.get(paramId).size() > 1) { - result = false; - } + /* + Currently update in database is happening in calculate phase . It should be seperated out + */ + protected Boolean updateDataBase(List jobExecutionParamSets) { + return true; + } + + /** + * This method update metrics for auto tuning monitoring for fitness compute daemon + * @param completedJobExecutionParamSets List of completed tuning job executions + */ + private Boolean updateMetrics(List completedJobExecutionParamSets) { + int fitnessNotUpdated = 0; + for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { + if (!completedJobExecutionParamSet.jobSuggestedParamSet.paramSetState.equals( + JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED)) { + fitnessNotUpdated++; + } else { + AutoTuningMetricsController.markFitnessComputedJobs(); } } + AutoTuningMetricsController.setFitnessComputeWaitJobs(fitnessNotUpdated); + return true; + } - if (result) { - logger.info("Switching off tuning for job: " + tuningJobExecutionParamSets.get( - 0).jobSuggestedParamSet.jobDefinition.jobName + " Reason: parameter set converged"); - } - return result; + private boolean isTuningEnabled(Integer jobDefinitionId) { + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() + .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinitionId) + .order() + // There can be multiple entries in tuningJobDefinition if the job is switch on/off multiple times. + // The latest entry gives the information regarding whether tuning is enabled or not + .desc(TuningJobDefinition.TABLE.createdTs) + .setMaxRows(1) + .findUnique(); + + return tuningJobDefinition != null && tuningJobDefinition.tuningEnabled; } /** @@ -206,7 +282,7 @@ private boolean isMedianGainNegative(List tuningJobE JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; if (jobExecution.executionState == JobExecution.ExecutionState.SUCCEEDED - && jobSuggestedParamSet.paramSetState == ParamSetStatus.FITNESS_COMPUTED) { + && jobSuggestedParamSet.paramSetState == JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED) { fitnessArray[entries] = jobSuggestedParamSet.fitness; entries += 1; if (entries == numFitnessForMedian) { @@ -236,21 +312,6 @@ private boolean isMedianGainNegative(List tuningJobE } } - /** - * Switches off tuning for the given job - * @param jobDefinition Job for which tuning is to be switched off - */ - private void disableTuning(JobDefinition jobDefinition, String reason) { - TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() - .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id) - .findUnique(); - if (tuningJobDefinition.tuningEnabled) { - tuningJobDefinition.tuningEnabled = false; - tuningJobDefinition.tuningDisabledReason = reason; - tuningJobDefinition.save(); - } - } - /** * Checks and disables tuning for the given job definitions. * Tuning can be disabled if: @@ -259,276 +320,183 @@ private void disableTuning(JobDefinition jobDefinition, String reason) { * - or number of tuning executions >= minTuningExecutions and median gain (in cost function) in last 6 executions is negative * @param jobDefinitionSet Set of jobs to check if tuning can be switched off for them */ - private void checkToDisableTuning(Set jobDefinitionSet) { + + protected void checkToDisableTuning(Set jobDefinitionSet) { for (JobDefinition jobDefinition : jobDefinitionSet) { - List tuningJobExecutionParamSets = - TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") - .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") - .where() - .eq(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet + '.' - + JobSuggestedParamSet.TABLE.jobDefinition + '.' + JobDefinition.TABLE.id, jobDefinition.id) - .order() - .desc("job_execution_id") - .findList(); - - if (tuningJobExecutionParamSets.size() >= minTuningExecutions) { - if (didParameterSetConverge(tuningJobExecutionParamSets)) { - logger.info("Parameters converged. Disabling tuning for job: " + jobDefinition.jobName); - disableTuning(jobDefinition, "Parameters converged"); - } else if (isMedianGainNegative(tuningJobExecutionParamSets)) { - logger.info("Unable to get gain while tuning. Disabling tuning for job: " + jobDefinition.jobName); - disableTuning(jobDefinition, "Unable to get gain"); - } else if (tuningJobExecutionParamSets.size() >= maxTuningExecutions) { - logger.info("Maximum tuning executions limit reached. Disabling tuning for job: " + jobDefinition.jobName); - disableTuning(jobDefinition, "Maximum executions reached"); - } + List tuningJobExecutionParamSets = + TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .where() + .eq(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.jobDefinition + + '.' + JobDefinition.TABLE.id, jobDefinition.id) + .order() + .desc("job_execution_id") + .findList(); + + if (reachToNumberOfThresholdIterations(tuningJobExecutionParamSets, jobDefinition)) { + disableTuning(jobDefinition, "User Specified Iterations reached"); + } + if (tuningJobExecutionParamSets.size() >= minTuningExecutions) { + if (didParameterSetConverge(tuningJobExecutionParamSets)) { + logger.info("Parameters converged. Disabling tuning for job: " + jobDefinition.jobName); + disableTuning(jobDefinition, "Parameters converged"); + } else if (isMedianGainNegative(tuningJobExecutionParamSets)) { + logger.info("Unable to get gain while tuning. Disabling tuning for job: " + jobDefinition.jobName); + disableTuning(jobDefinition, "Unable to get gain"); + } else if (tuningJobExecutionParamSets.size() >= maxTuningExecutions) { + logger.info("Maximum tuning executions limit reached. Disabling tuning for job: " + jobDefinition.jobName); + disableTuning(jobDefinition, "Maximum executions reached"); } + } } } - /** - * This method update metrics for auto tuning monitoring for fitness compute daemon - * @param completedJobExecutionParamSets List of completed tuning job executions - */ - private void updateMetrics(List completedJobExecutionParamSets) { - int fitnessNotUpdated = 0; - for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { - if (!completedJobExecutionParamSet.jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)) { - fitnessNotUpdated++; - } else { - AutoTuningMetricsController.markFitnessComputedJobs(); - } + + + private boolean reachToNumberOfThresholdIterations(List tuningJobExecutionParamSets, + JobDefinition jobDefinition) { + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() + .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id) + .findUnique(); + if (tuningJobExecutionParamSets != null + && tuningJobExecutionParamSets.size() == tuningJobDefinition.numberOfIterations) { + return true; } - AutoTuningMetricsController.setFitnessComputeWaitJobs(fitnessNotUpdated); + return false; } /** - * Returns the list of completed executions whose metrics are not computed - * @return List of job execution + * Switches off tuning for the given job + * @param jobDefinition Job for which tuning is to be switched off */ - private List getCompletedJobExecutionParamSets() { - logger.info("Fetching completed executions whose fitness are yet to be computed"); - List completedJobExecutionParamSet = new ArrayList(); - - List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.select("*") - .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") - .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") - .where() - .or(Expr.or(Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, - JobExecution.ExecutionState.SUCCEEDED), - Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, - JobExecution.ExecutionState.FAILED)), - Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, - JobExecution.ExecutionState.CANCELLED)) - .isNull(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.resourceUsage) - .findList(); - - logger.info("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); - - for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { - JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; - long diff = System.currentTimeMillis() - jobExecution.updatedTs.getTime(); - logger.info("Current Time in millis: " + System.currentTimeMillis() + ", Job execution last updated time " - + jobExecution.updatedTs.getTime()); - if (diff < fitnessComputeWaitInterval) { - logger.info("Delaying fitness compute for execution: " + jobExecution.jobExecId); - } else { - logger.info("Adding execution " + jobExecution.jobExecId + " to fitness computation queue"); - completedJobExecutionParamSet.add(tuningJobExecutionParamSet); - } - } - logger.info( - "Number of completed execution fetched for fitness computation: " + completedJobExecutionParamSet.size()); - return completedJobExecutionParamSet; + private void disableTuning(JobDefinition jobDefinition, String reason) { + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() + .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id) + .findUnique(); + if (tuningJobDefinition.tuningEnabled) { + tuningJobDefinition.tuningEnabled = false; + tuningJobDefinition.tuningDisabledReason = reason; + tuningJobDefinition.save(); + } } /** - * Updates the execution metrics - * @param completedJobExecutionParamSets List of completed executions + * Checks if the tuning parameters converge + * @param tuningJobExecutionParamSets List of previous executions and corresponding param sets + * @return true if the parameters converge, else false */ - private void updateExecutionMetrics(List completedJobExecutionParamSets) { - for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { + private boolean didParameterSetConverge(List tuningJobExecutionParamSets) { + boolean result = false; + int numParamSetForConvergence = 3; - JobExecution jobExecution = completedJobExecutionParamSet.jobExecution; - JobSuggestedParamSet jobSuggestedParamSet = completedJobExecutionParamSet.jobSuggestedParamSet; - JobDefinition job = jobExecution.job; + if (tuningJobExecutionParamSets.size() < numParamSetForConvergence) { + return false; + } - logger.info("Updating execution metrics and fitness for execution: " + jobExecution.jobExecId); - try { - TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") - .fetch(TuningJobDefinition.TABLE.job, "*") - .where() - .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, job.id) - .order() - .desc(TuningJobDefinition.TABLE.createdTs) - .findUnique(); - - List results = AppResult.find.select("*") - .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*") - .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, - "*") - .where() - .eq(AppResult.TABLE.FLOW_EXEC_ID, jobExecution.flowExecution.flowExecId) - .eq(AppResult.TABLE.JOB_EXEC_ID, jobExecution.jobExecId) - .findList(); + TuningAlgorithm.JobType jobType = tuningJobExecutionParamSets.get(0).jobSuggestedParamSet.tuningAlgorithm.jobType; - if (results != null && results.size() > 0) { - Double totalResourceUsed = 0D; - Double totalInputBytesInBytes = 0D; + if (jobType == TuningAlgorithm.JobType.PIG) { - for (AppResult appResult : results) { - totalResourceUsed += appResult.resourceUsed; - totalInputBytesInBytes += getTotalInputBytes(appResult); - } + Map> paramValueSet = new HashMap>(); - Long totalRunTime = Utils.getTotalRuntime(results); - Long totalDelay = Utils.getTotalWaittime(results); - Long totalExecutionTime = totalRunTime - totalDelay; - - if (totalExecutionTime != 0) { - jobExecution.executionTime = totalExecutionTime * 1.0 / (1000 * 60); - jobExecution.resourceUsage = totalResourceUsed * 1.0 / (1024 * 3600); - jobExecution.inputSizeInBytes = totalInputBytesInBytes; - jobExecution.update(); - logger.info( - "Metric Values for execution " + jobExecution.jobExecId + ": Execution time = " + totalExecutionTime - + ", Resource usage = " + totalResourceUsed + " and total input size = " + totalInputBytesInBytes); - } + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { - if (tuningJobDefinition.averageResourceUsage == null && totalExecutionTime != 0) { - tuningJobDefinition.averageResourceUsage = jobExecution.resourceUsage; - tuningJobDefinition.averageExecutionTime = jobExecution.executionTime; - tuningJobDefinition.averageInputSizeInBytes = jobExecution.inputSizeInBytes.longValue(); - tuningJobDefinition.update(); - } + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; - //Compute fitness + List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, + jobSuggestedParamSet.id) + .or(Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, + "mapreduce.map.memory.mb"), + Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, + "mapreduce.reduce.memory.mb")) + .findList(); - if (!jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)) { - if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { - logger.info("Execution id: " + jobExecution.id + " succeeded"); - updateJobSuggestedParamSetSucceededExecution(jobExecution, jobSuggestedParamSet, tuningJobDefinition); + // if jobSuggestedParamValueList contains both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb + // ie, if the size of jobSuggestedParamValueList is 2 + if (jobSuggestedParamValueList != null && jobSuggestedParamValueList.size() == 2) { + numParamSetForConvergence -= 1; + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { + Set tmp; + if (paramValueSet.containsKey(jobSuggestedParamValue.id)) { + tmp = paramValueSet.get(jobSuggestedParamValue.id); } else { - // Resetting param set to created state because this case captures the scenarios when - // either the job failed for reasons other than auto tuning or was killed/cancelled/skipped etc. - // In all the above scenarios, fitness cannot be computed for the param set correctly. - // Note that the penalty on failures caused by auto tuning is applied when the job execution is retried - // after failure. - logger.info("Execution id: " + jobExecution.id + " was not successful for reason other than tuning." - + "Resetting param set: " + jobSuggestedParamSet.id + " to CREATED state"); - resetParamSetToCreated(jobSuggestedParamSet); + tmp = new HashSet(); } - } - } else { - long diff = System.currentTimeMillis() - jobExecution.updatedTs.getTime(); - logger.debug("Current Time in millis: " + System.currentTimeMillis() + ", job execution last updated time " - + jobExecution.updatedTs.getTime()); - if (diff > ignoreExecutionWaitInterval) { - logger.info("Fitness of param set " + jobSuggestedParamSet.id + " corresponding to execution id: " + - jobExecution.id + " not computed for more than the maximum duration specified to compute fitness. " - + "Resetting the param set to CREATED state"); - resetParamSetToCreated(jobSuggestedParamSet); + tmp.add(jobSuggestedParamValue.paramValue); + paramValueSet.put(jobSuggestedParamValue.id, tmp); } } - } catch (Exception e) { - logger.error("Error updating fitness of execution: " + jobExecution.id + "\n Stacktrace: ", e); + + if (numParamSetForConvergence == 0) { + break; + } } - } - logger.info("Execution metrics updated"); - } - /** - * Resets the param set to CREATED state if its fitness is not already computed - * @param jobSuggestedParamSet Param set which is to be reset - */ - private void resetParamSetToCreated(JobSuggestedParamSet jobSuggestedParamSet) { - if (!jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.FITNESS_COMPUTED)) { - logger.info("Resetting parameter set to created: " + jobSuggestedParamSet.id); - jobSuggestedParamSet.paramSetState = ParamSetStatus.CREATED; - jobSuggestedParamSet.save(); + result = true; + for (Integer paramId : paramValueSet.keySet()) { + if (paramValueSet.get(paramId).size() > 1) { + result = false; + } + } } - } - /** - * Updates the job suggested param set when the corresponding execution was succeeded - * @param jobExecution JobExecution: succeeded job execution corresponding to the param set which is to be updated - * @param jobSuggestedParamSet param set which is to be updated - * @param tuningJobDefinition TuningJobDefinition of the job to which param set corresponds - */ - private void updateJobSuggestedParamSetSucceededExecution(JobExecution jobExecution, - JobSuggestedParamSet jobSuggestedParamSet, TuningJobDefinition tuningJobDefinition) { - int penaltyConstant = 3; - Double averageResourceUsagePerGBInput = - tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; - Double maxDesiredResourceUsagePerGBInput = - averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; - Double averageExecutionTimePerGBInput = - tuningJobDefinition.averageExecutionTime * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; - Double maxDesiredExecutionTimePerGBInput = - averageExecutionTimePerGBInput * tuningJobDefinition.allowedMaxExecutionTimePercent / 100.0; - Double resourceUsagePerGBInput = jobExecution.resourceUsage * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; - Double executionTimePerGBInput = jobExecution.executionTime * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; - - if (resourceUsagePerGBInput > maxDesiredResourceUsagePerGBInput - || executionTimePerGBInput > maxDesiredExecutionTimePerGBInput) { - logger.info("Execution " + jobExecution.jobExecId + " violates constraint on resource usage per GB input"); - jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; - } else { - jobSuggestedParamSet.fitness = resourceUsagePerGBInput; + if (result) { + logger.info("Switching off tuning for job: " + tuningJobExecutionParamSets.get( + 0).jobSuggestedParamSet.jobDefinition.jobName + " Reason: parameter set converged"); } - jobSuggestedParamSet.paramSetState = ParamSetStatus.FITNESS_COMPUTED; - jobSuggestedParamSet.fitnessJobExecution = jobExecution; - jobSuggestedParamSet = updateBestJobSuggestedParamSet(jobSuggestedParamSet); - jobSuggestedParamSet.update(); + return result; } - /** - * Updates the given job suggested param set to be the best param set if its fitness is less than the current best param set - * (since the objective is to minimize the fitness, the param set with the lowest fitness is the best) - * @param jobSuggestedParamSet JobSuggestedParamSet - */ - private JobSuggestedParamSet updateBestJobSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { - logger.info("Checking if a new best param set is found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId); - JobSuggestedParamSet currentBestJobSuggestedParamSet = JobSuggestedParamSet.find.where() - .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, - jobSuggestedParamSet.jobDefinition.id) - .eq(JobSuggestedParamSet.TABLE.isParamSetBest, 1) - .findUnique(); - if (currentBestJobSuggestedParamSet != null) { - if (currentBestJobSuggestedParamSet.fitness > jobSuggestedParamSet.fitness) { - logger.info("Param set: " + jobSuggestedParamSet.id + " is the new best param set for job: " + jobSuggestedParamSet.jobDefinition.jobDefId); - currentBestJobSuggestedParamSet.isParamSetBest = false; - jobSuggestedParamSet.isParamSetBest = true; - currentBestJobSuggestedParamSet.save(); + public final Boolean execute() { + logger.info("Executing Fitness Manager"); + Boolean calculateFitnessDone = false, databaseUpdateDone = false, updateMetricsDone = false; + List tuningJobExecutionParamSet = detectJobsForFitnessComputation(); + if (tuningJobExecutionParamSet != null && tuningJobExecutionParamSet.size() >= 1) { + logger.info("Calculating Fitness"); + calculateFitnessDone = calculateFitness(tuningJobExecutionParamSet); + } + /* + Update database is not implemented , since data base update is currently occuring in calculate fitnesss + */ + if (calculateFitnessDone) { + logger.info("Updating Database"); + databaseUpdateDone = updateDataBase(tuningJobExecutionParamSet); + } + if (databaseUpdateDone) { + logger.info("Updating Metrics"); + updateMetricsDone = updateMetrics(tuningJobExecutionParamSet); + } + logger.info("Disable Tuning if Required"); + if (tuningJobExecutionParamSet != null && tuningJobExecutionParamSet.size() >= 1) { + Set jobDefinitionSet = new HashSet(); + for (TuningJobExecutionParamSet completedJobExecutionParamSet : tuningJobExecutionParamSet) { + JobDefinition jobDefinition = completedJobExecutionParamSet.jobSuggestedParamSet.jobDefinition; + if (isTuningEnabled(jobDefinition.id)) { + jobDefinitionSet.add(jobDefinition); + } } - } else { - logger.info("No best param set found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId - + ". Marking current param set " + jobSuggestedParamSet.id + " as best"); - jobSuggestedParamSet.isParamSetBest = true; + checkToDisableTuning(jobDefinitionSet); } - return jobSuggestedParamSet; + logger.info("Fitness Computed"); + return true; } - /** - * Returns the total input size - * @param appResult appResult - * @return total input size - */ - private Long getTotalInputBytes(AppResult appResult) { - Long totalInputBytes = 0L; - if (appResult.yarnAppHeuristicResults != null) { - for (AppHeuristicResult appHeuristicResult : appResult.yarnAppHeuristicResults) { - if (appHeuristicResult.heuristicName.equals(CommonConstantsHeuristic.MAPPER_SPEED)) { - if (appHeuristicResult.yarnAppHeuristicResultDetails != null) { - for (AppHeuristicResultDetails appHeuristicResultDetails : appHeuristicResult.yarnAppHeuristicResultDetails) { - if (appHeuristicResultDetails.name.equals(CommonConstantsHeuristic.TOTAL_INPUT_SIZE_IN_MB)) { - totalInputBytes += Math.round(Double.parseDouble(appHeuristicResultDetails.value) * FileUtils.ONE_MB); - } - } - } - } + protected void getCompletedExecution(List tuningJobExecutionParamSets, + List completedJobExecutionParamSet) { + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; + long diff = System.currentTimeMillis() - jobExecution.updatedTs.getTime(); + logger.info("Current Time in millis: " + System.currentTimeMillis() + ", Job execution last updated time " + + jobExecution.updatedTs.getTime()); + if (diff < fitnessComputeWaitInterval) { + logger.info("Delaying fitness compute for execution: " + jobExecution.jobExecId); + } else { + logger.info("Adding execution " + jobExecution.jobExecId + " to fitness computation queue"); + completedJobExecutionParamSet.add(tuningJobExecutionParamSet); } } - return totalInputBytes; + logger.info( + "Number of completed execution fetched for fitness computation: " + completedJobExecutionParamSet.size()); } } diff --git a/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java b/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java new file mode 100644 index 000000000..75a7d2008 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java @@ -0,0 +1,90 @@ +package com.linkedin.drelephant.tuning; + +import controllers.AutoTuningMetricsController; +import java.util.List; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.TuningJobExecutionParamSet; +import org.apache.log4j.Logger; + + +public abstract class AbstractJobStatusManager implements Manager { + private final Logger logger = Logger.getLogger(getClass()); + protected abstract Boolean analyzeCompletedJobsExecution(List inProgressExecutionParamSet); + + protected List detectJobsExecutionInProgress() { + logger.info("Fetching the executions which are in progress"); + List tuningJobExecutionParamSets = + TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobExecution) + .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet) + .where() + .eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.IN_PROGRESS) + .findList(); + + + logger.info("Number of executions which are in progress: " + tuningJobExecutionParamSets.size()); + return tuningJobExecutionParamSets; + } + + + + protected Boolean updateDataBase(List jobs) { + for(TuningJobExecutionParamSet job : jobs){ + JobSuggestedParamSet jobSuggestedParamSet = job.jobSuggestedParamSet; + JobExecution jobExecution = job.jobExecution; + if (isJobCompleted(jobExecution)) { + jobExecution.update(); + jobSuggestedParamSet.update(); + logger.info("Execution " + jobExecution.jobExecId + " is completed"); + } else { + logger.info("Execution " + jobExecution.jobExecId + " is still in running state"); + } + } + return true; + } + + private Boolean isJobCompleted(JobExecution jobExecution){ + if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED) || jobExecution.executionState.equals( + JobExecution.ExecutionState.FAILED) || jobExecution.executionState.equals( + JobExecution.ExecutionState.CANCELLED)) { + return true; + } + else return false; + } + + protected Boolean updateMetrics(List completedJobs) { + for(TuningJobExecutionParamSet completedJob : completedJobs){ + JobExecution jobExecution = completedJob.jobExecution; + if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { + AutoTuningMetricsController.markSuccessfulJobs(); + } else if (jobExecution.executionState.equals(JobExecution.ExecutionState.FAILED)) { + AutoTuningMetricsController.markFailedJobs(); + } + } + return true; + } + + + + @Override + public final Boolean execute() { + logger.info("Executing Job Status Manager"); + Boolean calculateCompletedJobExDone = false, databaseUpdateDone = false, updateMetricsDone = false; + List tuningJobExecutionParamSet = detectJobsExecutionInProgress(); + if (tuningJobExecutionParamSet != null && tuningJobExecutionParamSet.size() >= 1) { + logger.info("Calculating Completed Jobs"); + calculateCompletedJobExDone = analyzeCompletedJobsExecution(tuningJobExecutionParamSet); + } + if (calculateCompletedJobExDone) { + logger.info("Updating Database"); + databaseUpdateDone = updateDataBase(tuningJobExecutionParamSet); + } + if (databaseUpdateDone) { + logger.info("Updating Metrics"); + updateMetricsDone = updateMetrics(tuningJobExecutionParamSet); + } + logger.info("Baseline Done"); + return updateMetricsDone; + } +} diff --git a/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java b/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java new file mode 100644 index 000000000..c1e7a9c25 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java @@ -0,0 +1,174 @@ +package com.linkedin.drelephant.tuning; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import models.AppResult; +import models.JobDefinition; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningJobDefinition; +import models.TuningParameter; +import org.apache.log4j.Logger; +import play.libs.Json; + +public abstract class AbstractTuningTypeManager implements Manager { + protected final String JSON_CURRENT_POPULATION_KEY = "current_population"; + private final Logger logger = Logger.getLogger(getClass()); + protected ExecutionEngine _executionEngine; + protected String tuningType = null; + protected String tuningAlgorithm = null; + + /** + * Fetches the list to job which need new parameter suggestion + * @return Job list + */ + + protected List detectJobsForParameterGeneration() { + List jobsForSwarmSuggestion = getJobsForParamSuggestion(); + List jobTuningInfoList = getJobsTuningInfo(jobsForSwarmSuggestion); + return jobTuningInfoList; + } + + /** + * Fetches the list to job which need new parameter suggestion + * @return Job list + */ + private List getJobsForParamSuggestion() { + // Todo: [Important] Change the logic. This is very rigid. Ideally you should look at the param set ids in the saved state, + // todo: [continuation] if their fitness is computed, pso can generate new params for the job + logger.info("Checking which jobs need new parameter suggestion"); + List jobsForParamSuggestion = new ArrayList(); + List pendingParamSetList = getPendingParamSets(); + //logger.info(" Jobs pending " + pendingParamSetList.size()); + List pendingParamJobList = new ArrayList(); + for (JobSuggestedParamSet pendingParamSet : pendingParamSetList) { + if (!pendingParamJobList.contains(pendingParamSet.jobDefinition)) { + pendingParamJobList.add(pendingParamSet.jobDefinition); + } + } + + List tuningJobDefinitionList = getTuningJobDefinitions(); + //logger.info("Total Jobs " + tuningJobDefinitionList.size()); + if (tuningJobDefinitionList.size() == 0) { + logger.error("No auto-tuning enabled jobs found"); + } + + for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitionList) { + if (!pendingParamJobList.contains(tuningJobDefinition.job)) { + logger.info("New parameter suggestion needed for job: " + tuningJobDefinition.job.jobName); + jobsForParamSuggestion.add(tuningJobDefinition); + } + } + logger.info("Number of job(s) which need new parameter suggestion: " + jobsForParamSuggestion.size()); + return jobsForParamSuggestion; + } + + protected List generateParameters(List jobsForParameterSuggestion) { + List updatedJobTuningInfoList = new ArrayList(); + for (JobTuningInfo jobTuningInfo : jobsForParameterSuggestion) { + JobTuningInfo newJobTuningInfo = generateParamSet(jobTuningInfo); + updatedJobTuningInfoList.add(newJobTuningInfo); + } + return updatedJobTuningInfoList; + } + + protected abstract JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo); + + protected abstract Boolean updateDatabase(List tuningJobDefinitions); + + public final Boolean execute() { + logger.info("Executing Tuning Algorithm"); + Boolean parameterGenerationDone = false, databaseUpdateDone = false, updateMetricsDone = false; + List jobTuningInfo = detectJobsForParameterGeneration(); + if (jobTuningInfo != null && jobTuningInfo.size() >= 1) { + logger.info("Generating Parameters "); + List updatedJobTuningInfoList = generateParameters(jobTuningInfo); + logger.info("Updating Database"); + databaseUpdateDone = updateDatabase(updatedJobTuningInfoList); + } + logger.info("Param Generation Done"); + return databaseUpdateDone; + } + + protected abstract List getPendingParamSets(); + + protected abstract List getTuningJobDefinitions(); + + /** + * Returns the tuning information for the jobs + * @param tuningJobs Job List + * @return Tuning information list + */ + protected List getJobsTuningInfo(List tuningJobs) { + List jobTuningInfoList = + new ArrayList(); + for (TuningJobDefinition tuningJobDefinition : tuningJobs) { + JobDefinition job = tuningJobDefinition.job; + List tuningParameterList = generateTuningParameterListWithDefaultValues(tuningJobDefinition); + // updating boundary constraints for the job + updateBoundryConstraint(tuningParameterList, job); + + com.linkedin.drelephant.tuning.JobTuningInfo jobTuningInfo = new com.linkedin.drelephant.tuning.JobTuningInfo(); + jobTuningInfo.setTuningJob(job); + jobTuningInfo.setJobType(tuningJobDefinition.tuningAlgorithm.jobType); + jobTuningInfo.setParametersToTune(tuningParameterList); + + saveJobState(jobTuningInfo, job); + + logger.info("Adding JobTuningInfo " + Json.toJson(jobTuningInfo)); + jobTuningInfoList.add(jobTuningInfo); + } + return jobTuningInfoList; + } + + protected abstract void saveJobState(JobTuningInfo jobTuningInfo, JobDefinition job); + + protected abstract void updateBoundryConstraint(List tuningParameterList,JobDefinition job); + + public abstract boolean isParamConstraintViolated(List jobSuggestedParamValues); + + + protected List generateTuningParameterListWithDefaultValues( + TuningJobDefinition tuningJobDefinition) { + JobDefinition job = tuningJobDefinition.job; + logger.info("Getting tuning information for job: " + job.jobDefId); + List tuningParameterList = TuningHelper.getTuningParameterList(tuningJobDefinition); + + logger.info("Fetching default parameter values for job " + tuningJobDefinition.job.jobDefId); + JobSuggestedParamSet defaultJobParamSet = TuningHelper.getDefaultParameterValuesforJob(tuningJobDefinition); + assignDefaultValues(defaultJobParamSet, tuningParameterList); + return tuningParameterList; + } + + protected void assignDefaultValues(JobSuggestedParamSet defaultJobParamSet, + List tuningParameterList) { + if (defaultJobParamSet != null) { + logger.info("Fetching default parameter values for job "); + List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + "." + JobExecution.TABLE.id, defaultJobParamSet.id) + .findList(); + + if (jobSuggestedParamValueList.size() > 0) { + logger.info("Giving default values "); + Map defaultExecutionParamMap = new HashMap(); + + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { + defaultExecutionParamMap.put(jobSuggestedParamValue.tuningParameter.id, jobSuggestedParamValue.paramValue); + } + + for (TuningParameter tuningParameter : tuningParameterList) { + logger.info("Giving default values "); + Integer paramId = tuningParameter.id; + if (defaultExecutionParamMap.containsKey(paramId)) { + logger.info("Updating value of param " + tuningParameter.paramName + " to " + defaultExecutionParamMap.get( + paramId)); + tuningParameter.defaultValue = defaultExecutionParamMap.get(paramId); + } + } + } + } + } +} diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java index b6b5b68ef..f498dd5e9 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java @@ -18,6 +18,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.drelephant.ElephantContext; +import com.linkedin.drelephant.tuning.obt.OptimizationAlgoFactory; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBT; import com.linkedin.drelephant.util.Utils; import controllers.AutoTuningMetricsController; @@ -80,6 +82,21 @@ private JobSuggestedParamSet getBestParamSet(String jobDefId) { return jobSuggestedParamSetBestParamSet; } + /** + * For a job, returns the Manually tuned param set + * @param jobDefId Sting JobDefId of the job + * @return JobSuggestedParamSet the best parameter set of the given job if it exists else the default parameter set + */ + private JobSuggestedParamSet getManuallyTunedParamSet(String jobDefId) { + JobSuggestedParamSet jobSuggestedParamSetBestParamSet = JobSuggestedParamSet.find.select("*") + .where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.jobDefId, jobDefId) + .eq(JobSuggestedParamSet.TABLE.isManuallyOverridenParameter, true) + .setMaxRows(1) + .findUnique(); + return jobSuggestedParamSetBestParamSet; + } + /** * Returns the param values corresponding to the given param set id * @param paramSetId Long parameter set id @@ -116,10 +133,13 @@ private void setMaxAllowedMetricIncreasePercentage(TuningInput tuningInput) { */ private void setTuningAlgorithm(TuningInput tuningInput) throws IllegalArgumentException { //Todo: Handle algorithm version later + + logger.info(" Optimization Algorithm Requested" + tuningInput.getOptimizationAlgo()); TuningAlgorithm tuningAlgorithm = TuningAlgorithm.find.select("*") .where() .eq(TuningAlgorithm.TABLE.jobType, tuningInput.getJobType()) .eq(TuningAlgorithm.TABLE.optimizationMetric, tuningInput.getOptimizationMetric()) + .eq(TuningAlgorithm.TABLE.optimizationAlgo, tuningInput.getOptimizationAlgo()) .findUnique(); if (tuningAlgorithm == null) { throw new IllegalArgumentException( @@ -230,6 +250,7 @@ private TuningJobDefinition addNewJobForTuning(TuningInput tuningInput) { Map defaultParams = null; try { defaultParams = tuningInput.getDefaultParams(); + logger.info(" Default parameters first time " + defaultParams); } catch (IOException e) { logger.error("Error in getting default parameters from request. ", e); } @@ -325,11 +346,54 @@ public Map getCurrentRunParameters(TuningInput tuningInput) thro setMaxAllowedMetricIncreasePercentage(tuningInput); } setTuningAlgorithm(tuningInput); + String jobDefId = tuningInput.getJobDefId(); - JobSuggestedParamSet jobSuggestedParamSet; + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") + .fetch(TuningJobDefinition.TABLE.job, "*") + .where() + .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, jobDefId) + .setMaxRows(1) + .orderBy(TuningJobDefinition.TABLE.createdTs + " desc") + .findUnique(); JobExecution jobExecution = getJobExecution(tuningInput); + if (tuningJobDefinition == null) { + return processForFirstExecution(tuningInput,jobExecution); + } else { + return processForSubsequentExecutions(tuningJobDefinition, tuningInput,jobExecution); + } + } + + private Map processForFirstExecution(TuningInput tuningInput,JobExecution jobExecution){ + logger.info(" New Job. Hence Not checking for AutoTuning . Running with default Parameters " + + tuningInput.getJobExecId()); + JobSuggestedParamSet jobSuggestedParamSet = getDefaultParameters(jobExecution.job); + markParameterSetSent(jobSuggestedParamSet); + addNewTuningJobExecutionParamSet(jobSuggestedParamSet, jobExecution); + List jobSuggestedParamValues = getParamSetValues(jobSuggestedParamSet.id); + logger.debug("Number of output parameters for execution " + tuningInput.getJobExecId() + " = " + + jobSuggestedParamValues.size()); + logger.info("Finishing getCurrentRunParameters"); + return jobSuggestedParamValueListToMap(jobSuggestedParamValues); + } + + private Map processForSubsequentExecutions (TuningJobDefinition tuningJobDefinition, TuningInput tuningInput,JobExecution jobExecution){ + logger.info(" Not a New Job . Hence check for autoTuning" + tuningInput.getJobExecId()); + if (tuningJobDefinition.autoApply) { + logger.info( " Auto Tuning Enabled . Hence Apply generated Parameter ,generated based on tuning algorithm"); + List jobSuggestedParamValues = processParameterTuningEnabled(tuningInput, jobExecution); + return jobSuggestedParamValueListToMap(jobSuggestedParamValues); + } else { + logger.info("Not a new Job . Auto Tuning Disabled . Send default parameters"); + return sendDefaultParameters(tuningInput, jobExecution); + } + } + + private List processParameterTuningEnabled(TuningInput tuningInput,JobExecution jobExecution) { + JobSuggestedParamSet jobSuggestedParamSet; + logger.debug("Finding parameter suggestion for job: " + jobExecution.job.jobName); if (tuningInput.getRetry()) { + logger.info(" Retry "); applyPenalty(tuningInput.getJobExecId()); jobSuggestedParamSet = getBestParamSet(tuningInput.getJobDefId()); } else { @@ -337,14 +401,81 @@ public Map getCurrentRunParameters(TuningInput tuningInput) thro jobSuggestedParamSet = getNewSuggestedParamSet(jobExecution.job); markParameterSetSent(jobSuggestedParamSet); } - addNewTuningJobExecutionParamSet(jobSuggestedParamSet, jobExecution); - List jobSuggestedParamValues = getParamSetValues(jobSuggestedParamSet.id); logger.debug("Number of output parameters for execution " + tuningInput.getJobExecId() + " = " + jobSuggestedParamValues.size()); logger.info("Finishing getCurrentRunParameters"); - return jobSuggestedParamValueListToMap(jobSuggestedParamValues); + return jobSuggestedParamValues; + } + + private Map sendDefaultParameters(TuningInput tuningInput, JobExecution jobExecution) { + JobSuggestedParamSet jobSuggestedParamSet; + logger.info(" Auto Tuning Disabled . Hence no parameter suggestion. Tagging execution with default values"); + updateDataBaseWithDefaultValues(tuningInput); + jobSuggestedParamSet = getDefaultParameters(jobExecution.job); + markParameterSetSent(jobSuggestedParamSet); + addNewTuningJobExecutionParamSet(jobSuggestedParamSet, jobExecution); + discardGeneratedParameter(jobExecution.job); + return new HashMap(); + } + + public void updateDataBaseWithDefaultValues(TuningInput tuningInput) { + JobDefinition job = + JobDefinition.find.select("*").where().eq(JobDefinition.TABLE.jobDefId, tuningInput.getJobDefId()).findUnique(); + Map defaultParams = null; + try { + defaultParams = tuningInput.getDefaultParams(); + logger.info("Default values " + defaultParams); + } catch (IOException e) { + logger.error("Error in getting default parameters from request. ", e); + } + insertParamSetForDefault(job, tuningInput.getTuningAlgorithm(), defaultParams); + } + + public JobSuggestedParamSet getDefaultParameters(JobDefinition jobDefinition){ + JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") + .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") + .where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefinition.id) + .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED) + .order() + .desc(JobSuggestedParamSet.TABLE.id) + .setMaxRows(1) + .findUnique(); + return jobSuggestedParamSet; + } + + public void discardGeneratedParameter(JobDefinition jobDefinition){ + List jobSuggestedParamSets = JobSuggestedParamSet.find.select("*") + .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") + .where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefinition.id) + .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED).findList(); + + if(jobSuggestedParamSets!=null && jobSuggestedParamSets.size()>=1){ + logger.info(" Discarding Generated Parameters , as auto tuning off."+jobSuggestedParamSets.size()); + for(JobSuggestedParamSet jobSuggestedParamSet : jobSuggestedParamSets){ + jobSuggestedParamSet.paramSetState=ParamSetStatus.DISCARDED; + jobSuggestedParamSet.update(); + } + } + } + + private void insertParamSetForDefault(JobDefinition job, TuningAlgorithm tuningAlgorithm, + Map paramValueMap) { + logger.debug("Inserting default parameter set for job: " + job.jobName); + JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); + jobSuggestedParamSet.jobDefinition = job; + jobSuggestedParamSet.tuningAlgorithm = tuningAlgorithm; + jobSuggestedParamSet.paramSetState = ParamSetStatus.CREATED; + jobSuggestedParamSet.isParamSetDefault = false; + jobSuggestedParamSet.areConstraintsViolated = false; + jobSuggestedParamSet.isParamSetBest = false; + jobSuggestedParamSet.isManuallyOverridenParameter = false; + jobSuggestedParamSet.save(); + insertParameterValues(jobSuggestedParamSet, paramValueMap); + logger.debug("Default parameter set inserted for job: " + job.jobName); } /** @@ -374,107 +505,120 @@ private void addNewTuningJobExecutionParamSet(JobSuggestedParamSet jobSuggestedP * @return JobSuggestedParamSet corresponding to the given job definition */ private JobSuggestedParamSet getNewSuggestedParamSet(JobDefinition jobDefinition) { - JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") - .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") - .where() - .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefinition.id) - .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED) - .order() - .asc(JobSuggestedParamSet.TABLE.id) - .setMaxRows(1) - .findUnique(); - + JobSuggestedParamSet jobSuggestedParamSet = getManuallyTunedParamSet(jobDefinition.jobDefId); if (jobSuggestedParamSet == null) { - //No new parameter set exists, returning the best parameter set - logger.info("Returning best parameter set as no parameter suggestion found for job: " + jobDefinition.jobName); - AutoTuningMetricsController.markParamSetNotFound(); - jobSuggestedParamSet = getBestParamSet(jobDefinition.jobDefId); + logger.info(" No Manually overriden parameter "); + jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") + .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") + .where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefinition.id) + .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED) + .order() + .asc(JobSuggestedParamSet.TABLE.id) + .setMaxRows(1) + .findUnique(); + if (jobSuggestedParamSet == null) { + logger.info("Returning best parameter set as no parameter suggestion found for job: " + jobDefinition.jobName); + jobSuggestedParamSet = getBestParamSet(jobDefinition.jobDefId); + } } return jobSuggestedParamSet; } - /** - * Returns the list of JobSuggestedParamValue as Map of String to Double - * @param jobSuggestedParamValues List of JobSuggestedParamValue - * @return Map of string to double containing the parameter name and corresponding value - */ - private Map jobSuggestedParamValueListToMap(List jobSuggestedParamValues) { - Map paramValues = new HashMap(); - if (jobSuggestedParamValues != null) { - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) { - logger.debug("Param Name is " + jobSuggestedParamValue.tuningParameter.paramName + " And value is " - + jobSuggestedParamValue.paramValue); - paramValues.put(jobSuggestedParamValue.tuningParameter.paramName, jobSuggestedParamValue.paramValue); + /** + * Returns the list of JobSuggestedParamValue as Map of String to Double + * @param jobSuggestedParamValues List of JobSuggestedParamValue + * @return Map of string to double containing the parameter name and corresponding value + */ + private Map jobSuggestedParamValueListToMap(List < JobSuggestedParamValue > jobSuggestedParamValues) { + Map paramValues = new HashMap(); + if (jobSuggestedParamValues != null) { + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) { + logger.debug("Param Name is " + jobSuggestedParamValue.tuningParameter.paramName + " And value is " + + jobSuggestedParamValue.paramValue); + paramValues.put(jobSuggestedParamValue.tuningParameter.paramName, jobSuggestedParamValue.paramValue); + } } + return paramValues; } - return paramValues; - } - /** - *Updates parameter set state to SENT if it is in CREATED state - * @param jobSuggestedParamSet JobSuggestedParamSet which is to be updated - */ - private void markParameterSetSent(JobSuggestedParamSet jobSuggestedParamSet) { - if (jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.CREATED)) { - logger.info("Marking paramSetID: " + jobSuggestedParamSet.id + " SENT"); - jobSuggestedParamSet.paramSetState = ParamSetStatus.SENT; + /** + *Updates parameter set state to SENT if it is in CREATED state + * @param jobSuggestedParamSet JobSuggestedParamSet which is to be updated + */ + private void markParameterSetSent (JobSuggestedParamSet jobSuggestedParamSet){ + if (jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.CREATED)) { + logger.info("Marking paramSetID: " + jobSuggestedParamSet.id + " SENT"); + jobSuggestedParamSet.paramSetState = ParamSetStatus.SENT; + jobSuggestedParamSet.save(); + } + } + + /** + * Inserts a parameter set in database + * @param job Job + */ + private void insertParamSet (JobDefinition job, TuningAlgorithm + tuningAlgorithm, Map < String, Double > paramValueMap){ + logger.debug("Inserting default parameter set for job: " + job.jobName); + JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); + jobSuggestedParamSet.jobDefinition = job; + jobSuggestedParamSet.tuningAlgorithm = tuningAlgorithm; + jobSuggestedParamSet.paramSetState = ParamSetStatus.CREATED; + jobSuggestedParamSet.isParamSetDefault = true; + jobSuggestedParamSet.areConstraintsViolated = false; + jobSuggestedParamSet.isParamSetBest = false; + jobSuggestedParamSet.isManuallyOverridenParameter = false; jobSuggestedParamSet.save(); + insertParameterValues(jobSuggestedParamSet, paramValueMap); + intializeOptimizationAlgoPrerequisite(tuningAlgorithm, jobSuggestedParamSet); + logger.debug("Default parameter set inserted for job: " + job.jobName); } - } - /** - * Inserts a parameter set in database - * @param job Job - */ - private void insertParamSet(JobDefinition job, TuningAlgorithm tuningAlgorithm, Map paramValueMap) { - logger.debug("Inserting default parameter set for job: " + job.jobName); - JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); - jobSuggestedParamSet.jobDefinition = job; - jobSuggestedParamSet.tuningAlgorithm = tuningAlgorithm; - jobSuggestedParamSet.paramSetState = ParamSetStatus.CREATED; - jobSuggestedParamSet.isParamSetDefault = true; - jobSuggestedParamSet.areConstraintsViolated = false; - jobSuggestedParamSet.isParamSetBest = false; - jobSuggestedParamSet.save(); - insertParameterValues(jobSuggestedParamSet, paramValueMap); - logger.debug("Default parameter set inserted for job: " + job.jobName); - } + /** + * Inserts parameter values in database + * @param jobSuggestedParamSet Set of the parameters which is to be inserted + * @param paramValueMap Map of parameter values as string + */ + @SuppressWarnings("unchecked") private void insertParameterValues (JobSuggestedParamSet + jobSuggestedParamSet, Map < String, Double > paramValueMap){ + ObjectMapper mapper = new ObjectMapper(); + if (paramValueMap != null) { + for (Map.Entry paramValue : paramValueMap.entrySet()) { + insertParameterValue(jobSuggestedParamSet, paramValue.getKey(), paramValue.getValue()); + } + } else { + logger.warn("ParamValueMap is null "); + } + } - /** - * Inserts parameter values in database - * @param jobSuggestedParamSet Set of the parameters which is to be inserted - * @param paramValueMap Map of parameter values as string - */ - @SuppressWarnings("unchecked") - private void insertParameterValues(JobSuggestedParamSet jobSuggestedParamSet, Map paramValueMap) { - ObjectMapper mapper = new ObjectMapper(); - if (paramValueMap != null) { - for (Map.Entry paramValue : paramValueMap.entrySet()) { - insertParameterValue(jobSuggestedParamSet, paramValue.getKey(), paramValue.getValue()); + private void intializeOptimizationAlgoPrerequisite (TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet + jobSuggestedParamSet){ + logger.info("Inserting parameter constraint " + tuningAlgorithm.optimizationAlgo.name()); + TuningTypeManagerOBT manager = OptimizationAlgoFactory.getOptimizationAlogrithm(tuningAlgorithm); + if (manager != null) { + manager.intializePrerequisite(tuningAlgorithm, jobSuggestedParamSet); } - } else { - logger.warn("ParamValueMap is null "); } - } - /** - * Inserts parameter value in database - * @param jobSuggestedParamSet Parameter set to which the parameter belongs - * @param paramName Parameter name - * @param paramValue Parameter value - */ - private void insertParameterValue(JobSuggestedParamSet jobSuggestedParamSet, String paramName, Double paramValue) { - logger.debug("Starting insertParameterValue"); - JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); - jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; - TuningParameter tuningParameter = - TuningParameter.find.where().eq(TuningParameter.TABLE.paramName, paramName).findUnique(); - if (tuningParameter != null) { - jobSuggestedParamValue.tuningParameter = tuningParameter; - jobSuggestedParamValue.paramValue = paramValue; - jobSuggestedParamValue.save(); - } else { - logger.warn("TuningAlgorithm param null " + paramName); + /** + * Inserts parameter value in database + * @param jobSuggestedParamSet Parameter set to which the parameter belongs + * @param paramName Parameter name + * @param paramValue Parameter value + */ + private void insertParameterValue (JobSuggestedParamSet jobSuggestedParamSet, String paramName, Double paramValue){ + logger.debug("Starting insertParameterValue"); + JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); + jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; + TuningParameter tuningParameter = + TuningParameter.find.where().eq(TuningParameter.TABLE.paramName, paramName).findUnique(); + if (tuningParameter != null) { + jobSuggestedParamValue.tuningParameter = tuningParameter; + jobSuggestedParamValue.paramValue = paramValue; + jobSuggestedParamValue.save(); + } else { + logger.warn("TuningAlgorithm param null " + paramName); + } } } -} diff --git a/app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java b/app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java deleted file mode 100644 index 4bd0113cc..000000000 --- a/app/com/linkedin/drelephant/tuning/AzkabanJobCompleteDetector.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.tuning; - -import com.linkedin.drelephant.clients.azkaban.AzkabanJobStatusUtil; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import models.JobExecution; -import models.JobExecution.ExecutionState; -import models.JobSuggestedParamSet; -import models.JobSuggestedParamSet.ParamSetStatus; -import models.TuningJobExecutionParamSet; -import org.apache.log4j.Logger; - - -/** - * Job completion detector for azkaban jobs. This utility uses azkaban rest api to find out if the jobs in a flow are - * completed or not. - */ -public class AzkabanJobCompleteDetector extends JobCompleteDetector { - - private static final Logger logger = Logger.getLogger(AzkabanJobCompleteDetector.class); - private AzkabanJobStatusUtil _azkabanJobStatusUtil; - - public enum AzkabanJobStatus { - FAILED, CANCELLED, KILLED, SUCCEEDED, SKIPPED - } - - /** - * Returns the list of completed executions - * @param inProgressExecutionParamSet List of executions (with corresponding param set) in progress - * @return List of completed executions - * @throws MalformedURLException MalformedURLException - * @throws URISyntaxException URISyntaxException - */ - protected List getCompletedExecutions(List inProgressExecutionParamSet) - throws MalformedURLException, URISyntaxException { - logger.info("Fetching the list of executions completed since last iteration"); - List completedExecutions = new ArrayList(); - try { - for (TuningJobExecutionParamSet tuningJobExecutionParamSet : inProgressExecutionParamSet) { - - JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; - JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; - - logger.info("Checking current status of started execution: " + jobExecution.jobExecId); - - if (_azkabanJobStatusUtil == null) { - logger.info("Initializing AzkabanJobStatusUtil"); - _azkabanJobStatusUtil = new AzkabanJobStatusUtil(); - } - - try { - Map jobStatus = _azkabanJobStatusUtil.getJobsFromFlow(jobExecution.flowExecution.flowExecId); - if (jobStatus != null) { - for (Map.Entry job : jobStatus.entrySet()) { - logger.info("Job Found:" + job.getKey() + ". Status: " + job.getValue()); - if (job.getKey().equals(jobExecution.job.jobName)) { - if (job.getValue().equals(AzkabanJobStatus.FAILED.toString())) { - if (jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.SENT)) { - jobSuggestedParamSet.paramSetState = ParamSetStatus.EXECUTED; - } - jobExecution.executionState = ExecutionState.FAILED; - } else if (job.getValue().equals(AzkabanJobStatus.SUCCEEDED.toString())) { - if (jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.SENT)) { - jobSuggestedParamSet.paramSetState = ParamSetStatus.EXECUTED; - } - jobExecution.executionState = ExecutionState.SUCCEEDED; - } else if (job.getValue().equals(AzkabanJobStatus.CANCELLED.toString()) || job.getValue() - .equals(AzkabanJobStatus.KILLED.toString()) || job.getValue() - .equals(AzkabanJobStatus.SKIPPED.toString())) { - if (jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.SENT)) { - jobSuggestedParamSet.paramSetState = ParamSetStatus.EXECUTED; - } - jobExecution.executionState = ExecutionState.CANCELLED; - } - - if (jobExecution.executionState.equals(ExecutionState.SUCCEEDED) || jobExecution.executionState.equals( - ExecutionState.FAILED) || jobExecution.executionState.equals(ExecutionState.CANCELLED)) { - jobExecution.update(); - jobSuggestedParamSet.update(); - completedExecutions.add(jobExecution); - logger.info("Execution " + jobExecution.jobExecId + " is completed"); - } else { - logger.info("Execution " + jobExecution.jobExecId + " is still in running state"); - } - } - } - } else { - logger.info("No jobs found for flow execution: " + jobExecution.flowExecution.flowExecId); - } - } catch (Exception e) { - logger.error("Error in checking status of execution: " + jobExecution.jobExecId, e); - } - } - } catch (Exception e) { - logger.error("Error in fetching list of completed executions", e); - e.printStackTrace(); - } - logger.info("Number of executions completed since last iteration: " + completedExecutions.size()); - return completedExecutions; - } -} diff --git a/app/com/linkedin/drelephant/tuning/BaselineComputeUtil.java b/app/com/linkedin/drelephant/tuning/BaselineComputeUtil.java deleted file mode 100644 index f906e1aa5..000000000 --- a/app/com/linkedin/drelephant/tuning/BaselineComputeUtil.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.tuning; - -import com.avaje.ebean.Ebean; -import com.avaje.ebean.SqlRow; -import com.linkedin.drelephant.ElephantContext; -import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; -import com.linkedin.drelephant.util.Utils; -import controllers.AutoTuningMetricsController; -import java.util.List; -import models.TuningJobDefinition; -import org.apache.commons.io.FileUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.log4j.Logger; - - -/** - * This class does baseline computation once the job is enabled for auto tuning. - * It takes average resource usage and execution time for last 30 jobs to get the baseline. - */ -public class BaselineComputeUtil { - - private static final Integer NUM_JOBS_FOR_BASELINE_DEFAULT = 30; - private static final String BASELINE_EXECUTION_COUNT = "baseline.execution.count"; - private final Logger logger = Logger.getLogger(getClass()); - private Integer _numJobsForBaseline = null; - - public BaselineComputeUtil() { - Configuration configuration = ElephantContext.instance().getAutoTuningConf(); - _numJobsForBaseline = - Utils.getNonNegativeInt(configuration, BASELINE_EXECUTION_COUNT, NUM_JOBS_FOR_BASELINE_DEFAULT); - } - - /** - * Computes baseline for the jobs new to auto tuning - * @return tuningJobDefinition - */ - public List computeBaseline() { - logger.info("Starting baseline computation"); - List tuningJobDefinitions = getJobForBaselineComputation(); - for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitions) { - try { - updateBaselineForJob(tuningJobDefinition); - } catch (Exception e) { - logger.error("Error in computing baseline for job: " + tuningJobDefinition.job.jobName, e); - } - } - updateMetrics(tuningJobDefinitions); - logger.info("Baseline computation complete"); - return tuningJobDefinitions; - } - - /** - * This method update metrics for auto tuning monitoring for baseline computation - * @param tuningJobDefinitions - */ - private void updateMetrics(List tuningJobDefinitions) { - int baselineComputeWaitJobs = 0; - for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitions) { - if (tuningJobDefinition.averageResourceUsage == null) { - baselineComputeWaitJobs++; - } else { - AutoTuningMetricsController.markBaselineComputed(); - } - } - AutoTuningMetricsController.setBaselineComputeWaitJobs(baselineComputeWaitJobs); - } - - /** - * Fetches the jobs whose baseline is to be computed. - * This is done by returning the jobs with null average resource usage - * @return List of jobs whose baseline needs to be added - */ - private List getJobForBaselineComputation() { - logger.info("Fetching jobs for which baseline metrics need to be computed"); - List tuningJobDefinitions = - TuningJobDefinition.find.where().eq(TuningJobDefinition.TABLE.averageResourceUsage, null).findList(); - return tuningJobDefinitions; - } - - /** - * Adds baseline metric values for a job - * @param tuningJobDefinition Job for which baseline is to be added - */ - private void updateBaselineForJob(TuningJobDefinition tuningJobDefinition) { - - logger.info("Computing and updating baseline metric values for job: " + tuningJobDefinition.job.jobName); - - String sql = "SELECT AVG(resource_used) AS resource_used, AVG(execution_time) AS execution_time FROM " - + "(SELECT job_exec_id, SUM(resource_used/(1024 * 3600)) AS resource_used, " - + "SUM((finish_time - start_time - total_delay)/(1000 * 60)) AS execution_time, " - + "MAX(start_time) AS start_time " + "FROM yarn_app_result WHERE job_def_id=:jobDefId " - + "GROUP BY job_exec_id " + "ORDER BY start_time DESC " + "LIMIT :num) temp"; - - logger.debug("Running query for baseline computation " + sql); - - SqlRow baseline = Ebean.createSqlQuery(sql) - .setParameter("jobDefId", tuningJobDefinition.job.jobDefId) - .setParameter("num", _numJobsForBaseline) - .findUnique(); - - Double avgResourceUsage = 0D; - Double avgExecutionTime = 0D; - avgResourceUsage = baseline.getDouble("resource_used"); - avgExecutionTime = baseline.getDouble("execution_time"); - tuningJobDefinition.averageExecutionTime = avgExecutionTime; - tuningJobDefinition.averageResourceUsage = avgResourceUsage; - tuningJobDefinition.averageInputSizeInBytes = getAvgInputSizeInBytes(tuningJobDefinition.job.jobDefId); - - logger.debug("Baseline metric values: Average resource usage=" + avgResourceUsage + " and Average execution time=" - + avgExecutionTime); - tuningJobDefinition.update(); - logger.info("Updated baseline metric value for job: " + tuningJobDefinition.job.jobName); - } - - /** - * Returns the average input size in bytes of a job (over last _numJobsForBaseline executions) - * @param jobDefId job definition id of the job - * @return average input size in bytes as long - */ - private Long getAvgInputSizeInBytes(String jobDefId) { - String sql = "SELECT AVG(inputSizeInBytes) as avgInputSizeInMB FROM " - + "(SELECT job_exec_id, SUM(cast(value as decimal)) inputSizeInBytes, MAX(start_time) AS start_time " - + "FROM yarn_app_result yar INNER JOIN yarn_app_heuristic_result yahr " + "ON yar.id=yahr.yarn_app_result_id " - + "INNER JOIN yarn_app_heuristic_result_details yahrd " + "ON yahr.id=yahrd.yarn_app_heuristic_result_id " - + "WHERE job_def_id=:jobDefId AND yahr.heuristic_name='" + CommonConstantsHeuristic.MAPPER_SPEED + "' " - + "AND yahrd.name='" + CommonConstantsHeuristic.TOTAL_INPUT_SIZE_IN_MB + "' " - + "GROUP BY job_exec_id ORDER BY start_time DESC LIMIT :num ) temp"; - - logger.debug("Running query for average input size computation " + sql); - - SqlRow baseline = Ebean.createSqlQuery(sql) - .setParameter("jobDefId", jobDefId) - .setParameter("num", _numJobsForBaseline) - .findUnique(); - Double avgInputSizeInBytes = baseline.getDouble("avgInputSizeInMB") * FileUtils.ONE_MB; - return avgInputSizeInBytes.longValue(); - } -} diff --git a/app/com/linkedin/drelephant/tuning/Constant.java b/app/com/linkedin/drelephant/tuning/Constant.java new file mode 100644 index 000000000..7a3a5a13d --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/Constant.java @@ -0,0 +1,13 @@ +package com.linkedin.drelephant.tuning; + +import java.util.HashMap; +import java.util.Map; + + +public class Constant { + public enum TuningType {HBT,OBT} + public enum AlgotihmType{PSO,PSO_IPSO} + public enum ExecutionEngineTypes{MR,SPARK} + public enum TypeofManagers{AbstractBaselineManager,AbstractFitnessManager,AbstractJobStatusManager,AbstractTuningTypeManager} + +} diff --git a/app/com/linkedin/drelephant/tuning/ExecutionEngine.java b/app/com/linkedin/drelephant/tuning/ExecutionEngine.java new file mode 100644 index 000000000..d21da01ec --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/ExecutionEngine.java @@ -0,0 +1,41 @@ +package com.linkedin.drelephant.tuning; + +import com.avaje.ebean.ExpressionList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import models.AppResult; +import models.JobDefinition; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; +import models.TuningParameterConstraint; + + +public interface ExecutionEngine { + + void computeValuesOfDerivedConfigurationParameters(List derivedParameterList, + List jobSuggestedParamValue); + + ExpressionList getPendingJobs(); + + ExpressionList getTuningJobDefinitionsForParameterSuggestion(); + /* + PSO related methods + */ + Boolean isParamConstraintViolatedPSO(List jobSuggestedParamValueList); + + + + Boolean isParamConstraintViolatedIPSO(List jobSuggestedParamValueList); + + public void parameterOptimizerIPSO(List results, JobExecution jobExecution); + + + + public String parameterGenerationsHBT(List results, List tuningParameters); + Boolean isParamConstraintViolatedHBT(List jobSuggestedParamValueList); +} diff --git a/app/com/linkedin/drelephant/tuning/Flow.java b/app/com/linkedin/drelephant/tuning/Flow.java new file mode 100644 index 000000000..769e4b33b --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/Flow.java @@ -0,0 +1,107 @@ +package com.linkedin.drelephant.tuning; + +import com.linkedin.drelephant.tuning.Schduler.AzkabanJobStatusManager; +import com.linkedin.drelephant.tuning.engine.MRExecutionEngine; +import com.linkedin.drelephant.tuning.engine.SparkExecutionEngine; +import com.linkedin.drelephant.tuning.hbt.BaselineManagerHBT; +import com.linkedin.drelephant.tuning.hbt.FitnessManagerHBT; +import com.linkedin.drelephant.tuning.hbt.TuningTypeManagerHBT; +import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoIPSO; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBT; +import com.linkedin.drelephant.tuning.obt.BaselineManagerOBT; +import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoPSO; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoIPSO; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoPSO; +import java.util.ArrayList; +import java.util.List; +import org.apache.log4j.Logger; + + +public class Flow { + List> pipelines = null; + private static final Logger logger = Logger.getLogger(Flow.class); + + public Flow() { + pipelines = new ArrayList>(); + createBaseLineManagersPipeline(); + createJobStatusManagersPipeline(); + createFitnessManagersPipeline(); + createTuningTypemManagersPipeline(); + } + + public List> getPipeline() { + return pipelines; + } + + public void executeFlow() throws InterruptedException { + for (final List pipelineType : this.pipelines) { + Thread t1 = new Thread(new Runnable() { + @Override + public void run() { + for (Manager manager : pipelineType) { + // logger.info(" Manager execution Status " + manager.getManagerName()); + if (manager.getClass().getSimpleName().toLowerCase().contains("hbt") || manager.getClass() + .getSimpleName() + .toLowerCase() + .contains("azkabanjob")) { + Boolean execute = manager.execute(); + } + // logger.info(" Manager execution Status " + execute + " " + manager.getManagerName()); + } + } + }); + t1.start(); + t1.join(); + } + } + + public void createBaseLineManagersPipeline() { + List baselineManagers = new ArrayList(); + for (Constant.TuningType tuningType : Constant.TuningType.values()) { + + baselineManagers.add( + ManagerFactory.getManager(tuningType.name(), null, null, AbstractBaselineManager.class.getSimpleName())); + } + this.pipelines.add(baselineManagers); + } + + public void createJobStatusManagersPipeline() { + List jobStatusManagers = new ArrayList(); + jobStatusManagers.add(ManagerFactory.getManager(null, null, null, AbstractJobStatusManager.class.getSimpleName())); + //jobStatusManagers.add(new JobStatusManagerOBT()); + this.pipelines.add(jobStatusManagers); + } + + public void createFitnessManagersPipeline() { + List fitnessManagers = new ArrayList(); + for (Constant.TuningType tuningType : Constant.TuningType.values()) { + for (Constant.AlgotihmType algotihmType : Constant.AlgotihmType.values()) { + logger.info(tuningType.name() + " " + algotihmType.name()); + Manager manager = ManagerFactory.getManager(tuningType.name(), algotihmType.name(), null, + AbstractFitnessManager.class.getSimpleName()); + if (manager != null) { + fitnessManagers.add(manager); + } + } + } + this.pipelines.add(fitnessManagers); + } + + public void createTuningTypemManagersPipeline() { + List algorithmManagers = new ArrayList(); + for (Constant.TuningType tuningType : Constant.TuningType.values()) { + for (Constant.AlgotihmType algotihmType : Constant.AlgotihmType.values()) { + for (Constant.ExecutionEngineTypes executionEngineTypes : Constant.ExecutionEngineTypes.values()) { + Manager manager = + ManagerFactory.getManager(tuningType.name(), algotihmType.name(), executionEngineTypes.name(), + AbstractTuningTypeManager.class.getSimpleName()); + if (manager != null) { + logger.info("Testing " + manager.getManagerName()); + algorithmManagers.add(manager); + } + } + } + } + this.pipelines.add(algorithmManagers); + } +} diff --git a/app/com/linkedin/drelephant/tuning/JobCompleteDetector.java b/app/com/linkedin/drelephant/tuning/JobCompleteDetector.java deleted file mode 100644 index 21b8a909b..000000000 --- a/app/com/linkedin/drelephant/tuning/JobCompleteDetector.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.tuning; - -import controllers.AutoTuningMetricsController; -import java.net.MalformedURLException; -import java.net.URISyntaxException; -import java.util.List; -import models.JobExecution; -import models.JobExecution.ExecutionState; -import models.TuningJobExecutionParamSet; -import org.apache.log4j.Logger; - - -/** - * This class pools the scheduler for completion status of execution and updates the database with current status - * of the job. - */ -public abstract class JobCompleteDetector { - private static final Logger logger = Logger.getLogger(JobCompleteDetector.class); - - /** - * Updates the status of completed executions - * @throws MalformedURLException MalformedURLException - * @throws URISyntaxException URISyntaxException - */ - public void updateCompletedExecutions() throws MalformedURLException, URISyntaxException { - logger.info("Updating execution status"); - List inProgressExecutionParamSet = getExecutionsInProgress(); - List completedExecutions = getCompletedExecutions(inProgressExecutionParamSet); - updateMetrics(completedExecutions); - logger.info("Finished updating execution status"); - } - - /** - * Updates metrics for auto tuning monitoring for job completion daemon - * @param completedExecutions List completed job executions - */ - private void updateMetrics(List completedExecutions) { - for (JobExecution jobExecution : completedExecutions) { - if (jobExecution.executionState.equals(ExecutionState.SUCCEEDED)) { - AutoTuningMetricsController.markSuccessfulJobs(); - } else if (jobExecution.executionState.equals(ExecutionState.FAILED)) { - AutoTuningMetricsController.markFailedJobs(); - } - } - } - - /** - * Returns the executions in progress - * @return JobExecution list - */ - private List getExecutionsInProgress() { - logger.info("Fetching the executions which are in progress"); - List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobExecution) - .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet) - .where() - .eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, - ExecutionState.IN_PROGRESS) - .findList(); - logger.info("Number of executions which are in progress: " + tuningJobExecutionParamSets.size()); - return tuningJobExecutionParamSets; - } - - /** - * Returns the list of completed executions. - * @param inProgressExecutionParamSet List of executions (with corresponding param set) in progress - * @return List of completed executions - * @throws MalformedURLException MalformedURLException - * @throws URISyntaxException URISyntaxException - */ - protected abstract List getCompletedExecutions( - List inProgressExecutionParamSet) throws MalformedURLException, URISyntaxException; -} diff --git a/app/com/linkedin/drelephant/tuning/JobTuningInfo.java b/app/com/linkedin/drelephant/tuning/JobTuningInfo.java index 36ce00f6a..bf8e15d73 100644 --- a/app/com/linkedin/drelephant/tuning/JobTuningInfo.java +++ b/app/com/linkedin/drelephant/tuning/JobTuningInfo.java @@ -24,7 +24,7 @@ /** * This class holds the tuning information for the job. */ -class JobTuningInfo { +public class JobTuningInfo { private JobDefinition _tuningJob; diff --git a/app/com/linkedin/drelephant/tuning/Manager.java b/app/com/linkedin/drelephant/tuning/Manager.java new file mode 100644 index 000000000..4503f2d7e --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/Manager.java @@ -0,0 +1,13 @@ +package com.linkedin.drelephant.tuning; + +public interface Manager { + /* + Use to execute the logic of all the managers . + */ + Boolean execute(); + /* + Manager Name + */ + String getManagerName(); + +} diff --git a/app/com/linkedin/drelephant/tuning/ManagerFactory.java b/app/com/linkedin/drelephant/tuning/ManagerFactory.java new file mode 100644 index 000000000..0290a470a --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/ManagerFactory.java @@ -0,0 +1,109 @@ +package com.linkedin.drelephant.tuning; + +import com.linkedin.drelephant.tuning.Schduler.AzkabanJobStatusManager; +import com.linkedin.drelephant.tuning.engine.MRExecutionEngine; +import com.linkedin.drelephant.tuning.engine.SparkExecutionEngine; +import com.linkedin.drelephant.tuning.hbt.BaselineManagerHBT; +import com.linkedin.drelephant.tuning.hbt.FitnessManagerHBT; +import com.linkedin.drelephant.tuning.hbt.TuningTypeManagerHBT; +import com.linkedin.drelephant.tuning.obt.BaselineManagerOBT; +import com.linkedin.drelephant.tuning.obt.FitnessManagerOBT; +import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoIPSO; +import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoPSO; +import com.linkedin.drelephant.tuning.obt.OptimizationAlgoFactory; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoIPSO; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoPSO; +import org.apache.log4j.Logger; + + + +public class ManagerFactory { + private static final Logger logger = Logger.getLogger(ManagerFactory.class); + + public static Manager getManager(String tuningType, String algorithmType, String executionEngineTypes, + String typeOfManagers) { + + if(typeOfManagers.equals(Constant.TypeofManagers.AbstractJobStatusManager.name())){ + logger.info("Manager Type Azkaban Job Status Manager"); + return new AzkabanJobStatusManager(); + } + + else if (tuningType.equals(Constant.TuningType.HBT.name())) { + return getMangersForHBT(algorithmType,executionEngineTypes,typeOfManagers); + } + else if (tuningType.equals(Constant.TuningType.OBT.name())) { + return getManagersForOBT(algorithmType,executionEngineTypes,typeOfManagers); + } + + return null; + } + private static Manager getMangersForHBT(String algorithmType, String executionEngineTypes, + String typeOfManagers){ + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractBaselineManager.name())) { + logger.info("Manager Type Base line Manager HBT"); + return new BaselineManagerHBT(); + } + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractFitnessManager.name())) { + logger.info("Manager Type Fitness Manager HBT"); + return new FitnessManagerHBT(); + } + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name()) + && executionEngineTypes.equals(Constant.ExecutionEngineTypes.MR.name())) { + logger.info("Manager Type TuningType Manager HBT MR"); + return new TuningTypeManagerHBT(new MRExecutionEngine()); + } + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name()) + && executionEngineTypes.equals(Constant.ExecutionEngineTypes.SPARK.name())) { + logger.info("Manager Type TuningType Manager HBT Spark"); + return new TuningTypeManagerHBT(new SparkExecutionEngine()); + } + return null; + } + + private static Manager getManagersForOBT(String algorithmType, String executionEngineTypes, + String typeOfManagers ){ + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractBaselineManager.name())) { + logger.info("Manager Type Base line Manager OBT"); + return new BaselineManagerOBT(); + } + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractFitnessManager.name()) && algorithmType.equals( + Constant.AlgotihmType.PSO.name())) { + logger.info("Manager Type Fitness Manager OBT PSO"); + return new FitnessManagerOBTAlgoPSO(); + } + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractFitnessManager.name()) && algorithmType.equals( + Constant.AlgotihmType.PSO_IPSO.name())) { + logger.info("Manager Type Fitness Manager OBT IPSO"); + return new FitnessManagerOBTAlgoIPSO(); + } + if(typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name())){ + return getManagerForOBTForTuningType(executionEngineTypes,algorithmType); + } + + return null; + } + + private static Manager getManagerForOBTForTuningType(String executionEngineTypes,String algorithmType){ + if (executionEngineTypes.equals(Constant.ExecutionEngineTypes.MR.name()) && algorithmType.equals( + Constant.AlgotihmType.PSO_IPSO.name())) { + logger.info("Manager Type TuningType Manager OBT IPSO MR"); + return new TuningTypeManagerOBTAlgoIPSO(new MRExecutionEngine()); + } + if (executionEngineTypes.equals(Constant.ExecutionEngineTypes.MR.name()) && algorithmType.equals( + Constant.AlgotihmType.PSO.name())) { + logger.info("Manager Type TuningType Manager OBT PSO MR"); + return new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); + } + if ( executionEngineTypes.equals(Constant.ExecutionEngineTypes.SPARK.name()) && algorithmType.equals( + Constant.AlgotihmType.PSO_IPSO.name())) { + logger.info("Manager Type TuningType Manager OBT IPSO Spark"); + return new TuningTypeManagerOBTAlgoIPSO(new SparkExecutionEngine()); + } + if (executionEngineTypes.equals(Constant.ExecutionEngineTypes.SPARK.name()) && algorithmType.equals( + Constant.AlgotihmType.PSO.name())) { + logger.info("Manager Type TuningType Manager OBT PSO Spark"); + return new TuningTypeManagerOBTAlgoPSO(new SparkExecutionEngine()); + } + return null; + } +} diff --git a/app/com/linkedin/drelephant/tuning/PSOParamGenerator.java b/app/com/linkedin/drelephant/tuning/PSOParamGenerator.java deleted file mode 100644 index 1d1651cad..000000000 --- a/app/com/linkedin/drelephant/tuning/PSOParamGenerator.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.tuning; - -import com.fasterxml.jackson.databind.JsonNode; -import com.linkedin.drelephant.ElephantContext; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.List; -import org.apache.hadoop.conf.Configuration; -import org.apache.log4j.Logger; -import play.libs.Json; - - -/** - * This class extends ParamGenerator class. It generates parameter suggestion using PSO algorithm, - */ -public class PSOParamGenerator extends ParamGenerator { - - private static final String PARAMS_TO_TUNE_FIELD_NAME = "parametersToTune"; - private static final String PYTHON_PATH_CONF = "python.path"; - private static final String PSO_DIR_PATH_ENV_VARIABLE = "PSO_DIR_PATH"; - private static final String PYTHON_PATH_ENV_VARIABLE = "PYTHONPATH"; - private final Logger logger = Logger.getLogger(PSOParamGenerator.class); - private String PYTHON_PATH = null; - private String TUNING_SCRIPT_PATH = null; - - public PSOParamGenerator() { - Configuration configuration = ElephantContext.instance().getAutoTuningConf(); - PYTHON_PATH = configuration.get(PYTHON_PATH_CONF); - if (PYTHON_PATH == null) { - PYTHON_PATH = System.getenv(PYTHON_PATH_ENV_VARIABLE); - } - String PSO_DIR_PATH = System.getenv(PSO_DIR_PATH_ENV_VARIABLE); - - if (PSO_DIR_PATH == null) { - throw new NullPointerException("Couldn't find directory containing PSO scripts"); - } - if (PYTHON_PATH == null) { - PYTHON_PATH = "python"; - } - TUNING_SCRIPT_PATH = PSO_DIR_PATH + "/pso_param_generation.py"; - logger.info("Tuning script path: " + TUNING_SCRIPT_PATH); - logger.info("Python path: " + PYTHON_PATH); - } - - /** - * Interacts with python scripts to generate new parameter suggestions - * @param jobTuningInfo Job tuning information - * @return Updated job tuning information - */ - public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { - logger.info("Generating param set for job: " + jobTuningInfo.getTuningJob().jobName); - - JobTuningInfo newJobTuningInfo = new JobTuningInfo(); - newJobTuningInfo.setTuningJob(jobTuningInfo.getTuningJob()); - newJobTuningInfo.setParametersToTune(jobTuningInfo.getParametersToTune()); - newJobTuningInfo.setJobType(jobTuningInfo.getJobType()); - - JsonNode jsonJobTuningInfo = Json.toJson(jobTuningInfo); - String parametersToTune = jsonJobTuningInfo.get(PARAMS_TO_TUNE_FIELD_NAME).toString(); - String stringTunerState = jobTuningInfo.getTunerState(); - stringTunerState = stringTunerState.replaceAll("\\s+", ""); - String jobType = jobTuningInfo.getJobType().toString(); - - List error = new ArrayList(); - - try { - Process p = Runtime.getRuntime() - .exec( - PYTHON_PATH + " " + TUNING_SCRIPT_PATH + " " + stringTunerState + " " + parametersToTune + " " + jobType); - BufferedReader inputStream = new BufferedReader(new InputStreamReader(p.getInputStream())); - BufferedReader errorStream = new BufferedReader(new InputStreamReader(p.getErrorStream())); - String updatedStringTunerState = inputStream.readLine(); - newJobTuningInfo.setTunerState(updatedStringTunerState); - String errorLine; - while ((errorLine = errorStream.readLine()) != null) { - error.add(errorLine); - } - if (error.size() != 0) { - logger.error("Error in python script running PSO: " + error.toString()); - } - } catch (IOException e) { - logger.error("Error in generateParamSet()", e); - } - return newJobTuningInfo; - } -} diff --git a/app/com/linkedin/drelephant/tuning/ParamGenerator.java b/app/com/linkedin/drelephant/tuning/ParamGenerator.java deleted file mode 100644 index 4b3ea3508..000000000 --- a/app/com/linkedin/drelephant/tuning/ParamGenerator.java +++ /dev/null @@ -1,522 +0,0 @@ -/* - * Copyright 2016 LinkedIn Corp. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.linkedin.drelephant.tuning; - -import com.avaje.ebean.Expr; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; -import controllers.AutoTuningMetricsController; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import models.JobDefinition; -import models.JobExecution; -import models.JobSavedState; -import models.JobSuggestedParamSet; -import models.JobSuggestedParamValue; -import models.TuningAlgorithm; -import models.TuningJobDefinition; -import models.TuningParameter; -import org.apache.commons.io.FileUtils; -import org.apache.log4j.Logger; -import play.libs.Json; - - -/** - * This is an abstract class for generating parameter suggestions for jobs - */ -public abstract class ParamGenerator { - - private static final String JSON_CURRENT_POPULATION_KEY = "current_population"; - private final Logger logger = Logger.getLogger(getClass()); - - /** - * Generates the parameters using tuningJobInfo and returns it in updated JobTuningInfo - * @param jobTuningInfo The tuning job information required to create new params - * @return The updated job tuning information containing the new params - */ - public abstract JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo); - - /** - * Converts a json to list of particles - * @param jsonParticleList A list of configurations (particles) in json - * @return Particle List - */ - private List jsonToParticleList(JsonNode jsonParticleList) { - - List particleList = new ArrayList(); - if (jsonParticleList == null) { - logger.info("Null json, empty particle list returned"); - } else { - for (JsonNode jsonParticle : jsonParticleList) { - Particle particle; - particle = Json.fromJson(jsonParticle, Particle.class); - if (particle != null) { - particleList.add(particle); - } - } - } - return particleList; - } - - /** - * Fetches the list to job which need new parameter suggestion - * @return Job list - */ - private List getJobsForParamSuggestion() { - // Todo: [Important] Change the logic. This is very rigid. Ideally you should look at the param set ids in the saved state, - // todo: [continuation] if their fitness is computed, pso can generate new params for the job - logger.info("Checking which jobs need new parameter suggestion"); - List jobsForParamSuggestion = new ArrayList(); - - List pendingParamSetList = JobSuggestedParamSet.find.select("*") - .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") - .where() - .or(Expr.or(Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.CREATED), - Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.SENT)), - Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.EXECUTED)) - .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) - .eq(JobSuggestedParamSet.TABLE.isParamSetBest, 0).findList(); - - List pendingParamJobList = new ArrayList(); - for (JobSuggestedParamSet pendingParamSet : pendingParamSetList) { - if (!pendingParamJobList.contains(pendingParamSet.jobDefinition)) { - pendingParamJobList.add(pendingParamSet.jobDefinition); - } - } - - List tuningJobDefinitionList = TuningJobDefinition.find.select("*") - .fetch(TuningJobDefinition.TABLE.job, "*") - .where() - .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) - .findList(); - - if (tuningJobDefinitionList.size() == 0) { - logger.error("No auto-tuning enabled jobs found"); - } - - for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitionList) { - if (!pendingParamJobList.contains(tuningJobDefinition.job)) { - logger.info("New parameter suggestion needed for job: " + tuningJobDefinition.job.jobName); - jobsForParamSuggestion.add(tuningJobDefinition); - } - } - logger.info("Number of job(s) which need new parameter suggestion: " + jobsForParamSuggestion.size()); - return jobsForParamSuggestion; - } - - /** - * Converts a list of particles to json - * @param particleList Particle List - * @return JsonNode - */ - private JsonNode particleListToJson(List particleList) { - JsonNode jsonNode; - - if (particleList == null) { - jsonNode = JsonNodeFactory.instance.objectNode(); - logger.info("Null particleList, returning empty json"); - } else { - jsonNode = Json.toJson(particleList); - } - return jsonNode; - } - - /** - * Returns the tuning information for the jobs - * @param tuningJobs Job List - * @return Tuning information list - */ - private List getJobsTuningInfo(List tuningJobs) { - - List jobTuningInfoList = new ArrayList(); - for (TuningJobDefinition tuningJobDefinition : tuningJobs) { - JobDefinition job = tuningJobDefinition.job; - logger.info("Getting tuning information for job: " + job.jobDefId); - List tuningParameterList = TuningParameter.find.where() - .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, - tuningJobDefinition.tuningAlgorithm.id) - .eq(TuningParameter.TABLE.isDerived, 0) - .findList(); - - logger.info("Fetching default parameter values for job " + tuningJobDefinition.job.jobDefId); - JobSuggestedParamSet defaultJobParamSet = JobSuggestedParamSet.find.where() - .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, tuningJobDefinition.job.id) - .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 1) - .order() - .desc(JobSuggestedParamSet.TABLE.id) - .setMaxRows(1) - .findUnique(); - - if (defaultJobParamSet != null) { - List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + "." + JobExecution.TABLE.id, - defaultJobParamSet.id) - .findList(); - - if (jobSuggestedParamValueList.size() > 0) { - Map defaultExecutionParamMap = new HashMap(); - - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - defaultExecutionParamMap.put(jobSuggestedParamValue.tuningParameter.id, - jobSuggestedParamValue.paramValue); - } - - for (TuningParameter tuningParameter : tuningParameterList) { - Integer paramId = tuningParameter.id; - if (defaultExecutionParamMap.containsKey(paramId)) { - logger.info( - "Updating value of param " + tuningParameter.paramName + " to " + defaultExecutionParamMap.get( - paramId)); - tuningParameter.defaultValue = defaultExecutionParamMap.get(paramId); - } - } - } - } - - JobTuningInfo jobTuningInfo = new JobTuningInfo(); - jobTuningInfo.setTuningJob(job); - jobTuningInfo.setJobType(tuningJobDefinition.tuningAlgorithm.jobType); - jobTuningInfo.setParametersToTune(tuningParameterList); - JobSavedState jobSavedState = JobSavedState.find.byId(job.id); - - boolean validSavedState = true; - if (jobSavedState != null && jobSavedState.isValid()) { - String savedState = new String(jobSavedState.savedState); - ObjectNode jsonSavedState = (ObjectNode) Json.parse(savedState); - JsonNode jsonCurrentPopulation = jsonSavedState.get(JSON_CURRENT_POPULATION_KEY); - List currentPopulation = jsonToParticleList(jsonCurrentPopulation); - for (Particle particle : currentPopulation) { - Long paramSetId = particle.getParamSetId(); - - logger.info("Param set id: " + paramSetId.toString()); - JobSuggestedParamSet jobSuggestedParamSet = - JobSuggestedParamSet.find.select("*").where().eq(JobSuggestedParamSet.TABLE.id, paramSetId).findUnique(); - - if (jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED) - && jobSuggestedParamSet.fitness != null) { - particle.setFitness(jobSuggestedParamSet.fitness); - } else { - validSavedState = false; - logger.error("Invalid saved state: Fitness of previous execution not computed."); - break; - } - } - - if (validSavedState) { - JsonNode updatedJsonCurrentPopulation = particleListToJson(currentPopulation); - jsonSavedState.set(JSON_CURRENT_POPULATION_KEY, updatedJsonCurrentPopulation); - savedState = Json.stringify(jsonSavedState); - jobTuningInfo.setTunerState(savedState); - } - } else { - logger.info("Saved state empty for job: " + job.jobDefId); - validSavedState = false; - } - - if (!validSavedState) { - jobTuningInfo.setTunerState("{}"); - } - - logger.info("Adding JobTuningInfo " + Json.toJson(jobTuningInfo)); - jobTuningInfoList.add(jobTuningInfo); - } - return jobTuningInfoList; - } - - /** - * Returns list of suggested parameters - * @param particle Particle (configuration) - * @param paramList Parameter List - * @return Suggested Param Value List - */ - private List getParamValueList(Particle particle, List paramList) { - logger.debug("Particle is: " + Json.toJson(particle)); - List jobSuggestedParamValueList = new ArrayList(); - - if (particle != null) { - List candidate = particle.getCandidate(); - - if (candidate != null) { - logger.debug("Candidate is:" + Json.toJson(candidate)); - for (int i = 0; i < candidate.size() && i < paramList.size(); i++) { - logger.info("Candidate is " + candidate); - - JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); - int paramId = paramList.get(i).id; - jobSuggestedParamValue.tuningParameter = TuningParameter.find.byId(paramId); - jobSuggestedParamValue.paramValue = candidate.get(i); - jobSuggestedParamValueList.add(jobSuggestedParamValue); - } - } else { - logger.info("Candidate is null"); - } - } else { - logger.info("Particle null"); - } - return jobSuggestedParamValueList; - } - - /** - * For every tuning info: - * For every new particle: - * From the tuner set extract the list of suggested parameters - * Check penalty - * Save the param in the job execution table by creating execution instance (Create an entry into param_set table) - * Update the execution instance in each of the suggested params (Update the param_set_id in each of the prams) - * save th suggested parameters - * update the paramsetid in the particle and add particle to a particlelist - * Update the tunerstate from the updated particles - * save the tuning info in db - * - * @param jobTuningInfoList JobTuningInfo List - */ - private void updateDatabase(List jobTuningInfoList) { - logger.info("Updating new parameter suggestion in database"); - if (jobTuningInfoList == null) { - logger.info("No new parameter suggestion to update"); - return; - } - - int paramSetNotGeneratedJobs = jobTuningInfoList.size(); - - for (JobTuningInfo jobTuningInfo : jobTuningInfoList) { - logger.info("Updating new parameter suggestion for job:" + jobTuningInfo.getTuningJob().jobDefId); - - JobDefinition job = jobTuningInfo.getTuningJob(); - List paramList = jobTuningInfo.getParametersToTune(); - String stringTunerState = jobTuningInfo.getTunerState(); - if (stringTunerState == null) { - logger.error("Suggested parameter suggestion is empty for job id: " + job.jobDefId); - continue; - } - - TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") - .fetch(TuningJobDefinition.TABLE.job, "*") - .where() - .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, job.id) - .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) - .findUnique(); - - List derivedParameterList = new ArrayList(); - derivedParameterList = TuningParameter.find.where() - .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, - tuningJobDefinition.tuningAlgorithm.id) - .eq(TuningParameter.TABLE.isDerived, 1) - .findList(); - - logger.info("No. of derived tuning params for job " + tuningJobDefinition.job.jobName + ": " - + derivedParameterList.size()); - - JsonNode jsonTunerState = Json.parse(stringTunerState); - JsonNode jsonSuggestedPopulation = jsonTunerState.get(JSON_CURRENT_POPULATION_KEY); - - if (jsonSuggestedPopulation == null) { - continue; - } - - paramSetNotGeneratedJobs--; - - List suggestedPopulation = jsonToParticleList(jsonSuggestedPopulation); - - for (Particle suggestedParticle : suggestedPopulation) { - AutoTuningMetricsController.markParamSetGenerated(); - List jobSuggestedParamValueList = getParamValueList(suggestedParticle, paramList); - - Map jobSuggestedParamValueMap = new HashMap(); - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - jobSuggestedParamValueMap.put(jobSuggestedParamValue.tuningParameter.paramName, - jobSuggestedParamValue.paramValue); - } - - for (TuningParameter derivedParameter : derivedParameterList) { - logger.info("Computing value of derived param: " + derivedParameter.paramName); - Double paramValue = null; - if (derivedParameter.paramName.equals("mapreduce.reduce.java.opts")) { - String parentParamName = "mapreduce.reduce.memory.mb"; - if (jobSuggestedParamValueMap.containsKey(parentParamName)) { - paramValue = 0.75 * jobSuggestedParamValueMap.get(parentParamName); - } - } else if (derivedParameter.paramName.equals("mapreduce.map.java.opts")) { - String parentParamName = "mapreduce.map.memory.mb"; - if (jobSuggestedParamValueMap.containsKey(parentParamName)) { - paramValue = 0.75 * jobSuggestedParamValueMap.get(parentParamName); - } - } else if (derivedParameter.paramName.equals("mapreduce.input.fileinputformat.split.maxsize")) { - String parentParamName = "pig.maxCombinedSplitSize"; - if (jobSuggestedParamValueMap.containsKey(parentParamName)) { - paramValue = jobSuggestedParamValueMap.get(parentParamName); - } - } - - if (paramValue != null) { - JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); - jobSuggestedParamValue.paramValue = paramValue; - jobSuggestedParamValue.tuningParameter = derivedParameter; - jobSuggestedParamValueList.add(jobSuggestedParamValue); - } - } - - JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); - jobSuggestedParamSet.jobDefinition = job; - jobSuggestedParamSet.tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; - jobSuggestedParamSet.isParamSetDefault = false; - jobSuggestedParamSet.isParamSetBest = false; - if (isParamConstraintViolated(jobSuggestedParamValueList, jobSuggestedParamSet.tuningAlgorithm.jobType)) { - logger.info("Parameter constraint violated. Applying penalty."); - int penaltyConstant = 3; - Double averageResourceUsagePerGBInput = - tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; - Double maxDesiredResourceUsagePerGBInput = - averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; - - jobSuggestedParamSet.areConstraintsViolated = true; - jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; - jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED; - } else { - jobSuggestedParamSet.areConstraintsViolated = false; - jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; - } - Long paramSetId = saveSuggestedParamSet(jobSuggestedParamSet); - - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; - } - suggestedParticle.setPramSetId(paramSetId); - saveSuggestedParams(jobSuggestedParamValueList); - } - - JsonNode updatedJsonSuggestedPopulation = particleListToJson(suggestedPopulation); - - ObjectNode updatedJsonTunerState = (ObjectNode) jsonTunerState; - updatedJsonTunerState.put(JSON_CURRENT_POPULATION_KEY, updatedJsonSuggestedPopulation); - String updatedStringTunerState = Json.stringify(updatedJsonTunerState); - jobTuningInfo.setTunerState(updatedStringTunerState); - } - AutoTuningMetricsController.setParamSetGenerateWaitJobs(paramSetNotGeneratedJobs); - saveTunerState(jobTuningInfoList); - } - - /** - * Check if the parameters violated constraints - * Constraint 1: sort.mb > 60% of map.memory: To avoid heap memory failure - * Constraint 2: map.memory - sort.mb < 768: To avoid heap memory failure - * Constraint 3: pig.maxCombinedSplitSize > 1.8*mapreduce.map.memory.mb - * @param jobSuggestedParamValueList List of suggested param values - * @param jobType Job type - * @return true if the constraint is violated, false otherwise - */ - private boolean isParamConstraintViolated(List jobSuggestedParamValueList, - TuningAlgorithm.JobType jobType) { - - logger.info("Checking whether parameter values are within constraints"); - Integer violations = 0; - - if (jobType.equals(TuningAlgorithm.JobType.PIG)) { - Double mrSortMemory = null; - Double mrMapMemory = null; - Double pigMaxCombinedSplitSize = null; - - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - if (jobSuggestedParamValue.tuningParameter.paramName.equals("mapreduce.task.io.sort.mb")) { - mrSortMemory = jobSuggestedParamValue.paramValue; - } else if (jobSuggestedParamValue.tuningParameter.paramName.equals("mapreduce.map.memory.mb")) { - mrMapMemory = jobSuggestedParamValue.paramValue; - } else if (jobSuggestedParamValue.tuningParameter.paramName.equals("pig.maxCombinedSplitSize")) { - pigMaxCombinedSplitSize = jobSuggestedParamValue.paramValue / FileUtils.ONE_MB; - } - } - - if (mrSortMemory != null && mrMapMemory != null) { - if (mrSortMemory > 0.6 * mrMapMemory) { - logger.info("Constraint violated: Sort memory > 60% of map memory"); - violations++; - } - if (mrMapMemory - mrSortMemory < 768) { - logger.info("Constraint violated: Map memory - sort memory < 768 mb"); - violations++; - } - } - - if (pigMaxCombinedSplitSize != null && mrMapMemory != null && (pigMaxCombinedSplitSize > 1.8 * mrMapMemory)) { - logger.info("Constraint violated: Pig max combined split size > 1.8 * map memory"); - violations++; - } - } - if (violations == 0) { - return false; - } else { - logger.info("Number of constraint(s) violated: " + violations); - return true; - } - } - - /** - * Save the tuning info list to the database - * @param jobTuningInfoList Tuning Info List - */ - private void saveTunerState(List jobTuningInfoList) { - for (JobTuningInfo jobTuningInfo : jobTuningInfoList) { - if (jobTuningInfo.getTunerState() == null) { - continue; - } - JobSavedState jobSavedState = JobSavedState.find.byId(jobTuningInfo.getTuningJob().id); - if (jobSavedState == null) { - jobSavedState = new JobSavedState(); - jobSavedState.jobDefinitionId = jobTuningInfo.getTuningJob().id; - } - jobSavedState.savedState = jobTuningInfo.getTunerState().getBytes(); - jobSavedState.save(); - } - } - - /** - * Saves the list of suggested parameter values to database - * @param jobSuggestedParamValueList Suggested Parameter Values List - */ - private void saveSuggestedParams(List jobSuggestedParamValueList) { - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - jobSuggestedParamValue.save(); - } - } - - /** - * Saves the suggested param set in the database and returns the param set id - * @param jobSuggestedParamSet JobExecution - * @return Param Set Id - */ - private Long saveSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { - jobSuggestedParamSet.save(); - return jobSuggestedParamSet.id; - } - - /** - * Fetches job which need parameters, generates parameters and stores it in the database - */ - public void getParams() { - List jobsForSwarmSuggestion = getJobsForParamSuggestion(); - List jobTuningInfoList = getJobsTuningInfo(jobsForSwarmSuggestion); - List updatedJobTuningInfoList = new ArrayList(); - for (JobTuningInfo jobTuningInfo : jobTuningInfoList) { - JobTuningInfo newJobTuningInfo = generateParamSet(jobTuningInfo); - updatedJobTuningInfoList.add(newJobTuningInfo); - } - updateDatabase(updatedJobTuningInfoList); - } -} diff --git a/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java b/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java new file mode 100644 index 000000000..553bddc99 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java @@ -0,0 +1,111 @@ +package com.linkedin.drelephant.tuning.Schduler; + +import com.linkedin.drelephant.clients.azkaban.AzkabanJobStatusUtil; +import com.linkedin.drelephant.tuning.AbstractJobStatusManager; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.TuningJobExecutionParamSet; +import org.apache.log4j.Logger; + + +public class AzkabanJobStatusManager extends AbstractJobStatusManager { + + private final Logger logger = Logger.getLogger(getClass()); + private AzkabanJobStatusUtil _azkabanJobStatusUtil; + + + public enum AzkabanJobStatus { + FAILED, CANCELLED, KILLED, SUCCEEDED, SKIPPED + } + + protected List detectJobsExecutionInProgress() { + logger.info("Fetching the executions which are in progress"); + List tuningJobExecutionParamSets = + TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobExecution) + .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet) + .where() + .eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.IN_PROGRESS) + .findList(); + + + logger.info("Number of executions which are in progress: " + tuningJobExecutionParamSets.size()); + return tuningJobExecutionParamSets; + } + + @Override + protected Boolean analyzeCompletedJobsExecution(List inProgressExecutionParamSet) { + logger.info("Fetching the list of executions completed since last iteration"); + List completedExecutions = new ArrayList(); + try { + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : inProgressExecutionParamSet) { + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; + logger.info("Checking current status of started execution: " + jobExecution.jobExecId); + assignAzkabanJobStatusUtil(); + analyzeJobExecution(jobExecution,jobSuggestedParamSet); + } + } catch (Exception e) { + logger.error("Error in fetching list of completed executions", e); + e.printStackTrace(); + } + logger.info("Number of executions completed since last iteration: " + completedExecutions.size()); + return true; + } + + private void assignAzkabanJobStatusUtil() { + if (_azkabanJobStatusUtil == null) { + logger.info("Initializing AzkabanJobStatusUtil"); + _azkabanJobStatusUtil = new AzkabanJobStatusUtil(); + } + } + + private void analyzeJobExecution(JobExecution jobExecution,JobSuggestedParamSet jobSuggestedParamSet){ + try { + Map jobStatus = _azkabanJobStatusUtil.getJobsFromFlow(jobExecution.flowExecution.flowExecId); + if (jobStatus != null) { + for (Map.Entry job : jobStatus.entrySet()) { + logger.info("Job Found:" + job.getKey() + ". Status: " + job.getValue()); + if (job.getKey().equals(jobExecution.job.jobName)) { + updateJobExecutionMetrics(job, jobSuggestedParamSet, jobExecution); + } + } + } else { + logger.info("No jobs found for flow execution: " + jobExecution.flowExecution.flowExecId); + } + } catch (Exception e) { + logger.error("Error in checking status of execution: " + jobExecution.jobExecId, e); + } + } + + private void updateJobExecutionMetrics(Map.Entry job, JobSuggestedParamSet jobSuggestedParamSet, + JobExecution jobExecution) { + if (job.getValue().equals(AzkabanJobStatus.FAILED.toString())) { + if (jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.SENT)) { + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.EXECUTED; + } + jobExecution.executionState = JobExecution.ExecutionState.FAILED; + } else if (job.getValue().equals(AzkabanJobStatus.SUCCEEDED.toString())) { + if (jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.SENT)) { + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.EXECUTED; + } + jobExecution.executionState = JobExecution.ExecutionState.SUCCEEDED; + } else if (job.getValue().equals(AzkabanJobStatus.CANCELLED.toString()) || job.getValue() + .equals(AzkabanJobStatus.KILLED.toString()) || job.getValue() + .equals(AzkabanJobStatus.SKIPPED.toString())) { + if (jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.SENT)) { + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.EXECUTED; + } + jobExecution.executionState = JobExecution.ExecutionState.CANCELLED; + } + } + + @Override + public String getManagerName() { + return "AzkabanJobStatusManager"; + } + +} diff --git a/app/com/linkedin/drelephant/tuning/TuningHelper.java b/app/com/linkedin/drelephant/tuning/TuningHelper.java new file mode 100644 index 000000000..4c4c10a01 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/TuningHelper.java @@ -0,0 +1,53 @@ +package com.linkedin.drelephant.tuning; + +import java.util.ArrayList; +import java.util.List; +import models.JobDefinition; +import models.JobSuggestedParamSet; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; + + +public class TuningHelper { + public static List getTuningParameterList(TuningJobDefinition tuningJobDefinition){ + List tuningParameterList = TuningParameter.find.where() + .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, + tuningJobDefinition.tuningAlgorithm.id) + .eq(TuningParameter.TABLE.isDerived, 0) + .findList(); + return tuningParameterList; + } + + public static JobSuggestedParamSet getDefaultParameterValuesforJob (TuningJobDefinition tuningJobDefinition){ + JobSuggestedParamSet defaultJobParamSet = JobSuggestedParamSet.find.where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, tuningJobDefinition.job.id) + .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 1) + .order() + .desc(JobSuggestedParamSet.TABLE.id) + .setMaxRows(1) + .findUnique(); + return defaultJobParamSet; + } + + public static TuningJobDefinition getTuningJobDefinition(JobDefinition job){ + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") + .fetch(TuningJobDefinition.TABLE.job, "*") + .where() + .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, job.id) + .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) + .findUnique(); + return tuningJobDefinition; + } + + public static List getDerivedParameterList(TuningJobDefinition tuningJobDefinition){ + List derivedParameterList = new ArrayList(); + derivedParameterList = TuningParameter.find.where() + .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, + tuningJobDefinition.tuningAlgorithm.id) + .eq(TuningParameter.TABLE.isDerived, 1) + .findList(); + return derivedParameterList; + } + +} diff --git a/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java b/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java new file mode 100644 index 000000000..da03aa143 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java @@ -0,0 +1,425 @@ +package com.linkedin.drelephant.tuning.engine; + +import com.avaje.ebean.Expr; +import com.avaje.ebean.ExpressionList; +import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; +import com.linkedin.drelephant.tuning.ExecutionEngine; +import com.linkedin.drelephant.util.MemoryFormatUtils; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import models.AppHeuristicResult; +import models.AppHeuristicResultDetails; +import models.AppResult; +import models.JobDefinition; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; +import models.TuningParameterConstraint; +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; + +import static java.lang.Math.*; + + +public class MRExecutionEngine implements ExecutionEngine { + private final Logger logger = Logger.getLogger(getClass()); + + enum UsageCounterSchema {USED_PHYSICAL_MEMORY, USED_VIRTUAL_MEMORY, USED_HEAP_MEMORY} + + private String functionTypes[] = {"map", "reduce"}; + + @Override + public void computeValuesOfDerivedConfigurationParameters(List derivedParameterList, + List jobSuggestedParamValues) { + Map jobSuggestedParamValueMap = new HashMap(); + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) { + jobSuggestedParamValueMap.put(jobSuggestedParamValue.tuningParameter.paramName, + jobSuggestedParamValue.paramValue); + } + + for (TuningParameter derivedParameter : derivedParameterList) { + logger.info("Computing value of derived param: " + derivedParameter.paramName); + Double paramValue = null; + if (derivedParameter.paramName.equals("mapreduce.reduce.java.opts")) { + String parentParamName = "mapreduce.reduce.memory.mb"; + if (jobSuggestedParamValueMap.containsKey(parentParamName)) { + paramValue = 0.75 * jobSuggestedParamValueMap.get(parentParamName); + } + } else if (derivedParameter.paramName.equals("mapreduce.map.java.opts")) { + String parentParamName = "mapreduce.map.memory.mb"; + if (jobSuggestedParamValueMap.containsKey(parentParamName)) { + paramValue = 0.75 * jobSuggestedParamValueMap.get(parentParamName); + } + } else if (derivedParameter.paramName.equals("mapreduce.input.fileinputformat.split.maxsize")) { + String parentParamName = "pig.maxCombinedSplitSize"; + if (jobSuggestedParamValueMap.containsKey(parentParamName)) { + paramValue = jobSuggestedParamValueMap.get(parentParamName); + } + } + + if (paramValue != null) { + JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); + jobSuggestedParamValue.paramValue = paramValue; + jobSuggestedParamValue.tuningParameter = derivedParameter; + jobSuggestedParamValues.add(jobSuggestedParamValue); + } + } + } + + /** + * Check if the parameters violated constraints + * Constraint 1: sort.mb > 60% of map.memory: To avoid heap memory failure + * Constraint 2: map.memory - sort.mb < 768: To avoid heap memory failure + * Constraint 3: pig.maxCombinedSplitSize > 1.8*mapreduce.map.memory.mb + * @param jobSuggestedParamValueList List of suggested param values + * @return true if the constraint is violated, false otherwise + */ + + @Override + public Boolean isParamConstraintViolatedIPSO(List jobSuggestedParamValueList) { + logger.info(" Constraint Violeted "); + Double mrSortMemory = null; + Double mrMapMemory = null; + Double mrReduceMemory = null; + Double mrMapXMX = null; + Double mrReduceXMX = null; + Double pigMaxCombinedSplitSize = null; + Integer violations = 0; + + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { + if (jobSuggestedParamValue.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.SORT_BUFFER_HADOOP_CONF.getValue())) { + mrSortMemory = jobSuggestedParamValue.paramValue; + } else if (jobSuggestedParamValue.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.MAPPER_MEMORY_HADOOP_CONF.getValue())) { + mrMapMemory = jobSuggestedParamValue.paramValue; + } else if (jobSuggestedParamValue.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.REDUCER_MEMORY_HADOOP_CONF.getValue())) { + mrReduceMemory = jobSuggestedParamValue.paramValue; + } else if (jobSuggestedParamValue.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.MAPPER_HEAP_HADOOP_CONF.getValue())) { + mrMapXMX = jobSuggestedParamValue.paramValue; + } else if (jobSuggestedParamValue.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.REDUCER_HEAP_HADOOP_CONF.getValue())) { + mrReduceXMX = jobSuggestedParamValue.paramValue; + } else if (jobSuggestedParamValue.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.PIG_SPLIT_SIZE_HADOOP_CONF.getValue())) { + pigMaxCombinedSplitSize = jobSuggestedParamValue.paramValue / FileUtils.ONE_MB; + } + } + + if (mrSortMemory != null && mrMapMemory != null) { + if (mrSortMemory > 0.6 * mrMapMemory) { + logger.debug("Sort Memory " + mrSortMemory); + logger.debug("Mapper Memory " + mrMapMemory); + logger.debug("Constraint violated: Sort memory > 60% of map memory"); + violations++; + } + if (mrMapMemory - mrSortMemory < 768) { + logger.debug("Sort Memory " + mrSortMemory); + logger.debug("Mapper Memory " + mrMapMemory); + logger.debug("Constraint violated: Map memory - sort memory < 768 mb"); + violations++; + } + } + if (mrMapXMX != null && mrMapMemory != null && mrMapXMX > 0.80 * mrMapMemory) { + logger.debug("Mapper Heap Max " + mrMapXMX); + logger.debug("Mapper Memory " + mrMapMemory); + logger.debug("Constraint violated: Mapper XMX > 0.8*mrMapMemory"); + violations++; + } + if (mrReduceMemory != null && mrReduceXMX != null && mrReduceXMX > 0.80 * mrReduceMemory) { + logger.debug("Reducer Heap Max " + mrMapXMX); + logger.debug("Reducer Memory " + mrMapMemory); + logger.debug("Constraint violated: Reducer XMX > 0.8*mrReducerMemory"); + violations++; + } + + if (pigMaxCombinedSplitSize != null && mrMapMemory != null && (pigMaxCombinedSplitSize > 1.8 * mrMapMemory)) { + logger.debug("Constraint violated: Pig max combined split size > 1.8 * map memory"); + violations++; + } + if (violations == 0) { + return false; + } else { + logger.info("Number of constraint(s) violated: " + violations); + return true; + } + } + + public Boolean isParamConstraintViolatedPSO(List jobSuggestedParamValueList) { + Double mrSortMemory = null; + Double mrMapMemory = null; + Double pigMaxCombinedSplitSize = null; + Integer violations = 0; + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { + if (jobSuggestedParamValue.tuningParameter.paramName.equals("mapreduce.task.io.sort.mb")) { + mrSortMemory = jobSuggestedParamValue.paramValue; + } else if (jobSuggestedParamValue.tuningParameter.paramName.equals("mapreduce.map.memory.mb")) { + mrMapMemory = jobSuggestedParamValue.paramValue; + } else if (jobSuggestedParamValue.tuningParameter.paramName.equals("pig.maxCombinedSplitSize")) { + pigMaxCombinedSplitSize = jobSuggestedParamValue.paramValue / FileUtils.ONE_MB; + } + } + if (mrSortMemory != null && mrMapMemory != null) { + if (mrSortMemory > 0.6 * mrMapMemory) { + logger.info("Constraint violated: Sort memory > 60% of map memory"); + violations++; + } + if (mrMapMemory - mrSortMemory < 768) { + logger.info("Constraint violated: Map memory - sort memory < 768 mb"); + violations++; + } + } + + if (pigMaxCombinedSplitSize != null && mrMapMemory != null && (pigMaxCombinedSplitSize > 1.8 * mrMapMemory)) { + logger.info("Constraint violated: Pig max combined split size > 1.8 * map memory"); + violations++; + } + if (violations == 0) { + return false; + } else { + logger.info("Number of constraint(s) violated: " + violations); + return true; + } + } + + @Override + public ExpressionList getPendingJobs() { + return JobSuggestedParamSet.find.select("*") + .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") + .where() + .or(Expr.or(Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.CREATED), + Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.SENT)), + Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.EXECUTED)) + .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.jobType, + TuningAlgorithm.JobType.PIG.name()) + .eq(JobSuggestedParamSet.TABLE.isParamSetBest, 0); + } + + @Override + public ExpressionList getTuningJobDefinitionsForParameterSuggestion() { + return TuningJobDefinition.find.select("*") + .fetch(TuningJobDefinition.TABLE.job, "*") + .where() + .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) + .eq(TuningJobDefinition.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.jobType, + TuningAlgorithm.JobType.PIG.name()); + } + + private void printInformation(Map> information) { + for (String functionType : information.keySet()) { + logger.debug("function Type " + functionType); + Map usage = information.get(functionType); + for (String data : usage.keySet()) { + logger.debug(data + " " + usage.get(data)); + } + } + } + + private void collectUsageDataPerApplicationForFunction(AppHeuristicResult appHeuristicResult, + Map counterData) { + if (appHeuristicResult.yarnAppHeuristicResultDetails != null) { + for (AppHeuristicResultDetails appHeuristicResultDetails : appHeuristicResult.yarnAppHeuristicResultDetails) { + for (CommonConstantsHeuristic.UtilizedParameterKeys value : CommonConstantsHeuristic.UtilizedParameterKeys.values()) { + if (appHeuristicResultDetails.name.equals(value.getValue())) { + counterData.put(value.getValue(), appHeuristicResultDetails.value == null ? 0 + : ((double) MemoryFormatUtils.stringToBytes(appHeuristicResultDetails.value))); + } + } + } + } + } + + private List extractUsageParameter(String functionType, Map> usageDataGlobal) { + Double usedPhysicalMemoryMB = 0.0, usedVirtualMemoryMB = 0.0, usedHeapMemoryMB = 0.0; + usedPhysicalMemoryMB = usageDataGlobal.get(functionType) + .get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_PHYSICAL_MEMORY.getValue()); + usedVirtualMemoryMB = usageDataGlobal.get(functionType) + .get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_VIRTUAL_MEMORY.getValue()); + usedHeapMemoryMB = usageDataGlobal.get(functionType) + .get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_TOTAL_COMMITTED_HEAP_USAGE_MEMORY.getValue()); + logger.debug(" Usage Stats " + functionType); + logger.debug(" Physical Memory Usage MB " + usedPhysicalMemoryMB); + logger.debug(" Virtual Memory Usage MB " + usedVirtualMemoryMB / 2.1); + logger.debug(" Heap Usage MB " + usedHeapMemoryMB); + List usageStats = new ArrayList(); + usageStats.add(usedPhysicalMemoryMB); + usageStats.add(usedVirtualMemoryMB); + usageStats.add(usedHeapMemoryMB); + return usageStats; + } + + private Map filterMemoryConstraint( + List parameterConstraints, String functionType) { + Map memoryConstraints = new HashMap(); + for (TuningParameterConstraint parameterConstraint : parameterConstraints) { + if (functionType.equals("map")) { + if (parameterConstraint.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.MAPPER_MEMORY_HADOOP_CONF.getValue())) { + memoryConstraints.put("CONTAINER_MEMORY", parameterConstraint); + } + if (parameterConstraint.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.MAPPER_HEAP_HADOOP_CONF.getValue())) { + memoryConstraints.put("CONTAINER_HEAP", parameterConstraint); + } + } + if (functionType.equals("reduce")) { + if (parameterConstraint.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.REDUCER_MEMORY_HADOOP_CONF.getValue())) { + memoryConstraints.put("CONTAINER_MEMORY", parameterConstraint); + } + if (parameterConstraint.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.REDUCER_HEAP_HADOOP_CONF.getValue())) { + memoryConstraints.put("CONTAINER_HEAP", parameterConstraint); + } + } + } + return memoryConstraints; + } + + private void memoryParameterIPSO(String trigger, Map constraints, + List usageStats) { + logger.info(" IPSO for " + trigger); + Double usagePhysicalMemory = usageStats.get(UsageCounterSchema.USED_PHYSICAL_MEMORY.ordinal()); + Double usageVirtualMemory = usageStats.get(UsageCounterSchema.USED_VIRTUAL_MEMORY.ordinal()); + Double usageHeapMemory = usageStats.get(UsageCounterSchema.USED_HEAP_MEMORY.ordinal()); + Double memoryMB = + applyContainerSizeFormula(constraints.get("CONTAINER_MEMORY"), usagePhysicalMemory, usageVirtualMemory); + applyHeapSizeFormula(constraints.get("CONTAINER_HEAP"), usageHeapMemory, memoryMB); + } + + @Override + public void parameterOptimizerIPSO(List results, JobExecution jobExecution) { + Map> previousUsedMetrics = extractParameterInformationIPSO(results); + List parameterConstraints = TuningParameterConstraint.find.where(). + eq("job_definition_id", jobExecution.job.id).findList(); + for (String function : functionTypes) { + logger.debug(" Optimizing Parameter Space " + function); + if (previousUsedMetrics.get(function) + .get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_PHYSICAL_MEMORY.getValue()) > 0.0) { + List usageStats = extractUsageParameter(function, previousUsedMetrics); + Map memoryConstraints = + filterMemoryConstraint(parameterConstraints, function); + memoryParameterIPSO(function, memoryConstraints, usageStats); + } + } + } + + @Override + public String parameterGenerationsHBT(List results, List tuningParameters) { + Map> previousUsedMetrics = extractParameterInformationIPSO(results); + StringBuffer idParameters = new StringBuffer(); + for (TuningParameter tuningParameter : tuningParameters) { + if (tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.MAPPER_MEMORY_HADOOP_CONF.getValue())) { + idParameters.append(tuningParameter.id) + .append("\t") + .append(previousUsedMetrics.get("map") + .get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_PHYSICAL_MEMORY.getValue())); + idParameters.append("\n"); + break; + } + } + logger.info(" New Suggested Parameter " + idParameters); + return idParameters.toString(); + } + + @Override + public Boolean isParamConstraintViolatedHBT(List jobSuggestedParamValueList) { + return false; + } + + private Double applyContainerSizeFormula(TuningParameterConstraint containerConstraint, Double usagePhysicalMemory, + Double usageVirtualMemory) { + Double memoryMB = max(usagePhysicalMemory, usageVirtualMemory / (2.1)); + Double containerSizeLower = getContainerSize(memoryMB); + Double containerSizeUpper = getContainerSize(1.2 * memoryMB); + logger.debug(" Previous Lower Bound Memory " + containerConstraint.lowerBound); + logger.debug(" Previous Upper Bound Memory " + containerConstraint.upperBound); + logger.debug(" Current Lower Bound Memory " + containerSizeLower); + logger.debug(" Current Upper Bound Memory " + containerSizeUpper); + containerConstraint.lowerBound = containerSizeLower; + containerConstraint.upperBound = containerSizeUpper; + containerConstraint.save(); + return memoryMB; + } + + private void applyHeapSizeFormula(TuningParameterConstraint containerHeapSizeConstraint, Double usageHeapMemory, + Double memoryMB) { + Double heapSizeLowerBound = min(0.75 * memoryMB, usageHeapMemory); + Double heapSizeUpperBound = heapSizeLowerBound * 1.2; + logger.debug(" Previous Lower Bound XMX " + containerHeapSizeConstraint.lowerBound); + logger.debug(" Previous Upper Bound XMX " + containerHeapSizeConstraint.upperBound); + logger.debug(" Current Lower Bound XMX " + heapSizeLowerBound); + logger.debug(" Current Upper Bound XMX " + heapSizeUpperBound); + containerHeapSizeConstraint.lowerBound = heapSizeLowerBound; + containerHeapSizeConstraint.upperBound = heapSizeUpperBound; + containerHeapSizeConstraint.save(); + } + + private Double getContainerSize(Double memory) { + return Math.ceil(memory / 1024.0) * 1024; + } + + private Map> extractParameterInformationIPSO(List appResults) { + logger.info(" Extract Parameter Information for MR IPSO"); + Map> usageDataGlobal = new HashMap>(); + intialize(usageDataGlobal); + for (AppResult appResult : appResults) { + Map> usageDataApplicationlocal = collectUsageDataPerApplicationIPSO(appResult); + for (String functionType : usageDataApplicationlocal.keySet()) { + Map usageDataForFunctionGlobal = usageDataGlobal.get(functionType); + Map usageDataForFunctionlocal = usageDataApplicationlocal.get(functionType); + for (String usageName : usageDataForFunctionlocal.keySet()) { + usageDataForFunctionGlobal.put(usageName, + max(usageDataForFunctionGlobal.get(usageName), usageDataForFunctionlocal.get(usageName))); + } + } + } + logger.debug("Usage Values Global "); + printInformation(usageDataGlobal); + return usageDataGlobal; + } + + private void intialize(Map> usageDataGlobal) { + for (String function : functionTypes) { + Map usageData = new HashMap(); + for (CommonConstantsHeuristic.UtilizedParameterKeys value : CommonConstantsHeuristic.UtilizedParameterKeys.values()) { + usageData.put(value.getValue(), 0.0); + } + usageDataGlobal.put(function, usageData); + } + } + + private Map> collectUsageDataPerApplicationIPSO(AppResult appResult) { + Map> usageData = null; + usageData = new HashMap>(); + if (appResult.yarnAppHeuristicResults != null) { + for (AppHeuristicResult appHeuristicResult : appResult.yarnAppHeuristicResults) { + + if (appHeuristicResult.heuristicName.equals("Mapper Memory")) { + Map counterData = new HashMap(); + collectUsageDataPerApplicationForFunction(appHeuristicResult, counterData); + usageData.put("map", counterData); + } + if (appHeuristicResult.heuristicName.equals("Reducer Memory")) { + Map counterData = new HashMap(); + collectUsageDataPerApplicationForFunction(appHeuristicResult, counterData); + usageData.put("reduce", counterData); + } + } + } + logger.debug("Usage Values local " + appResult.jobExecUrl); + printInformation(usageData); + + return usageData; + } +} + + diff --git a/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java b/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java new file mode 100644 index 000000000..a0c3cd901 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java @@ -0,0 +1,74 @@ +package com.linkedin.drelephant.tuning.engine; + +import com.avaje.ebean.Expr; +import com.avaje.ebean.ExpressionList; +import com.linkedin.drelephant.tuning.ExecutionEngine; +import java.util.List; +import java.util.Map; +import models.AppResult; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; +import models.TuningParameterConstraint; + + +public class SparkExecutionEngine implements ExecutionEngine { + + @Override + public void computeValuesOfDerivedConfigurationParameters(List derivedParameterList, + List jobSuggestedParamValue) { + + } + + @Override + public ExpressionList getPendingJobs() { + return JobSuggestedParamSet.find.select("*") + .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") + .where() + .or(Expr.or(Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.CREATED), + Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.SENT)), + Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.EXECUTED)) + .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) + .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm + + "." + TuningAlgorithm.TABLE.jobType, TuningAlgorithm.JobType.SPARK.name()) + .eq(JobSuggestedParamSet.TABLE.isParamSetBest, 0); + } + + @Override + public ExpressionList getTuningJobDefinitionsForParameterSuggestion() { + return TuningJobDefinition.find.select("*") + .fetch(TuningJobDefinition.TABLE.job, "*") + .where() + .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) + .eq(TuningJobDefinition.TABLE.tuningAlgorithm+"."+TuningAlgorithm.TABLE.jobType,TuningAlgorithm.JobType.SPARK); + } + + @Override + public Boolean isParamConstraintViolatedPSO(List jobSuggestedParamValueList) { + return null; + } + + @Override + public Boolean isParamConstraintViolatedIPSO(List jobSuggestedParamValueList) { + return null; + } + + @Override + public void parameterOptimizerIPSO(List results, JobExecution jobExecution) { + + } + + @Override + public String parameterGenerationsHBT(List results, List tuningParameters) { + return null; + } + + @Override + public Boolean isParamConstraintViolatedHBT(List jobSuggestedParamValueList) { + return null; + } +} + diff --git a/app/com/linkedin/drelephant/tuning/hbt/BaselineManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/BaselineManagerHBT.java new file mode 100644 index 000000000..444dcc46d --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/hbt/BaselineManagerHBT.java @@ -0,0 +1,39 @@ +package com.linkedin.drelephant.tuning.hbt; + +import com.avaje.ebean.Expr; +import com.linkedin.drelephant.tuning.AbstractBaselineManager; +import com.linkedin.drelephant.util.Utils; +import java.util.List; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import org.apache.log4j.Logger; + + +public class BaselineManagerHBT extends AbstractBaselineManager { + private final Logger logger = Logger.getLogger(getClass()); + public BaselineManagerHBT() { + NUM_JOBS_FOR_BASELINE_DEFAULT = 20; + _numJobsForBaseline = + Utils.getNonNegativeInt(configuration, super.BASELINE_EXECUTION_COUNT, NUM_JOBS_FOR_BASELINE_DEFAULT); + } + + @Override + protected List detectJobsForBaseLineComputation() { + + logger.info("Fetching jobs for HBT which baseline metrics need to be computed"); + List tuningJobDefinitions = TuningJobDefinition.find.where() + .eq(TuningJobDefinition.TABLE.averageResourceUsage, null) + .eq(TuningJobDefinition.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.optimizationAlgo, + TuningAlgorithm.OptimizationAlgo.HBT.name()) + .findList(); + if(tuningJobDefinitions!=null){ + logger.info("Total jobs for Baseline Computation in HBT " +tuningJobDefinitions.size()); + } + return tuningJobDefinitions; + } + + @Override + public String getManagerName() { + return "BaselineManagerHBT"; + } +} diff --git a/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java new file mode 100644 index 000000000..a7ead0cc3 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java @@ -0,0 +1,136 @@ +package com.linkedin.drelephant.tuning.hbt; + +import com.avaje.ebean.Expr; +import com.linkedin.drelephant.AutoTuner; +import com.linkedin.drelephant.ElephantContext; +import com.linkedin.drelephant.tuning.AbstractFitnessManager; +import com.linkedin.drelephant.util.Utils; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import models.AppResult; +import models.JobDefinition; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningJobExecutionParamSet; +import org.apache.log4j.Logger; +import org.apache.hadoop.conf.Configuration; + + +public class FitnessManagerHBT extends AbstractFitnessManager { + private final Logger logger = Logger.getLogger(getClass()); + + public FitnessManagerHBT() { + Configuration configuration = ElephantContext.instance().getAutoTuningConf(); + + // Time duration to wait for computing the fitness of a param set once the corresponding execution is completed + fitnessComputeWaitInterval = + Utils.getNonNegativeLong(configuration, FITNESS_COMPUTE_WAIT_INTERVAL, 5 * AutoTuner.ONE_MIN); + + // Time duration to wait for metrics (resource usage, execution time) of an execution to be computed before + // discarding it for fitness computation + ignoreExecutionWaitInterval = + Utils.getNonNegativeLong(configuration, IGNORE_EXECUTION_WAIT_INTERVAL, 2 * 60 * AutoTuner.ONE_MIN); + + // #executions after which tuning will stop even if parameters don't converge + maxTuningExecutions = Utils.getNonNegativeInt(configuration, MAX_TUNING_EXECUTIONS, 10); + + // #executions before which tuning cannot stop even if parameters converge + minTuningExecutions = Utils.getNonNegativeInt(configuration, MIN_TUNING_EXECUTIONS, 2); + } + + @Override + protected List detectJobsForFitnessComputation() { + logger.info("Fetching completed executions whose fitness are yet to be computed"); + List completedJobExecutionParamSet = new ArrayList(); + + List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.select("*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") + .where() + .or(Expr.or(Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.SUCCEEDED), + Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.FAILED)), + Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.CANCELLED)) + .isNull(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.resourceUsage) + .eq(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet + "." + JobSuggestedParamSet.TABLE.tuningAlgorithm + + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.HBT.name()) + .findList(); + + logger.info("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); + + getCompletedExecution(tuningJobExecutionParamSets, completedJobExecutionParamSet); + + return completedJobExecutionParamSet; + } + + @Override + protected void calculateAndUpdateFitness(JobExecution jobExecution, List results, + TuningJobDefinition tuningJobDefinition, JobSuggestedParamSet jobSuggestedParamSet) { + logger.info("calculateAndUpdateFitness"); + Double totalResourceUsed = 0D; + Double totalInputBytesInBytes = 0D; + for (AppResult appResult : results) { + totalResourceUsed += appResult.resourceUsed; + totalInputBytesInBytes += getTotalInputBytes(appResult); + } + + Long totalRunTime = Utils.getTotalRuntime(results); + Long totalDelay = Utils.getTotalWaittime(results); + Long totalExecutionTime = totalRunTime - totalDelay; + + if (totalExecutionTime != 0) { + updateJobExecution(jobExecution, totalResourceUsed, totalInputBytesInBytes, totalExecutionTime); + } + + if (tuningJobDefinition.averageResourceUsage == null && totalExecutionTime != 0) { + updateTuningJobDefinition(tuningJobDefinition, jobExecution); + } + + //Compute fitness + computeFitness(jobSuggestedParamSet, jobExecution, tuningJobDefinition, results); + } + + protected void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExecution jobExecution, + TuningJobDefinition tuningJobDefinition, List results) { + if (!jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED) + || !jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.DISCARDED)) { + if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { + logger.info("Execution id: " + jobExecution.id + " succeeded"); + updateJobSuggestedParamSetSucceededExecution(jobExecution, jobSuggestedParamSet, tuningJobDefinition); + } else { + // Resetting param set to created state because this case captures the scenarios when + // either the job failed for reasons other than auto tuning or was killed/cancelled/skipped etc. + // In all the above scenarios, fitness cannot be computed for the param set correctly. + // Note that the penalty on failures caused by auto tuning is applied when the job execution is retried + // after failure. + logger.info("HBT Execution id: " + jobExecution.id + " was not successful for reason other than tuning." + + "Resetting param set: " + jobSuggestedParamSet.id + " to CREATED state"); + resetParamSetToCreated(jobSuggestedParamSet); + } + } + } + + /** + * Resets the param set to CREATED state if its fitness is not already computed + * @param jobSuggestedParamSet Param set which is to be reset + */ + protected void resetParamSetToCreated(JobSuggestedParamSet jobSuggestedParamSet) { + if (!jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED) + && !jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.DISCARDED)) { + logger.info("Resetting parameter set to created: " + jobSuggestedParamSet.id); + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; + jobSuggestedParamSet.save(); + } + } + + + @Override + public String getManagerName() { + return "FitnessManagerHBT"; + } +} diff --git a/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java new file mode 100644 index 000000000..e01f1340a --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java @@ -0,0 +1,288 @@ +package com.linkedin.drelephant.tuning.hbt; + +import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; +import com.linkedin.drelephant.tuning.AbstractTuningTypeManager; +import com.linkedin.drelephant.tuning.JobTuningInfo; +import com.linkedin.drelephant.tuning.ExecutionEngine; +import com.linkedin.drelephant.tuning.Particle; +import com.linkedin.drelephant.tuning.TuningHelper; +import controllers.AutoTuningMetricsController; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import models.AppHeuristicResult; +import models.AppResult; +import models.JobDefinition; +import models.JobExecution; +import models.JobSavedState; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; +import org.apache.log4j.Logger; +import play.libs.Json; +import org.apache.commons.io.FileUtils; + +import static java.lang.Math.*; + +import com.avaje.ebean.Expr; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; + + +public class TuningTypeManagerHBT extends AbstractTuningTypeManager { + private final Logger logger = Logger.getLogger(getClass()); + + private Map> usageDataGlobal = null; + + public TuningTypeManagerHBT(ExecutionEngine executionEngine) { + tuningType = "HBT"; + this._executionEngine = executionEngine; + } + + @Override + protected List getPendingParamSets() { + List pendingParamSetList = _executionEngine.getPendingJobs() + .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.optimizationAlgo, + TuningAlgorithm.OptimizationAlgo.HBT.name()) + // .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) + .findList(); + logger.info( + " Number of Pending Jobs for parameter suggestion " + this._executionEngine + " " + pendingParamSetList.size()); + return pendingParamSetList; + } + + @Override + protected List getTuningJobDefinitions() { + List totalJobs = _executionEngine.getTuningJobDefinitionsForParameterSuggestion() + .eq(TuningJobDefinition.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.optimizationAlgo, + TuningAlgorithm.OptimizationAlgo.HBT.name()) + .findList(); + + logger.info(" Number of Total Jobs " + this._executionEngine + " " + totalJobs.size()); + return totalJobs; + } + + @Override + protected void saveJobState(JobTuningInfo jobTuningInfo, JobDefinition job) { + jobTuningInfo.setTunerState("{}"); + } + + @Override + public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { + logger.info("Generating param set for job: " + jobTuningInfo.getTuningJob().jobName); + String newTunedParameters = generateParamSet(jobTuningInfo.getParametersToTune(), jobTuningInfo.getTuningJob()); + jobTuningInfo.setTunerState(newTunedParameters); + return jobTuningInfo; + } + + private String generateParamSet(List tuningParameters, JobDefinition job) { + JobExecution jobExecution = JobExecution.find.select("*") + .where() + .eq(JobExecution.TABLE.job, job) + .order() + .desc(JobExecution.TABLE.updatedTs) + .setMaxRows(1) + .findUnique(); + logger.info("Job Status " + jobExecution.executionState.name()); + if (jobExecution.executionState.name().equals(JobExecution.ExecutionState.IN_PROGRESS.name()) + || jobExecution.executionState.name().equals(JobExecution.ExecutionState.NOT_STARTED.name())) { + logger.info(" Job is still running , cannot use for param generation "); + return ""; + } + + List results = getAppResults(jobExecution); + if (results == null) { + logger.info( + " Job is analyzing , cannot use for param generation " + jobExecution.id + " " + jobExecution.job.id); + return ""; + } + String idParameters = this._executionEngine.parameterGenerationsHBT(results, tuningParameters); + return idParameters.toString(); + } + + private List getAppResults(JobExecution jobExecution) { + List results = null; + try { + results = AppResult.find.select("*") + .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*") + .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, + "*") + .where() + .eq(AppResult.TABLE.FLOW_EXEC_ID, jobExecution.flowExecution.flowExecId) + .eq(AppResult.TABLE.JOB_EXEC_ID, jobExecution.jobExecId) + .findList(); + } catch (Exception e) { + logger.warn(" Job Analysis is not completed . "); + return results; + } + return results; + } + + /** + * For every tuning info: + * For every new particle: + * From the tuner set extract the list of suggested parameters + * Check penalty + * Save the param in the job execution table by creating execution instance (Create an entry into param_set table) + * Update the execution instance in each of the suggested params (Update the param_set_id in each of the prams) + * save th suggested parameters + * update the paramsetid in the particle and add particle to a particlelist + * Update the tunerstate from the updated particles + * save the tuning info in db + * + * @param jobTuningInfoList JobTuningInfo List + */ + protected Boolean updateDatabase(List jobTuningInfoList) { + logger.info("Updating new parameter suggestion in database HBT"); + if (jobTuningInfoList == null) { + logger.info("No new parameter suggestion to update"); + return false; + } + + for (JobTuningInfo jobTuningInfo : jobTuningInfoList) { + logger.info("Updating new parameter suggestion for job:" + jobTuningInfo.getTuningJob().jobDefId); + + JobDefinition job = jobTuningInfo.getTuningJob(); + List paramList = jobTuningInfo.getParametersToTune(); + String stringTunerState = jobTuningInfo.getTunerState(); + if (stringTunerState == null || stringTunerState.length() == 0) { + logger.error("Suggested parameter suggestion is empty for job id: " + job.jobDefId); + continue; + } + + TuningJobDefinition tuningJobDefinition = TuningHelper.getTuningJobDefinition(job); + + List derivedParameterList = TuningHelper.getDerivedParameterList(tuningJobDefinition); + + logger.info("No. of derived tuning params for job " + tuningJobDefinition.job.jobName + ": " + + derivedParameterList.size()); + + List jobSuggestedParamValueList = getParamValueList(stringTunerState); + _executionEngine.computeValuesOfDerivedConfigurationParameters(derivedParameterList, jobSuggestedParamValueList); + JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); + jobSuggestedParamSet.jobDefinition = job; + jobSuggestedParamSet.tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; + jobSuggestedParamSet.isParamSetDefault = false; + jobSuggestedParamSet.isParamSetBest = false; + if (isParamConstraintViolated(jobSuggestedParamValueList)) { + penaltyApplication(jobSuggestedParamSet, tuningJobDefinition); + } else { + logger.info(" Parameters constraints not violeted "); + jobSuggestedParamSet.areConstraintsViolated = false; + processParamSetStatus(jobSuggestedParamSet); + } + saveSuggestedParamSet(jobSuggestedParamSet); + + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { + jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; + } + logger.info(" Job Suggested list " + jobSuggestedParamValueList.size()); + saveSuggestedParams(jobSuggestedParamValueList); + } + + return true; + } + + private void processParamSetStatus(JobSuggestedParamSet jobSuggestedParamSet) { + TuningJobDefinition tuningJobDefinition1 = TuningJobDefinition.find.select("*") + .where() + .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, jobSuggestedParamSet.jobDefinition.id) + .setMaxRows(1) + .findUnique(); + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; + //handleDiscarding(tuningJobDefinition1, jobSuggestedParamSet); + } + + /* private void handleDiscarding(TuningJobDefinition tuningJobDefinition1, JobSuggestedParamSet jobSuggestedParamSet) { + if (tuningJobDefinition1.autoApply) { + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; + } else { + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.DISCARDED; + } + List tempJobSuggestedParamSet = JobSuggestedParamSet.find.select("*") + .where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, tuningJobDefinition1.job.id) + .findList(); + Boolean isManuallyOverriden = false; + for (JobSuggestedParamSet jobSuggestedParamSet1 : tempJobSuggestedParamSet) { + if (jobSuggestedParamSet1.isManuallyOverridenParameter) { + isManuallyOverriden = true; + } + } + if (isManuallyOverriden) { + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.DISCARDED; + } + }*/ + + private void penaltyApplication(JobSuggestedParamSet jobSuggestedParamSet, TuningJobDefinition tuningJobDefinition) { + logger.info("Parameter constraint violated. Applying penalty."); + int penaltyConstant = 4; + Double averageResourceUsagePerGBInput = + tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; + Double maxDesiredResourceUsagePerGBInput = + averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; + + jobSuggestedParamSet.areConstraintsViolated = true; + jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED; + } + + /** + * Saves the list of suggested parameter values to database + * @param jobSuggestedParamValueList Suggested Parameter Values List + */ + private void saveSuggestedParams(List jobSuggestedParamValueList) { + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { + jobSuggestedParamValue.save(); + } + } + + /** + * Saves the suggested param set in the database and returns the param set id + * @param jobSuggestedParamSet JobExecution + * @return Param Set Id + */ + private Long saveSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { + jobSuggestedParamSet.save(); + return jobSuggestedParamSet.id; + } + + private List getParamValueList(String tunerState) { + List jobSuggestedParamValueList = new ArrayList(); + for (String parameter : tunerState.split("\n")) { + logger.info(" Parameter values " + parameter); + String paramIDValues[] = parameter.split("\t"); + if (paramIDValues.length == 2) { + JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); + jobSuggestedParamValue.tuningParameter = TuningParameter.find.byId(Integer.parseInt(paramIDValues[0])); + jobSuggestedParamValue.paramValue = Double.parseDouble(paramIDValues[1]); + jobSuggestedParamValueList.add(jobSuggestedParamValue); + } + } + logger.info(" Job Suggested Values " + jobSuggestedParamValueList.size()); + return jobSuggestedParamValueList; + } + + @Override + protected void updateBoundryConstraint(List tuningParameterList, JobDefinition job) { + + } + + @Override + public boolean isParamConstraintViolated(List jobSuggestedParamValues) { + + return _executionEngine.isParamConstraintViolatedHBT(jobSuggestedParamValues); + } + + @Override + public String getManagerName() { + return "TuningTypeManagerHBT"; + } +} diff --git a/app/com/linkedin/drelephant/tuning/obt/BaselineManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/BaselineManagerOBT.java new file mode 100644 index 000000000..8c9e23262 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/obt/BaselineManagerOBT.java @@ -0,0 +1,43 @@ +package com.linkedin.drelephant.tuning.obt; + +import com.avaje.ebean.Expr; +import com.linkedin.drelephant.tuning.AbstractBaselineManager; +import com.linkedin.drelephant.util.Utils; +import java.util.List; +import models.JobExecution; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningJobExecutionParamSet; +import org.apache.log4j.Logger; + + +public class BaselineManagerOBT extends AbstractBaselineManager { + private final Logger logger = Logger.getLogger(getClass()); + + public BaselineManagerOBT() { + NUM_JOBS_FOR_BASELINE_DEFAULT = 30; + _numJobsForBaseline = + Utils.getNonNegativeInt(configuration, super.BASELINE_EXECUTION_COUNT, NUM_JOBS_FOR_BASELINE_DEFAULT); + } + + @Override + protected List detectJobsForBaseLineComputation() { + logger.info("Fetching jobs for which baseline metrics need to be computed"); + List tuningJobDefinitions = TuningJobDefinition.find.where() + .eq(TuningJobDefinition.TABLE.averageResourceUsage, null) + .or(Expr.eq(TuningJobDefinition.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.optimizationAlgo, + TuningAlgorithm.OptimizationAlgo.PSO.name()), + Expr.eq(TuningJobDefinition.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.optimizationAlgo, + TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name())) + .findList(); + if(tuningJobDefinitions!=null){ + logger.info("Total jobs for Baseline Computation in OBT"); + } + return tuningJobDefinitions; + } + + @Override + public String getManagerName() { + return "BaselineManagerOBT"; + } +} diff --git a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java new file mode 100644 index 000000000..527c4671b --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java @@ -0,0 +1,75 @@ +package com.linkedin.drelephant.tuning.obt; + +import com.linkedin.drelephant.AutoTuner; +import com.linkedin.drelephant.ElephantContext; +import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; +import com.linkedin.drelephant.tuning.AbstractFitnessManager; +import com.linkedin.drelephant.util.Utils; +import java.util.List; +import models.AppHeuristicResult; +import models.AppHeuristicResultDetails; +import models.AppResult; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.TuningJobDefinition; +import models.TuningJobExecutionParamSet; +import org.apache.log4j.Logger; +import org.apache.hadoop.conf.Configuration; + +public abstract class FitnessManagerOBT extends AbstractFitnessManager { + private final Logger logger = Logger.getLogger(getClass()); + + public FitnessManagerOBT() { + Configuration configuration = ElephantContext.instance().getAutoTuningConf(); + + // Time duration to wait for computing the fitness of a param set once the corresponding execution is completed + fitnessComputeWaitInterval = + Utils.getNonNegativeLong(configuration, FITNESS_COMPUTE_WAIT_INTERVAL, 5 * AutoTuner.ONE_MIN); + + // Time duration to wait for metrics (resource usage, execution time) of an execution to be computed before + // discarding it for fitness computation + ignoreExecutionWaitInterval = + Utils.getNonNegativeLong(configuration, IGNORE_EXECUTION_WAIT_INTERVAL, 2 * 60 * AutoTuner.ONE_MIN); + + // #executions after which tuning will stop even if parameters don't converge + maxTuningExecutions = Utils.getNonNegativeInt(configuration, MAX_TUNING_EXECUTIONS, 39); + + // #executions before which tuning cannot stop even if parameters converge + minTuningExecutions = Utils.getNonNegativeInt(configuration, MIN_TUNING_EXECUTIONS, 18); + } + + @Override + protected void calculateAndUpdateFitness(JobExecution jobExecution, List results, + TuningJobDefinition tuningJobDefinition, JobSuggestedParamSet jobSuggestedParamSet) { + Double totalResourceUsed = 0D; + Double totalInputBytesInBytes = 0D; + for (AppResult appResult : results) { + totalResourceUsed += appResult.resourceUsed; + totalInputBytesInBytes += getTotalInputBytes(appResult); + } + + Long totalRunTime = Utils.getTotalRuntime(results); + Long totalDelay = Utils.getTotalWaittime(results); + Long totalExecutionTime = totalRunTime - totalDelay; + + if (totalExecutionTime != 0) { + updateJobExecution(jobExecution, totalResourceUsed, totalInputBytesInBytes, totalExecutionTime); + } + + if (tuningJobDefinition.averageResourceUsage == null && totalExecutionTime != 0) { + updateTuningJobDefinition(tuningJobDefinition, jobExecution); + } + + //Compute fitness + computeFitness(jobSuggestedParamSet, jobExecution, tuningJobDefinition, results); + } + + + + protected abstract void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExecution jobExecution, + TuningJobDefinition tuningJobDefinition, List results); + + + + +} diff --git a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java new file mode 100644 index 000000000..06d01793c --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java @@ -0,0 +1,91 @@ +package com.linkedin.drelephant.tuning.obt; + +import com.avaje.ebean.Expr; +import com.linkedin.drelephant.AutoTuner; +import com.linkedin.drelephant.ElephantContext; +import com.linkedin.drelephant.tuning.AbstractFitnessManager; +import com.linkedin.drelephant.util.Utils; +import java.util.ArrayList; +import java.util.List; +import models.AppResult; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningJobExecutionParamSet; +import org.apache.log4j.Logger; +import org.apache.hadoop.conf.Configuration; + + +public class FitnessManagerOBTAlgoIPSO extends FitnessManagerOBT { + + private final Logger logger = Logger.getLogger(getClass()); + + public FitnessManagerOBTAlgoIPSO() { + super(); + } + + @Override + protected List detectJobsForFitnessComputation() { + logger.info("Fetching completed executions whose fitness are yet to be computed"); + List completedJobExecutionParamSet = new ArrayList(); + + List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.select("*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") + .where() + .or(Expr.or(Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.SUCCEEDED), + Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.FAILED)), + Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.CANCELLED)) + .isNull(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.resourceUsage) + .eq(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet + "." + JobSuggestedParamSet.TABLE.tuningAlgorithm + + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) + .findList(); + + logger.info("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); + + getCompletedExecution(tuningJobExecutionParamSets, completedJobExecutionParamSet); + + return completedJobExecutionParamSet; + } + + protected void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExecution jobExecution, + TuningJobDefinition tuningJobDefinition, List results) { + if (!jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED)) { + if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { + logger.info("Execution id: " + jobExecution.id + " succeeded"); + parameterSpaceOptimization(jobSuggestedParamSet, jobExecution, results); + updateJobSuggestedParamSetSucceededExecution(jobExecution, jobSuggestedParamSet, tuningJobDefinition); + } else { + // Resetting param set to created state because this case captures the scenarios when + // either the job failed for reasons other than auto tuning or was killed/cancelled/skipped etc. + // In all the above scenarios, fitness cannot be computed for the param set correctly. + // Note that the penalty on failures caused by auto tuning is applied when the job execution is retried + // after failure. + logger.info("Execution id: " + jobExecution.id + " was not successful for reason other than tuning." + + "Resetting param set: " + jobSuggestedParamSet.id + " to CREATED state"); + resetParamSetToCreated(jobSuggestedParamSet); + } + } + } + + /* + Todo : Move this method to TuningType + */ + private void parameterSpaceOptimization(JobSuggestedParamSet jobSuggestedParamSet, JobExecution jobExecution, + List results) { + TuningTypeManagerOBT optimizeManager = + OptimizationAlgoFactory.getOptimizationAlogrithm(jobSuggestedParamSet.tuningAlgorithm); + if (optimizeManager != null) { + optimizeManager.parameterOptimizer(results,jobExecution); + } + } + + @Override + public String getManagerName() { + return "FitnessManagerOBTAlgoIPSO"; + } +} diff --git a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoPSO.java b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoPSO.java new file mode 100644 index 000000000..c5fdb1a54 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoPSO.java @@ -0,0 +1,78 @@ +package com.linkedin.drelephant.tuning.obt; + +import com.avaje.ebean.Expr; +import com.linkedin.drelephant.AutoTuner; +import com.linkedin.drelephant.ElephantContext; +import com.linkedin.drelephant.tuning.AbstractFitnessManager; +import com.linkedin.drelephant.util.Utils; +import java.util.ArrayList; +import java.util.List; +import models.AppResult; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningJobExecutionParamSet; +import org.apache.log4j.Logger; +import org.apache.hadoop.conf.Configuration; + + +public class FitnessManagerOBTAlgoPSO extends FitnessManagerOBT { + private final Logger logger = Logger.getLogger(getClass()); + + public FitnessManagerOBTAlgoPSO() { + super(); + } + + @Override + protected List detectJobsForFitnessComputation() { + logger.info("Fetching completed executions whose fitness are yet to be computed"); + List completedJobExecutionParamSet = new ArrayList(); + + List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.select("*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") + .where() + .or(Expr.or(Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.SUCCEEDED), + Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.FAILED)), + Expr.eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, + JobExecution.ExecutionState.CANCELLED)) + .isNull(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.resourceUsage) + .eq(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet + "." + JobSuggestedParamSet.TABLE.tuningAlgorithm + + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.PSO.name()) + .findList(); + + logger.info("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); + + getCompletedExecution(tuningJobExecutionParamSets,completedJobExecutionParamSet); + + return completedJobExecutionParamSet; + } + + @Override + protected void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExecution jobExecution, + TuningJobDefinition tuningJobDefinition, List results) { + if (!jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED)) { + if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { + logger.info("Execution id: " + jobExecution.id + " succeeded"); + updateJobSuggestedParamSetSucceededExecution(jobExecution, jobSuggestedParamSet, tuningJobDefinition); + } else { + // Resetting param set to created state because this case captures the scenarios when + // either the job failed for reasons other than auto tuning or was killed/cancelled/skipped etc. + // In all the above scenarios, fitness cannot be computed for the param set correctly. + // Note that the penalty on failures caused by auto tuning is applied when the job execution is retried + // after failure. + logger.info("Execution id: " + jobExecution.id + " was not successful for reason other than tuning." + + "Resetting param set: " + jobSuggestedParamSet.id + " to CREATED state"); + resetParamSetToCreated(jobSuggestedParamSet); + } + } + } + + @Override + public String getManagerName() { + return "FitnessManagerOBTAlgoPSO"; + } +} diff --git a/app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java b/app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java new file mode 100644 index 000000000..cf94c88f5 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java @@ -0,0 +1,36 @@ +package com.linkedin.drelephant.tuning.obt; + +import com.linkedin.drelephant.tuning.engine.MRExecutionEngine; +import com.linkedin.drelephant.tuning.engine.SparkExecutionEngine; +import models.TuningAlgorithm; +import org.apache.log4j.Logger; + + +public class OptimizationAlgoFactory { + private static final Logger logger = Logger.getLogger(OptimizationAlgoFactory.class); + + public static TuningTypeManagerOBT getOptimizationAlogrithm(TuningAlgorithm tuningAlgorithm) { + if (tuningAlgorithm.optimizationAlgo.name().equals(TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) + && tuningAlgorithm.jobType.name().equals(TuningAlgorithm.JobType.PIG.name())) { + logger.info("OPTIMIZATION ALGORITHM PSO_IPSO MR"); + return new TuningTypeManagerOBTAlgoIPSO(new MRExecutionEngine()); + } + if (tuningAlgorithm.optimizationAlgo.name().equals(TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) + && tuningAlgorithm.jobType.name().equals(TuningAlgorithm.JobType.SPARK.name())) { + logger.info("OPTIMIZATION ALGORITHM PSO_IPSO SPARK"); + return new TuningTypeManagerOBTAlgoIPSO(new SparkExecutionEngine()); + } + if (tuningAlgorithm.optimizationAlgo.name().equals(TuningAlgorithm.OptimizationAlgo.PSO.name()) + && tuningAlgorithm.jobType.name().equals(TuningAlgorithm.JobType.PIG.name())) { + logger.info("OPTIMIZATION ALGORITHM PSO PIG"); + return new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); + } + if (tuningAlgorithm.optimizationAlgo.name().equals(TuningAlgorithm.OptimizationAlgo.PSO.name()) + && tuningAlgorithm.jobType.name().equals(TuningAlgorithm.JobType.SPARK.name())) { + logger.info("OPTIMIZATION ALGORITHM PSO SPARK"); + return new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); + } + logger.info("OPTIMIZATION ALGORITHM PSO"); + return null; + } +} diff --git a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java new file mode 100644 index 000000000..91c75bf09 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java @@ -0,0 +1,403 @@ +package com.linkedin.drelephant.tuning.obt; + +import com.avaje.ebean.Expr; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkedin.drelephant.ElephantContext; +import com.linkedin.drelephant.tuning.AbstractTuningTypeManager; +import com.linkedin.drelephant.tuning.JobTuningInfo; +import com.linkedin.drelephant.tuning.Particle; +import com.linkedin.drelephant.tuning.ExecutionEngine; +import com.linkedin.drelephant.tuning.TuningHelper; +import controllers.AutoTuningMetricsController; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import models.AppResult; +import models.JobDefinition; +import models.JobExecution; +import models.JobSavedState; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; +import play.libs.Json; +import org.apache.hadoop.conf.Configuration; + + +public abstract class TuningTypeManagerOBT extends AbstractTuningTypeManager { + + private static final String PARAMS_TO_TUNE_FIELD_NAME = "parametersToTune"; + private static final String PYTHON_PATH_CONF = "python.path"; + private static final String PSO_DIR_PATH_ENV_VARIABLE = "PSO_DIR_PATH"; + private static final String PYTHON_PATH_ENV_VARIABLE = "PYTHONPATH"; + private String PYTHON_PATH = null; + private String TUNING_SCRIPT_PATH = null; + private final Logger logger = Logger.getLogger(getClass()); + + public TuningTypeManagerOBT() { + tuningType = "OBT"; + Configuration configuration = ElephantContext.instance().getAutoTuningConf(); + + PYTHON_PATH = configuration.get(PYTHON_PATH_CONF); + if (PYTHON_PATH == null) { + PYTHON_PATH = System.getenv(PYTHON_PATH_ENV_VARIABLE); + } + String PSO_DIR_PATH = System.getenv(PSO_DIR_PATH_ENV_VARIABLE); + + if (PSO_DIR_PATH == null) { + throw new NullPointerException("Couldn't find directory containing PSO scripts"); + } + if (PYTHON_PATH == null) { + PYTHON_PATH = "python"; + } + TUNING_SCRIPT_PATH = PSO_DIR_PATH + "/pso_param_generation.py"; + logger.info("Tuning script path: " + TUNING_SCRIPT_PATH); + logger.info("Python path: " + PYTHON_PATH); + } + + @Override + protected void saveJobState(JobTuningInfo jobTuningInfo, JobDefinition job) { + boolean validSavedState = true; + JobSavedState jobSavedState = JobSavedState.find.byId(job.id); + if (jobSavedState != null && jobSavedState.isValid()) { + String savedState = new String(jobSavedState.savedState); + ObjectNode jsonSavedState = (ObjectNode) Json.parse(savedState); + JsonNode jsonCurrentPopulation = jsonSavedState.get(JSON_CURRENT_POPULATION_KEY); + List currentPopulation = jsonToParticleList(jsonCurrentPopulation); + for (Particle particle : currentPopulation) { + Long paramSetId = particle.getParamSetId(); + + logger.info("Param set id: " + paramSetId.toString()); + JobSuggestedParamSet jobSuggestedParamSet = + JobSuggestedParamSet.find.select("*").where().eq(JobSuggestedParamSet.TABLE.id, paramSetId).findUnique(); + + if (jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED) + && jobSuggestedParamSet.fitness != null) { + particle.setFitness(jobSuggestedParamSet.fitness); + } else { + validSavedState = false; + logger.error("Invalid saved state: Fitness of previous execution not computed."); + break; + } + } + + if (validSavedState) { + JsonNode updatedJsonCurrentPopulation = particleListToJson(currentPopulation); + jsonSavedState.set(JSON_CURRENT_POPULATION_KEY, updatedJsonCurrentPopulation); + savedState = Json.stringify(jsonSavedState); + jobTuningInfo.setTunerState(savedState); + } + } else { + logger.info("Saved state empty for job: " + job.jobDefId); + validSavedState = false; + } + + if (!validSavedState) { + jobTuningInfo.setTunerState("{}"); + } + } + + /** + * Converts a json to list of particles + * @param jsonParticleList A list of configurations (particles) in json + * @return Particle List + */ + private List jsonToParticleList(JsonNode jsonParticleList) { + + List particleList = new ArrayList(); + if (jsonParticleList == null) { + logger.info("Null json, empty particle list returned"); + } else { + for (JsonNode jsonParticle : jsonParticleList) { + Particle particle; + particle = Json.fromJson(jsonParticle, Particle.class); + if (particle != null) { + particleList.add(particle); + } + } + } + return particleList; + } + + /** + * Converts a list of particles to json + * @param particleList Particle List + * @return JsonNode + */ + private JsonNode particleListToJson(List particleList) { + JsonNode jsonNode; + + if (particleList == null) { + jsonNode = JsonNodeFactory.instance.objectNode(); + logger.info("Null particleList, returning empty json"); + } else { + jsonNode = Json.toJson(particleList); + } + return jsonNode; + } + + /** + * Interacts with python scripts to generate new parameter suggestions + * @param jobTuningInfo Job tuning information + * @return Updated job tuning information + */ + + @Override + public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { + logger.info("Generating param set for job: " + jobTuningInfo.getTuningJob().jobName); + + JobTuningInfo newJobTuningInfo = new JobTuningInfo(); + newJobTuningInfo.setTuningJob(jobTuningInfo.getTuningJob()); + newJobTuningInfo.setParametersToTune(jobTuningInfo.getParametersToTune()); + newJobTuningInfo.setJobType(jobTuningInfo.getJobType()); + + JsonNode jsonJobTuningInfo = Json.toJson(jobTuningInfo); + String parametersToTune = jsonJobTuningInfo.get(PARAMS_TO_TUNE_FIELD_NAME).toString(); + String stringTunerState = jobTuningInfo.getTunerState(); + stringTunerState = stringTunerState.replaceAll("\\s+", ""); + String jobType = jobTuningInfo.getJobType().toString(); + + logger.info(" ID " + jobTuningInfo.getTuningJob().id); + + int swarmSize = getSwarmSize(); + + List error = new ArrayList(); + logger.debug("String State " + stringTunerState); + try { + + Process p = Runtime.getRuntime() + .exec(PYTHON_PATH + " " + TUNING_SCRIPT_PATH + " " + stringTunerState + " " + parametersToTune + " " + jobType + + " " + swarmSize); + logger.info( + PYTHON_PATH + " " + TUNING_SCRIPT_PATH + " " + stringTunerState + " " + parametersToTune + " " + jobType + " " + + swarmSize); + + BufferedReader inputStream = new BufferedReader(new InputStreamReader(p.getInputStream())); + BufferedReader errorStream = new BufferedReader(new InputStreamReader(p.getErrorStream())); + String updatedStringTunerState = inputStream.readLine(); + logger.info("Param Generator Testing" + updatedStringTunerState); + newJobTuningInfo.setTunerState(updatedStringTunerState); + String errorLine; + while ((errorLine = errorStream.readLine()) != null) { + error.add(errorLine); + } + if (error.size() != 0) { + logger.error("Error in python script running PSO: " + error.toString()); + } + } catch (IOException e) { + logger.error("Error in generateParamSet()", e); + } + return newJobTuningInfo; + } + + /** + * For every tuning info: + * For every new particle: + * From the tuner set extract the list of suggested parameters + * Check penalty + * Save the param in the job execution table by creating execution instance (Create an entry into param_set table) + * Update the execution instance in each of the suggested params (Update the param_set_id in each of the prams) + * save th suggested parameters + * update the paramsetid in the particle and add particle to a particlelist + * Update the tunerstate from the updated particles + * save the tuning info in db + * + * @param jobTuningInfoList JobTuningInfo List + */ + protected Boolean updateDatabase(List jobTuningInfoList) { + logger.info("Updating new parameter suggestion in database"); + if (jobTuningInfoList == null) { + logger.info("No new parameter suggestion to update"); + return false; + } + + int paramSetNotGeneratedJobs = jobTuningInfoList.size(); + + for (JobTuningInfo jobTuningInfo : jobTuningInfoList) { + logger.info("Updating new parameter suggestion for job:" + jobTuningInfo.getTuningJob().jobDefId); + + JobDefinition job = jobTuningInfo.getTuningJob(); + List paramList = jobTuningInfo.getParametersToTune(); + String stringTunerState = jobTuningInfo.getTunerState(); + if (stringTunerState == null) { + logger.error("Suggested parameter suggestion is empty for job id: " + job.jobDefId); + continue; + } + + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") + .fetch(TuningJobDefinition.TABLE.job, "*") + .where() + .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, job.id) + .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) + .findUnique(); + + List derivedParameterList = new ArrayList(); + derivedParameterList = TuningParameter.find.where() + .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, + tuningJobDefinition.tuningAlgorithm.id) + .eq(TuningParameter.TABLE.isDerived, 1) + .findList(); + + logger.info("No. of derived tuning params for job " + tuningJobDefinition.job.jobName + ": " + + derivedParameterList.size()); + + JsonNode jsonTunerState = Json.parse(stringTunerState); + JsonNode jsonSuggestedPopulation = jsonTunerState.get(JSON_CURRENT_POPULATION_KEY); + + if (jsonSuggestedPopulation == null) { + continue; + } + + paramSetNotGeneratedJobs--; + + List suggestedPopulation = jsonToParticleList(jsonSuggestedPopulation); + + for (Particle suggestedParticle : suggestedPopulation) { + AutoTuningMetricsController.markParamSetGenerated(); + List jobSuggestedParamValueList = getParamValueList(suggestedParticle, paramList); + _executionEngine.computeValuesOfDerivedConfigurationParameters(derivedParameterList, + jobSuggestedParamValueList); + JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); + jobSuggestedParamSet.jobDefinition = job; + jobSuggestedParamSet.tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; + jobSuggestedParamSet.isParamSetDefault = false; + jobSuggestedParamSet.isParamSetBest = false; + if (isParamConstraintViolated(jobSuggestedParamValueList)) { + logger.info("Parameter constraint violated. Applying penalty."); + int penaltyConstant = 3; + Double averageResourceUsagePerGBInput = + tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; + Double maxDesiredResourceUsagePerGBInput = + averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; + + jobSuggestedParamSet.areConstraintsViolated = true; + jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED; + } else { + jobSuggestedParamSet.areConstraintsViolated = false; + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; + } + Long paramSetId = saveSuggestedParamSet(jobSuggestedParamSet); + + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { + jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; + } + suggestedParticle.setPramSetId(paramSetId); + saveSuggestedParams(jobSuggestedParamValueList); + } + + JsonNode updatedJsonSuggestedPopulation = particleListToJson(suggestedPopulation); + + ObjectNode updatedJsonTunerState = (ObjectNode) jsonTunerState; + updatedJsonTunerState.put(JSON_CURRENT_POPULATION_KEY, updatedJsonSuggestedPopulation); + String updatedStringTunerState = Json.stringify(updatedJsonTunerState); + jobTuningInfo.setTunerState(updatedStringTunerState); + } + AutoTuningMetricsController.setParamSetGenerateWaitJobs(paramSetNotGeneratedJobs); + saveTunerState(jobTuningInfoList); + return true; + } + + /** + * Returns list of suggested parameters + * @param particle Particle (configuration) + * @param paramList Parameter List + * @return Suggested Param Value List + */ + private List getParamValueList(Particle particle, List paramList) { + logger.debug("Particle is: " + Json.toJson(particle)); + List jobSuggestedParamValueList = new ArrayList(); + + if (particle != null) { + List candidate = particle.getCandidate(); + + if (candidate != null) { + logger.debug("Candidate is:" + Json.toJson(candidate)); + for (int i = 0; i < candidate.size() && i < paramList.size(); i++) { + logger.info("Candidate is " + candidate); + + JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); + int paramId = paramList.get(i).id; + jobSuggestedParamValue.tuningParameter = TuningParameter.find.byId(paramId); + jobSuggestedParamValue.paramValue = candidate.get(i); + jobSuggestedParamValueList.add(jobSuggestedParamValue); + } + } else { + logger.info("Candidate is null"); + } + } else { + logger.info("Particle null"); + } + return jobSuggestedParamValueList; + } + + /** + * Save the tuning info list to the database + * @param jobTuningInfoList Tuning Info List + */ + private void saveTunerState(List jobTuningInfoList) { + for (JobTuningInfo jobTuningInfo : jobTuningInfoList) { + if (jobTuningInfo.getTunerState() == null) { + continue; + } + JobSavedState jobSavedState = JobSavedState.find.byId(jobTuningInfo.getTuningJob().id); + if (jobSavedState == null) { + jobSavedState = new JobSavedState(); + jobSavedState.jobDefinitionId = jobTuningInfo.getTuningJob().id; + } + jobSavedState.savedState = jobTuningInfo.getTunerState().getBytes(); + jobSavedState.save(); + } + } + + /** + * Saves the list of suggested parameter values to database + * @param jobSuggestedParamValueList Suggested Parameter Values List + */ + private void saveSuggestedParams(List jobSuggestedParamValueList) { + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { + jobSuggestedParamValue.save(); + } + } + + /** + * Saves the suggested param set in the database and returns the param set id + * @param jobSuggestedParamSet JobExecution + * @return Param Set Id + */ + private Long saveSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { + jobSuggestedParamSet.save(); + return jobSuggestedParamSet.id; + } + + @Override + public String getManagerName() { + return "TuningTypeManagerOBT"; + } + + /* + Intialize any prequisite require for Optimizer + Calls once in lifetime of the flow + */ + public abstract void intializePrerequisite(TuningAlgorithm tuningAlgorithm, + JobSuggestedParamSet jobSuggestedParamSet); + + /* + Optimize search space + call after each execution of flow + */ + public abstract void parameterOptimizer(List appResults, JobExecution jobExecution); + + + + protected abstract int getSwarmSize(); +} diff --git a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoIPSO.java b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoIPSO.java new file mode 100644 index 000000000..5e0655f97 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoIPSO.java @@ -0,0 +1,135 @@ +package com.linkedin.drelephant.tuning.obt; + +import com.avaje.ebean.Expr; +import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; +import com.linkedin.drelephant.tuning.ExecutionEngine; +import com.linkedin.drelephant.tuning.engine.MRExecutionEngine; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import models.AppResult; +import models.JobDefinition; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; +import models.TuningParameterConstraint; +import org.apache.log4j.Logger; + +import static java.lang.Math.*; + + +public class TuningTypeManagerOBTAlgoIPSO extends TuningTypeManagerOBT { + + private static final Logger logger = Logger.getLogger(TuningTypeManagerOBTAlgoIPSO.class); + private Map> usageDataGlobal = null; + + + enum UsageCounterSchema {USED_PHYSICAL_MEMORY, USED_VIRTUAL_MEMORY, USED_HEAP_MEMORY} + + public TuningTypeManagerOBTAlgoIPSO(ExecutionEngine executionEngine) { + tuningAlgorithm = TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name(); + this._executionEngine = executionEngine; + } + + @Override + protected void updateBoundryConstraint(List tuningParameterList, JobDefinition job) { + applyIntelligenceOnParameter(tuningParameterList, job); + } + + @Override + public boolean isParamConstraintViolated(List jobSuggestedParamValues) { + return _executionEngine.isParamConstraintViolatedIPSO(jobSuggestedParamValues); + } + + + + @Override + protected List getPendingParamSets() { + List pendingParamSetList = _executionEngine.getPendingJobs() + .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm + + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) + .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) + .findList(); + return pendingParamSetList; + } + + @Override + protected List getTuningJobDefinitions() { + return _executionEngine.getTuningJobDefinitionsForParameterSuggestion() + .eq(TuningJobDefinition.TABLE.tuningAlgorithm + + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) + .findList(); + } + + @Override + public void intializePrerequisite(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { + logger.info(" Intialize Prerequisite "); + setDefaultParameterValues(tuningAlgorithm, jobSuggestedParamSet); + } + + private void setDefaultParameterValues(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { + List tuningParameters = + TuningParameter.find.where().eq(TuningParameter.TABLE.tuningAlgorithm, tuningAlgorithm).findList(); + for (TuningParameter tuningParameter : tuningParameters) { + TuningParameterConstraint tuningParameterConstraint = new TuningParameterConstraint(); + tuningParameterConstraint.jobDefinition = jobSuggestedParamSet.jobDefinition; + tuningParameterConstraint.tuningParameter = tuningParameter; + tuningParameterConstraint.lowerBound = tuningParameter.minValue; + tuningParameterConstraint.upperBound = tuningParameter.maxValue; + tuningParameterConstraint.constraintType = TuningParameterConstraint.ConstraintType.BOUNDARY; + tuningParameterConstraint.save(); + } + } + + + + + + @Override + public void parameterOptimizer(List appResults, JobExecution jobExecution) { + logger.info(" IPSO Optimizer"); + _executionEngine.parameterOptimizerIPSO(appResults,jobExecution); + } + + + public void applyIntelligenceOnParameter(List tuningParameterList, JobDefinition job) { + logger.info(" Apply Intelligence"); + List tuningParameterConstraintList = new ArrayList(); + try { + tuningParameterConstraintList = TuningParameterConstraint.find.where() + .eq("job_definition_id", job.id) + .eq(TuningParameterConstraint.TABLE.constraintType, TuningParameterConstraint.ConstraintType.BOUNDARY) + .findList(); + } catch (NullPointerException e) { + logger.info("No boundary constraints found for job: " + job.jobName); + } + + Map paramConstrainIndexMap = new HashMap(); + int i = 0; + for (TuningParameterConstraint tuningParameterConstraint : tuningParameterConstraintList) { + paramConstrainIndexMap.put(tuningParameterConstraint.tuningParameter.id, i); + i += 1; + } + + for (TuningParameter tuningParameter : tuningParameterList) { + if (paramConstrainIndexMap.containsKey(tuningParameter.id)) { + int index = paramConstrainIndexMap.get(tuningParameter.id); + tuningParameter.minValue = tuningParameterConstraintList.get(index).lowerBound; + tuningParameter.maxValue = tuningParameterConstraintList.get(index).upperBound; + } + } + } + + @Override + public int getSwarmSize() { + return 2; + } + + public String getManagerName() { + return "TuningTypeManagerOBTAlgoIPSO" + this._executionEngine.getClass().getSimpleName(); + } +} diff --git a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoPSO.java b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoPSO.java new file mode 100644 index 000000000..ae2e819a5 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoPSO.java @@ -0,0 +1,71 @@ +package com.linkedin.drelephant.tuning.obt; + +import com.avaje.ebean.Expr; +import com.linkedin.drelephant.tuning.ExecutionEngine; +import java.util.List; +import java.util.Map; +import models.AppResult; +import models.JobDefinition; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; + + +public class TuningTypeManagerOBTAlgoPSO extends TuningTypeManagerOBT{ + public TuningTypeManagerOBTAlgoPSO(ExecutionEngine executionEngine) { + tuningAlgorithm = TuningAlgorithm.OptimizationAlgo.PSO.name(); + this._executionEngine=executionEngine; + } + + @Override + protected void updateBoundryConstraint(List tuningParameterList, JobDefinition job) { + + } + + @Override + public boolean isParamConstraintViolated(List jobSuggestedParamValues) { + return _executionEngine.isParamConstraintViolatedPSO(jobSuggestedParamValues); + } + + + + @Override + protected List getPendingParamSets() { + List pendingParamSetList = _executionEngine.getPendingJobs() + .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm + + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.PSO.name()) + .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) + .findList(); + return pendingParamSetList; + } + + @Override + protected List getTuningJobDefinitions() { + return _executionEngine.getTuningJobDefinitionsForParameterSuggestion() + .eq(TuningJobDefinition.TABLE.tuningAlgorithm + + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.PSO.name()) + .findList(); + } + + @Override + public void intializePrerequisite(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { + + } + + @Override + public void parameterOptimizer(List appResults, JobExecution jobExecution) { + + } + + @Override + public int getSwarmSize() { + return 3; + } + + public String getManagerName() { + return "TuningTypeManagerOBTAlgoPSO" + this._executionEngine.getClass().getSimpleName(); + } +} diff --git a/app/controllers/Application.java b/app/controllers/Application.java index 1c1a60900..bf46b679d 100644 --- a/app/controllers/Application.java +++ b/app/controllers/Application.java @@ -44,6 +44,7 @@ import models.AppHeuristicResult; import models.AppResult; +import models.TuningAlgorithm; import org.apache.commons.collections.map.ListOrderedMap; import org.apache.http.client.utils.URLEncodedUtils; import org.apache.http.message.BasicNameValuePair; @@ -928,7 +929,9 @@ public static Result getCurrentRunParameters() { skipExecutionForOptimization = Boolean.parseBoolean(paramValueMap.get("skipExecutionForOptimization")); } String jobType = paramValueMap.get("autoTuningJobType"); - String optimizationAlgo = paramValueMap.get("optimizationAlgo"); + String optimizationAlgo = + paramValueMap.get("optimizationAlgo") == null ? TuningAlgorithm.OptimizationAlgo.HBT.name() + : paramValueMap.get("optimizationAlgo"); String optimizationAlgoVersion = paramValueMap.get("optimizationAlgoVersion"); String optimizationMetric = paramValueMap.get("optimizationMetric"); diff --git a/app/models/JobSuggestedParamSet.java b/app/models/JobSuggestedParamSet.java index bad279932..45a03c681 100644 --- a/app/models/JobSuggestedParamSet.java +++ b/app/models/JobSuggestedParamSet.java @@ -55,6 +55,7 @@ public static class TABLE { public static final String fitness = "fitness"; public static final String fitnessJobExecution = "fitnessJobExecution"; public static final String isParamSetBest = "isParamSetBest"; + public static final String isManuallyOverridenParameter = "isManuallyOverridenParameter"; public static final String areConstraintsViolated = "areConstraintsViolated"; public static final String createdTs = "createdTs"; public static final String updatedTs = "updatedTs"; @@ -91,6 +92,9 @@ public static class TABLE { @Column(nullable = false) public Boolean isParamSetBest; + @Column(nullable = false) + public Boolean isManuallyOverridenParameter=false; + @Column(nullable = false) public Boolean areConstraintsViolated; diff --git a/app/models/TuningAlgorithm.java b/app/models/TuningAlgorithm.java index 69b94c9b7..3258cd8b2 100644 --- a/app/models/TuningAlgorithm.java +++ b/app/models/TuningAlgorithm.java @@ -43,7 +43,7 @@ public enum JobType { } public enum OptimizationAlgo { - PSO + PSO, PSO_IPSO, HBT } public enum OptimizationMetric { diff --git a/app/models/TuningJobDefinition.java b/app/models/TuningJobDefinition.java index 27c7c61d3..c905b563c 100644 --- a/app/models/TuningJobDefinition.java +++ b/app/models/TuningJobDefinition.java @@ -44,6 +44,7 @@ public static class TABLE { public static final String client = "client"; public static final String tuningAlgorithm = "tuningAlgorithm"; public static final String tuningEnabled = "tuningEnabled"; + public static final String autoApply = "autoApply"; public static final String averageResourceUsage = "averageResourceUsage"; public static final String averageExecutionTime = "averageExecutionTime"; public static final String averageInputSizeInBytes = "averageInputSizeInBytes"; @@ -51,8 +52,10 @@ public static class TABLE { public static final String allowedMaxExecutionTimePercent = "allowedMaxExecutionTimePercent"; public static final String job = "job"; public static final String tuningDisabledReason = "tuningDisabledReason"; + public static final String numberOfIterations = "numberOfIterations"; public static final String createdTs = "createdTs"; public static final String updatedTs = "updatedTs"; + } @ManyToOne(cascade = CascadeType.ALL) @@ -69,6 +72,9 @@ public static class TABLE { @Column(nullable = false) public boolean tuningEnabled; + @Column(nullable = true) + public boolean autoApply = false; + @Column(nullable = true) public Double averageResourceUsage; @@ -84,6 +90,9 @@ public static class TABLE { @Column(nullable = true) public Double allowedMaxExecutionTimePercent; + @Column(nullable = true) + public Integer numberOfIterations=10; + public Double getAverageInputSizeInGB() { if (averageInputSizeInBytes != null) { return averageInputSizeInBytes * 1.0 / (1024 * 1024 * 1024); diff --git a/app/models/TuningParameterConstraint.java b/app/models/TuningParameterConstraint.java new file mode 100644 index 000000000..52a7d4832 --- /dev/null +++ b/app/models/TuningParameterConstraint.java @@ -0,0 +1,105 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package models; + +import java.sql.Timestamp; + +import javax.persistence.CascadeType; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EnumType; +import javax.persistence.Enumerated; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.JoinTable; +import javax.persistence.ManyToOne; +import javax.persistence.Table; + +import com.avaje.ebean.annotation.UpdatedTimestamp; + +import play.db.ebean.Model; + + +@Entity +@Table(name = "tuning_parameter_constraint") +public class TuningParameterConstraint extends Model { + + private static final long serialVersionUID = 1L; + + public enum ConstraintType { + BOUNDARY, INTERDEPENDENT + } + + public static class TABLE { + public static final String TABLE_NAME = "tuning_parameter_constraint"; + public static final String id = "id"; + public static final String jobDefinitionId = "jobDefinitionId"; + public static final String constraintType = "constraintType"; + public static final String tuningParameterId = "tuningParameterId"; + public static final String lowerBound = "lowerBound"; + public static final String upperBound = "upperBound"; + public static final String createdTs = "createdTs"; + public static final String updatedTs = "updatedTs"; + } + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + public Integer id; + + @ManyToOne(cascade = CascadeType.ALL) + @JoinTable(name = "job_definition", joinColumns = {@JoinColumn(name = "job_definition_id", referencedColumnName = "id")}) + public JobDefinition jobDefinition; + + @Enumerated(EnumType.STRING) + @Column(nullable = false) + public ConstraintType constraintType; + + @ManyToOne(cascade = CascadeType.ALL) + @JoinTable(name = "tuning_parameter", joinColumns = {@JoinColumn(name = "tuning_parameter_id", referencedColumnName = "id")}) + public TuningParameter tuningParameter; + + @Column(nullable = false) + public Double lowerBound; + + @Column(nullable = false) + public Double upperBound; + + @Column(nullable = false) + public Timestamp createdTs; + + @Column(nullable = false) + @UpdatedTimestamp + public Timestamp updatedTs; + + + public static Finder find = + new Finder(Integer.class, TuningParameterConstraint.class); + + @Override + public void save() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.save(); + } + + @Override + public void update() { + this.updatedTs = new Timestamp(System.currentTimeMillis()); + super.update(); + } +} diff --git a/conf/evolutions/default/5.sql b/conf/evolutions/default/5.sql index 640c292be..4a49663e7 100644 --- a/conf/evolutions/default/5.sql +++ b/conf/evolutions/default/5.sql @@ -110,12 +110,14 @@ CREATE TABLE IF NOT EXISTS tuning_job_definition ( client varchar(100) NOT NULL COMMENT 'client who is using this. sometime same as scheduler.', tuning_algorithm_id int(10) unsigned NOT NULL COMMENT 'foreign key from tuning_algorithm table. algorithm to be used for tuning this job', tuning_enabled tinyint(4) NOT NULL COMMENT 'auto tuning is enabled or not ', + auto_apply tinyint(4) NOT NULL COMMENT 'auto apply is enabled or not ', average_resource_usage double DEFAULT NULL COMMENT 'average resource usage when optimization started on this job', average_execution_time double DEFAULT NULL COMMENT 'average execution time (excluding delay) when optimization started on this job', average_input_size_in_bytes bigint(20) DEFAULT NULL COMMENT 'Average input size in bytes when optimization started on this job', allowed_max_resource_usage_percent double DEFAULT NULL COMMENT 'Limit on resource usage, For ex 150 means it should not go beyond 150% ', allowed_max_execution_time_percent double DEFAULT NULL COMMENT 'Limit on execution time, For ex 150 means it should not go beyond 150% ', tuning_disabled_reason text NULL COMMENT 'reason for disabling tuning, if any', + number_of_iterations double DEFAULT 10 COMMENT 'number of iterations', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, CONSTRAINT tuning_job_definition_ibfk_1 FOREIGN KEY (job_definition_id) REFERENCES job_definition (id), @@ -180,6 +182,7 @@ CREATE TABLE IF NOT EXISTS job_suggested_param_set ( are_constraints_violated tinyint(4) default 0 NOT NULL COMMENT 'are constraints violated for the parameter set', is_param_set_default tinyint(4) DEFAULT 0 NOT NULL COMMENT 'Is parameter set default', is_param_set_best tinyint(4) DEFAULT 0 NOT NULL COMMENT 'Is parameter set best', + is_manually_overriden_parameter tinyint(4) DEFAULT 0 NOT NULL COMMENT 'Is parameter set is manually overriden', fitness double DEFAULT NULL COMMENT 'fitness of this parameter set', fitness_job_execution_id int(10) unsigned NULL COMMENT 'foreign key from job_execution table', created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, diff --git a/conf/evolutions/default/6.sql b/conf/evolutions/default/6.sql new file mode 100644 index 000000000..4cb59fc33 --- /dev/null +++ b/conf/evolutions/default/6.sql @@ -0,0 +1,60 @@ +# +# Copyright 2016 LinkedIn Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# + +# --- !Ups + +/** + * This query make neccesssary steps to run IPSO + */ + +ALTER TABLE tuning_algorithm MODIFY COLUMN optimization_algo enum('PSO','PSO_IPSO','HBT') NOT NULL COMMENT 'optimization algorithm name e.g. PSO' ; + +INSERT INTO tuning_algorithm VALUES (3, 'PIG', 'PSO_IPSO', '3', 'RESOURCE', current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_algorithm VALUES (4, 'PIG', 'HBT', '4', 'RESOURCE', current_timestamp(0), current_timestamp(0)); + +INSERT INTO tuning_parameter VALUES (10,'mapreduce.task.io.sort.mb',3,100,50,1920,50, 0, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (11,'mapreduce.map.memory.mb',3,2048,1024,8192,1024, 0, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (12,'mapreduce.task.io.sort.factor',3,10,10,150,10 ,0, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (13,'mapreduce.map.sort.spill.percent',3,0.8,0.6,0.9,0.1, 0, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (14,'mapreduce.reduce.memory.mb',3,2048,1024,8192,1024, 0, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (15,'mapreduce.reduce.java.opts',3,1536,500,6144,64, 0, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (16,'mapreduce.map.java.opts',3,1536,500,6144,64, 0, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (17,'mapreduce.input.fileinputformat.split.maxsize',3,536870912,536870912,536870912,128, 1, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (18,'pig.maxCombinedSplitSize',3,536870912,536870912,536870912,128, 0, current_timestamp(0), current_timestamp(0)); + +INSERT INTO tuning_parameter VALUES (19,'mapreduce.map.memory.mb',4,2048,1024,8192,1024, 0, current_timestamp(0), current_timestamp(0)); + +CREATE TABLE IF NOT EXISTS tuning_parameter_constraint ( + id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', + job_definition_id int(10) unsigned NOT NULL COMMENT 'Job Definition ID', + tuning_parameter_id int(100) unsigned NOT NULL COMMENT 'tuning_parameter_id ', + constraint_type enum('BOUNDARY','INTERDEPENDENT') NOT NULL COMMENT 'Constraint ID', + lower_bound double COMMENT 'Lower bound of parameter', + upper_bound double COMMENT 'Upper bound of parameter', + created_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_ts timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id), + CONSTRAINT tuning_parameter_constraint_ibfk_1 FOREIGN KEY (job_definition_id) REFERENCES tuning_job_definition (job_definition_id), + CONSTRAINT tuning_parameter_constraint_ibfk_2 FOREIGN KEY (tuning_parameter_id) REFERENCES tuning_parameter (id) +) ENGINE=InnoDB; + +create index index_tuning_parameter_constraint on tuning_parameter_constraint (job_definition_id); + +# --- !Downs +DELETE FROM tuning_parameter WHERE tuning_algorithm_id=3; +DELETE FROM tuning_algorithm WHERE optimization_algo='PSO_IPSO' ; +ALTER TABLE tuning_algorithm MODIFY COLUMN optimization_algo enum('PSO') NOT NULL COMMENT 'optimization algorithm name e.g. PSO' ; +DROP TABLE tuning_parameter_constraint; diff --git a/test/com/linkedin/drelephant/tuning/BaselineManagerTestRunner.java b/test/com/linkedin/drelephant/tuning/BaselineManagerTestRunner.java new file mode 100644 index 000000000..37bafe7cb --- /dev/null +++ b/test/com/linkedin/drelephant/tuning/BaselineManagerTestRunner.java @@ -0,0 +1,37 @@ +package com.linkedin.drelephant.tuning; + +import com.linkedin.drelephant.tuning.hbt.BaselineManagerHBT; +import com.linkedin.drelephant.tuning.obt.BaselineManagerOBT; +import java.util.List; + +import static common.DBTestUtil.*; +import static org.junit.Assert.*; +import static play.test.Helpers.*; + + +public class BaselineManagerTestRunner implements Runnable{ + + private void populateTestData() { + try { + initDBBaseline(); + } catch (Exception e) { + e.printStackTrace(); + } + } + @Override + public void run() { + populateTestData(); + Flow flow = new Flow(); + flow.createBaseLineManagersPipeline(); + List> pipeline = flow.getPipeline(); + List baseLineManagers = pipeline.get(0); + testManagerCreations(baseLineManagers); + } + + private void testManagerCreations(List baseLineManagers){ + assertTrue("Base line Manager HBT ", baseLineManagers.get(0) instanceof BaselineManagerHBT); + assertTrue("Base line Manager OBT ", baseLineManagers.get(1) instanceof BaselineManagerOBT); + BaselineManagerHBT baselineManagerHBT = (BaselineManagerHBT)baseLineManagers.get(0); + BaselineManagerOBT baselineManagerOBT = (BaselineManagerOBT)baseLineManagers.get(1); + } +} diff --git a/test/com/linkedin/drelephant/tuning/IPSOManagerTest.java b/test/com/linkedin/drelephant/tuning/IPSOManagerTest.java new file mode 100644 index 000000000..fcbb47a31 --- /dev/null +++ b/test/com/linkedin/drelephant/tuning/IPSOManagerTest.java @@ -0,0 +1,56 @@ +package com.linkedin.drelephant.tuning; + +import com.linkedin.drelephant.DrElephant; +import com.linkedin.drelephant.ElephantContext; +import java.util.HashMap; +import java.util.Map; + +import static common.DBTestUtil.*; +import static common.TestConstants.*; + +import org.slf4j.LoggerFactory; +import play.Application; +import play.GlobalSettings; +import play.test.FakeApplication; +import org.apache.hadoop.conf.Configuration; + +import static org.junit.Assert.*; +import static play.test.Helpers.*; + +import org.junit.Before; +import org.junit.Test; + + +public class IPSOManagerTest { + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(IPSOManagerTest.class); + private static FakeApplication fakeApp; + private int numParametersToTune; + + @Before + public void setup() { + Map dbConn = new HashMap(); + dbConn.put(DB_DEFAULT_DRIVER_KEY, DB_DEFAULT_DRIVER_VALUE); + dbConn.put(DB_DEFAULT_URL_KEY, DB_DEFAULT_URL_VALUE); + dbConn.put(EVOLUTION_PLUGIN_KEY, EVOLUTION_PLUGIN_VALUE); + dbConn.put(APPLY_EVOLUTIONS_DEFAULT_KEY, APPLY_EVOLUTIONS_DEFAULT_VALUE); + + GlobalSettings gs = new GlobalSettings() { + @Override + public void onStart(Application app) { + LOGGER.info("Starting FakeApplication"); + } + }; + + fakeApp = fakeApplication(dbConn, gs); + Configuration configuration = ElephantContext.instance().getAutoTuningConf(); + Boolean autoTuningEnabled = configuration.getBoolean(DrElephant.AUTO_TUNING_ENABLED, false); + // org.junit.Assume.assumeTrue(autoTuningEnabled); + } + + @Test + public void testIPSOManager() { + running(testServer(TEST_SERVER_PORT, fakeApp), new IPSOManagerTestRunner()); + //running(testServer(TEST_SERVER_PORT, fakeApp), new BaselineManagerTestRunner()); + + } +} diff --git a/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java b/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java new file mode 100644 index 000000000..2940d38db --- /dev/null +++ b/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java @@ -0,0 +1,211 @@ +package com.linkedin.drelephant.tuning; + +import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; +import com.linkedin.drelephant.tuning.obt.OptimizationAlgoFactory; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBT; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoIPSO; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import models.AppHeuristicResult; +import models.AppResult; +import models.JobDefinition; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; +import models.TuningParameterConstraint; + +import static org.junit.Assert.*; +import static play.test.Helpers.*; +import static common.DBTestUtil.*; + + +public class IPSOManagerTestRunner implements Runnable { + + private void populateTestData() { + try { + initDBIPSO(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public void run() { + populateTestData(); + JobDefinition jobDefinition = JobDefinition.find.byId(100003); + TuningJobDefinition tuningJobDefinition = + TuningJobDefinition.find.where().eq("job.id", jobDefinition.id).findUnique(); + TuningAlgorithm tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; + JobSuggestedParamSet jobSuggestedParamSet = + JobSuggestedParamSet.find.where().eq("fitness_job_execution_id", 1541).findUnique(); + JobExecution jobExecution = JobExecution.find.byId(1541L); + TuningTypeManagerOBT optimizeManager = checkIPSOManager(tuningAlgorithm); + testIPSOIntializePrerequisite(optimizeManager, tuningAlgorithm, jobSuggestedParamSet); + //testIPSOExtractParameterInformation(jobExecution, optimizeManager); + testIPSOParameterOptimizer(jobExecution, optimizeManager); + testIPSOApplyIntelligenceOnParameter(tuningJobDefinition, jobDefinition, optimizeManager); + testIPSONumberOfConstraintsViolated(optimizeManager); + } + + private TuningTypeManagerOBT checkIPSOManager(TuningAlgorithm tuningAlgorithm) { + TuningTypeManagerOBT optimizeManager = OptimizationAlgoFactory.getOptimizationAlogrithm(tuningAlgorithm); + assertTrue("Optimization Algorithm type ", optimizeManager instanceof TuningTypeManagerOBTAlgoIPSO); + return optimizeManager; + } + + private void testIPSOIntializePrerequisite(TuningTypeManagerOBT optimizeManager, TuningAlgorithm tuningAlgorithm, + JobSuggestedParamSet jobSuggestedParamSet) { + optimizeManager.intializePrerequisite(tuningAlgorithm, jobSuggestedParamSet); + List tuningParameterConstraint = + TuningParameterConstraint.find.where().eq("job_definition_id", 100003).findList(); + assertTrue(" Parameters Constraint Size ", tuningParameterConstraint.size() == 9); + } + + /* private void testIPSOExtractParameterInformation(JobExecution jobExecution, TuningTypeManagerOBT optimizeManager) { + List results = AppResult.find.select("*") + .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*") + .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, "*") + .where() + .eq(AppResult.TABLE.FLOW_EXEC_ID, jobExecution.flowExecution.flowExecId) + .eq(AppResult.TABLE.JOB_EXEC_ID, jobExecution.jobExecId) + .findList(); + assertTrue(" Apps for Jobs ", results.size() > 0); + *//* Map> usageData = optimizeManager.extractParameterInformation(results); + testMapData(usageData); + testReduceData(usageData); +*//* + }*/ + + /*private void testMapData(Map> usageData) { + assertTrue(" Usage data ", + usageData.get("map").get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_PHYSICAL_MEMORY.getValue()) == 595); + assertTrue(" Usage data ", usageData.get("map") + .get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_TOTAL_COMMITTED_HEAP_USAGE_MEMORY.getValue()) == 427); + assertTrue(" Usage data ", + usageData.get("map").get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_VIRTUAL_MEMORY.getValue()) == 2200); + } + + private void testReduceData(Map> usageData) { + assertTrue(" Usage data ", + usageData.get("reduce").get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_PHYSICAL_MEMORY.getValue()) + == 497); + assertTrue(" Usage data ", usageData.get("reduce") + .get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_TOTAL_COMMITTED_HEAP_USAGE_MEMORY.getValue()) == 300); + assertTrue(" Usage data ", + usageData.get("reduce").get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_VIRTUAL_MEMORY.getValue()) + == 2100); + }*/ + + private void testIPSOParameterOptimizer(JobExecution jobExecution, TuningTypeManagerOBT optimizeManager) { + List results = AppResult.find.select("*") + .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*") + .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, "*") + .where() + .eq(AppResult.TABLE.FLOW_EXEC_ID, jobExecution.flowExecution.flowExecId) + .eq(AppResult.TABLE.JOB_EXEC_ID, jobExecution.jobExecId) + .findList(); + assertTrue(" Apps for Jobs ", results.size() > 0); + optimizeManager.parameterOptimizer(results,jobExecution); + List parameterConstraints = TuningParameterConstraint.find.where(). + eq("job_definition_id", 100003).findList(); + for (TuningParameterConstraint parameterConstraint : parameterConstraints) { + testMapParameterBoundries(parameterConstraint); + testReduceParameterBoundries(parameterConstraint); + } + } + + private void testMapParameterBoundries(TuningParameterConstraint parameterConstraint) { + if (parameterConstraint.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.MAPPER_MEMORY_HADOOP_CONF.getValue())) { + assertTrue("Mapper Memory Lower Bound ", parameterConstraint.lowerBound == 2048.0); + assertTrue("Mapper Memory Upper Bound ", parameterConstraint.upperBound == 2048.0); + } + if (parameterConstraint.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.MAPPER_HEAP_HADOOP_CONF.getValue())) { + assertTrue("Mapper Heap Memory Lower Bound ", parameterConstraint.lowerBound == 427.0); + assertTrue("Mapper Heap Memory Upper Bound ", parameterConstraint.upperBound == 512.4); + } + } + + private void testReduceParameterBoundries(TuningParameterConstraint parameterConstraint) { + if (parameterConstraint.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.REDUCER_MEMORY_HADOOP_CONF.getValue())) { + assertTrue("Mapper Memory Lower Bound ", parameterConstraint.lowerBound == 1024.0); + assertTrue("Mapper Memory Upper Bound ", parameterConstraint.upperBound == 2048.0); + } + if (parameterConstraint.tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.REDUCER_HEAP_HADOOP_CONF.getValue())) { + assertTrue("Reducer Heap Memory Lower Bound ", parameterConstraint.lowerBound == 300.0); + assertTrue("Reducer Heap Memory Upper Bound ", parameterConstraint.upperBound == 360.0); + } + } + + private void testIPSOApplyIntelligenceOnParameter(TuningJobDefinition tuningJobDefinition, + JobDefinition jobDefinition, TuningTypeManagerOBT optimizeManager) { + List tuningParameterList = TuningParameter.find.where() + .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, + tuningJobDefinition.tuningAlgorithm.id) + .eq(TuningParameter.TABLE.isDerived, 0) + .findList(); + optimizeManager.updateBoundryConstraint(tuningParameterList, jobDefinition); + for (TuningParameter tuningParameter : tuningParameterList) { + testMapMinMaxValue(tuningParameter); + testReduceMinMaxValue(tuningParameter); + testNonIPSOParameter(tuningParameter); + } + } + + private void testMapMinMaxValue(TuningParameter tuningParameter) { + if (tuningParameter.paramName.equals(CommonConstantsHeuristic.ParameterKeys.MAPPER_MEMORY_HADOOP_CONF.getValue())) { + assertTrue("Mapper Memory Lower Bound ", tuningParameter.minValue == 2048.0); + assertTrue("Mapper Memory Upper Bound ", tuningParameter.maxValue == 2048.0); + } + if (tuningParameter.paramName.equals(CommonConstantsHeuristic.ParameterKeys.MAPPER_HEAP_HADOOP_CONF.getValue())) { + assertTrue("Mapper Heap Memory Lower Bound ", tuningParameter.minValue == 427.0); + assertTrue("Mapper Heap Memory Upper Bound ", tuningParameter.maxValue == 512.4); + } + } + + private void testReduceMinMaxValue(TuningParameter tuningParameter) { + if (tuningParameter.paramName.equals( + CommonConstantsHeuristic.ParameterKeys.REDUCER_MEMORY_HADOOP_CONF.getValue())) { + assertTrue("Mapper Memory Lower Bound ", tuningParameter.minValue == 1024.0); + assertTrue("Mapper Memory Upper Bound ", tuningParameter.maxValue == 2048.0); + } + if (tuningParameter.paramName.equals(CommonConstantsHeuristic.ParameterKeys.REDUCER_HEAP_HADOOP_CONF.getValue())) { + assertTrue("Reducer Heap Memory Lower Bound ", tuningParameter.minValue == 300.0); + assertTrue("Reducer Heap Memory Upper Bound ", tuningParameter.maxValue == 360.0); + } + } + + private void testNonIPSOParameter(TuningParameter tuningParameter) { + if (tuningParameter.paramName.equals("mapreduce.task.io.sort.factor")) { + assertTrue("Task Sort Factor Min Value ", tuningParameter.minValue == 10.0); + assertTrue("Task Sort Factor Max Value ", tuningParameter.maxValue == 150.0); + } + } + + private void testIPSONumberOfConstraintsViolated(TuningTypeManagerOBT optimizeManager) { + JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); + jobSuggestedParamValue.tuningParameter = TuningParameter.find.byId(11); + jobSuggestedParamValue.paramValue = 2048.0; + JobSuggestedParamValue jobSuggestedParamValue1 = new JobSuggestedParamValue(); + jobSuggestedParamValue1.tuningParameter = TuningParameter.find.byId(16); + jobSuggestedParamValue1.paramValue = 2048.0; + List jobSuggestedParamValueList = new ArrayList(); + jobSuggestedParamValueList.add(jobSuggestedParamValue); + jobSuggestedParamValueList.add(jobSuggestedParamValue1); + Boolean violations = optimizeManager.isParamConstraintViolated(jobSuggestedParamValueList); + assertTrue("Parameter constraint violeted " + violations, violations); + jobSuggestedParamValueList.clear(); + jobSuggestedParamValue1.paramValue = 1024.0; + jobSuggestedParamValueList.add(jobSuggestedParamValue); + jobSuggestedParamValueList.add(jobSuggestedParamValue1); + violations = optimizeManager.isParamConstraintViolated(jobSuggestedParamValueList); + assertTrue("Parameter constraint violeted " + violations, violations == false); + } +} diff --git a/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java b/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java index 255268255..32f10e1a5 100644 --- a/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java +++ b/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java @@ -20,11 +20,13 @@ import com.fasterxml.jackson.databind.node.JsonNodeType; import com.linkedin.drelephant.DrElephant; import com.linkedin.drelephant.ElephantContext; +import com.linkedin.drelephant.tuning.engine.MRExecutionEngine; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBT; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoPSO; import java.util.HashMap; import java.util.List; import java.util.Map; import models.JobDefinition; -import models.JobExecution; import models.JobSuggestedParamSet; import models.JobSuggestedParamValue; import models.TuningAlgorithm; @@ -101,15 +103,18 @@ public void run() { LOGGER.info("PSOParamGeneratorTest parameter list: " + Json.toJson(tuningParameterList)); - JobTuningInfo jobTuningInfo = new JobTuningInfo(); + JobTuningInfo + jobTuningInfo = new JobTuningInfo(); jobTuningInfo.setTuningJob(jobDefinition); jobTuningInfo.setParametersToTune(tuningParameterList); jobTuningInfo.setTunerState("{}"); jobTuningInfo.setJobType(TuningAlgorithm.JobType.PIG); - PSOParamGenerator psoParamGenerator = new PSOParamGenerator(); + //PSOParamGenerator psoParamGenerator = new PSOParamGenerator(); + TuningTypeManagerOBT tuningTypeManagerOBT = new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); + //algorithmManagerOBT.generateParamSet() - JobTuningInfo updatedJobTuningInfo = psoParamGenerator.generateParamSet(jobTuningInfo); + JobTuningInfo updatedJobTuningInfo = tuningTypeManagerOBT.generateParamSet(jobTuningInfo); assertTrue("Updated JobTuningInfo: Job definition mismatch", updatedJobTuningInfo.getTuningJob().equals(jobDefinition)); assertTrue("Updated JobTuningInfo: Parameter list mismatch", @@ -151,7 +156,7 @@ public void run() { assertEquals("Random number state not of type string", JsonNodeType.STRING, randomNumberState.getNodeType()); jobTuningInfo.setTunerState(updatedJobTuningInfo.getTunerState()); - updatedJobTuningInfo = psoParamGenerator.generateParamSet(jobTuningInfo); + updatedJobTuningInfo = tuningTypeManagerOBT.generateParamSet(jobTuningInfo); assertTrue("Updated JobTuningInfo: Job definition mismatch", updatedJobTuningInfo.getTuningJob().equals(jobDefinition)); assertTrue("Updated JobTuningInfo: Parameter list mismatch", @@ -200,8 +205,11 @@ public void getParamsTest() { running(testServer(TEST_SERVER_PORT, fakeApp), new Runnable() { public void run() { populateTestData(); - PSOParamGenerator psoParamGenerator = new PSOParamGenerator(); - psoParamGenerator.getParams(); + /* PSOParamGenerator psoParamGenerator = new PSOParamGenerator(); + psoParamGenerator.getParams();*/ + + Manager manager = new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); + manager.execute(); List jobSuggestedParamSetList = JobSuggestedParamSet.find.where() .eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.CREATED) diff --git a/test/common/DBTestUtil.java b/test/common/DBTestUtil.java index f15455ee4..04e5f0193 100644 --- a/test/common/DBTestUtil.java +++ b/test/common/DBTestUtil.java @@ -24,12 +24,19 @@ import org.apache.commons.io.IOUtils; import play.db.DB; -import static common.TestConstants.TEST_DATA_FILE; -import static common.TestConstants.TEST_AUTO_TUNING_DATA_FILE1;; +import static common.TestConstants.*; public class DBTestUtil { + public static void initDBBaseline() throws IOException, SQLException { + initDBUtil(TEST_BASELINE_DATA_FILE); + } + + public static void initDBIPSO() throws IOException, SQLException { + initDBUtil(TEST_IPSO_DATA_FILE); + } + public static void initDB() throws IOException, SQLException { initDBUtil(TEST_DATA_FILE); } diff --git a/test/common/TestConstants.java b/test/common/TestConstants.java index cb58cc1c0..ab13f13b4 100644 --- a/test/common/TestConstants.java +++ b/test/common/TestConstants.java @@ -22,6 +22,8 @@ public class TestConstants { public static final int TEST_SERVER_PORT = 9001; public static final String BASE_URL = "http://localhost:" + TEST_SERVER_PORT; public static final String TEST_DATA_FILE = "test/resources/test-init.sql"; + public static final String TEST_IPSO_DATA_FILE = "test/resources/test-init-ipso.sql"; + public static final String TEST_BASELINE_DATA_FILE = "test/resources/test-init-baseline.sql"; public static final String TEST_AUTO_TUNING_DATA_FILE1 = "test/resources/tunein-test1.sql"; public static final int RESPONSE_TIMEOUT = 3000; // milliseconds diff --git a/test/resources/test-init-baseline.sql b/test/resources/test-init-baseline.sql new file mode 100644 index 000000000..47830484f --- /dev/null +++ b/test/resources/test-init-baseline.sql @@ -0,0 +1,163 @@ +insert into yarn_app_result + (id,name,username,queue_name,start_time,finish_time,tracking_url,job_type,severity,score,workflow_depth,scheduler,job_name,job_exec_id,flow_exec_id,job_def_id,flow_def_id,job_exec_url,flow_exec_url,job_def_url,flow_def_url,resource_used,resource_wasted,total_delay) values + ('application_1458194917883_1453361','Email Overwriter','growth','misc_default',1460980616502,1460980723925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453361','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654676&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654676','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 100, 30, 20), + ('application_1458194917883_1453362','Email Overwriter','metrics','misc_default',1460980823925,1460980923925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453362','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654677&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654677','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 200, 40, 10); + +insert into yarn_app_heuristic_result(id,yarn_app_result_id,heuristic_class,heuristic_name,severity,score) values +(137594512,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), +(137594513,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), +(137594516,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), +(137594520,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), +(137594523,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), +(137594525,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), +(137594530,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), +(137594531,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), +(137594534,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), +(137594537,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), +(137594540,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0), +(137594612,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), +(137594613,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), +(137594616,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), +(137594620,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), +(137594623,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), +(137594625,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), +(137594630,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), +(137594631,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), +(137594634,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), +(137594637,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), +(137594640,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0); + +insert into yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name,value,details) values +(137594512,'Group A','1 tasks @ 4 MB avg','NULL'), +(137594512,'Group B','1 tasks @ 79 MB avg','NULL'), +(137594512,'Number of tasks','2','NULL'), +(137594513,'Avg task CPU time (ms)','11510','NULL'), + (137594513,'Avg task GC time (ms)','76','NULL'), + (137594513,'Avg task runtime (ms)','11851','NULL'), + (137594513,'Number of tasks','2','NULL'), + (137594513,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), + (137594516,'Average task input size','42 MB','NULL'), + (137594516,'Average task runtime','11 sec','NULL'), + (137594516,'Max task runtime','12 sec','NULL'), + (137594516,'Min task runtime','11 sec','NULL'), + (137594516,'Number of tasks','2','NULL'), + (137594520,'Median task input size','42 MB','NULL'), + (137594520,'Median task runtime','11 sec','NULL'), + (137594520,'Median task speed','3 MB/s','NULL'), + (137594520,'Number of tasks','2','NULL'), + (137594523,'Avg output records per task','56687','NULL'), + (137594523,'Avg spilled records per task','79913','NULL'), + (137594523,'Number of tasks','2','NULL'), + (137594523,'Ratio of spilled records to output records','1.4097111356119074','NULL'), + (137594525,'Avg Physical Memory (MB)','522','NULL'), + (137594525,'Avg task runtime','11 sec','NULL'), + (137594525,'Avg Virtual Memory (MB)','3307','NULL'), + (137594525,'Max Physical Memory (MB)','595','NULL'), + (137594525,'Max Total Committed Heap Usage Memory (MB)','427','NULL'), + (137594525,'Max Virtual Memory (MB)','1426','NULL'), + (137594525,'Min Physical Memory (MB)','449','NULL'), + (137594525,'Number of tasks','2','NULL'), + (137594525,'Requested Container Memory','2 GB','NULL'), + (137594530,'Group A','11 tasks @ 868 KB avg','NULL'), + (137594530,'Group B','9 tasks @ 883 KB avg ','NULL'), + (137594530,'Number of tasks','20','NULL'), + (137594531,'Avg task CPU time (ms)','8912','NULL'), + (137594531,'Avg task GC time (ms)','73','NULL'), + (137594531,'Avg task runtime (ms)','11045','NULL'), + (137594531,'Number of tasks','20','NULL'), + (137594531,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), + (137594534,'Average task runtime','11 sec','NULL'), + (137594534,'Max task runtime','14 sec','NULL'), + (137594534,'Min task runtime','8 sec','NULL'), + (137594534,'Number of tasks','20','NULL'), + (137594537,'Avg Physical Memory (MB)','416','NULL'), + (137594537,'Avg task runtime','11 sec','NULL'), + (137594537,'Avg Virtual Memory (MB)','3326','NULL'), + (137594537,'Max Physical Memory (MB)','497','NULL'), + (137594537,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594537,'Max Virtual Memory (MB)','1350','NULL'), + (137594537,'Min Physical Memory (MB)','354','NULL'), + (137594537,'Number of tasks','20','NULL'), + (137594537,'Requested Container Memory','2 GB','NULL'), + (137594540,'Average code runtime','1 sec','NULL'), + (137594540,'Average shuffle time','9 sec (5.49x)','NULL'), + (137594540,'Average sort time','(0.04x)','NULL'), + (137594540,'Number of tasks','20','NULL'), + (137594612,'Group A','1 tasks @ 4 MB avg','NULL'), + (137594612,'Group B','1 tasks @ 79 MB avg','NULL'), + (137594612,'Number of tasks','2','NULL'), + (137594613,'Avg task CPU time (ms)','11510','NULL'), + (137594613,'Avg task GC time (ms)','76','NULL'), + (137594613,'Avg task runtime (ms)','11851','NULL'), + (137594613,'Number of tasks','2','NULL'), + (137594613,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), + (137594616,'Average task input size','42 MB','NULL'), + (137594616,'Average task runtime','11 sec','NULL'), + (137594616,'Max task runtime','12 sec','NULL'), + (137594616,'Min task runtime','11 sec','NULL'), + (137594616,'Number of tasks','2','NULL'), + (137594620,'Median task input size','42 MB','NULL'), + (137594620,'Median task runtime','11 sec','NULL'), + (137594620,'Median task speed','3 MB/s','NULL'), + (137594620,'Number of tasks','2','NULL'), + (137594623,'Avg output records per task','56687','NULL'), + (137594623,'Avg spilled records per task','79913','NULL'), + (137594623,'Number of tasks','2','NULL'), + (137594623,'Ratio of spilled records to output records','1.4097111356119074','NULL'), + (137594625,'Avg Physical Memory (MB)','522','NULL'), + (137594625,'Avg task runtime','11 sec','NULL'), + (137594625,'Avg Virtual Memory (MB)','3307','NULL'), + (137594625,'Max Physical Memory (MB)','595','NULL'), + (137594625,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594625,'Max Virtual Memory (MB)','2200','NULL'), + (137594625,'Min Physical Memory (MB)','449','NULL'), + (137594625,'Number of tasks','2','NULL'), + (137594625,'Requested Container Memory','2 GB','NULL'), + (137594630,'Group A','11 tasks @ 868 KB avg','NULL'), + (137594630,'Group B','9 tasks @ 883 KB avg ','NULL'), + (137594630,'Number of tasks','20','NULL'), + (137594631,'Avg task CPU time (ms)','8912','NULL'), + (137594631,'Avg task GC time (ms)','73','NULL'), + (137594631,'Avg task runtime (ms)','11045','NULL'), + (137594631,'Number of tasks','20','NULL'), + (137594631,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), + (137594634,'Average task runtime','11 sec','NULL'), + (137594634,'Max task runtime','14 sec','NULL'), + (137594634,'Min task runtime','8 sec','NULL'), + (137594634,'Number of tasks','20','NULL'), + (137594637,'Avg Physical Memory (MB)','416','NULL'), + (137594637,'Avg task runtime','11 sec','NULL'), + (137594637,'Avg Virtual Memory (MB)','3326','NULL'), + (137594637,'Max Physical Memory (MB)','497','NULL'), + (137594637,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594637,'Max Virtual Memory (MB)','2100','NULL'), + (137594637,'Min Physical Memory (MB)','354','NULL'), + (137594637,'Number of tasks','20','NULL'), + (137594637,'Requested Container Memory','2 GB','NULL'), + (137594640,'Average code runtime','1 sec','NULL'), + (137594640,'Average shuffle time','9 sec (5.49x)','NULL'), + (137594640,'Average sort time','(0.04x)','NULL'), + (137594640,'Number of tasks','20','NULL'); + +INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES (10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'); + +INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES (100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','mkumar1','2018-02-12 08:40:42','2018-02-12 08:40:43'); + +INSERT INTO tuning_job_definition(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason, number_of_iterations, auto_apply) +VALUES +(100003,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true), +(100004,'azkaban',3,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true), +(100005,'azkaban',1,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true), +(100003,'azkaban',4,1,5.178423333333334,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true); + + + + +INSERT INTO flow_execution(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003); + +INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1541,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'); + + + diff --git a/test/resources/test-init-ipso.sql b/test/resources/test-init-ipso.sql new file mode 100644 index 000000000..6b559764e --- /dev/null +++ b/test/resources/test-init-ipso.sql @@ -0,0 +1,167 @@ +insert into yarn_app_result + (id,name,username,queue_name,start_time,finish_time,tracking_url,job_type,severity,score,workflow_depth,scheduler,job_name,job_exec_id,flow_exec_id,job_def_id,flow_def_id,job_exec_url,flow_exec_url,job_def_url,flow_def_url,resource_used,resource_wasted,total_delay) values + ('application_1458194917883_1453361','Email Overwriter','growth','misc_default',1460980616502,1460980723925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453361','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654676&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654676','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 100, 30, 20), + ('application_1458194917883_1453362','Email Overwriter','metrics','misc_default',1460980823925,1460980923925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453362','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654677&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654677','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 200, 40, 10); + +insert into yarn_app_heuristic_result(id,yarn_app_result_id,heuristic_class,heuristic_name,severity,score) values +(137594512,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), +(137594513,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), +(137594516,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), +(137594520,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), +(137594523,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), +(137594525,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), +(137594530,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), +(137594531,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), +(137594534,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), +(137594537,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), +(137594540,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0), +(137594612,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), +(137594613,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), +(137594616,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), +(137594620,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), +(137594623,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), +(137594625,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), +(137594630,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), +(137594631,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), +(137594634,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), +(137594637,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), +(137594640,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0); + +insert into yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name,value,details) values +(137594512,'Group A','1 tasks @ 4 MB avg','NULL'), +(137594512,'Group B','1 tasks @ 79 MB avg','NULL'), +(137594512,'Number of tasks','2','NULL'), +(137594513,'Avg task CPU time (ms)','11510','NULL'), + (137594513,'Avg task GC time (ms)','76','NULL'), + (137594513,'Avg task runtime (ms)','11851','NULL'), + (137594513,'Number of tasks','2','NULL'), + (137594513,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), + (137594516,'Average task input size','42 MB','NULL'), + (137594516,'Average task runtime','11 sec','NULL'), + (137594516,'Max task runtime','12 sec','NULL'), + (137594516,'Min task runtime','11 sec','NULL'), + (137594516,'Number of tasks','2','NULL'), + (137594520,'Median task input size','42 MB','NULL'), + (137594520,'Median task runtime','11 sec','NULL'), + (137594520,'Median task speed','3 MB/s','NULL'), + (137594520,'Number of tasks','2','NULL'), + (137594523,'Avg output records per task','56687','NULL'), + (137594523,'Avg spilled records per task','79913','NULL'), + (137594523,'Number of tasks','2','NULL'), + (137594523,'Ratio of spilled records to output records','1.4097111356119074','NULL'), + (137594525,'Avg Physical Memory (MB)','522','NULL'), + (137594525,'Avg task runtime','11 sec','NULL'), + (137594525,'Avg Virtual Memory (MB)','3307','NULL'), + (137594525,'Max Physical Memory (MB)','595','NULL'), + (137594525,'Max Total Committed Heap Usage Memory (MB)','427','NULL'), + (137594525,'Max Virtual Memory (MB)','1426','NULL'), + (137594525,'Min Physical Memory (MB)','449','NULL'), + (137594525,'Number of tasks','2','NULL'), + (137594525,'Requested Container Memory','2 GB','NULL'), + (137594530,'Group A','11 tasks @ 868 KB avg','NULL'), + (137594530,'Group B','9 tasks @ 883 KB avg ','NULL'), + (137594530,'Number of tasks','20','NULL'), + (137594531,'Avg task CPU time (ms)','8912','NULL'), + (137594531,'Avg task GC time (ms)','73','NULL'), + (137594531,'Avg task runtime (ms)','11045','NULL'), + (137594531,'Number of tasks','20','NULL'), + (137594531,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), + (137594534,'Average task runtime','11 sec','NULL'), + (137594534,'Max task runtime','14 sec','NULL'), + (137594534,'Min task runtime','8 sec','NULL'), + (137594534,'Number of tasks','20','NULL'), + (137594537,'Avg Physical Memory (MB)','416','NULL'), + (137594537,'Avg task runtime','11 sec','NULL'), + (137594537,'Avg Virtual Memory (MB)','3326','NULL'), + (137594537,'Max Physical Memory (MB)','497','NULL'), + (137594537,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594537,'Max Virtual Memory (MB)','1350','NULL'), + (137594537,'Min Physical Memory (MB)','354','NULL'), + (137594537,'Number of tasks','20','NULL'), + (137594537,'Requested Container Memory','2 GB','NULL'), + (137594540,'Average code runtime','1 sec','NULL'), + (137594540,'Average shuffle time','9 sec (5.49x)','NULL'), + (137594540,'Average sort time','(0.04x)','NULL'), + (137594540,'Number of tasks','20','NULL'), + (137594612,'Group A','1 tasks @ 4 MB avg','NULL'), + (137594612,'Group B','1 tasks @ 79 MB avg','NULL'), + (137594612,'Number of tasks','2','NULL'), + (137594613,'Avg task CPU time (ms)','11510','NULL'), + (137594613,'Avg task GC time (ms)','76','NULL'), + (137594613,'Avg task runtime (ms)','11851','NULL'), + (137594613,'Number of tasks','2','NULL'), + (137594613,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), + (137594616,'Average task input size','42 MB','NULL'), + (137594616,'Average task runtime','11 sec','NULL'), + (137594616,'Max task runtime','12 sec','NULL'), + (137594616,'Min task runtime','11 sec','NULL'), + (137594616,'Number of tasks','2','NULL'), + (137594620,'Median task input size','42 MB','NULL'), + (137594620,'Median task runtime','11 sec','NULL'), + (137594620,'Median task speed','3 MB/s','NULL'), + (137594620,'Number of tasks','2','NULL'), + (137594623,'Avg output records per task','56687','NULL'), + (137594623,'Avg spilled records per task','79913','NULL'), + (137594623,'Number of tasks','2','NULL'), + (137594623,'Ratio of spilled records to output records','1.4097111356119074','NULL'), + (137594625,'Avg Physical Memory (MB)','522','NULL'), + (137594625,'Avg task runtime','11 sec','NULL'), + (137594625,'Avg Virtual Memory (MB)','3307','NULL'), + (137594625,'Max Physical Memory (MB)','595','NULL'), + (137594625,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594625,'Max Virtual Memory (MB)','2200','NULL'), + (137594625,'Min Physical Memory (MB)','449','NULL'), + (137594625,'Number of tasks','2','NULL'), + (137594625,'Requested Container Memory','2 GB','NULL'), + (137594630,'Group A','11 tasks @ 868 KB avg','NULL'), + (137594630,'Group B','9 tasks @ 883 KB avg ','NULL'), + (137594630,'Number of tasks','20','NULL'), + (137594631,'Avg task CPU time (ms)','8912','NULL'), + (137594631,'Avg task GC time (ms)','73','NULL'), + (137594631,'Avg task runtime (ms)','11045','NULL'), + (137594631,'Number of tasks','20','NULL'), + (137594631,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), + (137594634,'Average task runtime','11 sec','NULL'), + (137594634,'Max task runtime','14 sec','NULL'), + (137594634,'Min task runtime','8 sec','NULL'), + (137594634,'Number of tasks','20','NULL'), + (137594637,'Avg Physical Memory (MB)','416','NULL'), + (137594637,'Avg task runtime','11 sec','NULL'), + (137594637,'Avg Virtual Memory (MB)','3326','NULL'), + (137594637,'Max Physical Memory (MB)','497','NULL'), + (137594637,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594637,'Max Virtual Memory (MB)','2100','NULL'), + (137594637,'Min Physical Memory (MB)','354','NULL'), + (137594637,'Number of tasks','20','NULL'), + (137594637,'Requested Container Memory','2 GB','NULL'), + (137594640,'Average code runtime','1 sec','NULL'), + (137594640,'Average shuffle time','9 sec (5.49x)','NULL'), + (137594640,'Average sort time','(0.04x)','NULL'), + (137594640,'Number of tasks','20','NULL'); + +INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES (10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'); + +INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES (100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','mkumar1','2018-02-12 08:40:42','2018-02-12 08:40:43'); + +INSERT INTO tuning_job_definition(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason) +VALUES (100003,'azkaban',3,1,40.29456456163195,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL); + +INSERT INTO flow_execution(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003); + +INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1541,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'); + + +INSERT INTO job_suggested_param_set(fitness_job_execution_id, tuning_algorithm_id, param_set_state, is_param_set_default, fitness, is_param_set_best, are_constraints_violated, job_definition_id,id) VALUES +(1541,3,'FITNESS_COMPUTED',0,0.06987967161749142,0,0,100003,1541); + +INSERT INTO job_suggested_param_value(id, job_suggested_param_set_id, tuning_parameter_id, param_value, created_ts, updated_ts) VALUES +(3209,1541,10,149.5239493606563,'2018-02-14 05:30:42','2018-02-14 05:30:42'), +(3210,1541,11,1536,'2018-02-14 05:30:42','2018-02-14 05:30:42'), +(3211,1541,12,10,'2018-02-14 05:30:42','2018-02-14 05:30:42'), +(3212,1541,13,0.761466551875019,'2018-02-14 05:30:42','2018-02-14 05:30:42'), +(3213,1541,14,2844.365182469904,'2018-02-14 05:30:42','2018-02-14 05:30:42'), +(3215,1541,15,2133.273886852428,'2018-02-14 05:30:42','2018-02-14 05:30:42'), +(3216,1541,16,1152,'2018-02-14 05:30:42','2018-02-14 05:30:42'); + diff --git a/test/rest/RestAPITest.java b/test/rest/RestAPITest.java index 5ac9b93f6..d619e9648 100644 --- a/test/rest/RestAPITest.java +++ b/test/rest/RestAPITest.java @@ -21,8 +21,9 @@ import com.google.gson.Gson; import com.linkedin.drelephant.DrElephant; import com.linkedin.drelephant.ElephantContext; -import com.linkedin.drelephant.tuning.BaselineComputeUtil; -import com.linkedin.drelephant.tuning.FitnessComputeUtil; +import com.linkedin.drelephant.tuning.Manager; +import com.linkedin.drelephant.tuning.obt.BaselineManagerOBT; +import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoPSO; import com.linkedin.drelephant.util.Utils; import java.util.HashMap; @@ -163,8 +164,11 @@ public void run() { jobSuggestedParamSet.paramSetState = ParamSetStatus.EXECUTED; jobSuggestedParamSet.update(); - FitnessComputeUtil fitnessComputeUtil = new FitnessComputeUtil(); - fitnessComputeUtil.updateFitness(); + /*FitnessComputeUtil fitnessComputeUtil = new FitnessComputeUtil(); + fitnessComputeUtil.updateFitness();*/ + + Manager manager = new FitnessManagerOBTAlgoPSO(); + manager.execute(); jobSuggestedParamSet = JobSuggestedParamSet.find.byId(jobSuggestedParamSet.id); @@ -205,8 +209,10 @@ public void run() { assertTrue("New Job Not created ", tuningJobDefinition.job.jobName.equals("countByCountryFlowSmallNew_countByCountry")); - BaselineComputeUtil baselineComputeUtil = new BaselineComputeUtil(); - baselineComputeUtil.computeBaseline(); + //BaselineComputeUtil baselineComputeUtil = new BaselineComputeUtil(); + Manager manager = new BaselineManagerOBT(); + manager.execute(); + //baselineComputeUtil.computeBaseline(); tuningJobDefinition = TuningJobDefinition.find.select("*") .where() @@ -250,6 +256,7 @@ private JsonNode getTestGetCurrentRunParameterNewData() { params.put("client", "azkaban"); params.put("autoTuningJobType", "PIG"); params.put("optimizationMetric", "RESOURCE"); + params.put("optimizationAlgo","PSO"); params.put("userName", "mkumar1"); params.put("isRetry", "false"); params.put("skipExecutionForOptimization", "false"); @@ -286,6 +293,7 @@ private JsonNode getTestGetCurrentRunParameterData() { params.put("client", "azkaban"); params.put("autoTuningJobType", "PIG"); params.put("optimizationMetric", "RESOURCE"); + params.put("optimizationAlgo","PSO"); params.put("userName", "mkumar1"); params.put("isRetry", "false"); params.put("skipExecutionForOptimization", "false"); @@ -1014,6 +1022,7 @@ private void populateTestData() { private void populateAutoTuningTestData1() { try { initAutoTuningDB1(); + //initDB(); } catch (Exception e) { e.printStackTrace(); } From 5a1f63bafdfb68b95819942b670800f1251d6083 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Fri, 31 Aug 2018 17:39:32 +0530 Subject: [PATCH 02/22] Ipso configuration changes for Unified Architecture --- app-conf/HeuristicConf.xml | 6 ++ .../heuristics/CommonConstantsHeuristic.java | 55 +++++++++++- .../heuristics/ConfigurationHeuristic.java | 86 +++++++++++++++++++ .../heuristics/GenericMemoryHeuristic.java | 78 +++++++++++------ 4 files changed, 198 insertions(+), 27 deletions(-) create mode 100644 app/com/linkedin/drelephant/mapreduce/heuristics/ConfigurationHeuristic.java diff --git a/app-conf/HeuristicConf.xml b/app-conf/HeuristicConf.xml index e2a83689b..1f75292c7 100644 --- a/app-conf/HeuristicConf.xml +++ b/app-conf/HeuristicConf.xml @@ -320,6 +320,12 @@ + + mapreduce + MapReduceConfiguration + com.linkedin.drelephant.mapreduce.heuristics.ConfigurationHeuristic + views.html.help.mapreduce.helpMapperSpeed + diff --git a/app/com/linkedin/drelephant/mapreduce/heuristics/CommonConstantsHeuristic.java b/app/com/linkedin/drelephant/mapreduce/heuristics/CommonConstantsHeuristic.java index af20d4a12..f9864806a 100644 --- a/app/com/linkedin/drelephant/mapreduce/heuristics/CommonConstantsHeuristic.java +++ b/app/com/linkedin/drelephant/mapreduce/heuristics/CommonConstantsHeuristic.java @@ -2,6 +2,57 @@ public class CommonConstantsHeuristic { - public static final String MAPPER_SPEED="Mapper Speed"; - public static final String TOTAL_INPUT_SIZE_IN_MB="Total input size in MB"; + public static final String MAPPER_SPEED = "Mapper Speed"; + public static final String TOTAL_INPUT_SIZE_IN_MB = "Total input size in MB"; + + public enum UtilizedParameterKeys { + AVG_PHYSICAL_MEMORY("Avg Physical Memory (MB)"), + MAX_PHYSICAL_MEMORY("Max Physical Memory (MB)"), + MIN_PHYSICAL_MEMORY("Min Physical Memory (MB)"), + AVG_VIRTUAL_MEMORY("Avg Virtual Memory (MB)"), + MAX_VIRTUAL_MEMORY("Max Virtual Memory (MB)"), + MIN_VIRTUAL_MEMORY("Min Virtual Memory (MB)"), + AVG_TOTAL_COMMITTED_HEAP_USAGE_MEMORY("Avg Total Committed Heap Usage Memory (MB)"), + MAX_TOTAL_COMMITTED_HEAP_USAGE_MEMORY("Max Total Committed Heap Usage Memory (MB)"), + MIN_TOTAL_COMMITTED_HEAP_USAGE_MEMORY("Min Total Committed Heap Usage Memory (MB)"); + private String value; + + UtilizedParameterKeys(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + } + + public enum ParameterKeys { + MAPPER_MEMORY_HADOOP_CONF("mapreduce.map.memory.mb"), + MAPPER_HEAP_HADOOP_CONF("mapreduce.map.java.opts"), + SORT_BUFFER_HADOOP_CONF("mapreduce.task.io.sort.mb"), + SORT_FACTOR_HADOOP_CONF("mapreduce.task.io.sort.factor"), + SORT_SPILL_HADOOP_CONF("mapreduce.map.sort.spill.percent"), + REDUCER_MEMORY_HADOOP_CONF("mapreduce.reduce.memory.mb"), + REDUCER_HEAP_HADOOP_CONF("mapreduce.reduce.java.opts"), + SPLIT_SIZE_HADOOP_CONF("mapreduce.input.fileinputformat.split.maxsize"), + CHILD_HEAP_SIZE_HADOOP_CONF("mapred.child.java.opts"), + PIG_SPLIT_SIZE_HADOOP_CONF("pig.maxCombinedSplitSize"), + MAPPER_MEMORY_HEURISTICS_CONF("Mapper Memory"), + MAPPER_HEAP_HEURISTICS_CONF("Mapper Heap"), + REDUCER_MEMORY_HEURISTICS_CONF("Reducer Memory"), + REDUCER_HEAP_HEURISTICS_CONF("Reducer heap"), + SORT_BUFFER_HEURISTICS_CONF("Sort Buffer"), + SORT_FACTOR_HEURISTICS_CONF("Sort Factor"), + SORT_SPILL_HEURISTICS_CONF("Sort Spill"), + SPLIT_SIZE_HEURISTICS_CONF("Split Size"), + PIG_MAX_SPLIT_SIZE_HEURISTICS_CONF("Pig Max Split Size"); + private String value; + ParameterKeys(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + } } diff --git a/app/com/linkedin/drelephant/mapreduce/heuristics/ConfigurationHeuristic.java b/app/com/linkedin/drelephant/mapreduce/heuristics/ConfigurationHeuristic.java new file mode 100644 index 000000000..5a10a8dd7 --- /dev/null +++ b/app/com/linkedin/drelephant/mapreduce/heuristics/ConfigurationHeuristic.java @@ -0,0 +1,86 @@ +/* + * Copyright 2016 LinkedIn Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.linkedin.drelephant.mapreduce.heuristics; + +import com.linkedin.drelephant.analysis.Heuristic; +import com.linkedin.drelephant.analysis.HeuristicResult; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; +import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; + +import static com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic.ParameterKeys.*; + +import org.apache.log4j.Logger; + + +/* +Heuristics to collect memory data/counter values about application previous exeution. + */ +public class ConfigurationHeuristic implements Heuristic { + private static final Logger logger = Logger.getLogger(ConfigurationHeuristic.class); + + private HeuristicConfigurationData _heuristicConfData; + + public ConfigurationHeuristic(HeuristicConfigurationData heuristicConfData) { + this._heuristicConfData = heuristicConfData; + } + + @Override + public HeuristicConfigurationData getHeuristicConfData() { + return _heuristicConfData; + } + + @Override + public HeuristicResult apply(MapReduceApplicationData data) { + if (!data.getSucceeded()) { + return null; + } + String mapperMemory = data.getConf().getProperty(MAPPER_MEMORY_HADOOP_CONF.getValue()); + String mapperHeap = data.getConf().getProperty(MAPPER_HEAP_HADOOP_CONF.getValue()); + if (mapperHeap == null) { + mapperHeap = data.getConf().getProperty(CHILD_HEAP_SIZE_HADOOP_CONF.getValue()); + } + String sortBuffer = data.getConf().getProperty(SORT_BUFFER_HADOOP_CONF.getValue()); + String sortFactor = data.getConf().getProperty(SORT_FACTOR_HADOOP_CONF.getValue()); + String sortSplill = data.getConf().getProperty(SORT_SPILL_HADOOP_CONF.getValue()); + String reducerMemory = data.getConf().getProperty(REDUCER_MEMORY_HADOOP_CONF.getValue()); + String reducerHeap = data.getConf().getProperty(REDUCER_HEAP_HADOOP_CONF.getValue()); + if (reducerHeap == null) { + reducerHeap = data.getConf().getProperty(CHILD_HEAP_SIZE_HADOOP_CONF.getValue()); + } + String splitSize = data.getConf().getProperty(SPLIT_SIZE_HADOOP_CONF.getValue()); + String pigSplitSize = data.getConf().getProperty(PIG_SPLIT_SIZE_HADOOP_CONF.getValue()); + HeuristicResult result = + new HeuristicResult(_heuristicConfData.getClassName(), _heuristicConfData.getHeuristicName(), Severity.LOW, 0); + + result.addResultDetail(MAPPER_MEMORY_HEURISTICS_CONF.getValue(), mapperMemory); + result.addResultDetail(MAPPER_HEAP_HEURISTICS_CONF.getValue(), mapperHeap.replaceAll("\\s+", "\n")); + result.addResultDetail(REDUCER_MEMORY_HEURISTICS_CONF.getValue(), reducerMemory); + result.addResultDetail(REDUCER_HEAP_HEURISTICS_CONF.getValue(), reducerHeap.replaceAll("\\s+", "\n")); + result.addResultDetail(SORT_BUFFER_HEURISTICS_CONF.getValue(), sortBuffer); + result.addResultDetail(SORT_FACTOR_HEURISTICS_CONF.getValue(), sortFactor); + result.addResultDetail(SORT_SPILL_HEURISTICS_CONF.getValue(), sortSplill); + if (splitSize != null) { + result.addResultDetail(SPLIT_SIZE_HEURISTICS_CONF.getValue(), splitSize); + } + if (pigSplitSize != null) { + result.addResultDetail(PIG_MAX_SPLIT_SIZE_HEURISTICS_CONF.getValue(), pigSplitSize); + } + + return result; + } +} diff --git a/app/com/linkedin/drelephant/mapreduce/heuristics/GenericMemoryHeuristic.java b/app/com/linkedin/drelephant/mapreduce/heuristics/GenericMemoryHeuristic.java index 04644dc51..488e93537 100644 --- a/app/com/linkedin/drelephant/mapreduce/heuristics/GenericMemoryHeuristic.java +++ b/app/com/linkedin/drelephant/mapreduce/heuristics/GenericMemoryHeuristic.java @@ -33,6 +33,8 @@ import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; +import static com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic.UtilizedParameterKeys.*; + /** * This heuristic deals with the efficiency of container size @@ -59,8 +61,7 @@ private long getContainerMemDefaultMBytes() { String strValue = paramMap.get(CONTAINER_MEM_DEFAULT_MB); try { return Long.valueOf(strValue); - } - catch (NumberFormatException e) { + } catch (NumberFormatException e) { logger.warn(CONTAINER_MEM_DEFAULT_MB + ": expected number [" + strValue + "]"); } } @@ -80,7 +81,7 @@ private void loadParameters() { long containerMemDefaultBytes = getContainerMemDefaultMBytes() * FileUtils.ONE_MB; logger.info(heuristicName + " will use " + CONTAINER_MEM_DEFAULT_MB + " with the following threshold setting: " - + containerMemDefaultBytes); + + containerMemDefaultBytes); double[] confMemoryLimits = Utils.getParam(paramMap.get(CONTAINER_MEM_SEVERITY), memoryLimits.length); if (confMemoryLimits != null) { @@ -110,7 +111,7 @@ public HeuristicConfigurationData getHeuristicConfData() { @Override public HeuristicResult apply(MapReduceApplicationData data) { - if(!data.getSucceeded()) { + if (!data.getSucceeded()) { return null; } @@ -122,14 +123,13 @@ public HeuristicResult apply(MapReduceApplicationData data) { containerMem = Long.parseLong(containerSizeStr); } catch (NumberFormatException e0) { // Some job has a string var like "${VAR}" for this config. - if(containerSizeStr.startsWith("$")) { - String realContainerConf = containerSizeStr.substring(containerSizeStr.indexOf("{")+1, - containerSizeStr.indexOf("}")); + if (containerSizeStr.startsWith("$")) { + String realContainerConf = + containerSizeStr.substring(containerSizeStr.indexOf("{") + 1, containerSizeStr.indexOf("}")); String realContainerSizeStr = data.getConf().getProperty(realContainerConf); try { containerMem = Long.parseLong(realContainerSizeStr); - } - catch (NumberFormatException e1) { + } catch (NumberFormatException e1) { logger.warn(realContainerConf + ": expected number [" + realContainerSizeStr + "]"); } } else { @@ -146,26 +146,40 @@ public HeuristicResult apply(MapReduceApplicationData data) { List taskPMems = new ArrayList(); List taskVMems = new ArrayList(); List runtimesMs = new ArrayList(); - long taskPMin = Long.MAX_VALUE; - long taskPMax = 0; + List taskHeapUsages = new ArrayList(); + long taskPMin = Long.MAX_VALUE, taskVMin = Long.MAX_VALUE, taskHUMin = Long.MAX_VALUE; + long taskPMax = 0, taskVMax = 0, taskHUMax = 0; for (MapReduceTaskData task : tasks) { if (task.isTimeAndCounterDataPresent()) { runtimesMs.add(task.getTotalRunTimeMs()); long taskPMem = task.getCounters().get(MapReduceCounterData.CounterName.PHYSICAL_MEMORY_BYTES); long taskVMem = task.getCounters().get(MapReduceCounterData.CounterName.VIRTUAL_MEMORY_BYTES); - taskPMems.add(taskPMem); + long taskHeapUsage = task.getCounters().get(MapReduceCounterData.CounterName.COMMITTED_HEAP_BYTES); taskPMin = Math.min(taskPMin, taskPMem); taskPMax = Math.max(taskPMax, taskPMem); + taskVMin = Math.min(taskVMin, taskVMem); + taskVMax = Math.max(taskVMax, taskVMem); + taskHUMin = Math.min(taskHUMin, taskHeapUsage); + taskHUMax = Math.max(taskHUMax, taskHeapUsage); taskVMems.add(taskVMem); + taskPMems.add(taskPMem); + taskHeapUsages.add(taskHeapUsage); } } - if(taskPMin == Long.MAX_VALUE) { + if (taskPMin == Long.MAX_VALUE) { taskPMin = 0; } + if (taskVMin == Long.MAX_VALUE) { + taskVMin = 0; + } + if (taskHUMin == Long.MAX_VALUE) { + taskHUMin = 0; + } long taskPMemAvg = Statistics.average(taskPMems); long taskVMemAvg = Statistics.average(taskVMems); + long taskHeapUsageAvg = Statistics.average(taskHeapUsages); long averageTimeMs = Statistics.average(runtimesMs); Severity severity; @@ -175,22 +189,37 @@ public HeuristicResult apply(MapReduceApplicationData data) { severity = getTaskMemoryUtilSeverity(taskPMemAvg, containerMem); } - HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), - _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length)); + HeuristicResult result = + new HeuristicResult(_heuristicConfData.getClassName(), _heuristicConfData.getHeuristicName(), severity, + Utils.getHeuristicScore(severity, tasks.length)); result.addResultDetail("Number of tasks", Integer.toString(tasks.length)); result.addResultDetail("Avg task runtime", Statistics.readableTimespan(averageTimeMs)); - result.addResultDetail("Avg Physical Memory (MB)", Long.toString(taskPMemAvg / FileUtils.ONE_MB)); - result.addResultDetail("Max Physical Memory (MB)", Long.toString(taskPMax / FileUtils.ONE_MB)); - result.addResultDetail("Min Physical Memory (MB)", Long.toString(taskPMin / FileUtils.ONE_MB)); - result.addResultDetail("Avg Virtual Memory (MB)", Long.toString(taskVMemAvg / FileUtils.ONE_MB)); + result.addResultDetail(AVG_PHYSICAL_MEMORY.getValue(), + Long.toString(taskPMemAvg / FileUtils.ONE_MB)); + result.addResultDetail(MAX_PHYSICAL_MEMORY.getValue(), + Long.toString(taskPMax / FileUtils.ONE_MB)); + result.addResultDetail(MIN_PHYSICAL_MEMORY.getValue(), + Long.toString(taskPMin / FileUtils.ONE_MB)); + result.addResultDetail(AVG_VIRTUAL_MEMORY.getValue(), + Long.toString(taskVMemAvg / FileUtils.ONE_MB)); + result.addResultDetail(MAX_VIRTUAL_MEMORY.getValue(), + Long.toString(taskVMax / FileUtils.ONE_MB)); + result.addResultDetail(MIN_VIRTUAL_MEMORY.getValue(), + Long.toString(taskVMin / FileUtils.ONE_MB)); + result.addResultDetail(AVG_TOTAL_COMMITTED_HEAP_USAGE_MEMORY.getValue(), + Long.toString(taskHeapUsageAvg / FileUtils.ONE_MB)); + result.addResultDetail(MAX_TOTAL_COMMITTED_HEAP_USAGE_MEMORY.getValue(), + Long.toString(taskHUMax / FileUtils.ONE_MB)); + result.addResultDetail(MIN_TOTAL_COMMITTED_HEAP_USAGE_MEMORY.getValue(), + Long.toString(taskHUMin / FileUtils.ONE_MB)); result.addResultDetail("Requested Container Memory", FileUtils.byteCountToDisplaySize(containerMem)); return result; } private Severity getTaskMemoryUtilSeverity(long taskMemAvg, long taskMemMax) { - double ratio = ((double)taskMemAvg) / taskMemMax; + double ratio = ((double) taskMemAvg) / taskMemMax; Severity sevRatio = getMemoryRatioSeverity(ratio); // Severity is reduced if the requested container memory is close to default Severity sevMax = getContainerMemorySeverity(taskMemMax); @@ -198,14 +227,13 @@ private Severity getTaskMemoryUtilSeverity(long taskMemAvg, long taskMemMax) { return Severity.min(sevRatio, sevMax); } - private Severity getContainerMemorySeverity(long taskMemMax) { - return Severity.getSeverityAscending( - taskMemMax, memoryLimits[0], memoryLimits[1], memoryLimits[2], memoryLimits[3]); + return Severity.getSeverityAscending(taskMemMax, memoryLimits[0], memoryLimits[1], memoryLimits[2], + memoryLimits[3]); } private Severity getMemoryRatioSeverity(double ratio) { - return Severity.getSeverityDescending( - ratio, memRatioLimits[0], memRatioLimits[1], memRatioLimits[2], memRatioLimits[3]); + return Severity.getSeverityDescending(ratio, memRatioLimits[0], memRatioLimits[1], memRatioLimits[2], + memRatioLimits[3]); } } From 44f5d8c27399093bf5fe480d76761527e977489f Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Tue, 4 Sep 2018 15:37:58 +0530 Subject: [PATCH 03/22] Added Documentation ; Added is_suggested_param_set ; Added logic for disable tuning for HBT --- .../tuning/AbstractBaselineManager.java | 12 +- .../tuning/AbstractFitnessManager.java | 177 +++-------------- .../tuning/AbstractJobStatusManager.java | 27 ++- .../tuning/AbstractTuningTypeManager.java | 77 ++++++-- .../tuning/AutoTuningAPIHelper.java | 15 +- .../drelephant/tuning/ExecutionEngine.java | 54 +++++- app/com/linkedin/drelephant/tuning/Flow.java | 11 +- .../drelephant/tuning/ManagerFactory.java | 42 ++-- .../tuning/engine/MRExecutionEngine.java | 17 +- .../tuning/engine/SparkExecutionEngine.java | 10 +- .../tuning/hbt/FitnessManagerHBT.java | 87 +++++++++ .../tuning/hbt/TuningTypeManagerHBT.java | 1 + .../tuning/obt/FitnessManagerOBT.java | 182 +++++++++++++++++- .../tuning/obt/OptimizationAlgoFactory.java | 6 +- .../tuning/obt/TuningTypeManagerOBT.java | 37 ++-- app/models/JobSuggestedParamSet.java | 4 + conf/evolutions/default/5.sql | 1 + conf/evolutions/default/6.sql | 2 + 18 files changed, 521 insertions(+), 241 deletions(-) diff --git a/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java b/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java index 7336efb4c..cf1f53fc7 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java @@ -13,6 +13,11 @@ import controllers.AutoTuningMetricsController; import org.apache.hadoop.conf.Configuration; + +/** + * This class is abstract class for BaselineManager. Each tuning type will have its own specific implementation. + * for e.g BaselineManagerOBT, BaselineManagerHBT + */ public abstract class AbstractBaselineManager implements Manager { protected final String BASELINE_EXECUTION_COUNT = "baseline.execution.count"; protected Integer NUM_JOBS_FOR_BASELINE_DEFAULT = 30; @@ -33,7 +38,7 @@ public abstract class AbstractBaselineManager implements Manager { protected Integer _numJobsForBaseline = null; protected Configuration configuration = null; - public AbstractBaselineManager(){ + public AbstractBaselineManager() { configuration = ElephantContext.instance().getAutoTuningConf(); } @@ -111,7 +116,8 @@ protected Boolean calculateBaseLine(List tuningJobDefinitio * @return average input size in bytes as long */ private Long getAvgInputSizeInBytes(String jobDefId) { - logger.debug("Running query for average input size computation " + avgInputSizeSQL + " Job definintion ID "+jobDefId); + logger.debug( + "Running query for average input size computation " + avgInputSizeSQL + " Job definintion ID " + jobDefId); SqlRow baseline = Ebean.createSqlQuery(avgInputSizeSQL) .setParameter("jobDefId", jobDefId) @@ -160,6 +166,4 @@ protected Boolean updateMetrics(List tuningJobDefinitions) } return true; } - - } diff --git a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java index 98295b8c1..248cc9d76 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java @@ -43,8 +43,28 @@ public abstract class AbstractFitnessManager implements Manager { protected Long fitnessComputeWaitInterval; protected Long ignoreExecutionWaitInterval; + /** + * Detects & return the jobs for which fitness computation have to be done. + * @return + */ protected abstract List detectJobsForFitnessComputation(); + /** + * This method is used to calculate the fitness and update into DB. + * @param jobExecution : Job Execution for which fitness need to be computed + * @param results : Heuristics App results for the same. + * @param tuningJobDefinition : Tuning job defination of the job. + * @param jobSuggestedParamSet : Status of the suggested param set , so that fitness can be computed. + */ + protected abstract void calculateAndUpdateFitness(JobExecution jobExecution, List results, + TuningJobDefinition tuningJobDefinition, JobSuggestedParamSet jobSuggestedParamSet); + + /** + * This methods disable tuning , if certain prereuistes matched. + * @param jobDefinitionSet + */ + protected abstract void checkToDisableTuning(Set jobDefinitionSet); + protected Boolean calculateFitness(List completedJobExecutionParamSets) { for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { JobExecution jobExecution = completedJobExecutionParamSet.jobExecution; @@ -95,8 +115,7 @@ private void handleFitnessCalculation(JobExecution jobExecution, List } } - protected abstract void calculateAndUpdateFitness(JobExecution jobExecution, List results, - TuningJobDefinition tuningJobDefinition, JobSuggestedParamSet jobSuggestedParamSet); + protected void updateJobExecution(JobExecution jobExecution, Double totalResourceUsed, Double totalInputBytesInBytes, Long totalExecutionTime) { @@ -262,98 +281,14 @@ private boolean isTuningEnabled(Integer jobDefinitionId) { return tuningJobDefinition != null && tuningJobDefinition.tuningEnabled; } - /** - * Checks if the median gain (from tuning) during the last 6 executions is negative - * Last 6 executions constitutes 2 iterations of PSO (given the swarm size is three). Negative average gains in - * latest 2 algorithm iterations (after a fixed number of minimum iterations) imply that either the algorithm hasn't - * converged or there isn't enough scope for tuning. In both the cases, switching tuning off is desired - * @param tuningJobExecutionParamSets List of previous executions - * @return true if the median gain is negative, else false - */ - private boolean isMedianGainNegative(List tuningJobExecutionParamSets) { - int numFitnessForMedian = 6; - Double[] fitnessArray = new Double[numFitnessForMedian]; - int entries = 0; - if (tuningJobExecutionParamSets.size() < numFitnessForMedian) { - return false; - } - for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { - JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; - JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; - if (jobExecution.executionState == JobExecution.ExecutionState.SUCCEEDED - && jobSuggestedParamSet.paramSetState == JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED) { - fitnessArray[entries] = jobSuggestedParamSet.fitness; - entries += 1; - if (entries == numFitnessForMedian) { - break; - } - } - } - Arrays.sort(fitnessArray); - double medianFitness; - if (fitnessArray.length % 2 == 0) { - medianFitness = (fitnessArray[fitnessArray.length / 2] + fitnessArray[fitnessArray.length / 2 - 1]) / 2; - } else { - medianFitness = fitnessArray[fitnessArray.length / 2]; - } - JobDefinition jobDefinition = tuningJobExecutionParamSets.get(0).jobSuggestedParamSet.jobDefinition; - TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where(). - eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id).findUnique(); - double baselineFitness = - tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; - if (medianFitness > baselineFitness) { - logger.info("Switching off tuning for job: " + jobDefinition.jobName + " Reason: unable to tune enough"); - return true; - } else { - return false; - } - } - /** - * Checks and disables tuning for the given job definitions. - * Tuning can be disabled if: - * - Number of tuning executions >= maxTuningExecutions - * - or number of tuning executions >= minTuningExecutions and parameters converge - * - or number of tuning executions >= minTuningExecutions and median gain (in cost function) in last 6 executions is negative - * @param jobDefinitionSet Set of jobs to check if tuning can be switched off for them - */ - - protected void checkToDisableTuning(Set jobDefinitionSet) { - for (JobDefinition jobDefinition : jobDefinitionSet) { - List tuningJobExecutionParamSets = - TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") - .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") - .where() - .eq(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.jobDefinition - + '.' + JobDefinition.TABLE.id, jobDefinition.id) - .order() - .desc("job_execution_id") - .findList(); - - if (reachToNumberOfThresholdIterations(tuningJobExecutionParamSets, jobDefinition)) { - disableTuning(jobDefinition, "User Specified Iterations reached"); - } - if (tuningJobExecutionParamSets.size() >= minTuningExecutions) { - if (didParameterSetConverge(tuningJobExecutionParamSets)) { - logger.info("Parameters converged. Disabling tuning for job: " + jobDefinition.jobName); - disableTuning(jobDefinition, "Parameters converged"); - } else if (isMedianGainNegative(tuningJobExecutionParamSets)) { - logger.info("Unable to get gain while tuning. Disabling tuning for job: " + jobDefinition.jobName); - disableTuning(jobDefinition, "Unable to get gain"); - } else if (tuningJobExecutionParamSets.size() >= maxTuningExecutions) { - logger.info("Maximum tuning executions limit reached. Disabling tuning for job: " + jobDefinition.jobName); - disableTuning(jobDefinition, "Maximum executions reached"); - } - } - } - } - private boolean reachToNumberOfThresholdIterations(List tuningJobExecutionParamSets, + public boolean reachToNumberOfThresholdIterations(List tuningJobExecutionParamSets, JobDefinition jobDefinition) { TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id) @@ -369,7 +304,7 @@ private boolean reachToNumberOfThresholdIterations(List tuningJobExecutionParamSets) { - boolean result = false; - int numParamSetForConvergence = 3; - - if (tuningJobExecutionParamSets.size() < numParamSetForConvergence) { - return false; - } - - TuningAlgorithm.JobType jobType = tuningJobExecutionParamSets.get(0).jobSuggestedParamSet.tuningAlgorithm.jobType; - - if (jobType == TuningAlgorithm.JobType.PIG) { - - Map> paramValueSet = new HashMap>(); - - for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { - - JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; - List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, - jobSuggestedParamSet.id) - .or(Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, - "mapreduce.map.memory.mb"), - Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, - "mapreduce.reduce.memory.mb")) - .findList(); - - // if jobSuggestedParamValueList contains both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb - // ie, if the size of jobSuggestedParamValueList is 2 - if (jobSuggestedParamValueList != null && jobSuggestedParamValueList.size() == 2) { - numParamSetForConvergence -= 1; - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { - Set tmp; - if (paramValueSet.containsKey(jobSuggestedParamValue.id)) { - tmp = paramValueSet.get(jobSuggestedParamValue.id); - } else { - tmp = new HashSet(); - } - tmp.add(jobSuggestedParamValue.paramValue); - paramValueSet.put(jobSuggestedParamValue.id, tmp); - } - } - - if (numParamSetForConvergence == 0) { - break; - } - } - - result = true; - for (Integer paramId : paramValueSet.keySet()) { - if (paramValueSet.get(paramId).size() > 1) { - result = false; - } - } - } - - if (result) { - logger.info("Switching off tuning for job: " + tuningJobExecutionParamSets.get( - 0).jobSuggestedParamSet.jobDefinition.jobName + " Reason: parameter set converged"); - } - return result; - } public final Boolean execute() { logger.info("Executing Fitness Manager"); diff --git a/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java b/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java index 75a7d2008..36ea82eb6 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java @@ -8,9 +8,20 @@ import org.apache.log4j.Logger; +/** + * Abstract Job Status Manager provides the functionality to check the job status of the job. This should have implementation specific to schduler + */ public abstract class AbstractJobStatusManager implements Manager { private final Logger logger = Logger.getLogger(getClass()); - protected abstract Boolean analyzeCompletedJobsExecution(List inProgressExecutionParamSet); + + /** + * This method find out the completed execution of the job and update the database. + * This method is schduler dependent. + * @param inProgressExecutionParamSet : Jobs for which status have to find out. + * @return + */ + protected abstract Boolean analyzeCompletedJobsExecution( + List inProgressExecutionParamSet); protected List detectJobsExecutionInProgress() { logger.info("Fetching the executions which are in progress"); @@ -22,15 +33,12 @@ protected List detectJobsExecutionInProgress() { JobExecution.ExecutionState.IN_PROGRESS) .findList(); - logger.info("Number of executions which are in progress: " + tuningJobExecutionParamSets.size()); return tuningJobExecutionParamSets; } - - protected Boolean updateDataBase(List jobs) { - for(TuningJobExecutionParamSet job : jobs){ + for (TuningJobExecutionParamSet job : jobs) { JobSuggestedParamSet jobSuggestedParamSet = job.jobSuggestedParamSet; JobExecution jobExecution = job.jobExecution; if (isJobCompleted(jobExecution)) { @@ -44,17 +52,18 @@ protected Boolean updateDataBase(List jobs) { return true; } - private Boolean isJobCompleted(JobExecution jobExecution){ + private Boolean isJobCompleted(JobExecution jobExecution) { if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED) || jobExecution.executionState.equals( JobExecution.ExecutionState.FAILED) || jobExecution.executionState.equals( JobExecution.ExecutionState.CANCELLED)) { return true; + } else { + return false; } - else return false; } protected Boolean updateMetrics(List completedJobs) { - for(TuningJobExecutionParamSet completedJob : completedJobs){ + for (TuningJobExecutionParamSet completedJob : completedJobs) { JobExecution jobExecution = completedJob.jobExecution; if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { AutoTuningMetricsController.markSuccessfulJobs(); @@ -65,8 +74,6 @@ protected Boolean updateMetrics(List completedJobs) return true; } - - @Override public final Boolean execute() { logger.info("Executing Job Status Manager"); diff --git a/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java b/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java index c1e7a9c25..dd8061b90 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java @@ -14,6 +14,10 @@ import org.apache.log4j.Logger; import play.libs.Json; + +/** + * This class is used to generate/suggest parameters . Based on TuningType , Algorithm Type & execution engine. + */ public abstract class AbstractTuningTypeManager implements Manager { protected final String JSON_CURRENT_POPULATION_KEY = "current_population"; private final Logger logger = Logger.getLogger(getClass()); @@ -21,6 +25,64 @@ public abstract class AbstractTuningTypeManager implements Manager { protected String tuningType = null; protected String tuningAlgorithm = null; + /** + * This method will generate the Param Set and update in JobTuningInfo in saved state. + * @param jobTuningInfo + * @return + */ + protected abstract JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo); + + /** + * This method is used to update the database as required by param generator + * @param tuningJobDefinitions + * @return + */ + + protected abstract Boolean updateDatabase(List tuningJobDefinitions); + + /** + * Get pending jobs for which param generation cannot be done + * @return + */ + protected abstract List getPendingParamSets(); + + /** + * Get Jobs for which tuning enabled. Subtraction of above one from this will give , the jobs for which param + * generation have to done + * @return + */ + + protected abstract List getTuningJobDefinitions(); + + /** + * If the algorithm requires to save previous state ,then this method should be implemented + * @param jobTuningInfo + * @param job + */ + + protected abstract void saveJobState(JobTuningInfo jobTuningInfo, JobDefinition job); + + /** + * + * @param tuningParameterList + * @param job + */ + + /** + * Update the boundry constraint based on the previous executions + * @param tuningParameterList + * @param job + */ + + protected abstract void updateBoundryConstraint(List tuningParameterList, JobDefinition job); + + /** + * Check if the suggested parameter violetes the constraint. + * @param jobSuggestedParamValues + * @return + */ + public abstract boolean isParamConstraintViolated(List jobSuggestedParamValues); + /** * Fetches the list to job which need new parameter suggestion * @return Job list @@ -75,10 +137,6 @@ protected List generateParameters(List jobsForPara return updatedJobTuningInfoList; } - protected abstract JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo); - - protected abstract Boolean updateDatabase(List tuningJobDefinitions); - public final Boolean execute() { logger.info("Executing Tuning Algorithm"); Boolean parameterGenerationDone = false, databaseUpdateDone = false, updateMetricsDone = false; @@ -93,10 +151,6 @@ public final Boolean execute() { return databaseUpdateDone; } - protected abstract List getPendingParamSets(); - - protected abstract List getTuningJobDefinitions(); - /** * Returns the tuning information for the jobs * @param tuningJobs Job List @@ -124,13 +178,6 @@ protected List getJobsTuningInfo(List tuning return jobTuningInfoList; } - protected abstract void saveJobState(JobTuningInfo jobTuningInfo, JobDefinition job); - - protected abstract void updateBoundryConstraint(List tuningParameterList,JobDefinition job); - - public abstract boolean isParamConstraintViolated(List jobSuggestedParamValues); - - protected List generateTuningParameterListWithDefaultValues( TuningJobDefinition tuningJobDefinition) { JobDefinition job = tuningJobDefinition.job; diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java index f498dd5e9..8701e381c 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java @@ -473,8 +473,9 @@ private void insertParamSetForDefault(JobDefinition job, TuningAlgorithm tuningA jobSuggestedParamSet.areConstraintsViolated = false; jobSuggestedParamSet.isParamSetBest = false; jobSuggestedParamSet.isManuallyOverridenParameter = false; + jobSuggestedParamSet.isParamSetSuggested = false; jobSuggestedParamSet.save(); - insertParameterValues(jobSuggestedParamSet, paramValueMap); + insertParameterValues(jobSuggestedParamSet, paramValueMap,tuningAlgorithm); logger.debug("Default parameter set inserted for job: " + job.jobName); } @@ -569,8 +570,9 @@ private void insertParamSet (JobDefinition job, TuningAlgorithm jobSuggestedParamSet.areConstraintsViolated = false; jobSuggestedParamSet.isParamSetBest = false; jobSuggestedParamSet.isManuallyOverridenParameter = false; + jobSuggestedParamSet.isParamSetSuggested = false; jobSuggestedParamSet.save(); - insertParameterValues(jobSuggestedParamSet, paramValueMap); + insertParameterValues(jobSuggestedParamSet, paramValueMap,tuningAlgorithm); intializeOptimizationAlgoPrerequisite(tuningAlgorithm, jobSuggestedParamSet); logger.debug("Default parameter set inserted for job: " + job.jobName); } @@ -581,11 +583,11 @@ private void insertParamSet (JobDefinition job, TuningAlgorithm * @param paramValueMap Map of parameter values as string */ @SuppressWarnings("unchecked") private void insertParameterValues (JobSuggestedParamSet - jobSuggestedParamSet, Map < String, Double > paramValueMap){ + jobSuggestedParamSet, Map < String, Double > paramValueMap, TuningAlgorithm tuningAlgorithm){ ObjectMapper mapper = new ObjectMapper(); if (paramValueMap != null) { for (Map.Entry paramValue : paramValueMap.entrySet()) { - insertParameterValue(jobSuggestedParamSet, paramValue.getKey(), paramValue.getValue()); + insertParameterValue(jobSuggestedParamSet, paramValue.getKey(), paramValue.getValue(),tuningAlgorithm); } } else { logger.warn("ParamValueMap is null "); @@ -607,12 +609,13 @@ private void intializeOptimizationAlgoPrerequisite (TuningAlgorithm tuningAlgori * @param paramName Parameter name * @param paramValue Parameter value */ - private void insertParameterValue (JobSuggestedParamSet jobSuggestedParamSet, String paramName, Double paramValue){ + private void insertParameterValue (JobSuggestedParamSet jobSuggestedParamSet, String paramName, Double paramValue,TuningAlgorithm tuningAlgorithm){ logger.debug("Starting insertParameterValue"); JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; TuningParameter tuningParameter = - TuningParameter.find.where().eq(TuningParameter.TABLE.paramName, paramName).findUnique(); + TuningParameter.find.where().eq(TuningParameter.TABLE.paramName, paramName) + .eq(TuningParameter.TABLE.tuningAlgorithm,tuningAlgorithm).findUnique(); if (tuningParameter != null) { jobSuggestedParamValue.tuningParameter = tuningParameter; jobSuggestedParamValue.paramValue = paramValue; diff --git a/app/com/linkedin/drelephant/tuning/ExecutionEngine.java b/app/com/linkedin/drelephant/tuning/ExecutionEngine.java index d21da01ec..a8e018ec1 100644 --- a/app/com/linkedin/drelephant/tuning/ExecutionEngine.java +++ b/app/com/linkedin/drelephant/tuning/ExecutionEngine.java @@ -15,27 +15,69 @@ import models.TuningParameterConstraint; +/** + * Exeuction Engine is the interface of different execution engines. Currently it have two implementations + * MRExecutionEngine : Handle parameters releated to Map Reduce. + * SparkExecutionEngine : Handle parameters releated to Spark + */ + public interface ExecutionEngine { + /** + * This method is used to compute the values of derived parameters . These parameter values have not suggested by tuning algorithm + * + * @param derivedParameterList Derived Parameter List + * @param jobSuggestedParamValue Update job suggested param value with the derived parameter value list. + * + */ void computeValuesOfDerivedConfigurationParameters(List derivedParameterList, List jobSuggestedParamValue); + /** + * This method is used to get the pending JobSuggestedParamSet , param set which are not in FitnessCompute & discarded jobs + * + * @return Return pending jobs + */ + ExpressionList getPendingJobs(); + /** + * This method is used to get jobs for which tuning enabled. Subtraction of this set with the above set + * gives us the jobs for which parameter have to be generated + * + * @return Return Tuning enabled jobs + */ ExpressionList getTuningJobDefinitionsForParameterSuggestion(); - /* - PSO related methods + + /** + * Check if the suggested parameters violets any constraint for PSO . */ Boolean isParamConstraintViolatedPSO(List jobSuggestedParamValueList); - - + /** + * Check if the suggested parameters violets any constraint for IPSO . + */ Boolean isParamConstraintViolatedIPSO(List jobSuggestedParamValueList); + /** + * Optimizes parameter boundries based on the Heuristics. This is only applicable for IPSO + * @param results : App Results + * @param jobExecution : Job Execution on the basis of which parameter boundries will be changed. + */ public void parameterOptimizerIPSO(List results, JobExecution jobExecution); - - + /** + * Parameter Suggestion + * @param results : Based on the previous execution results + * @param tuningParameters : Tuning parameters + * @return : Suggest parameters new values. + */ public String parameterGenerationsHBT(List results, List tuningParameters); + + /** + * Check if the suggested parameter violets any constraint. + * @param jobSuggestedParamValueList + * @return + */ Boolean isParamConstraintViolatedHBT(List jobSuggestedParamValueList); } diff --git a/app/com/linkedin/drelephant/tuning/Flow.java b/app/com/linkedin/drelephant/tuning/Flow.java index 769e4b33b..eea3fc3dd 100644 --- a/app/com/linkedin/drelephant/tuning/Flow.java +++ b/app/com/linkedin/drelephant/tuning/Flow.java @@ -17,6 +17,15 @@ import org.apache.log4j.Logger; +/** + * This is the Flow , which create four pipeline and each pipeline executes in seperate thread. + * Four pipeline are + * BaseLineManager : This will have two manager BaselineManagerOBT, BaselineManagerHBT + * JobStatusManagerPipeline: This will have one manager , AzkabanJobStatusManager + * FitnessManager : This will have three managers FitnessManagerHBT,FitnessManagerOBTAlgoIPSO,FitnessManagerOBTPSO + * TuningTypeManager : This will have Combinations of TuningType , AlgorithmType and execution engine. for e.g + * TuningTypeManagerOBTAlgoIPSO for Map Reduce and Spark ... + */ public class Flow { List> pipelines = null; private static final Logger logger = Logger.getLogger(Flow.class); @@ -39,7 +48,7 @@ public void executeFlow() throws InterruptedException { @Override public void run() { for (Manager manager : pipelineType) { - // logger.info(" Manager execution Status " + manager.getManagerName()); + // logger.info(" Manager execution Status " + manager.getManagerName()); if (manager.getClass().getSimpleName().toLowerCase().contains("hbt") || manager.getClass() .getSimpleName() .toLowerCase() diff --git a/app/com/linkedin/drelephant/tuning/ManagerFactory.java b/app/com/linkedin/drelephant/tuning/ManagerFactory.java index 0290a470a..be1faf04e 100644 --- a/app/com/linkedin/drelephant/tuning/ManagerFactory.java +++ b/app/com/linkedin/drelephant/tuning/ManagerFactory.java @@ -16,6 +16,10 @@ import org.apache.log4j.Logger; +/** + This is the factory of the Managers and return manager object based on input type + + */ public class ManagerFactory { private static final Logger logger = Logger.getLogger(ManagerFactory.class); @@ -23,22 +27,19 @@ public class ManagerFactory { public static Manager getManager(String tuningType, String algorithmType, String executionEngineTypes, String typeOfManagers) { - if(typeOfManagers.equals(Constant.TypeofManagers.AbstractJobStatusManager.name())){ - logger.info("Manager Type Azkaban Job Status Manager"); + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractJobStatusManager.name())) { + logger.info("Manager Type Azkaban Job Status Manager"); return new AzkabanJobStatusManager(); - } - - else if (tuningType.equals(Constant.TuningType.HBT.name())) { - return getMangersForHBT(algorithmType,executionEngineTypes,typeOfManagers); - } - else if (tuningType.equals(Constant.TuningType.OBT.name())) { - return getManagersForOBT(algorithmType,executionEngineTypes,typeOfManagers); + } else if (tuningType.equals(Constant.TuningType.HBT.name())) { + return getMangersForHBT(algorithmType, executionEngineTypes, typeOfManagers); + } else if (tuningType.equals(Constant.TuningType.OBT.name())) { + return getManagersForOBT(algorithmType, executionEngineTypes, typeOfManagers); } return null; } - private static Manager getMangersForHBT(String algorithmType, String executionEngineTypes, - String typeOfManagers){ + + private static Manager getMangersForHBT(String algorithmType, String executionEngineTypes, String typeOfManagers) { if (typeOfManagers.equals(Constant.TypeofManagers.AbstractBaselineManager.name())) { logger.info("Manager Type Base line Manager HBT"); return new BaselineManagerHBT(); @@ -47,21 +48,20 @@ private static Manager getMangersForHBT(String algorithmType, String executionEn logger.info("Manager Type Fitness Manager HBT"); return new FitnessManagerHBT(); } - if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name()) - && executionEngineTypes.equals(Constant.ExecutionEngineTypes.MR.name())) { + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name()) && executionEngineTypes.equals( + Constant.ExecutionEngineTypes.MR.name())) { logger.info("Manager Type TuningType Manager HBT MR"); return new TuningTypeManagerHBT(new MRExecutionEngine()); } - if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name()) - && executionEngineTypes.equals(Constant.ExecutionEngineTypes.SPARK.name())) { + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name()) && executionEngineTypes.equals( + Constant.ExecutionEngineTypes.SPARK.name())) { logger.info("Manager Type TuningType Manager HBT Spark"); return new TuningTypeManagerHBT(new SparkExecutionEngine()); } return null; } - private static Manager getManagersForOBT(String algorithmType, String executionEngineTypes, - String typeOfManagers ){ + private static Manager getManagersForOBT(String algorithmType, String executionEngineTypes, String typeOfManagers) { if (typeOfManagers.equals(Constant.TypeofManagers.AbstractBaselineManager.name())) { logger.info("Manager Type Base line Manager OBT"); return new BaselineManagerOBT(); @@ -76,14 +76,14 @@ private static Manager getManagersForOBT(String algorithmType, String executionE logger.info("Manager Type Fitness Manager OBT IPSO"); return new FitnessManagerOBTAlgoIPSO(); } - if(typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name())){ - return getManagerForOBTForTuningType(executionEngineTypes,algorithmType); + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name())) { + return getManagerForOBTForTuningType(executionEngineTypes, algorithmType); } return null; } - private static Manager getManagerForOBTForTuningType(String executionEngineTypes,String algorithmType){ + private static Manager getManagerForOBTForTuningType(String executionEngineTypes, String algorithmType) { if (executionEngineTypes.equals(Constant.ExecutionEngineTypes.MR.name()) && algorithmType.equals( Constant.AlgotihmType.PSO_IPSO.name())) { logger.info("Manager Type TuningType Manager OBT IPSO MR"); @@ -94,7 +94,7 @@ private static Manager getManagerForOBTForTuningType(String executionEngineTypes logger.info("Manager Type TuningType Manager OBT PSO MR"); return new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); } - if ( executionEngineTypes.equals(Constant.ExecutionEngineTypes.SPARK.name()) && algorithmType.equals( + if (executionEngineTypes.equals(Constant.ExecutionEngineTypes.SPARK.name()) && algorithmType.equals( Constant.AlgotihmType.PSO_IPSO.name())) { logger.info("Manager Type TuningType Manager OBT IPSO Spark"); return new TuningTypeManagerOBTAlgoIPSO(new SparkExecutionEngine()); diff --git a/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java b/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java index da03aa143..0c15f8473 100644 --- a/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java +++ b/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java @@ -26,6 +26,10 @@ import static java.lang.Math.*; +/** + * This class represents Map Reduce Exectuion Engine. It handles all the cases releated to Map Reduce Engine + */ + public class MRExecutionEngine implements ExecutionEngine { private final Logger logger = Logger.getLogger(getClass()); @@ -318,12 +322,21 @@ public String parameterGenerationsHBT(List results, List getPendingJobs() { Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.SENT)), Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.EXECUTED)) .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) - .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm - + "." + TuningAlgorithm.TABLE.jobType, TuningAlgorithm.JobType.SPARK.name()) + .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.jobType, + TuningAlgorithm.JobType.SPARK.name()) .eq(JobSuggestedParamSet.TABLE.isParamSetBest, 0); } @@ -43,7 +46,8 @@ public ExpressionList getTuningJobDefinitionsForParameterSu .fetch(TuningJobDefinition.TABLE.job, "*") .where() .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) - .eq(TuningJobDefinition.TABLE.tuningAlgorithm+"."+TuningAlgorithm.TABLE.jobType,TuningAlgorithm.JobType.SPARK); + .eq(TuningJobDefinition.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.jobType, + TuningAlgorithm.JobType.SPARK); } @Override diff --git a/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java index a7ead0cc3..76250f0a1 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Set; +import models.AppHeuristicResult; import models.AppResult; import models.JobDefinition; import models.JobExecution; @@ -17,6 +18,7 @@ import models.TuningJobExecutionParamSet; import org.apache.log4j.Logger; import org.apache.hadoop.conf.Configuration; +import scala.App; public class FitnessManagerHBT extends AbstractFitnessManager { @@ -128,6 +130,91 @@ protected void resetParamSetToCreated(JobSuggestedParamSet jobSuggestedParamSet) } } + @Override + protected void checkToDisableTuning(Set jobDefinitionSet) { + for (JobDefinition jobDefinition : jobDefinitionSet) { + List tuningJobExecutionParamSets = + TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .where() + .eq(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.jobDefinition + + '.' + JobDefinition.TABLE.id, jobDefinition.id) + .order() + .desc("job_execution_id") + .findList(); + + if (reachToNumberOfThresholdIterations(tuningJobExecutionParamSets, jobDefinition)) { + disableTuning(jobDefinition, "User Specified Iterations reached"); + } + if (areHeuristicsPassed(tuningJobExecutionParamSets)) { + disableTuning(jobDefinition, "All Heuristics Passed"); + } + } + } + + private boolean areHeuristicsPassed(List tuningJobExecutionParamSets) { + if (tuningJobExecutionParamSets != null && tuningJobExecutionParamSets.size() >= 1) { + TuningJobExecutionParamSet tuningJobExecutionParamSet = tuningJobExecutionParamSets.get(0); + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; + List results = getAppResult(jobExecution); + if (results != null) { + return areAppResultsHaveSeverity(results); + } else { + logger.info(" App Results are null "); + return false; + } + } else { + logger.info(" Tuning Job Execution Param Set is null "); + return false; + } + } + + private boolean areAppResultsHaveSeverity(List results) { + List heuristicsWithHighSeverity = new ArrayList(); + for (AppResult appResult : results) { + if (appResult.yarnAppHeuristicResults != null) { + for (AppHeuristicResult appHeuristicResult : appResult.yarnAppHeuristicResults) { + if (appHeuristicResult.severity.getValue() == 3 || appHeuristicResult.severity.getValue() == 4) { + heuristicsWithHighSeverity.add( + appResult.id + "\t" + appHeuristicResult.heuristicName + "\t" + "have high severity" + "\t" + + appHeuristicResult.severity.getValue()); + } + } + } else { + logger.info(appResult.id + " " + appResult.jobDefId + " have yarn app result null "); + return false; + } + } + return checkHeuriticsforSeverity(heuristicsWithHighSeverity); + } + + private boolean checkHeuriticsforSeverity(List heuristicsWithHighSeverity) { + if (heuristicsWithHighSeverity.size() == 0) { + return true; + } else { + for (String failedHeuristics : heuristicsWithHighSeverity) { + logger.info(failedHeuristics); + } + return false; + } + } + + private List getAppResult(JobExecution jobExecution) { + List results = null; + try { + results = AppResult.find.select("*") + .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*") + .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, + "*") + .where() + .eq(AppResult.TABLE.FLOW_EXEC_ID, jobExecution.flowExecution.flowExecId) + .eq(AppResult.TABLE.JOB_EXEC_ID, jobExecution.jobExecId) + .findList(); + } catch (Exception e) { + logger.warn(" Job Analysis is not completed . "); + } + return results; + } @Override public String getManagerName() { diff --git a/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java index e01f1340a..3a00a581c 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java @@ -171,6 +171,7 @@ protected Boolean updateDatabase(List jobTuningInfoList) { jobSuggestedParamSet.tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; jobSuggestedParamSet.isParamSetDefault = false; jobSuggestedParamSet.isParamSetBest = false; + jobSuggestedParamSet.isParamSetSuggested = true; if (isParamConstraintViolated(jobSuggestedParamValueList)) { penaltyApplication(jobSuggestedParamSet, tuningJobDefinition); } else { diff --git a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java index 527c4671b..6e8cae0e8 100644 --- a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java +++ b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java @@ -1,18 +1,29 @@ package com.linkedin.drelephant.tuning.obt; +import com.avaje.ebean.Expr; import com.linkedin.drelephant.AutoTuner; import com.linkedin.drelephant.ElephantContext; import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; import com.linkedin.drelephant.tuning.AbstractFitnessManager; import com.linkedin.drelephant.util.Utils; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import models.AppHeuristicResult; import models.AppHeuristicResultDetails; import models.AppResult; +import models.JobDefinition; import models.JobExecution; import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; import models.TuningJobDefinition; import models.TuningJobExecutionParamSet; +import models.TuningParameter; +import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.apache.hadoop.conf.Configuration; @@ -38,6 +49,17 @@ public FitnessManagerOBT() { minTuningExecutions = Utils.getNonNegativeInt(configuration, MIN_TUNING_EXECUTIONS, 18); } + /** + * Used to compute the fitness. It will be algorithm dependent. + * @param jobSuggestedParamSet + * @param jobExecution + * @param tuningJobDefinition + * @param results + */ + protected abstract void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExecution jobExecution, + TuningJobDefinition tuningJobDefinition, List results); + + @Override protected void calculateAndUpdateFitness(JobExecution jobExecution, List results, TuningJobDefinition tuningJobDefinition, JobSuggestedParamSet jobSuggestedParamSet) { @@ -66,10 +88,166 @@ protected void calculateAndUpdateFitness(JobExecution jobExecution, List results); + /** + * Checks and disables tuning for the given job definitions. + * Tuning can be disabled if: + * - Number of tuning executions >= maxTuningExecutions + * - or number of tuning executions >= minTuningExecutions and parameters converge + * - or number of tuning executions >= minTuningExecutions and median gain (in cost function) in last 6 executions is negative + * @param jobDefinitionSet Set of jobs to check if tuning can be switched off for them + */ +@Override + protected void checkToDisableTuning(Set jobDefinitionSet) { + for (JobDefinition jobDefinition : jobDefinitionSet) { + List tuningJobExecutionParamSets = + TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") + .fetch(TuningJobExecutionParamSet.TABLE.jobExecution, "*") + .where() + .eq(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.jobDefinition + + '.' + JobDefinition.TABLE.id, jobDefinition.id) + .order() + .desc("job_execution_id") + .findList(); + + if (reachToNumberOfThresholdIterations(tuningJobExecutionParamSets, jobDefinition)) { + disableTuning(jobDefinition, "User Specified Iterations reached"); + } + if (tuningJobExecutionParamSets.size() >= minTuningExecutions) { + if (didParameterSetConverge(tuningJobExecutionParamSets)) { + logger.info("Parameters converged. Disabling tuning for job: " + jobDefinition.jobName); + disableTuning(jobDefinition, "Parameters converged"); + } else if (isMedianGainNegative(tuningJobExecutionParamSets)) { + logger.info("Unable to get gain while tuning. Disabling tuning for job: " + jobDefinition.jobName); + disableTuning(jobDefinition, "Unable to get gain"); + } else if (tuningJobExecutionParamSets.size() >= maxTuningExecutions) { + logger.info("Maximum tuning executions limit reached. Disabling tuning for job: " + jobDefinition.jobName); + disableTuning(jobDefinition, "Maximum executions reached"); + } + } + } + } + + /** + * Checks if the tuning parameters converge + * @param tuningJobExecutionParamSets List of previous executions and corresponding param sets + * @return true if the parameters converge, else false + */ + private boolean didParameterSetConverge(List tuningJobExecutionParamSets) { + boolean result = false; + int numParamSetForConvergence = 3; + + if (tuningJobExecutionParamSets.size() < numParamSetForConvergence) { + return false; + } + + TuningAlgorithm.JobType jobType = tuningJobExecutionParamSets.get(0).jobSuggestedParamSet.tuningAlgorithm.jobType; + + if (jobType == TuningAlgorithm.JobType.PIG) { + + Map> paramValueSet = new HashMap>(); + + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { + + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + + List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, + jobSuggestedParamSet.id) + .or(Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, + "mapreduce.map.memory.mb"), + Expr.eq(JobSuggestedParamValue.TABLE.tuningParameter + '.' + TuningParameter.TABLE.paramName, + "mapreduce.reduce.memory.mb")) + .findList(); + + // if jobSuggestedParamValueList contains both mapreduce.map.memory.mb and mapreduce.reduce.memory.mb + // ie, if the size of jobSuggestedParamValueList is 2 + if (jobSuggestedParamValueList != null && jobSuggestedParamValueList.size() == 2) { + numParamSetForConvergence -= 1; + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { + Set tmp; + if (paramValueSet.containsKey(jobSuggestedParamValue.id)) { + tmp = paramValueSet.get(jobSuggestedParamValue.id); + } else { + tmp = new HashSet(); + } + tmp.add(jobSuggestedParamValue.paramValue); + paramValueSet.put(jobSuggestedParamValue.id, tmp); + } + } + + if (numParamSetForConvergence == 0) { + break; + } + } + + result = true; + for (Integer paramId : paramValueSet.keySet()) { + if (paramValueSet.get(paramId).size() > 1) { + result = false; + } + } + } + + if (result) { + logger.info("Switching off tuning for job: " + tuningJobExecutionParamSets.get( + 0).jobSuggestedParamSet.jobDefinition.jobName + " Reason: parameter set converged"); + } + return result; + } + + + /** + * Checks if the median gain (from tuning) during the last 6 executions is negative + * Last 6 executions constitutes 2 iterations of PSO (given the swarm size is three). Negative average gains in + * latest 2 algorithm iterations (after a fixed number of minimum iterations) imply that either the algorithm hasn't + * converged or there isn't enough scope for tuning. In both the cases, switching tuning off is desired + * @param tuningJobExecutionParamSets List of previous executions + * @return true if the median gain is negative, else false + */ + private boolean isMedianGainNegative(List tuningJobExecutionParamSets) { + int numFitnessForMedian = 6; + Double[] fitnessArray = new Double[numFitnessForMedian]; + int entries = 0; + + if (tuningJobExecutionParamSets.size() < numFitnessForMedian) { + return false; + } + for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { + JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; + JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; + if (jobExecution.executionState == JobExecution.ExecutionState.SUCCEEDED + && jobSuggestedParamSet.paramSetState == JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED) { + fitnessArray[entries] = jobSuggestedParamSet.fitness; + entries += 1; + if (entries == numFitnessForMedian) { + break; + } + } + } + Arrays.sort(fitnessArray); + double medianFitness; + if (fitnessArray.length % 2 == 0) { + medianFitness = (fitnessArray[fitnessArray.length / 2] + fitnessArray[fitnessArray.length / 2 - 1]) / 2; + } else { + medianFitness = fitnessArray[fitnessArray.length / 2]; + } + + JobDefinition jobDefinition = tuningJobExecutionParamSets.get(0).jobSuggestedParamSet.jobDefinition; + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where(). + eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinition.id).findUnique(); + double baselineFitness = + tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; + + if (medianFitness > baselineFitness) { + logger.info("Switching off tuning for job: " + jobDefinition.jobName + " Reason: unable to tune enough"); + return true; + } else { + return false; + } + } + } diff --git a/app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java b/app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java index cf94c88f5..a2782aa7c 100644 --- a/app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java +++ b/app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java @@ -6,6 +6,10 @@ import org.apache.log4j.Logger; +/** + * Optimization Algo Factory . Based on the Tuning Type , algorithm type & execution engine type. + * Factory will produce one of the tuningtype manager object + */ public class OptimizationAlgoFactory { private static final Logger logger = Logger.getLogger(OptimizationAlgoFactory.class); @@ -30,7 +34,7 @@ public static TuningTypeManagerOBT getOptimizationAlogrithm(TuningAlgorithm tuni logger.info("OPTIMIZATION ALGORITHM PSO SPARK"); return new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); } - logger.info("OPTIMIZATION ALGORITHM PSO"); + logger.info("OPTIMIZATION ALGORITHM HBT"); return null; } } diff --git a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java index 91c75bf09..93def8674 100644 --- a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java +++ b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java @@ -43,6 +43,26 @@ public abstract class TuningTypeManagerOBT extends AbstractTuningTypeManager { private String TUNING_SCRIPT_PATH = null; private final Logger logger = Logger.getLogger(getClass()); + + /* + Intialize any prequisite require for Optimizer + Calls once in lifetime of the flow + */ + public abstract void intializePrerequisite(TuningAlgorithm tuningAlgorithm, + JobSuggestedParamSet jobSuggestedParamSet); + + /* + Optimize search space + call after each execution of flow + */ + public abstract void parameterOptimizer(List appResults, JobExecution jobExecution); + + /** + * Swarm size specific to OBT + * @return + */ + protected abstract int getSwarmSize(); + public TuningTypeManagerOBT() { tuningType = "OBT"; Configuration configuration = ElephantContext.instance().getAutoTuningConf(); @@ -271,6 +291,7 @@ protected Boolean updateDatabase(List jobTuningInfoList) { jobSuggestedParamSet.tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; jobSuggestedParamSet.isParamSetDefault = false; jobSuggestedParamSet.isParamSetBest = false; + jobSuggestedParamSet.isParamSetSuggested = true; if (isParamConstraintViolated(jobSuggestedParamValueList)) { logger.info("Parameter constraint violated. Applying penalty."); int penaltyConstant = 3; @@ -384,20 +405,4 @@ public String getManagerName() { return "TuningTypeManagerOBT"; } - /* - Intialize any prequisite require for Optimizer - Calls once in lifetime of the flow - */ - public abstract void intializePrerequisite(TuningAlgorithm tuningAlgorithm, - JobSuggestedParamSet jobSuggestedParamSet); - - /* - Optimize search space - call after each execution of flow - */ - public abstract void parameterOptimizer(List appResults, JobExecution jobExecution); - - - - protected abstract int getSwarmSize(); } diff --git a/app/models/JobSuggestedParamSet.java b/app/models/JobSuggestedParamSet.java index 45a03c681..b8e8b6fdb 100644 --- a/app/models/JobSuggestedParamSet.java +++ b/app/models/JobSuggestedParamSet.java @@ -52,6 +52,7 @@ public static class TABLE { public static final String tuningAlgorithm = "tuningAlgorithm"; public static final String paramSetState = "paramSetState"; public static final String isParamSetDefault = "isParamSetDefault"; + public static final String isParamSetSuggested = "isParamSetSuggested"; public static final String fitness = "fitness"; public static final String fitnessJobExecution = "fitnessJobExecution"; public static final String isParamSetBest = "isParamSetBest"; @@ -87,6 +88,9 @@ public static class TABLE { @Column(nullable = false) public Boolean isParamSetDefault; + @Column(nullable = true) + public Boolean isParamSetSuggested ; + public Double fitness; @Column(nullable = false) diff --git a/conf/evolutions/default/5.sql b/conf/evolutions/default/5.sql index 4a49663e7..cd0cead95 100644 --- a/conf/evolutions/default/5.sql +++ b/conf/evolutions/default/5.sql @@ -181,6 +181,7 @@ CREATE TABLE IF NOT EXISTS job_suggested_param_set ( param_set_state enum('CREATED','SENT','EXECUTED','FITNESS_COMPUTED','DISCARDED') NOT NULL COMMENT 'state of this execution parameter set', are_constraints_violated tinyint(4) default 0 NOT NULL COMMENT 'are constraints violated for the parameter set', is_param_set_default tinyint(4) DEFAULT 0 NOT NULL COMMENT 'Is parameter set default', + is_param_set_suggested tinyint(4) DEFAULT 0 NOT NULL COMMENT 'Is parameter set suggested', is_param_set_best tinyint(4) DEFAULT 0 NOT NULL COMMENT 'Is parameter set best', is_manually_overriden_parameter tinyint(4) DEFAULT 0 NOT NULL COMMENT 'Is parameter set is manually overriden', fitness double DEFAULT NULL COMMENT 'fitness of this parameter set', diff --git a/conf/evolutions/default/6.sql b/conf/evolutions/default/6.sql index 4cb59fc33..207fa7645 100644 --- a/conf/evolutions/default/6.sql +++ b/conf/evolutions/default/6.sql @@ -36,6 +36,8 @@ INSERT INTO tuning_parameter VALUES (17,'mapreduce.input.fileinputformat.split.m INSERT INTO tuning_parameter VALUES (18,'pig.maxCombinedSplitSize',3,536870912,536870912,536870912,128, 0, current_timestamp(0), current_timestamp(0)); INSERT INTO tuning_parameter VALUES (19,'mapreduce.map.memory.mb',4,2048,1024,8192,1024, 0, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (20,'mapreduce.map.java.opts',4,1536,500,6144,64, 0, current_timestamp(0), current_timestamp(0)); + CREATE TABLE IF NOT EXISTS tuning_parameter_constraint ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', From 10d5c154e4cba63eb2d13d3217293a59b04c8613 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Fri, 7 Sep 2018 17:07:20 +0530 Subject: [PATCH 04/22] Changed to get tuning type from DB instead of Azkaban : Changed to work for HBT to OBT toggle : Added unit test cases : Enabled debug log for debugging --- .../tuning/AutoTuningAPIHelper.java | 288 ++++++++++-------- .../linkedin/drelephant/tuning/Constant.java | 2 +- app/com/linkedin/drelephant/tuning/Flow.java | 30 +- .../drelephant/tuning/ManagerFactory.java | 9 +- .../tuning/engine/MRExecutionEngine.java | 82 +++-- .../tuning/hbt/FitnessManagerHBT.java | 9 +- .../tuning/hbt/TuningTypeManagerHBT.java | 5 +- .../tuning/obt/TuningTypeManagerOBT.java | 2 +- .../obt/TuningTypeManagerOBTAlgoIPSO.java | 4 +- .../obt/TuningTypeManagerOBTAlgoPSO.java | 2 +- app/controllers/Application.java | 4 +- scripts/pso/pso_param_generation.py | 7 + .../drelephant/tuning/FlowTestRunner.java | 94 ++++++ .../tuning/IPSOManagerTestRunner.java | 6 +- ...anagerTest.java => TuningManagerTest.java} | 10 +- 15 files changed, 366 insertions(+), 188 deletions(-) create mode 100644 test/com/linkedin/drelephant/tuning/FlowTestRunner.java rename test/com/linkedin/drelephant/tuning/{IPSOManagerTest.java => TuningManagerTest.java} (89%) diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java index 8701e381c..df90aaa65 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java @@ -16,6 +16,7 @@ package com.linkedin.drelephant.tuning; +import com.avaje.ebean.Expr; import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.drelephant.ElephantContext; import com.linkedin.drelephant.tuning.obt.OptimizationAlgoFactory; @@ -130,24 +131,11 @@ private void setMaxAllowedMetricIncreasePercentage(TuningInput tuningInput) { /** * Sets the tuning algorithm based on the job type and optimization metric * @param tuningInput TuningInput for which tuning algorithm is to be set - */ - private void setTuningAlgorithm(TuningInput tuningInput) throws IllegalArgumentException { - //Todo: Handle algorithm version later - - logger.info(" Optimization Algorithm Requested" + tuningInput.getOptimizationAlgo()); - TuningAlgorithm tuningAlgorithm = TuningAlgorithm.find.select("*") - .where() - .eq(TuningAlgorithm.TABLE.jobType, tuningInput.getJobType()) - .eq(TuningAlgorithm.TABLE.optimizationMetric, tuningInput.getOptimizationMetric()) - .eq(TuningAlgorithm.TABLE.optimizationAlgo, tuningInput.getOptimizationAlgo()) - .findUnique(); - if (tuningAlgorithm == null) { - throw new IllegalArgumentException( - "Wrong job type or optimization metric. Job Type " + tuningInput.getJobType() + ". Optimization Metrics: " - + tuningInput.getOptimizationMetric()); - } + *//* + private void setTuningAlgorithm(TuningInput tuningInput,TuningJobDefinition tuningJobDefinition) throws IllegalArgumentException { + TuningAlgorithm tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; tuningInput.setTuningAlgorithm(tuningAlgorithm); - } + }*/ /** * Applies penalty to the param set corresponding to the given execution @@ -257,7 +245,7 @@ private TuningJobDefinition addNewJobForTuning(TuningInput tuningInput) { TuningJobDefinition tuningJobDefinition = new TuningJobDefinition(); tuningJobDefinition.job = job; tuningJobDefinition.client = client; - tuningJobDefinition.tuningAlgorithm = tuningInput.getTuningAlgorithm(); + tuningJobDefinition.tuningAlgorithm = getTuningAlgorithmForfirstTime(tuningInput); tuningJobDefinition.tuningEnabled = true; tuningJobDefinition.allowedMaxExecutionTimePercent = tuningInput.getAllowedMaxExecutionTimePercent(); tuningJobDefinition.allowedMaxResourceUsagePercent = tuningInput.getAllowedMaxResourceUsagePercent(); @@ -269,6 +257,21 @@ private TuningJobDefinition addNewJobForTuning(TuningInput tuningInput) { return tuningJobDefinition; } + private TuningAlgorithm getTuningAlgorithmForfirstTime(TuningInput tuningInput) { + logger.info(" Since its new job . Algorithm type is HBT "); + TuningAlgorithm tuningAlgorithm = TuningAlgorithm.find.select("*") + .where() + .eq(TuningAlgorithm.TABLE.jobType, tuningInput.getJobType()) + .eq(TuningAlgorithm.TABLE.optimizationMetric, "RESOURCE") + .eq(TuningAlgorithm.TABLE.optimizationAlgo, "HBT") + .findUnique(); + if (tuningAlgorithm == null) { + throw new IllegalArgumentException("Wrong job type " + tuningInput.getJobType()); + } + tuningInput.setTuningAlgorithm(tuningAlgorithm); + return tuningAlgorithm; + } + /** * Returns the job definition corresponding to the given tuning input if it exists, else creates one and returns it * @param tuningInput Tuning Input corresponding to which job definition is to be returned @@ -339,15 +342,14 @@ private JobExecution getJobExecution(TuningInput tuningInput) { * @return Parameter Suggestion */ public Map getCurrentRunParameters(TuningInput tuningInput) throws Exception { + /* try {*/ logger.info("Parameter set request received from execution: " + tuningInput.getJobExecId()); - if (tuningInput.getAllowedMaxExecutionTimePercent() == null || tuningInput.getAllowedMaxResourceUsagePercent() == null) { setMaxAllowedMetricIncreasePercentage(tuningInput); } - setTuningAlgorithm(tuningInput); - String jobDefId = tuningInput.getJobDefId(); + String jobDefId = tuningInput.getJobDefId(); TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") .fetch(TuningJobDefinition.TABLE.job, "*") .where() @@ -355,17 +357,29 @@ public Map getCurrentRunParameters(TuningInput tuningInput) thro .setMaxRows(1) .orderBy(TuningJobDefinition.TABLE.createdTs + " desc") .findUnique(); + JobExecution jobExecution = getJobExecution(tuningInput); + //setTuningAlgorithm(tuningInput,tuningJobDefinition); if (tuningJobDefinition == null) { - return processForFirstExecution(tuningInput,jobExecution); + tuningInput.setTuningAlgorithm(getTuningAlgorithmForfirstTime(tuningInput)); + logger.info(" Tuning Algorithm Type " + tuningInput.getTuningAlgorithm().optimizationAlgo.name()); + return processForFirstExecution(tuningInput, jobExecution); } else { - return processForSubsequentExecutions(tuningJobDefinition, tuningInput,jobExecution); + tuningInput.setTuningAlgorithm(tuningJobDefinition.tuningAlgorithm); + logger.info(" Tuning Algorithm Type " + tuningInput.getTuningAlgorithm().optimizationAlgo.name()); + return processForSubsequentExecutions(tuningJobDefinition, tuningInput, jobExecution); } - } + } /*catch (Exception e) { + logger.info(" Exception occured " + e.getMessage()); + logger.info(" Running Default parameters "); + return new HashMap(); + }*/ - private Map processForFirstExecution(TuningInput tuningInput,JobExecution jobExecution){ - logger.info(" New Job. Hence Not checking for AutoTuning . Running with default Parameters " - + tuningInput.getJobExecId()); +//} + + private Map processForFirstExecution(TuningInput tuningInput, JobExecution jobExecution) { + logger.info( + " New Job. Hence Not checking for AutoTuning . Running with default Parameters " + tuningInput.getJobExecId()); JobSuggestedParamSet jobSuggestedParamSet = getDefaultParameters(jobExecution.job); markParameterSetSent(jobSuggestedParamSet); addNewTuningJobExecutionParamSet(jobSuggestedParamSet, jobExecution); @@ -376,11 +390,11 @@ private Map processForFirstExecution(TuningInput tuningInput,Job return jobSuggestedParamValueListToMap(jobSuggestedParamValues); } - - private Map processForSubsequentExecutions (TuningJobDefinition tuningJobDefinition, TuningInput tuningInput,JobExecution jobExecution){ + private Map processForSubsequentExecutions(TuningJobDefinition tuningJobDefinition, + TuningInput tuningInput, JobExecution jobExecution) { logger.info(" Not a New Job . Hence check for autoTuning" + tuningInput.getJobExecId()); if (tuningJobDefinition.autoApply) { - logger.info( " Auto Tuning Enabled . Hence Apply generated Parameter ,generated based on tuning algorithm"); + logger.info(" Auto Tuning Enabled . Hence Apply generated Parameter ,generated based on tuning algorithm"); List jobSuggestedParamValues = processParameterTuningEnabled(tuningInput, jobExecution); return jobSuggestedParamValueListToMap(jobSuggestedParamValues); } else { @@ -389,7 +403,8 @@ private Map processForSubsequentExecutions (TuningJobDefinition } } - private List processParameterTuningEnabled(TuningInput tuningInput,JobExecution jobExecution) { + private List processParameterTuningEnabled(TuningInput tuningInput, + JobExecution jobExecution) { JobSuggestedParamSet jobSuggestedParamSet; logger.debug("Finding parameter suggestion for job: " + jobExecution.job.jobName); if (tuningInput.getRetry()) { @@ -398,7 +413,7 @@ private List processParameterTuningEnabled(TuningInput t jobSuggestedParamSet = getBestParamSet(tuningInput.getJobDefId()); } else { logger.debug("Finding parameter suggestion for job: " + jobExecution.job.jobName); - jobSuggestedParamSet = getNewSuggestedParamSet(jobExecution.job); + jobSuggestedParamSet = getNewSuggestedParamSet(jobExecution.job, tuningInput.getTuningAlgorithm()); markParameterSetSent(jobSuggestedParamSet); } addNewTuningJobExecutionParamSet(jobSuggestedParamSet, jobExecution); @@ -433,7 +448,7 @@ public void updateDataBaseWithDefaultValues(TuningInput tuningInput) { insertParamSetForDefault(job, tuningInput.getTuningAlgorithm(), defaultParams); } - public JobSuggestedParamSet getDefaultParameters(JobDefinition jobDefinition){ + public JobSuggestedParamSet getDefaultParameters(JobDefinition jobDefinition) { JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") .where() @@ -446,17 +461,18 @@ public JobSuggestedParamSet getDefaultParameters(JobDefinition jobDefinition){ return jobSuggestedParamSet; } - public void discardGeneratedParameter(JobDefinition jobDefinition){ + public void discardGeneratedParameter(JobDefinition jobDefinition) { List jobSuggestedParamSets = JobSuggestedParamSet.find.select("*") .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") .where() .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefinition.id) - .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED).findList(); + .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED) + .findList(); - if(jobSuggestedParamSets!=null && jobSuggestedParamSets.size()>=1){ - logger.info(" Discarding Generated Parameters , as auto tuning off."+jobSuggestedParamSets.size()); - for(JobSuggestedParamSet jobSuggestedParamSet : jobSuggestedParamSets){ - jobSuggestedParamSet.paramSetState=ParamSetStatus.DISCARDED; + if (jobSuggestedParamSets != null && jobSuggestedParamSets.size() >= 1) { + logger.info(" Discarding Generated Parameters , as auto tuning off." + jobSuggestedParamSets.size()); + for (JobSuggestedParamSet jobSuggestedParamSet : jobSuggestedParamSets) { + jobSuggestedParamSet.paramSetState = ParamSetStatus.DISCARDED; jobSuggestedParamSet.update(); } } @@ -475,7 +491,7 @@ private void insertParamSetForDefault(JobDefinition job, TuningAlgorithm tuningA jobSuggestedParamSet.isManuallyOverridenParameter = false; jobSuggestedParamSet.isParamSetSuggested = false; jobSuggestedParamSet.save(); - insertParameterValues(jobSuggestedParamSet, paramValueMap,tuningAlgorithm); + insertParameterValues(jobSuggestedParamSet, paramValueMap, tuningAlgorithm); logger.debug("Default parameter set inserted for job: " + job.jobName); } @@ -505,19 +521,23 @@ private void addNewTuningJobExecutionParamSet(JobSuggestedParamSet jobSuggestedP * @param jobDefinition jobDefinition for which param set is to be returned * @return JobSuggestedParamSet corresponding to the given job definition */ - private JobSuggestedParamSet getNewSuggestedParamSet(JobDefinition jobDefinition) { + private JobSuggestedParamSet getNewSuggestedParamSet(JobDefinition jobDefinition, TuningAlgorithm tuningAlgorithm) { JobSuggestedParamSet jobSuggestedParamSet = getManuallyTunedParamSet(jobDefinition.jobDefId); if (jobSuggestedParamSet == null) { logger.info(" No Manually overriden parameter "); + makeOtherAlgoParamGeneratedDiscarded(jobDefinition, tuningAlgorithm); + jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") .where() .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefinition.id) .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED) + .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm, tuningAlgorithm) .order() .asc(JobSuggestedParamSet.TABLE.id) .setMaxRows(1) .findUnique(); + if (jobSuggestedParamSet == null) { logger.info("Returning best parameter set as no parameter suggestion found for job: " + jobDefinition.jobName); jobSuggestedParamSet = getBestParamSet(jobDefinition.jobDefId); @@ -526,102 +546,122 @@ private JobSuggestedParamSet getNewSuggestedParamSet(JobDefinition jobDefinition return jobSuggestedParamSet; } - /** - * Returns the list of JobSuggestedParamValue as Map of String to Double - * @param jobSuggestedParamValues List of JobSuggestedParamValue - * @return Map of string to double containing the parameter name and corresponding value - */ - private Map jobSuggestedParamValueListToMap(List < JobSuggestedParamValue > jobSuggestedParamValues) { - Map paramValues = new HashMap(); - if (jobSuggestedParamValues != null) { - for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) { - logger.debug("Param Name is " + jobSuggestedParamValue.tuningParameter.paramName + " And value is " - + jobSuggestedParamValue.paramValue); - paramValues.put(jobSuggestedParamValue.tuningParameter.paramName, jobSuggestedParamValue.paramValue); - } + private void makeOtherAlgoParamGeneratedDiscarded(JobDefinition jobDefinition, TuningAlgorithm tuningAlgorithm) { + logger.info("Making other tuning type suggested as discarded , if tuning type is changed"); + List jobSuggestedParamSets = JobSuggestedParamSet.find.select("*") + .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") + .where() + .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefinition.id) + .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED) + .not(Expr.eq(JobSuggestedParamSet.TABLE.tuningAlgorithm, tuningAlgorithm)) + .findList(); + if(jobSuggestedParamSets != null){ + logger.info(" other algorithm parameter created " + jobSuggestedParamSets.size()); + for(JobSuggestedParamSet jobSuggestedParamSet : jobSuggestedParamSets){ + jobSuggestedParamSet.paramSetState = ParamSetStatus.DISCARDED; + jobSuggestedParamSet.save(); } - return paramValues; } + } - /** - *Updates parameter set state to SENT if it is in CREATED state - * @param jobSuggestedParamSet JobSuggestedParamSet which is to be updated - */ - private void markParameterSetSent (JobSuggestedParamSet jobSuggestedParamSet){ - if (jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.CREATED)) { - logger.info("Marking paramSetID: " + jobSuggestedParamSet.id + " SENT"); - jobSuggestedParamSet.paramSetState = ParamSetStatus.SENT; - jobSuggestedParamSet.save(); + /** + * Returns the list of JobSuggestedParamValue as Map of String to Double + * @param jobSuggestedParamValues List of JobSuggestedParamValue + * @return Map of string to double containing the parameter name and corresponding value + */ + private Map jobSuggestedParamValueListToMap(List jobSuggestedParamValues) { + Map paramValues = new HashMap(); + if (jobSuggestedParamValues != null) { + for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) { + logger.debug("Param Name is " + jobSuggestedParamValue.tuningParameter.paramName + " And value is " + + jobSuggestedParamValue.paramValue); + paramValues.put(jobSuggestedParamValue.tuningParameter.paramName, jobSuggestedParamValue.paramValue); } } + return paramValues; + } - /** - * Inserts a parameter set in database - * @param job Job - */ - private void insertParamSet (JobDefinition job, TuningAlgorithm - tuningAlgorithm, Map < String, Double > paramValueMap){ - logger.debug("Inserting default parameter set for job: " + job.jobName); - JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); - jobSuggestedParamSet.jobDefinition = job; - jobSuggestedParamSet.tuningAlgorithm = tuningAlgorithm; - jobSuggestedParamSet.paramSetState = ParamSetStatus.CREATED; - jobSuggestedParamSet.isParamSetDefault = true; - jobSuggestedParamSet.areConstraintsViolated = false; - jobSuggestedParamSet.isParamSetBest = false; - jobSuggestedParamSet.isManuallyOverridenParameter = false; - jobSuggestedParamSet.isParamSetSuggested = false; + /** + *Updates parameter set state to SENT if it is in CREATED state + * @param jobSuggestedParamSet JobSuggestedParamSet which is to be updated + */ + private void markParameterSetSent(JobSuggestedParamSet jobSuggestedParamSet) { + if (jobSuggestedParamSet.paramSetState.equals(ParamSetStatus.CREATED)) { + logger.info("Marking paramSetID: " + jobSuggestedParamSet.id + " SENT"); + jobSuggestedParamSet.paramSetState = ParamSetStatus.SENT; jobSuggestedParamSet.save(); - insertParameterValues(jobSuggestedParamSet, paramValueMap,tuningAlgorithm); - intializeOptimizationAlgoPrerequisite(tuningAlgorithm, jobSuggestedParamSet); - logger.debug("Default parameter set inserted for job: " + job.jobName); } + } + + /** + * Inserts a parameter set in database + * @param job Job + */ + private void insertParamSet(JobDefinition job, TuningAlgorithm tuningAlgorithm, Map paramValueMap) { + logger.debug("Inserting default parameter set for job: " + job.jobName); + JobSuggestedParamSet jobSuggestedParamSet = new JobSuggestedParamSet(); + jobSuggestedParamSet.jobDefinition = job; + jobSuggestedParamSet.tuningAlgorithm = tuningAlgorithm; + jobSuggestedParamSet.paramSetState = ParamSetStatus.CREATED; + jobSuggestedParamSet.isParamSetDefault = true; + jobSuggestedParamSet.areConstraintsViolated = false; + jobSuggestedParamSet.isParamSetBest = false; + jobSuggestedParamSet.isManuallyOverridenParameter = false; + jobSuggestedParamSet.isParamSetSuggested = false; + jobSuggestedParamSet.save(); + insertParameterValues(jobSuggestedParamSet, paramValueMap, tuningAlgorithm); + initializeOptimizationAlgoPrerequisite(tuningAlgorithm, jobSuggestedParamSet); + logger.debug("Default parameter set inserted for job: " + job.jobName); + } - /** - * Inserts parameter values in database - * @param jobSuggestedParamSet Set of the parameters which is to be inserted - * @param paramValueMap Map of parameter values as string - */ - @SuppressWarnings("unchecked") private void insertParameterValues (JobSuggestedParamSet - jobSuggestedParamSet, Map < String, Double > paramValueMap, TuningAlgorithm tuningAlgorithm){ - ObjectMapper mapper = new ObjectMapper(); - if (paramValueMap != null) { - for (Map.Entry paramValue : paramValueMap.entrySet()) { - insertParameterValue(jobSuggestedParamSet, paramValue.getKey(), paramValue.getValue(),tuningAlgorithm); - } - } else { - logger.warn("ParamValueMap is null "); + /** + * Inserts parameter values in database + * @param jobSuggestedParamSet Set of the parameters which is to be inserted + * @param paramValueMap Map of parameter values as string + */ + @SuppressWarnings("unchecked") + private void insertParameterValues(JobSuggestedParamSet jobSuggestedParamSet, Map paramValueMap, + TuningAlgorithm tuningAlgorithm) { + ObjectMapper mapper = new ObjectMapper(); + if (paramValueMap != null) { + for (Map.Entry paramValue : paramValueMap.entrySet()) { + insertParameterValue(jobSuggestedParamSet, paramValue.getKey(), paramValue.getValue(), tuningAlgorithm); } + } else { + logger.warn("ParamValueMap is null "); } + } - private void intializeOptimizationAlgoPrerequisite (TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet - jobSuggestedParamSet){ - logger.info("Inserting parameter constraint " + tuningAlgorithm.optimizationAlgo.name()); - TuningTypeManagerOBT manager = OptimizationAlgoFactory.getOptimizationAlogrithm(tuningAlgorithm); - if (manager != null) { - manager.intializePrerequisite(tuningAlgorithm, jobSuggestedParamSet); - } + private void initializeOptimizationAlgoPrerequisite(TuningAlgorithm tuningAlgorithm, + JobSuggestedParamSet jobSuggestedParamSet) { + logger.info("Inserting parameter constraint " + tuningAlgorithm.optimizationAlgo.name()); + TuningTypeManagerOBT manager = OptimizationAlgoFactory.getOptimizationAlogrithm(tuningAlgorithm); + if (manager != null) { + manager.initializePrerequisite(tuningAlgorithm, jobSuggestedParamSet); } + } - /** - * Inserts parameter value in database - * @param jobSuggestedParamSet Parameter set to which the parameter belongs - * @param paramName Parameter name - * @param paramValue Parameter value - */ - private void insertParameterValue (JobSuggestedParamSet jobSuggestedParamSet, String paramName, Double paramValue,TuningAlgorithm tuningAlgorithm){ - logger.debug("Starting insertParameterValue"); - JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); - jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; - TuningParameter tuningParameter = - TuningParameter.find.where().eq(TuningParameter.TABLE.paramName, paramName) - .eq(TuningParameter.TABLE.tuningAlgorithm,tuningAlgorithm).findUnique(); - if (tuningParameter != null) { - jobSuggestedParamValue.tuningParameter = tuningParameter; - jobSuggestedParamValue.paramValue = paramValue; - jobSuggestedParamValue.save(); - } else { - logger.warn("TuningAlgorithm param null " + paramName); - } + /** + * Inserts parameter value in database + * @param jobSuggestedParamSet Parameter set to which the parameter belongs + * @param paramName Parameter name + * @param paramValue Parameter value + */ + private void insertParameterValue(JobSuggestedParamSet jobSuggestedParamSet, String paramName, Double paramValue, + TuningAlgorithm tuningAlgorithm) { + logger.debug("Starting insertParameterValue"); + JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); + jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; + TuningParameter tuningParameter = TuningParameter.find.where() + .eq(TuningParameter.TABLE.paramName, paramName) + .eq(TuningParameter.TABLE.tuningAlgorithm, tuningAlgorithm) + .findUnique(); + if (tuningParameter != null) { + jobSuggestedParamValue.tuningParameter = tuningParameter; + jobSuggestedParamValue.paramValue = paramValue; + jobSuggestedParamValue.save(); + } else { + logger.warn("TuningAlgorithm param null " + paramName); } } +} diff --git a/app/com/linkedin/drelephant/tuning/Constant.java b/app/com/linkedin/drelephant/tuning/Constant.java index 7a3a5a13d..513f0c135 100644 --- a/app/com/linkedin/drelephant/tuning/Constant.java +++ b/app/com/linkedin/drelephant/tuning/Constant.java @@ -6,7 +6,7 @@ public class Constant { public enum TuningType {HBT,OBT} - public enum AlgotihmType{PSO,PSO_IPSO} + public enum AlgotihmType{PSO,PSO_IPSO,HBT} public enum ExecutionEngineTypes{MR,SPARK} public enum TypeofManagers{AbstractBaselineManager,AbstractFitnessManager,AbstractJobStatusManager,AbstractTuningTypeManager} diff --git a/app/com/linkedin/drelephant/tuning/Flow.java b/app/com/linkedin/drelephant/tuning/Flow.java index eea3fc3dd..c4343bf8a 100644 --- a/app/com/linkedin/drelephant/tuning/Flow.java +++ b/app/com/linkedin/drelephant/tuning/Flow.java @@ -35,7 +35,7 @@ public Flow() { createBaseLineManagersPipeline(); createJobStatusManagersPipeline(); createFitnessManagersPipeline(); - createTuningTypemManagersPipeline(); + createTuningTypeManagersPipeline(); } public List> getPipeline() { @@ -48,14 +48,9 @@ public void executeFlow() throws InterruptedException { @Override public void run() { for (Manager manager : pipelineType) { - // logger.info(" Manager execution Status " + manager.getManagerName()); - if (manager.getClass().getSimpleName().toLowerCase().contains("hbt") || manager.getClass() - .getSimpleName() - .toLowerCase() - .contains("azkabanjob")) { + logger.info(" Manager execution Status " + manager.getManagerName()); Boolean execute = manager.execute(); - } - // logger.info(" Manager execution Status " + execute + " " + manager.getManagerName()); + logger.info(" Manager execution Status " + execute + " " + manager.getManagerName()); } } }); @@ -85,18 +80,22 @@ public void createFitnessManagersPipeline() { List fitnessManagers = new ArrayList(); for (Constant.TuningType tuningType : Constant.TuningType.values()) { for (Constant.AlgotihmType algotihmType : Constant.AlgotihmType.values()) { - logger.info(tuningType.name() + " " + algotihmType.name()); Manager manager = ManagerFactory.getManager(tuningType.name(), algotihmType.name(), null, AbstractFitnessManager.class.getSimpleName()); if (manager != null) { + logger.info(manager.getManagerName()); fitnessManagers.add(manager); } } } + /*fitnessManagers.add(new FitnessManagerHBT()); + fitnessManagers.add(new FitnessManagerOBTAlgoIPSO()); + fitnessManagers.add(new FitnessManagerOBTAlgoPSO());*/ + //this.pipelines.add(new FitnessManagerHBT()) this.pipelines.add(fitnessManagers); } - public void createTuningTypemManagersPipeline() { + public void createTuningTypeManagersPipeline() { List algorithmManagers = new ArrayList(); for (Constant.TuningType tuningType : Constant.TuningType.values()) { for (Constant.AlgotihmType algotihmType : Constant.AlgotihmType.values()) { @@ -105,12 +104,21 @@ public void createTuningTypemManagersPipeline() { ManagerFactory.getManager(tuningType.name(), algotihmType.name(), executionEngineTypes.name(), AbstractTuningTypeManager.class.getSimpleName()); if (manager != null) { - logger.info("Testing " + manager.getManagerName()); + logger.info(manager.getManagerName()); algorithmManagers.add(manager); } } } } + /*algorithmManagers.add(new TuningTypeManagerHBT(new MRExecutionEngine())); + algorithmManagers.add(new TuningTypeManagerHBT(new SparkExecutionEngine())); + algorithmManagers.add(new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine())); + algorithmManagers.add(new TuningTypeManagerOBTAlgoPSO(new SparkExecutionEngine())); + algorithmManagers.add(new TuningTypeManagerOBTAlgoIPSO(new MRExecutionEngine())); + algorithmManagers.add(new TuningTypeManagerOBTAlgoIPSO(new SparkExecutionEngine())); +*/ this.pipelines.add(algorithmManagers); } + + } diff --git a/app/com/linkedin/drelephant/tuning/ManagerFactory.java b/app/com/linkedin/drelephant/tuning/ManagerFactory.java index be1faf04e..41ddeb76d 100644 --- a/app/com/linkedin/drelephant/tuning/ManagerFactory.java +++ b/app/com/linkedin/drelephant/tuning/ManagerFactory.java @@ -44,17 +44,20 @@ private static Manager getMangersForHBT(String algorithmType, String executionEn logger.info("Manager Type Base line Manager HBT"); return new BaselineManagerHBT(); } - if (typeOfManagers.equals(Constant.TypeofManagers.AbstractFitnessManager.name())) { + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractFitnessManager.name()) && algorithmType.equals( + Constant.AlgotihmType.HBT.name())) { logger.info("Manager Type Fitness Manager HBT"); return new FitnessManagerHBT(); } if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name()) && executionEngineTypes.equals( - Constant.ExecutionEngineTypes.MR.name())) { + Constant.ExecutionEngineTypes.MR.name()) && algorithmType.equals( + Constant.AlgotihmType.HBT.name())) { logger.info("Manager Type TuningType Manager HBT MR"); return new TuningTypeManagerHBT(new MRExecutionEngine()); } if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name()) && executionEngineTypes.equals( - Constant.ExecutionEngineTypes.SPARK.name())) { + Constant.ExecutionEngineTypes.SPARK.name()) && algorithmType.equals( + Constant.AlgotihmType.HBT.name())) { logger.info("Manager Type TuningType Manager HBT Spark"); return new TuningTypeManagerHBT(new SparkExecutionEngine()); } diff --git a/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java b/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java index 0c15f8473..d054bd4ad 100644 --- a/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java +++ b/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java @@ -32,7 +32,7 @@ public class MRExecutionEngine implements ExecutionEngine { private final Logger logger = Logger.getLogger(getClass()); - + boolean debugEnabled = logger.isDebugEnabled(); enum UsageCounterSchema {USED_PHYSICAL_MEMORY, USED_VIRTUAL_MEMORY, USED_HEAP_MEMORY} private String functionTypes[] = {"map", "reduce"}; @@ -119,33 +119,43 @@ public Boolean isParamConstraintViolatedIPSO(List jobSug if (mrSortMemory != null && mrMapMemory != null) { if (mrSortMemory > 0.6 * mrMapMemory) { - logger.debug("Sort Memory " + mrSortMemory); - logger.debug("Mapper Memory " + mrMapMemory); - logger.debug("Constraint violated: Sort memory > 60% of map memory"); + if (debugEnabled) { + logger.debug("Sort Memory " + mrSortMemory); + logger.debug("Mapper Memory " + mrMapMemory); + logger.debug("Constraint violated: Sort memory > 60% of map memory"); + } violations++; } if (mrMapMemory - mrSortMemory < 768) { - logger.debug("Sort Memory " + mrSortMemory); - logger.debug("Mapper Memory " + mrMapMemory); - logger.debug("Constraint violated: Map memory - sort memory < 768 mb"); + if (debugEnabled) { + logger.debug("Sort Memory " + mrSortMemory); + logger.debug("Mapper Memory " + mrMapMemory); + logger.debug("Constraint violated: Map memory - sort memory < 768 mb"); + } violations++; } } if (mrMapXMX != null && mrMapMemory != null && mrMapXMX > 0.80 * mrMapMemory) { - logger.debug("Mapper Heap Max " + mrMapXMX); - logger.debug("Mapper Memory " + mrMapMemory); - logger.debug("Constraint violated: Mapper XMX > 0.8*mrMapMemory"); + if (debugEnabled) { + logger.debug("Mapper Heap Max " + mrMapXMX); + logger.debug("Mapper Memory " + mrMapMemory); + logger.debug("Constraint violated: Mapper XMX > 0.8*mrMapMemory"); + } violations++; } if (mrReduceMemory != null && mrReduceXMX != null && mrReduceXMX > 0.80 * mrReduceMemory) { - logger.debug("Reducer Heap Max " + mrMapXMX); - logger.debug("Reducer Memory " + mrMapMemory); - logger.debug("Constraint violated: Reducer XMX > 0.8*mrReducerMemory"); + if (debugEnabled) { + logger.debug("Reducer Heap Max " + mrMapXMX); + logger.debug("Reducer Memory " + mrMapMemory); + logger.debug("Constraint violated: Reducer XMX > 0.8*mrReducerMemory"); + } violations++; } if (pigMaxCombinedSplitSize != null && mrMapMemory != null && (pigMaxCombinedSplitSize > 1.8 * mrMapMemory)) { - logger.debug("Constraint violated: Pig max combined split size > 1.8 * map memory"); + if (debugEnabled) { + logger.debug("Constraint violated: Pig max combined split size > 1.8 * map memory"); + } violations++; } if (violations == 0) { @@ -218,10 +228,14 @@ public ExpressionList getTuningJobDefinitionsForParameterSu private void printInformation(Map> information) { for (String functionType : information.keySet()) { - logger.debug("function Type " + functionType); + if (debugEnabled) { + logger.debug("function Type " + functionType); + } Map usage = information.get(functionType); for (String data : usage.keySet()) { - logger.debug(data + " " + usage.get(data)); + if (debugEnabled) { + logger.debug(data + " " + usage.get(data)); + } } } } @@ -248,10 +262,12 @@ private List extractUsageParameter(String functionType, Map usageStats = new ArrayList(); usageStats.add(usedPhysicalMemoryMB); usageStats.add(usedVirtualMemoryMB); @@ -304,7 +320,7 @@ public void parameterOptimizerIPSO(List results, JobExecution jobExec List parameterConstraints = TuningParameterConstraint.find.where(). eq("job_definition_id", jobExecution.job.id).findList(); for (String function : functionTypes) { - logger.debug(" Optimizing Parameter Space " + function); + logger.info(" Optimizing Parameter Space " + function); if (previousUsedMetrics.get(function) .get(CommonConstantsHeuristic.UtilizedParameterKeys.MAX_PHYSICAL_MEMORY.getValue()) > 0.0) { List usageStats = extractUsageParameter(function, previousUsedMetrics); @@ -353,10 +369,12 @@ private Double applyContainerSizeFormula(TuningParameterConstraint containerCons Double memoryMB = max(usagePhysicalMemory, usageVirtualMemory / (2.1)); Double containerSizeLower = getContainerSize(memoryMB); Double containerSizeUpper = getContainerSize(1.2 * memoryMB); - logger.debug(" Previous Lower Bound Memory " + containerConstraint.lowerBound); - logger.debug(" Previous Upper Bound Memory " + containerConstraint.upperBound); - logger.debug(" Current Lower Bound Memory " + containerSizeLower); - logger.debug(" Current Upper Bound Memory " + containerSizeUpper); + if (debugEnabled) { + logger.debug(" Previous Lower Bound Memory " + containerConstraint.lowerBound); + logger.debug(" Previous Upper Bound Memory " + containerConstraint.upperBound); + logger.debug(" Current Lower Bound Memory " + containerSizeLower); + logger.debug(" Current Upper Bound Memory " + containerSizeUpper); + } containerConstraint.lowerBound = containerSizeLower; containerConstraint.upperBound = containerSizeUpper; containerConstraint.save(); @@ -367,10 +385,12 @@ private void applyHeapSizeFormula(TuningParameterConstraint containerHeapSizeCon Double memoryMB) { Double heapSizeLowerBound = min(0.75 * memoryMB, usageHeapMemory); Double heapSizeUpperBound = heapSizeLowerBound * 1.2; - logger.debug(" Previous Lower Bound XMX " + containerHeapSizeConstraint.lowerBound); - logger.debug(" Previous Upper Bound XMX " + containerHeapSizeConstraint.upperBound); - logger.debug(" Current Lower Bound XMX " + heapSizeLowerBound); - logger.debug(" Current Upper Bound XMX " + heapSizeUpperBound); + if (debugEnabled) { + logger.debug(" Previous Lower Bound XMX " + containerHeapSizeConstraint.lowerBound); + logger.debug(" Previous Upper Bound XMX " + containerHeapSizeConstraint.upperBound); + logger.debug(" Current Lower Bound XMX " + heapSizeLowerBound); + logger.debug(" Current Upper Bound XMX " + heapSizeUpperBound); + } containerHeapSizeConstraint.lowerBound = heapSizeLowerBound; containerHeapSizeConstraint.upperBound = heapSizeUpperBound; containerHeapSizeConstraint.save(); @@ -395,7 +415,7 @@ private Map> extractParameterInformationIPSO(List> collectUsageDataPerApplicationIPSO(AppR } } } - logger.debug("Usage Values local " + appResult.jobExecUrl); + logger.info("Usage Values local " + appResult.jobExecUrl); printInformation(usageData); return usageData; diff --git a/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java index 76250f0a1..87f53f19e 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java @@ -147,17 +147,20 @@ protected void checkToDisableTuning(Set jobDefinitionSet) { disableTuning(jobDefinition, "User Specified Iterations reached"); } if (areHeuristicsPassed(tuningJobExecutionParamSets)) { - disableTuning(jobDefinition, "All Heuristics Passed"); + disableTuning(jobDefinition, "All Heuristics Passed1"); } } } private boolean areHeuristicsPassed(List tuningJobExecutionParamSets) { + logger.info(" Testing All Heuristics "); if (tuningJobExecutionParamSets != null && tuningJobExecutionParamSets.size() >= 1) { + logger.info("tuningJobExecutionParamSets have some values"); TuningJobExecutionParamSet tuningJobExecutionParamSet = tuningJobExecutionParamSets.get(0); JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; List results = getAppResult(jobExecution); if (results != null) { + logger.info(" Results are not null "); return areAppResultsHaveSeverity(results); } else { logger.info(" App Results are null "); @@ -180,9 +183,6 @@ private boolean areAppResultsHaveSeverity(List results) { + appHeuristicResult.severity.getValue()); } } - } else { - logger.info(appResult.id + " " + appResult.jobDefId + " have yarn app result null "); - return false; } } return checkHeuriticsforSeverity(heuristicsWithHighSeverity); @@ -190,6 +190,7 @@ private boolean areAppResultsHaveSeverity(List results) { private boolean checkHeuriticsforSeverity(List heuristicsWithHighSeverity) { if (heuristicsWithHighSeverity.size() == 0) { + logger.info(" No sever heursitics "); return true; } else { for (String failedHeuristics : heuristicsWithHighSeverity) { diff --git a/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java index 3a00a581c..871fba8a2 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java @@ -177,7 +177,8 @@ protected Boolean updateDatabase(List jobTuningInfoList) { } else { logger.info(" Parameters constraints not violeted "); jobSuggestedParamSet.areConstraintsViolated = false; - processParamSetStatus(jobSuggestedParamSet); + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; + //processParamSetStatus(jobSuggestedParamSet); } saveSuggestedParamSet(jobSuggestedParamSet); @@ -197,7 +198,7 @@ private void processParamSetStatus(JobSuggestedParamSet jobSuggestedParamSet) { .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, jobSuggestedParamSet.jobDefinition.id) .setMaxRows(1) .findUnique(); - jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; + //jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; //handleDiscarding(tuningJobDefinition1, jobSuggestedParamSet); } diff --git a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java index 93def8674..1746ac068 100644 --- a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java +++ b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java @@ -48,7 +48,7 @@ public abstract class TuningTypeManagerOBT extends AbstractTuningTypeManager { Intialize any prequisite require for Optimizer Calls once in lifetime of the flow */ - public abstract void intializePrerequisite(TuningAlgorithm tuningAlgorithm, + public abstract void initializePrerequisite(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet); /* diff --git a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoIPSO.java b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoIPSO.java index 5e0655f97..aca1bfa5c 100644 --- a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoIPSO.java +++ b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoIPSO.java @@ -24,7 +24,7 @@ public class TuningTypeManagerOBTAlgoIPSO extends TuningTypeManagerOBT { - private static final Logger logger = Logger.getLogger(TuningTypeManagerOBTAlgoIPSO.class); + private final Logger logger = Logger.getLogger(getClass()); private Map> usageDataGlobal = null; @@ -66,7 +66,7 @@ protected List getTuningJobDefinitions() { } @Override - public void intializePrerequisite(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { + public void initializePrerequisite(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { logger.info(" Intialize Prerequisite "); setDefaultParameterValues(tuningAlgorithm, jobSuggestedParamSet); } diff --git a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoPSO.java b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoPSO.java index ae2e819a5..bf6c54996 100644 --- a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoPSO.java +++ b/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoPSO.java @@ -51,7 +51,7 @@ protected List getTuningJobDefinitions() { } @Override - public void intializePrerequisite(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { + public void initializePrerequisite(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { } diff --git a/app/controllers/Application.java b/app/controllers/Application.java index bf46b679d..b18931c6c 100644 --- a/app/controllers/Application.java +++ b/app/controllers/Application.java @@ -930,8 +930,8 @@ public static Result getCurrentRunParameters() { } String jobType = paramValueMap.get("autoTuningJobType"); String optimizationAlgo = - paramValueMap.get("optimizationAlgo") == null ? TuningAlgorithm.OptimizationAlgo.HBT.name() - : paramValueMap.get("optimizationAlgo"); + paramValueMap.get("optimizationAlgo") /*== null ? TuningAlgorithm.OptimizationAlgo.HBT.name() + : paramValueMap.get("optimizationAlgo")*/; String optimizationAlgoVersion = paramValueMap.get("optimizationAlgoVersion"); String optimizationMetric = paramValueMap.get("optimizationMetric"); diff --git a/scripts/pso/pso_param_generation.py b/scripts/pso/pso_param_generation.py index 3d1581886..8f1ed72fa 100644 --- a/scripts/pso/pso_param_generation.py +++ b/scripts/pso/pso_param_generation.py @@ -26,6 +26,7 @@ param_name = [] iteration = 0 job_type = "" +swarm_size = "" LARGE_DUMMY_FITNESS = 10000 @@ -42,6 +43,8 @@ ARG_TUNING_STATE_KEY = 'json_tuning_state' ARG_PARAMETERS_TO_TUNE_KEY = 'parameters_to_tune' ARG_JOB_TYPE = "job_type" +ARG_SWARM_SIZE = "swarm_size" + TUNING_STATE_ARCHIVE_KEY = 'archive' TUNING_STATE_PREV_POPULATION_KEY = 'prev_population' @@ -247,6 +250,8 @@ def main(json_tuning_state, display=False): pseudo_random_number_generator = Random() args = {} + POPULATION_SIZE = int(swarm_size) + if TUNING_STATE_ARCHIVE_KEY not in tuning_state: pseudo_random_number_generator.seed(time.time()) pso = restartable_pso.restartable_pso(pseudo_random_number_generator) @@ -293,10 +298,12 @@ def main(json_tuning_state, display=False): parser.add_argument(ARG_TUNING_STATE_KEY, help='Saved tuning state object') parser.add_argument(ARG_PARAMETERS_TO_TUNE_KEY) parser.add_argument(ARG_JOB_TYPE) + parser.add_argument(ARG_SWARM_SIZE) args = parser.parse_args() json_tuning_state = args.json_tuning_state parameters_to_tune = args.parameters_to_tune job_type = args.job_type + swarm_size = args.swarm_size parameters_to_tune = json.loads(parameters_to_tune) initialize_params(parameters_to_tune) main(json_tuning_state) diff --git a/test/com/linkedin/drelephant/tuning/FlowTestRunner.java b/test/com/linkedin/drelephant/tuning/FlowTestRunner.java new file mode 100644 index 000000000..2ef479644 --- /dev/null +++ b/test/com/linkedin/drelephant/tuning/FlowTestRunner.java @@ -0,0 +1,94 @@ +package com.linkedin.drelephant.tuning; + +import com.linkedin.drelephant.tuning.Schduler.AzkabanJobStatusManager; +import com.linkedin.drelephant.tuning.engine.MRExecutionEngine; +import com.linkedin.drelephant.tuning.engine.SparkExecutionEngine; +import com.linkedin.drelephant.tuning.hbt.BaselineManagerHBT; +import com.linkedin.drelephant.tuning.hbt.FitnessManagerHBT; +import com.linkedin.drelephant.tuning.hbt.TuningTypeManagerHBT; +import com.linkedin.drelephant.tuning.obt.BaselineManagerOBT; +import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoIPSO; +import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoPSO; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoIPSO; +import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoPSO; +import java.util.List; + +import static org.junit.Assert.*; +import static play.test.Helpers.*; + + +public class FlowTestRunner implements Runnable { + @Override + public void run() { + Flow flow = new Flow(); + testPipeline(flow); + testCreateBaseLineManagersPipeline(flow); + testCreateJobStatusManagersPipeline(flow); + testCreateFitnessManagersPipeline(flow); + testCreateTuningTypeManagersPipeline(flow); + testCreateTuningTypeManagersPipeline(flow); + } + + private void testPipeline(Flow flow) { + List> pipelines = flow.getPipeline(); + assertTrue(" Total Number of pipeline ", pipelines.size() == 4); + } + + private void testCreateBaseLineManagersPipeline(Flow flow) { + List> pipelines = flow.getPipeline(); + List baseLineManagers = pipelines.get(0); + assertTrue(" Total Number of Base line Managers ", baseLineManagers.size() == 2); + assertTrue(" HBT Baseline Manager ", baseLineManagers.get(0) instanceof BaselineManagerHBT); + assertTrue(" OBT Baseline Manager ", baseLineManagers.get(1) instanceof BaselineManagerOBT); + } + + private void testCreateJobStatusManagersPipeline(Flow flow) { + List> pipelines = flow.getPipeline(); + List jobStatusManagers = pipelines.get(1); + assertTrue(" Total Number of Base line Managers ", jobStatusManagers.size() == 1); + assertTrue(" Azkaban Job Status Manager ", jobStatusManagers.get(0) instanceof AzkabanJobStatusManager); + } + + private void testCreateFitnessManagersPipeline(Flow flow) { + List> pipelines = flow.getPipeline(); + List fitnessManagers = pipelines.get(2); + assertTrue(" Total Number of fitness Managers ", fitnessManagers.size() == 3); + assertTrue(" FitnessManagerHBT ", fitnessManagers.get(0) instanceof FitnessManagerHBT); + assertTrue(" FitnessManagerOBTPSO ", fitnessManagers.get(1) instanceof FitnessManagerOBTAlgoPSO); + assertTrue(" FitnessManagerOBTIPSO ", fitnessManagers.get(2) instanceof FitnessManagerOBTAlgoIPSO); + } + + private void testCreateTuningTypeManagersPipeline(Flow flow) { + List> pipelines = flow.getPipeline(); + List tuningTypeManagers = pipelines.get(3); + assertTrue(" Total Number of tuningType Managers ", tuningTypeManagers.size() == 6); + + assertTrue(" TuningTypeManagerHBT ", tuningTypeManagers.get(0) instanceof TuningTypeManagerHBT); + assertTrue(" TuningTypeManagerHBTMR ", + ((TuningTypeManagerHBT) tuningTypeManagers.get(0))._executionEngine instanceof MRExecutionEngine); + + assertTrue(" TuningTypeManagerHBT ", tuningTypeManagers.get(1) instanceof TuningTypeManagerHBT); + assertTrue(" TuningTypeManagerHBTSpark ", + ((TuningTypeManagerHBT) tuningTypeManagers.get(1))._executionEngine instanceof SparkExecutionEngine); + + assertTrue(" TuningTypeManagerOBTPSO ", tuningTypeManagers.get(2) instanceof TuningTypeManagerOBTAlgoPSO); + assertTrue(" TuningTypeManagerOBTPSOMR ", + ((TuningTypeManagerOBTAlgoPSO) tuningTypeManagers.get(2))._executionEngine instanceof MRExecutionEngine); + + assertTrue(" TuningTypeManagerOBTPSO ", tuningTypeManagers.get(3) instanceof TuningTypeManagerOBTAlgoPSO); + assertTrue(" TuningTypeManagerOBTPSOSpark ", + ((TuningTypeManagerOBTAlgoPSO) tuningTypeManagers.get(3))._executionEngine instanceof SparkExecutionEngine); + + assertTrue(" TuningTypeManagerOBTIPSO ", tuningTypeManagers.get(4) instanceof TuningTypeManagerOBTAlgoIPSO); + assertTrue(" TuningTypeManagerOBTIPSOMR ", + ((TuningTypeManagerOBTAlgoIPSO) tuningTypeManagers.get(4))._executionEngine instanceof MRExecutionEngine); + + assertTrue(" TuningTypeManagerOBTIPSO ", tuningTypeManagers.get(5) instanceof TuningTypeManagerOBTAlgoIPSO); + assertTrue(" TuningTypeManagerOBTPSOSpark ", + ((TuningTypeManagerOBTAlgoIPSO) tuningTypeManagers.get(5))._executionEngine instanceof SparkExecutionEngine); + } +} + /* StringBuffer stringBuffer = new StringBuffer(); + for(Manager manager : tuningTypeManagers){ + stringBuffer.append(manager.getClass().getName()).append(" "); + }*/ diff --git a/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java b/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java index 2940d38db..0ed363678 100644 --- a/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java +++ b/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java @@ -44,7 +44,7 @@ public void run() { JobSuggestedParamSet.find.where().eq("fitness_job_execution_id", 1541).findUnique(); JobExecution jobExecution = JobExecution.find.byId(1541L); TuningTypeManagerOBT optimizeManager = checkIPSOManager(tuningAlgorithm); - testIPSOIntializePrerequisite(optimizeManager, tuningAlgorithm, jobSuggestedParamSet); + testIPSOInitializePrerequisite(optimizeManager, tuningAlgorithm, jobSuggestedParamSet); //testIPSOExtractParameterInformation(jobExecution, optimizeManager); testIPSOParameterOptimizer(jobExecution, optimizeManager); testIPSOApplyIntelligenceOnParameter(tuningJobDefinition, jobDefinition, optimizeManager); @@ -57,9 +57,9 @@ private TuningTypeManagerOBT checkIPSOManager(TuningAlgorithm tuningAlgorithm) { return optimizeManager; } - private void testIPSOIntializePrerequisite(TuningTypeManagerOBT optimizeManager, TuningAlgorithm tuningAlgorithm, + private void testIPSOInitializePrerequisite(TuningTypeManagerOBT optimizeManager, TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { - optimizeManager.intializePrerequisite(tuningAlgorithm, jobSuggestedParamSet); + optimizeManager.initializePrerequisite(tuningAlgorithm, jobSuggestedParamSet); List tuningParameterConstraint = TuningParameterConstraint.find.where().eq("job_definition_id", 100003).findList(); assertTrue(" Parameters Constraint Size ", tuningParameterConstraint.size() == 9); diff --git a/test/com/linkedin/drelephant/tuning/IPSOManagerTest.java b/test/com/linkedin/drelephant/tuning/TuningManagerTest.java similarity index 89% rename from test/com/linkedin/drelephant/tuning/IPSOManagerTest.java rename to test/com/linkedin/drelephant/tuning/TuningManagerTest.java index fcbb47a31..f39e5975c 100644 --- a/test/com/linkedin/drelephant/tuning/IPSOManagerTest.java +++ b/test/com/linkedin/drelephant/tuning/TuningManagerTest.java @@ -21,8 +21,8 @@ import org.junit.Test; -public class IPSOManagerTest { - private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(IPSOManagerTest.class); +public class TuningManagerTest { + private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(TuningManagerTest.class); private static FakeApplication fakeApp; private int numParametersToTune; @@ -50,7 +50,11 @@ public void onStart(Application app) { @Test public void testIPSOManager() { running(testServer(TEST_SERVER_PORT, fakeApp), new IPSOManagerTestRunner()); - //running(testServer(TEST_SERVER_PORT, fakeApp), new BaselineManagerTestRunner()); } + + @Test + public void testFlowTestRunner(){ + running(testServer(TEST_SERVER_PORT, fakeApp), new FlowTestRunner()); + } } From 27c04114e9b219feb15d0bc08d6e88a56fe6a7f3 Mon Sep 17 00:00:00 2001 From: "Manoj Kumar(Bangalore Engineering)" Date: Mon, 10 Sep 2018 11:51:29 +0530 Subject: [PATCH 05/22] New Spark HBT algorithm for specific suggestion --- .../spark/heuristics/DriverHeuristic.scala | 9 +- .../heuristics/JvmUsedMemoryHeuristic.scala | 4 +- .../heuristics/UnifiedMemoryHeuristic.scala | 3 +- .../tuning/AbstractFitnessManager.java | 2 +- .../tuning/AbstractTuningTypeManager.java | 47 +-- .../engine/SparkConfigurationConstants.java | 17 + .../tuning/engine/SparkExecutionEngine.java | 42 ++- .../engine/SparkHBTParamRecommender.java | 326 ++++++++++++++++++ .../tuning/hbt/FitnessManagerHBT.java | 33 +- .../tuning/hbt/TuningTypeManagerHBT.java | 9 +- app/models/AppResult.java | 38 +- app/models/JobExecution.java | 7 +- conf/evolutions/default/6.sql | 9 +- 13 files changed, 487 insertions(+), 59 deletions(-) create mode 100644 app/com/linkedin/drelephant/tuning/engine/SparkConfigurationConstants.java create mode 100644 app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java diff --git a/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala index 8a1b71cc5..0510dcef8 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala @@ -23,8 +23,9 @@ import scala.util.Try import com.linkedin.drelephant.analysis._ import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData import com.linkedin.drelephant.spark.data.SparkApplicationData -import com.linkedin.drelephant.util.{MemoryFormatUtils, Utils} +import com.linkedin.drelephant.util.MemoryFormatUtils import com.linkedin.drelephant.spark.fetchers.statusapiv1.ExecutorSummary +import com.linkedin.drelephant.util.Utils /** * A heuristic based the driver's configurations and memory used. @@ -73,11 +74,11 @@ class DriverHeuristic(private val heuristicConfigurationData: HeuristicConfigura SPARK_YARN_DRIVER_MEMORY_OVERHEAD, evaluator.sparkYarnDriverMemoryOverhead ), - new HeuristicResultDetails("Max driver peak JVM used memory", MemoryFormatUtils.bytesToString(evaluator.maxDriverPeakJvmUsedMemory)) + new HeuristicResultDetails(DRIVER_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME, MemoryFormatUtils.bytesToString(evaluator.maxDriverPeakJvmUsedMemory)) ) if(evaluator.severityJvmUsedMemory != Severity.NONE) { resultDetails = resultDetails :+ new HeuristicResultDetails("Driver Peak JVM used Memory", "The allocated memory for the driver (in " + SPARK_DRIVER_MEMORY_KEY + ") is much more than the peak JVM used memory by the driver.") - resultDetails = resultDetails :+ new HeuristicResultDetails("Suggested spark.driver.memory", MemoryFormatUtils.roundOffMemoryStringToNextInteger(MemoryFormatUtils.bytesToString(((1 + BUFFER_FRACTION) * (evaluator.maxDriverPeakJvmUsedMemory + reservedMemory)).toLong))) + resultDetails = resultDetails :+ new HeuristicResultDetails(SUGGESTED_SPARK_DRIVER_MEMORY_HEURISTIC_NAME, MemoryFormatUtils.roundOffMemoryStringToNextInteger(MemoryFormatUtils.bytesToString(((1 + BUFFER_FRACTION) * (evaluator.maxDriverPeakJvmUsedMemory + reservedMemory)).toLong))) } if (evaluator.severityGc != Severity.NONE) { resultDetails = resultDetails :+ new HeuristicResultDetails("Gc ratio high", "The driver is spending too much time on GC. We recommend increasing the driver memory.") @@ -115,6 +116,8 @@ object DriverHeuristic { val EXECUTION_MEMORY = "executionMemory" val STORAGE_MEMORY = "storageMemory" val JVM_USED_MEMORY = "jvmUsedMemory" + val DRIVER_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME="Max driver peak JVM used memory" + val SUGGESTED_SPARK_DRIVER_MEMORY_HEURISTIC_NAME="Suggested spark.driver.memory" val BUFFER_FRACTION = 0.2 // 300 * FileUtils.ONE_MB (300 * 1024 * 1024) diff --git a/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala index 929f8af50..3c9202838 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala @@ -44,7 +44,7 @@ class JvmUsedMemoryHeuristic(private val heuristicConfigurationData: HeuristicCo val evaluator = new Evaluator(this, data) var resultDetails = Seq( - new HeuristicResultDetails("Max executor peak JVM used memory", MemoryFormatUtils.bytesToString(evaluator.maxExecutorPeakJvmUsedMemory)), + new HeuristicResultDetails(MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME, MemoryFormatUtils.bytesToString(evaluator.maxExecutorPeakJvmUsedMemory)), new HeuristicResultDetails("spark.executor.memory", MemoryFormatUtils.bytesToString(evaluator.sparkExecutorMemory)) ) @@ -73,6 +73,8 @@ object JvmUsedMemoryHeuristic { val reservedMemory: Long = 314572800 val BUFFER_FRACTION: Double = 0.2 val MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLD_KEY = "executor_peak_jvm_memory_threshold" + val MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME = "Max executor peak JVM used memory" + lazy val DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD = "2G" class Evaluator(jvmUsedMemoryHeuristic: JvmUsedMemoryHeuristic, data: SparkApplicationData) { diff --git a/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala index 80dfba54f..9a16d8bf4 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala @@ -47,7 +47,7 @@ class UnifiedMemoryHeuristic(private val heuristicConfigurationData: HeuristicCo var resultDetails = Seq( new HeuristicResultDetails("Unified Memory Space Allocated", MemoryFormatUtils.bytesToString(evaluator.maxMemory)), new HeuristicResultDetails("Mean peak unified memory", MemoryFormatUtils.bytesToString(evaluator.meanUnifiedMemory)), - new HeuristicResultDetails("Max peak unified memory", MemoryFormatUtils.bytesToString(evaluator.maxUnifiedMemory)), + new HeuristicResultDetails(MAX_PEAK_UNIFIED_MEMORY_HEURISTIC_NAME, MemoryFormatUtils.bytesToString(evaluator.maxUnifiedMemory)), new HeuristicResultDetails("spark.executor.memory", MemoryFormatUtils.bytesToString(evaluator.sparkExecutorMemory)), new HeuristicResultDetails("spark.memory.fraction", evaluator.sparkMemoryFraction.toString) ) @@ -74,6 +74,7 @@ object UnifiedMemoryHeuristic { val DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD = "2G" val UNIFIED_MEMORY_ALLOCATED_THRESHOLD = "256M" val SPARK_MEMORY_FRACTION_THRESHOLD : Double = 0.05 + val MAX_PEAK_UNIFIED_MEMORY_HEURISTIC_NAME="Max peak unified memory"; class Evaluator(unifiedMemoryHeuristic: UnifiedMemoryHeuristic, data: SparkApplicationData) { lazy val appConfigurationProperties: Map[String, String] = diff --git a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java index 248cc9d76..7b598c4d0 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java @@ -220,7 +220,7 @@ protected void updateJobSuggestedParamSetSucceededExecution(JobExecution jobExec * (since the objective is to minimize the fitness, the param set with the lowest fitness is the best) * @param jobSuggestedParamSet JobSuggestedParamSet */ - private JobSuggestedParamSet updateBestJobSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { + protected JobSuggestedParamSet updateBestJobSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { logger.info("Checking if a new best param set is found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId); JobSuggestedParamSet currentBestJobSuggestedParamSet = JobSuggestedParamSet.find.where() .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, diff --git a/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java b/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java index dd8061b90..2e5ac26ba 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java @@ -4,14 +4,16 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import models.AppResult; + import models.JobDefinition; import models.JobExecution; import models.JobSuggestedParamSet; import models.JobSuggestedParamValue; import models.TuningJobDefinition; import models.TuningParameter; + import org.apache.log4j.Logger; + import play.libs.Json; @@ -138,17 +140,22 @@ protected List generateParameters(List jobsForPara } public final Boolean execute() { - logger.info("Executing Tuning Algorithm"); - Boolean parameterGenerationDone = false, databaseUpdateDone = false, updateMetricsDone = false; - List jobTuningInfo = detectJobsForParameterGeneration(); - if (jobTuningInfo != null && jobTuningInfo.size() >= 1) { - logger.info("Generating Parameters "); - List updatedJobTuningInfoList = generateParameters(jobTuningInfo); - logger.info("Updating Database"); - databaseUpdateDone = updateDatabase(updatedJobTuningInfoList); + try { + logger.info("Executing Tuning Algorithm"); + Boolean parameterGenerationDone = false, databaseUpdateDone = false, updateMetricsDone = false; + List jobTuningInfo = detectJobsForParameterGeneration(); + if (jobTuningInfo != null && jobTuningInfo.size() >= 1) { + logger.info("Generating Parameters "); + List updatedJobTuningInfoList = generateParameters(jobTuningInfo); + logger.info("Updating Database"); + databaseUpdateDone = updateDatabase(updatedJobTuningInfoList); + } + logger.info("Param Generation Done"); + return databaseUpdateDone; + } catch (Exception e) { + logger.info("Exception in generating parameters ", e); } - logger.info("Param Generation Done"); - return databaseUpdateDone; + return false; } /** @@ -178,8 +185,7 @@ protected List getJobsTuningInfo(List tuning return jobTuningInfoList; } - protected List generateTuningParameterListWithDefaultValues( - TuningJobDefinition tuningJobDefinition) { + protected List generateTuningParameterListWithDefaultValues(TuningJobDefinition tuningJobDefinition) { JobDefinition job = tuningJobDefinition.job; logger.info("Getting tuning information for job: " + job.jobDefId); List tuningParameterList = TuningHelper.getTuningParameterList(tuningJobDefinition); @@ -190,13 +196,14 @@ protected List generateTuningParameterListWithDefaultValues( return tuningParameterList; } - protected void assignDefaultValues(JobSuggestedParamSet defaultJobParamSet, - List tuningParameterList) { + protected void assignDefaultValues(JobSuggestedParamSet defaultJobParamSet, List tuningParameterList) { if (defaultJobParamSet != null) { logger.info("Fetching default parameter values for job "); - List jobSuggestedParamValueList = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + "." + JobExecution.TABLE.id, defaultJobParamSet.id) - .findList(); + List jobSuggestedParamValueList = + JobSuggestedParamValue.find + .where() + .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + "." + JobExecution.TABLE.id, + defaultJobParamSet.id).findList(); if (jobSuggestedParamValueList.size() > 0) { logger.info("Giving default values "); @@ -210,8 +217,8 @@ protected void assignDefaultValues(JobSuggestedParamSet defaultJobParamSet, logger.info("Giving default values "); Integer paramId = tuningParameter.id; if (defaultExecutionParamMap.containsKey(paramId)) { - logger.info("Updating value of param " + tuningParameter.paramName + " to " + defaultExecutionParamMap.get( - paramId)); + logger.info("Updating value of param " + tuningParameter.paramName + " to " + + defaultExecutionParamMap.get(paramId)); tuningParameter.defaultValue = defaultExecutionParamMap.get(paramId); } } diff --git a/app/com/linkedin/drelephant/tuning/engine/SparkConfigurationConstants.java b/app/com/linkedin/drelephant/tuning/engine/SparkConfigurationConstants.java new file mode 100644 index 000000000..abe3ca392 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/engine/SparkConfigurationConstants.java @@ -0,0 +1,17 @@ +package com.linkedin.drelephant.tuning.engine; + +public class SparkConfigurationConstants { + public final static String SPARK_EXECUTOR_MEMORY_KEY = "spark.executor.memory"; + public final static String SPARK_MEMORY_FRACTION_KEY = "spark.memory.fraction"; + public final static String SPARK_DRIVER_MEMORY_KEY = "spark.driver.memory"; + public final static String SPARK_EXECUTOR_INSTANCES_KEY = "spark.executor.instances"; + public final static String SPARK_EXECUTOR_CORES_KEY = "spark.executor.cores"; + public final static String SPARK_SERIALIZER_KEY = "spark.serializer"; + public final static String SPARK_APPLICATION_DURATION = "spark.application.duration"; + public final static String SPARK_SHUFFLE_SERVICE_ENABLED = "spark.shuffle.service.enabled"; + public final static String SPARK_DYNAMIC_ALLOCATION_ENABLED = "spark.dynamicAllocation.enabled"; + public final static String SPARK_DYNAMIC_ALLOCATION_MIN_EXECUTORS = "spark.dynamicAllocation.minExecutors"; + public final static String SPARK_DYNAMIC_ALLOCATION_MAX_EXECUTORS = "spark.dynamicAllocation.maxExecutors"; + public final static String SPARK_YARN_JARS = "spark.yarn.secondary.jars"; + public final static String SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD = "spark.yarn.executor.memoryOverhead"; +} diff --git a/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java b/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java index ad8235f8d..890d4304e 100644 --- a/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java +++ b/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java @@ -1,10 +1,7 @@ package com.linkedin.drelephant.tuning.engine; -import com.avaje.ebean.Expr; -import com.avaje.ebean.ExpressionList; -import com.linkedin.drelephant.tuning.ExecutionEngine; +import java.util.HashMap; import java.util.List; -import java.util.Map; import models.AppResult; import models.JobExecution; import models.JobSuggestedParamSet; @@ -12,13 +9,22 @@ import models.TuningAlgorithm; import models.TuningJobDefinition; import models.TuningParameter; -import models.TuningParameterConstraint; +import org.apache.log4j.Logger; +import com.avaje.ebean.Expr; +import com.avaje.ebean.ExpressionList; +import com.linkedin.drelephant.tuning.ExecutionEngine; /** * This class represents Spark Exectuion Engine. It handles all the cases releated to Spark Engine */ public class SparkExecutionEngine implements ExecutionEngine { + private final Logger logger = Logger.getLogger(getClass()); + private String executionEngineName = "SPARK"; + + public String getExecutionEngineName() { + return this.executionEngineName; + } @Override public void computeValuesOfDerivedConfigurationParameters(List derivedParameterList, @@ -28,7 +34,8 @@ public void computeValuesOfDerivedConfigurationParameters(List @Override public ExpressionList getPendingJobs() { - return JobSuggestedParamSet.find.select("*") + return JobSuggestedParamSet.find + .select("*") .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") .where() .or(Expr.or(Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.CREATED), @@ -36,13 +43,13 @@ public ExpressionList getPendingJobs() { Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.EXECUTED)) .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.jobType, - TuningAlgorithm.JobType.SPARK.name()) - .eq(JobSuggestedParamSet.TABLE.isParamSetBest, 0); + TuningAlgorithm.JobType.SPARK.name()).eq(JobSuggestedParamSet.TABLE.isParamSetBest, 0); } @Override public ExpressionList getTuningJobDefinitionsForParameterSuggestion() { - return TuningJobDefinition.find.select("*") + return TuningJobDefinition.find + .select("*") .fetch(TuningJobDefinition.TABLE.job, "*") .where() .eq(TuningJobDefinition.TABLE.tuningEnabled, 1) @@ -67,12 +74,25 @@ public void parameterOptimizerIPSO(List results, JobExecution jobExec @Override public String parameterGenerationsHBT(List results, List tuningParameters) { + if (results != null && results.size() > 0) { + SparkHBTParamRecommender sparkHBTParamRecommender = new SparkHBTParamRecommender(results.get(0)); + HashMap suggestedParameters = sparkHBTParamRecommender.getHBTSuggestion(); + StringBuffer idParameters = new StringBuffer(); + for (TuningParameter tuningParameter : tuningParameters) { + if (suggestedParameters.containsKey(tuningParameter.paramName)) { + idParameters.append(tuningParameter.id).append("\t") + .append(suggestedParameters.get(tuningParameter.paramName)); + idParameters.append("\n"); + } + return idParameters.toString(); + } + } return null; } @Override public Boolean isParamConstraintViolatedHBT(List jobSuggestedParamValueList) { - return null; + return false; } -} +} diff --git a/app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java b/app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java new file mode 100644 index 000000000..fe61d5248 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java @@ -0,0 +1,326 @@ +package com.linkedin.drelephant.tuning.engine; + +import java.util.HashMap; +import java.util.Map; +import models.AppResult; +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; +import com.linkedin.drelephant.spark.heuristics.ConfigurationHeuristic; +import com.linkedin.drelephant.spark.heuristics.DriverHeuristic; +import com.linkedin.drelephant.spark.heuristics.JvmUsedMemoryHeuristic; +import com.linkedin.drelephant.spark.heuristics.UnifiedMemoryHeuristic; +import com.linkedin.drelephant.util.MemoryFormatUtils; + + +/** + * This class recommends spark parameters based on previous run heuristics. For spark it expects one + */ +public class SparkHBTParamRecommender { + private final Logger logger = Logger.getLogger(SparkHBTParamRecommender.class); + + AppResult appResult; + + // TODos Move these to configuration + public static final int MAX_EXECUTOR_CORE = 4; + public static final long MAX_EXECUTOR_MEMORY = 10 * FileUtils.ONE_GB; + public static final int CLUSTER_DEFAULT_EXECUTOR_CORE = 1; + public static final long RESERVED_MEMORY = 300 * FileUtils.ONE_MB; + + private static final long EXECUTOR_MEMORY_BUFFER_PER_CORE = 5; + private static final long EXECUTOR_MEMORY_BUFFER_OVERALL = 5; + private static final long DRIVER_MEMORY_BUFFER = 20; + + private long maxPeakUnifiedMemory; + private long maxPeakJVMUsedMemory; + private long driverMaxPeakJVMUsedMemory; + private long suggestedSparkDriverMemory; + + private long lastRunExecutorMemory; + private int lastRunExecutorCore; + private long lastRunExecutorMemoryOverhead; + private long lastRunDriverMemoryOverhead; + + private Long suggestedExecutorMemory; + private Integer suggestedCore; + private Double suggestedMemoryFactor; + private Long suggestedDriverMemory; + + public SparkHBTParamRecommender(AppResult appResult) { + this.appResult = appResult; + Map appHeuristicsResultDetailsMap = appResult.getHeuristicsResultDetailsMap(); + + maxPeakUnifiedMemory = + MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(UnifiedMemoryHeuristic.class + .getCanonicalName() + "_" + UnifiedMemoryHeuristic.MAX_PEAK_UNIFIED_MEMORY_HEURISTIC_NAME())); + + maxPeakJVMUsedMemory = + MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(JvmUsedMemoryHeuristic.class + .getCanonicalName() + "_" + JvmUsedMemoryHeuristic.MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME())); + + lastRunExecutorMemoryOverhead = + MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(ConfigurationHeuristic.class + .getCanonicalName() + "_" + ConfigurationHeuristic.SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD())); + + String coreConfigStr = + appHeuristicsResultDetailsMap.get(ConfigurationHeuristic.class.getCanonicalName() + "_" + + ConfigurationHeuristic.SPARK_EXECUTOR_CORES_KEY()); + + lastRunExecutorCore = CLUSTER_DEFAULT_EXECUTOR_CORE; + try { + lastRunExecutorCore = Integer.parseInt(coreConfigStr); + } catch (NumberFormatException e) { + // Do Nothing + } + + driverMaxPeakJVMUsedMemory = + MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(DriverHeuristic.class.getCanonicalName() + + "_" + DriverHeuristic.DRIVER_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME())); + + lastRunDriverMemoryOverhead = + MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(DriverHeuristic.class.getCanonicalName() + + "_" + DriverHeuristic.SPARK_YARN_DRIVER_MEMORY_OVERHEAD())); + + suggestedSparkDriverMemory = + MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(DriverHeuristic.class.getCanonicalName() + + "_" + DriverHeuristic.SUGGESTED_SPARK_DRIVER_MEMORY_HEURISTIC_NAME())); + + logger.info("Following are the heuristics values for last run : "); + logger.info("maxPeakUnifiedMemory: " + maxPeakUnifiedMemory); + logger.info("maxPeakJVMUsedMemory: " + maxPeakJVMUsedMemory); + logger.info("lastRunExecutorCore: " + lastRunExecutorCore); + logger.info("driverMaxPeakJVMUsedMemory: " + driverMaxPeakJVMUsedMemory); + logger.info("suggestedSparkDriverMemory: " + suggestedSparkDriverMemory); + + } + + public long getLastRunExecutorMemoryOverhead() { + return lastRunExecutorMemoryOverhead; + } + + public void setLastRunExecutorMemoryOverhead(long lastRunExecutorMemoryOverhead) { + this.lastRunExecutorMemoryOverhead = lastRunExecutorMemoryOverhead; + } + + public Long getSuggestedExecutorMemory() { + return suggestedExecutorMemory; + } + + public void setSuggestedExecutorMemory(Long suggestedExecutorMemory) { + this.suggestedExecutorMemory = suggestedExecutorMemory; + } + + public Integer getSuggestedCore() { + return suggestedCore; + } + + public void setSuggestedCore(Integer suggestedCore) { + this.suggestedCore = suggestedCore; + } + + public Double getSuggestedMemoryFactor() { + return suggestedMemoryFactor; + } + + public void setSuggestedMemoryFactor(Double suggestedMemoryFactor) { + this.suggestedMemoryFactor = suggestedMemoryFactor; + } + + public Long getSuggestedDriverMemory() { + return suggestedDriverMemory; + } + + public void setSuggestedDriverMemory(Long suggestedDriverMemory) { + this.suggestedDriverMemory = suggestedDriverMemory; + } + + public long getMaxPeakUnifiedMemory() { + return maxPeakUnifiedMemory; + } + + public void setMaxPeakUnifiedMemory(long maxPeakUnifiedMemory) { + this.maxPeakUnifiedMemory = maxPeakUnifiedMemory; + } + + public long getMaxPeakJVMUsedMemory() { + return maxPeakJVMUsedMemory; + } + + public void setMaxPeakJVMUsedMemory(long maxPeakJVMUsedMemory) { + this.maxPeakJVMUsedMemory = maxPeakJVMUsedMemory; + } + + public long getLastRunExecutorMemory() { + return lastRunExecutorMemory; + } + + public void setLastRunExecutorMemory(long lastRunExecutorMemory) { + this.lastRunExecutorMemory = lastRunExecutorMemory; + } + + public int getLastRunExecutorCore() { + return lastRunExecutorCore; + } + + public void setLastRunExecutorCore(int lastRunExecutorCore) { + this.lastRunExecutorCore = lastRunExecutorCore; + } + + public long getDriverMaxPeakJVMUsedMemory() { + return driverMaxPeakJVMUsedMemory; + } + + public void setDriverMaxPeakJVMUsedMemory(long driverMaxPeakJVMUsedMemory) { + this.driverMaxPeakJVMUsedMemory = driverMaxPeakJVMUsedMemory; + } + + public long getSuggestedSparkDriverMemory() { + return suggestedSparkDriverMemory; + } + + public void setSuggestedSparkDriverMemory(long suggestedSparkDriverMemory) { + this.suggestedSparkDriverMemory = suggestedSparkDriverMemory; + } + + /** + * This method returns suggested parameter for the job. Currently it runs following parameters: + * executor memory, core, driver memory and memory factor + * + * @return HashMap with property name as key and double value + */ + public HashMap getHBTSuggestion() { + HashMap suggestedParameters = new HashMap(); + suggestExecutorMemoryCore(); + suggestMemoryFactor(); + suggestDriverMemory(); + suggestedParameters.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY, (double) suggestedExecutorMemory + / FileUtils.ONE_MB); + suggestedParameters.put(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY, suggestedCore.doubleValue()); + if (suggestedMemoryFactor < UnifiedMemoryHeuristic.SPARK_MEMORY_FRACTION_THRESHOLD()) { + suggestedMemoryFactor = UnifiedMemoryHeuristic.SPARK_MEMORY_FRACTION_THRESHOLD(); + } + suggestedParameters.put(SparkConfigurationConstants.SPARK_MEMORY_FRACTION_KEY, suggestedMemoryFactor); + suggestedParameters.put(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY, (double) suggestedDriverMemory + / FileUtils.ONE_MB); + logger.info("Following are the suggestions for spark parameters for app id : " + appResult.flowExecId); + logger.info("suggestedExecutorMemory " + suggestedExecutorMemory); + logger.info("suggestedCore " + suggestedCore); + logger.info("suggestedMemoryFactor " + suggestedMemoryFactor); + logger.info("suggestedDriverMemory " + suggestedDriverMemory); + return suggestedParameters; + } + + private Long getMaxPeakJVMUsedMemoryPerCore() { + return maxPeakJVMUsedMemory / lastRunExecutorCore; + } + + private Long getMaxPeakUnifiedMemoryPerCore() { + return maxPeakUnifiedMemory / lastRunExecutorCore; + } + + /** + * Suggest executor memory and core We calculate per core peak JVM used memory. Then we start from + * max executor core and compute memory requirement for max executor core If this is below + * threshold of max executor core then good, otherwise reduce the core and check memory + * requirement. + */ + private void suggestExecutorMemoryCore() { + boolean validSuggestion = false; + long maxPeakJVMUsedMemoryPerCore = getMaxPeakJVMUsedMemoryPerCore(); + for (int core = MAX_EXECUTOR_CORE; core > 0; core--) { + long currSuggestedMemory = suggestExecutorMemory(maxPeakJVMUsedMemoryPerCore, core); + if (currSuggestedMemory < MAX_EXECUTOR_MEMORY || core == 1) { + suggestedExecutorMemory = currSuggestedMemory; + suggestedCore = core; + validSuggestion = true; + break; + } + } + if (!validSuggestion) { + suggestedExecutorMemory = lastRunExecutorMemory; + suggestedCore = lastRunExecutorCore; + } + suggestedExecutorMemory = getRoundedExecutorMemory(); + } + + /** + * This method is to get the rounded executor memory. We don't want to round executor memory to + * multiple of 1GB as container size is decided by executor memory + memoryOverhead. So round the + * executor memory + memoryOverhead and then distribute extra memory to executor memory + * + * @return rounded executor memory + */ + private long getRoundedExecutorMemory() { + long memoryOverhead = lastRunExecutorMemoryOverhead; + if (lastRunExecutorMemoryOverhead == 0) { + memoryOverhead = Math.max(suggestedExecutorMemory / 10, 384); + } + + long roundedContainerSize = getRoundedContainerSize(suggestedExecutorMemory + memoryOverhead + FileUtils.ONE_MB); + long executorMemory = + (long) (suggestedExecutorMemory + (roundedContainerSize - suggestedExecutorMemory - memoryOverhead - FileUtils.ONE_MB) * 0.9); + + return executorMemory; + } + + /** + * This method is to get the rounded executor memory. We don't want to round executor memory to + * multiple of 1GB as container size is decided by executor memory + memoryOverhead. So round the + * executor memory + memoryOverhead and then distribute extra memory to executor memory + * + * @return rounded driver memory + */ + private long getRoundedDriverMemory() { + long memoryOverhead = lastRunDriverMemoryOverhead; + if (lastRunDriverMemoryOverhead == 0) { + memoryOverhead = Math.max(suggestedDriverMemory / 10, 384); + } + + long roundedContainerSize = getRoundedContainerSize(suggestedDriverMemory + memoryOverhead + FileUtils.ONE_MB); + long driverMemory = + (long) (suggestedDriverMemory + (roundedContainerSize - suggestedDriverMemory - memoryOverhead - FileUtils.ONE_MB) * 0.9); + + return driverMemory; + } + + /** + * Suggest executor memory based on peak JVM used memory per core and number of core after adding + * buffer + * + * @param maxPeakJVMUsedMemoryPerCore + * @param core + * @return suggested executor memory + */ + private long suggestExecutorMemory(long maxPeakJVMUsedMemoryPerCore, int core) { + return (maxPeakJVMUsedMemoryPerCore * (100 + EXECUTOR_MEMORY_BUFFER_PER_CORE) / 100) * core + * (100 + EXECUTOR_MEMORY_BUFFER_OVERALL) / 100; + } + + /** + * Suggest memory factor using peak unified memory per core and number of core after adding buffer + */ + private void suggestMemoryFactor() { + long unifiedMemoryRequirement = + (getMaxPeakUnifiedMemoryPerCore() * (100 + EXECUTOR_MEMORY_BUFFER_PER_CORE) / 100) * suggestedCore + * (100 + EXECUTOR_MEMORY_BUFFER_OVERALL) / 100; + suggestedMemoryFactor = unifiedMemoryRequirement * 1.0 / suggestedExecutorMemory; + } + + /** + * Suggest driver memory based on driver peak JVM used memory + */ + private void suggestDriverMemory() { + suggestedDriverMemory = driverMaxPeakJVMUsedMemory * (100 + DRIVER_MEMORY_BUFFER) / 100; + suggestedDriverMemory = getRoundedDriverMemory(); + } + + /** + * Rounded container size in multiple of 1GB + * + * @param memory + * @return rounded container size + */ + private long getRoundedContainerSize(long memory) { + return ((long) Math.ceil(memory * 1.0 / FileUtils.ONE_GB)) * FileUtils.ONE_GB; + } + +} diff --git a/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java index 87f53f19e..642ab91c5 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java @@ -1,10 +1,5 @@ package com.linkedin.drelephant.tuning.hbt; -import com.avaje.ebean.Expr; -import com.linkedin.drelephant.AutoTuner; -import com.linkedin.drelephant.ElephantContext; -import com.linkedin.drelephant.tuning.AbstractFitnessManager; -import com.linkedin.drelephant.util.Utils; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -16,9 +11,13 @@ import models.TuningAlgorithm; import models.TuningJobDefinition; import models.TuningJobExecutionParamSet; -import org.apache.log4j.Logger; import org.apache.hadoop.conf.Configuration; -import scala.App; +import org.apache.log4j.Logger; +import com.avaje.ebean.Expr; +import com.linkedin.drelephant.AutoTuner; +import com.linkedin.drelephant.ElephantContext; +import com.linkedin.drelephant.tuning.AbstractFitnessManager; +import com.linkedin.drelephant.util.Utils; public class FitnessManagerHBT extends AbstractFitnessManager { @@ -76,9 +75,11 @@ protected void calculateAndUpdateFitness(JobExecution jobExecution, List results) { + appHeuristicResult.severity.getValue()); } } + } else { + logger.info(appResult.id + " " + appResult.jobDefId + " have yarn app result null "); + return true; } } return checkHeuriticsforSeverity(heuristicsWithHighSeverity); diff --git a/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java index 871fba8a2..11d750f22 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java @@ -103,7 +103,14 @@ private String generateParamSet(List tuningParameters, JobDefin " Job is analyzing , cannot use for param generation " + jobExecution.id + " " + jobExecution.job.id); return ""; } - String idParameters = this._executionEngine.parameterGenerationsHBT(results, tuningParameters); + String idParameters = null; + try + { + idParameters=this._executionEngine.parameterGenerationsHBT(results, tuningParameters); + }catch(Exception e) + { + logger.error("Exception in getting specific parameters ", e); + } return idParameters.toString(); } diff --git a/app/models/AppResult.java b/app/models/AppResult.java index e7c9c7f6c..0d31dd544 100644 --- a/app/models/AppResult.java +++ b/app/models/AppResult.java @@ -16,14 +16,9 @@ package models; -import com.fasterxml.jackson.annotation.JsonManagedReference; -import com.linkedin.drelephant.analysis.Severity; - -import com.linkedin.drelephant.util.Utils; -import java.util.Date; -import play.db.ebean.Model; - +import java.util.HashMap; import java.util.List; +import java.util.Map; import javax.persistence.CascadeType; import javax.persistence.Column; @@ -32,6 +27,12 @@ import javax.persistence.OneToMany; import javax.persistence.Table; +import play.db.ebean.Model; + +import com.fasterxml.jackson.annotation.JsonManagedReference; +import com.linkedin.drelephant.analysis.Severity; +import com.linkedin.drelephant.util.Utils; + @Entity @Table(name = "yarn_app_result") @@ -57,7 +58,7 @@ public static class TABLE { public static final String ID = "id"; public static final String NAME = "name"; public static final String USERNAME = "username"; - public static final String QUEUE_NAME = "queueName"; + public static final String QUEUE_NAME = "queueName"; public static final String START_TIME = "startTime"; public static final String FINISH_TIME = "finishTime"; public static final String TRACKING_URL = "trackingUrl"; @@ -82,8 +83,8 @@ public static class TABLE { } public static String getSearchFields() { - return Utils.commaSeparated(AppResult.TABLE.NAME, AppResult.TABLE.USERNAME, TABLE.QUEUE_NAME, AppResult.TABLE.JOB_TYPE, - AppResult.TABLE.SEVERITY, AppResult.TABLE.FINISH_TIME); + return Utils.commaSeparated(AppResult.TABLE.NAME, AppResult.TABLE.USERNAME, TABLE.QUEUE_NAME, + AppResult.TABLE.JOB_TYPE, AppResult.TABLE.SEVERITY, AppResult.TABLE.FINISH_TIME); } @Id @@ -164,4 +165,21 @@ public static String getSearchFields() { public List yarnAppHeuristicResults; public static Finder find = new Finder(String.class, AppResult.class); + + public Map getHeuristicsResultDetailsMap() { + Map heuristicsDetailsMap = new HashMap(); + if (yarnAppHeuristicResults != null) { + List appHeuristicResultDetails; + for (AppHeuristicResult appHeuristicResult : yarnAppHeuristicResults) { + appHeuristicResultDetails = appHeuristicResult.yarnAppHeuristicResultDetails; + if (appHeuristicResultDetails != null) { + for (AppHeuristicResultDetails appHeuristicResultDetail : appHeuristicResultDetails) { + heuristicsDetailsMap.put(appHeuristicResult.heuristicClass + "_" + appHeuristicResultDetail.name, + appHeuristicResultDetail.value); + } + } + } + } + return heuristicsDetailsMap; + } } diff --git a/app/models/JobExecution.java b/app/models/JobExecution.java index 2f67f0c88..0da6f3c2f 100644 --- a/app/models/JobExecution.java +++ b/app/models/JobExecution.java @@ -17,7 +17,6 @@ package models; import java.sql.Timestamp; - import javax.persistence.CascadeType; import javax.persistence.Column; import javax.persistence.Entity; @@ -30,9 +29,8 @@ import javax.persistence.JoinTable; import javax.persistence.ManyToOne; import javax.persistence.Table; - +import javax.persistence.Transient; import com.avaje.ebean.annotation.UpdatedTimestamp; - import play.db.ebean.Model; @@ -95,6 +93,9 @@ public static class TABLE { @UpdatedTimestamp public Timestamp updatedTs; + @Transient + public Double score; + @Override public void save() { this.updatedTs = new Timestamp(System.currentTimeMillis()); diff --git a/conf/evolutions/default/6.sql b/conf/evolutions/default/6.sql index 207fa7645..a9ef947a5 100644 --- a/conf/evolutions/default/6.sql +++ b/conf/evolutions/default/6.sql @@ -24,6 +24,7 @@ ALTER TABLE tuning_algorithm MODIFY COLUMN optimization_algo enum('PSO','PSO_IPS INSERT INTO tuning_algorithm VALUES (3, 'PIG', 'PSO_IPSO', '3', 'RESOURCE', current_timestamp(0), current_timestamp(0)); INSERT INTO tuning_algorithm VALUES (4, 'PIG', 'HBT', '4', 'RESOURCE', current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_algorithm VALUES (5, 'SPARK', 'HBT', '1', 'RESOURCE', current_timestamp(0), current_timestamp(0)); INSERT INTO tuning_parameter VALUES (10,'mapreduce.task.io.sort.mb',3,100,50,1920,50, 0, current_timestamp(0), current_timestamp(0)); INSERT INTO tuning_parameter VALUES (11,'mapreduce.map.memory.mb',3,2048,1024,8192,1024, 0, current_timestamp(0), current_timestamp(0)); @@ -34,11 +35,17 @@ INSERT INTO tuning_parameter VALUES (15,'mapreduce.reduce.java.opts',3,1536,500, INSERT INTO tuning_parameter VALUES (16,'mapreduce.map.java.opts',3,1536,500,6144,64, 0, current_timestamp(0), current_timestamp(0)); INSERT INTO tuning_parameter VALUES (17,'mapreduce.input.fileinputformat.split.maxsize',3,536870912,536870912,536870912,128, 1, current_timestamp(0), current_timestamp(0)); INSERT INTO tuning_parameter VALUES (18,'pig.maxCombinedSplitSize',3,536870912,536870912,536870912,128, 0, current_timestamp(0), current_timestamp(0)); - INSERT INTO tuning_parameter VALUES (19,'mapreduce.map.memory.mb',4,2048,1024,8192,1024, 0, current_timestamp(0), current_timestamp(0)); INSERT INTO tuning_parameter VALUES (20,'mapreduce.map.java.opts',4,1536,500,6144,64, 0, current_timestamp(0), current_timestamp(0)); + +INSERT INTO tuning_parameter VALUES (21,'spark.executor.memory',5,2048,1024,8192,1024, 0, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (22,'spark.driver.memory',5,2048,1024,8192,1024, 0, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (23,'spark.executor.cores',5,1,1,5,1,0, current_timestamp(0), current_timestamp(0)); +INSERT INTO tuning_parameter VALUES (24,'spark.memory.fraction',5,0.05,0.05,0.9,0.01,0, current_timestamp(0), current_timestamp(0)); + + CREATE TABLE IF NOT EXISTS tuning_parameter_constraint ( id int(10) unsigned NOT NULL AUTO_INCREMENT COMMENT 'Auto increment unique id', job_definition_id int(10) unsigned NOT NULL COMMENT 'Job Definition ID', From a87748484e61fb00fb476081284311bf408c8a34 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Mon, 10 Sep 2018 17:02:14 +0530 Subject: [PATCH 06/22] Renamed TuningType to ParamGenerator , Generalized ParameterGenerateManagerOBT , minor changes in naming variables/metohds --- .../tuning/AbstractBaselineManager.java | 10 +-- .../tuning/AbstractFitnessManager.java | 10 +-- .../tuning/AbstractJobStatusManager.java | 10 +-- ... => AbstractParameterGenerateManager.java} | 11 ++- .../tuning/AutoTuningAPIHelper.java | 4 +- .../linkedin/drelephant/tuning/Constant.java | 2 +- app/com/linkedin/drelephant/tuning/Flow.java | 14 +-- .../linkedin/drelephant/tuning/Manager.java | 2 +- .../drelephant/tuning/ManagerFactory.java | 27 +++--- .../Schduler/AzkabanJobStatusManager.java | 4 +- ....java => ParameterGenerateManagerHBT.java} | 19 +--- .../tuning/obt/FitnessManagerOBTAlgoIPSO.java | 6 +- .../tuning/obt/OptimizationAlgoFactory.java | 10 +-- .../obt/ParameterGenerateManagerOBT.java | 60 +++++++++++++ ...> ParameterGenerateManagerOBTAlgoPSO.java} | 47 +++------- ...terGenerateManagerOBTAlgoPSOIPSOImpl.java} | 90 +++++++++---------- ...rameterGenerateManagerOBTAlgoPSOImpl.java} | 11 ++- .../drelephant/tuning/FlowTestRunner.java | 31 +++---- .../tuning/IPSOManagerTestRunner.java | 21 +++-- .../tuning/PSOParamGeneratorTest.java | 9 +- 20 files changed, 203 insertions(+), 195 deletions(-) rename app/com/linkedin/drelephant/tuning/{AbstractTuningTypeManager.java => AbstractParameterGenerateManager.java} (96%) rename app/com/linkedin/drelephant/tuning/hbt/{TuningTypeManagerHBT.java => ParameterGenerateManagerHBT.java} (94%) create mode 100644 app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBT.java rename app/com/linkedin/drelephant/tuning/obt/{TuningTypeManagerOBT.java => ParameterGenerateManagerOBTAlgoPSO.java} (93%) rename app/com/linkedin/drelephant/tuning/obt/{TuningTypeManagerOBTAlgoIPSO.java => ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java} (52%) rename app/com/linkedin/drelephant/tuning/obt/{TuningTypeManagerOBTAlgoPSO.java => ParameterGenerateManagerOBTAlgoPSOImpl.java} (83%) diff --git a/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java b/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java index cf1f53fc7..1bf5e2a12 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java @@ -50,9 +50,9 @@ public AbstractBaselineManager() { 4) Update the metrics . */ @Override - public final Boolean execute() { + public final boolean execute() { logger.info("Executing BaseLine"); - Boolean baseLineComputationDone = false, databaseUpdateDone = false, updateMetricsDone = false; + boolean baseLineComputationDone = false, databaseUpdateDone = false, updateMetricsDone = false; List tuningJobDefinitions = detectJobsForBaseLineComputation(); if (tuningJobDefinitions != null && tuningJobDefinitions.size() >= 1) { logger.info("Computing BaseLine"); @@ -83,7 +83,7 @@ public final Boolean execute() { * @param tuningJobDefinitions Job for which baseline is to be computed */ - protected Boolean calculateBaseLine(List tuningJobDefinitions) { + protected boolean calculateBaseLine(List tuningJobDefinitions) { for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitions) { try { logger.info("Computing and updating baseline metric values for job: " + tuningJobDefinition.job.jobName); @@ -131,7 +131,7 @@ private Long getAvgInputSizeInBytes(String jobDefId) { * This method update database for auto tuning monitoring for baseline computation * @param tuningJobDefinitions */ - protected Boolean updateDataBase(List tuningJobDefinitions) { + protected boolean updateDataBase(List tuningJobDefinitions) { try { for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitions) { tuningJobDefinition.update(); @@ -149,7 +149,7 @@ protected Boolean updateDataBase(List tuningJobDefinitions) * @param tuningJobDefinitions */ - protected Boolean updateMetrics(List tuningJobDefinitions) { + protected boolean updateMetrics(List tuningJobDefinitions) { try { int baselineComputeWaitJobs = 0; for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitions) { diff --git a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java index 7b598c4d0..26c265bb5 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java @@ -65,7 +65,7 @@ protected abstract void calculateAndUpdateFitness(JobExecution jobExecution, Lis */ protected abstract void checkToDisableTuning(Set jobDefinitionSet); - protected Boolean calculateFitness(List completedJobExecutionParamSets) { + protected boolean calculateFitness(List completedJobExecutionParamSets) { for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { JobExecution jobExecution = completedJobExecutionParamSet.jobExecution; JobSuggestedParamSet jobSuggestedParamSet = completedJobExecutionParamSet.jobSuggestedParamSet; @@ -246,7 +246,7 @@ protected JobSuggestedParamSet updateBestJobSuggestedParamSet(JobSuggestedParamS /* Currently update in database is happening in calculate phase . It should be seperated out */ - protected Boolean updateDataBase(List jobExecutionParamSets) { + protected boolean updateDataBase(List jobExecutionParamSets) { return true; } @@ -254,7 +254,7 @@ protected Boolean updateDataBase(List jobExecutionPa * This method update metrics for auto tuning monitoring for fitness compute daemon * @param completedJobExecutionParamSets List of completed tuning job executions */ - private Boolean updateMetrics(List completedJobExecutionParamSets) { + private boolean updateMetrics(List completedJobExecutionParamSets) { int fitnessNotUpdated = 0; for (TuningJobExecutionParamSet completedJobExecutionParamSet : completedJobExecutionParamSets) { if (!completedJobExecutionParamSet.jobSuggestedParamSet.paramSetState.equals( @@ -317,9 +317,9 @@ public void disableTuning(JobDefinition jobDefinition, String reason) { - public final Boolean execute() { + public final boolean execute() { logger.info("Executing Fitness Manager"); - Boolean calculateFitnessDone = false, databaseUpdateDone = false, updateMetricsDone = false; + boolean calculateFitnessDone = false, databaseUpdateDone = false, updateMetricsDone = false; List tuningJobExecutionParamSet = detectJobsForFitnessComputation(); if (tuningJobExecutionParamSet != null && tuningJobExecutionParamSet.size() >= 1) { logger.info("Calculating Fitness"); diff --git a/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java b/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java index 36ea82eb6..68f7007c1 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java @@ -20,7 +20,7 @@ public abstract class AbstractJobStatusManager implements Manager { * @param inProgressExecutionParamSet : Jobs for which status have to find out. * @return */ - protected abstract Boolean analyzeCompletedJobsExecution( + protected abstract boolean analyzeCompletedJobsExecution( List inProgressExecutionParamSet); protected List detectJobsExecutionInProgress() { @@ -37,7 +37,7 @@ protected List detectJobsExecutionInProgress() { return tuningJobExecutionParamSets; } - protected Boolean updateDataBase(List jobs) { + protected boolean updateDataBase(List jobs) { for (TuningJobExecutionParamSet job : jobs) { JobSuggestedParamSet jobSuggestedParamSet = job.jobSuggestedParamSet; JobExecution jobExecution = job.jobExecution; @@ -62,7 +62,7 @@ private Boolean isJobCompleted(JobExecution jobExecution) { } } - protected Boolean updateMetrics(List completedJobs) { + protected boolean updateMetrics(List completedJobs) { for (TuningJobExecutionParamSet completedJob : completedJobs) { JobExecution jobExecution = completedJob.jobExecution; if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { @@ -75,9 +75,9 @@ protected Boolean updateMetrics(List completedJobs) } @Override - public final Boolean execute() { + public final boolean execute() { logger.info("Executing Job Status Manager"); - Boolean calculateCompletedJobExDone = false, databaseUpdateDone = false, updateMetricsDone = false; + boolean calculateCompletedJobExDone = false, databaseUpdateDone = false, updateMetricsDone = false; List tuningJobExecutionParamSet = detectJobsExecutionInProgress(); if (tuningJobExecutionParamSet != null && tuningJobExecutionParamSet.size() >= 1) { logger.info("Calculating Completed Jobs"); diff --git a/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java b/app/com/linkedin/drelephant/tuning/AbstractParameterGenerateManager.java similarity index 96% rename from app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java rename to app/com/linkedin/drelephant/tuning/AbstractParameterGenerateManager.java index 2e5ac26ba..1591d43b8 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractTuningTypeManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractParameterGenerateManager.java @@ -20,12 +20,11 @@ /** * This class is used to generate/suggest parameters . Based on TuningType , Algorithm Type & execution engine. */ -public abstract class AbstractTuningTypeManager implements Manager { +public abstract class AbstractParameterGenerateManager implements Manager { protected final String JSON_CURRENT_POPULATION_KEY = "current_population"; private final Logger logger = Logger.getLogger(getClass()); protected ExecutionEngine _executionEngine; - protected String tuningType = null; - protected String tuningAlgorithm = null; + /** * This method will generate the Param Set and update in JobTuningInfo in saved state. @@ -40,7 +39,7 @@ public abstract class AbstractTuningTypeManager implements Manager { * @return */ - protected abstract Boolean updateDatabase(List tuningJobDefinitions); + protected abstract boolean updateDatabase(List tuningJobDefinitions); /** * Get pending jobs for which param generation cannot be done @@ -139,10 +138,10 @@ protected List generateParameters(List jobsForPara return updatedJobTuningInfoList; } - public final Boolean execute() { + public final boolean execute() { try { logger.info("Executing Tuning Algorithm"); - Boolean parameterGenerationDone = false, databaseUpdateDone = false, updateMetricsDone = false; + boolean parameterGenerationDone = false, databaseUpdateDone = false, updateMetricsDone = false; List jobTuningInfo = detectJobsForParameterGeneration(); if (jobTuningInfo != null && jobTuningInfo.size() >= 1) { logger.info("Generating Parameters "); diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java index df90aaa65..b01c64cd2 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java @@ -20,7 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.linkedin.drelephant.ElephantContext; import com.linkedin.drelephant.tuning.obt.OptimizationAlgoFactory; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBT; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBT; import com.linkedin.drelephant.util.Utils; import controllers.AutoTuningMetricsController; @@ -635,7 +635,7 @@ private void insertParameterValues(JobSuggestedParamSet jobSuggestedParamSet, Ma private void initializeOptimizationAlgoPrerequisite(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { logger.info("Inserting parameter constraint " + tuningAlgorithm.optimizationAlgo.name()); - TuningTypeManagerOBT manager = OptimizationAlgoFactory.getOptimizationAlogrithm(tuningAlgorithm); + ParameterGenerateManagerOBT manager = OptimizationAlgoFactory.getOptimizationAlogrithm(tuningAlgorithm); if (manager != null) { manager.initializePrerequisite(tuningAlgorithm, jobSuggestedParamSet); } diff --git a/app/com/linkedin/drelephant/tuning/Constant.java b/app/com/linkedin/drelephant/tuning/Constant.java index 513f0c135..72e267142 100644 --- a/app/com/linkedin/drelephant/tuning/Constant.java +++ b/app/com/linkedin/drelephant/tuning/Constant.java @@ -8,6 +8,6 @@ public class Constant { public enum TuningType {HBT,OBT} public enum AlgotihmType{PSO,PSO_IPSO,HBT} public enum ExecutionEngineTypes{MR,SPARK} - public enum TypeofManagers{AbstractBaselineManager,AbstractFitnessManager,AbstractJobStatusManager,AbstractTuningTypeManager} + public enum TypeofManagers{AbstractBaselineManager,AbstractFitnessManager,AbstractJobStatusManager,AbstractParameterGenerateManager} } diff --git a/app/com/linkedin/drelephant/tuning/Flow.java b/app/com/linkedin/drelephant/tuning/Flow.java index c4343bf8a..cc272d5d0 100644 --- a/app/com/linkedin/drelephant/tuning/Flow.java +++ b/app/com/linkedin/drelephant/tuning/Flow.java @@ -1,17 +1,5 @@ package com.linkedin.drelephant.tuning; -import com.linkedin.drelephant.tuning.Schduler.AzkabanJobStatusManager; -import com.linkedin.drelephant.tuning.engine.MRExecutionEngine; -import com.linkedin.drelephant.tuning.engine.SparkExecutionEngine; -import com.linkedin.drelephant.tuning.hbt.BaselineManagerHBT; -import com.linkedin.drelephant.tuning.hbt.FitnessManagerHBT; -import com.linkedin.drelephant.tuning.hbt.TuningTypeManagerHBT; -import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoIPSO; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBT; -import com.linkedin.drelephant.tuning.obt.BaselineManagerOBT; -import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoPSO; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoIPSO; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoPSO; import java.util.ArrayList; import java.util.List; import org.apache.log4j.Logger; @@ -102,7 +90,7 @@ public void createTuningTypeManagersPipeline() { for (Constant.ExecutionEngineTypes executionEngineTypes : Constant.ExecutionEngineTypes.values()) { Manager manager = ManagerFactory.getManager(tuningType.name(), algotihmType.name(), executionEngineTypes.name(), - AbstractTuningTypeManager.class.getSimpleName()); + AbstractParameterGenerateManager.class.getSimpleName()); if (manager != null) { logger.info(manager.getManagerName()); algorithmManagers.add(manager); diff --git a/app/com/linkedin/drelephant/tuning/Manager.java b/app/com/linkedin/drelephant/tuning/Manager.java index 4503f2d7e..c910965fe 100644 --- a/app/com/linkedin/drelephant/tuning/Manager.java +++ b/app/com/linkedin/drelephant/tuning/Manager.java @@ -4,7 +4,7 @@ public interface Manager { /* Use to execute the logic of all the managers . */ - Boolean execute(); + boolean execute(); /* Manager Name */ diff --git a/app/com/linkedin/drelephant/tuning/ManagerFactory.java b/app/com/linkedin/drelephant/tuning/ManagerFactory.java index 41ddeb76d..41fd8fc80 100644 --- a/app/com/linkedin/drelephant/tuning/ManagerFactory.java +++ b/app/com/linkedin/drelephant/tuning/ManagerFactory.java @@ -5,14 +5,13 @@ import com.linkedin.drelephant.tuning.engine.SparkExecutionEngine; import com.linkedin.drelephant.tuning.hbt.BaselineManagerHBT; import com.linkedin.drelephant.tuning.hbt.FitnessManagerHBT; -import com.linkedin.drelephant.tuning.hbt.TuningTypeManagerHBT; +import com.linkedin.drelephant.tuning.hbt.ParameterGenerateManagerHBT; import com.linkedin.drelephant.tuning.obt.BaselineManagerOBT; -import com.linkedin.drelephant.tuning.obt.FitnessManagerOBT; import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoIPSO; import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoPSO; -import com.linkedin.drelephant.tuning.obt.OptimizationAlgoFactory; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoIPSO; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoPSO; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBTAlgoPSOIPSOImpl; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBTAlgoPSO; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBTAlgoPSOImpl; import org.apache.log4j.Logger; @@ -49,17 +48,17 @@ private static Manager getMangersForHBT(String algorithmType, String executionEn logger.info("Manager Type Fitness Manager HBT"); return new FitnessManagerHBT(); } - if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name()) && executionEngineTypes.equals( + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractParameterGenerateManager.name()) && executionEngineTypes.equals( Constant.ExecutionEngineTypes.MR.name()) && algorithmType.equals( Constant.AlgotihmType.HBT.name())) { logger.info("Manager Type TuningType Manager HBT MR"); - return new TuningTypeManagerHBT(new MRExecutionEngine()); + return new ParameterGenerateManagerHBT(new MRExecutionEngine()); } - if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name()) && executionEngineTypes.equals( + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractParameterGenerateManager.name()) && executionEngineTypes.equals( Constant.ExecutionEngineTypes.SPARK.name()) && algorithmType.equals( Constant.AlgotihmType.HBT.name())) { logger.info("Manager Type TuningType Manager HBT Spark"); - return new TuningTypeManagerHBT(new SparkExecutionEngine()); + return new ParameterGenerateManagerHBT(new SparkExecutionEngine()); } return null; } @@ -79,7 +78,7 @@ private static Manager getManagersForOBT(String algorithmType, String executionE logger.info("Manager Type Fitness Manager OBT IPSO"); return new FitnessManagerOBTAlgoIPSO(); } - if (typeOfManagers.equals(Constant.TypeofManagers.AbstractTuningTypeManager.name())) { + if (typeOfManagers.equals(Constant.TypeofManagers.AbstractParameterGenerateManager.name())) { return getManagerForOBTForTuningType(executionEngineTypes, algorithmType); } @@ -90,22 +89,22 @@ private static Manager getManagerForOBTForTuningType(String executionEngineTypes if (executionEngineTypes.equals(Constant.ExecutionEngineTypes.MR.name()) && algorithmType.equals( Constant.AlgotihmType.PSO_IPSO.name())) { logger.info("Manager Type TuningType Manager OBT IPSO MR"); - return new TuningTypeManagerOBTAlgoIPSO(new MRExecutionEngine()); + return new ParameterGenerateManagerOBTAlgoPSOIPSOImpl(new MRExecutionEngine()); } if (executionEngineTypes.equals(Constant.ExecutionEngineTypes.MR.name()) && algorithmType.equals( Constant.AlgotihmType.PSO.name())) { logger.info("Manager Type TuningType Manager OBT PSO MR"); - return new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); + return new ParameterGenerateManagerOBTAlgoPSOImpl(new MRExecutionEngine()); } if (executionEngineTypes.equals(Constant.ExecutionEngineTypes.SPARK.name()) && algorithmType.equals( Constant.AlgotihmType.PSO_IPSO.name())) { logger.info("Manager Type TuningType Manager OBT IPSO Spark"); - return new TuningTypeManagerOBTAlgoIPSO(new SparkExecutionEngine()); + return new ParameterGenerateManagerOBTAlgoPSOIPSOImpl(new SparkExecutionEngine()); } if (executionEngineTypes.equals(Constant.ExecutionEngineTypes.SPARK.name()) && algorithmType.equals( Constant.AlgotihmType.PSO.name())) { logger.info("Manager Type TuningType Manager OBT PSO Spark"); - return new TuningTypeManagerOBTAlgoPSO(new SparkExecutionEngine()); + return new ParameterGenerateManagerOBTAlgoPSOImpl(new SparkExecutionEngine()); } return null; } diff --git a/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java b/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java index 553bddc99..c4093753c 100644 --- a/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java +++ b/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java @@ -37,7 +37,7 @@ protected List detectJobsExecutionInProgress() { } @Override - protected Boolean analyzeCompletedJobsExecution(List inProgressExecutionParamSet) { + protected boolean analyzeCompletedJobsExecution(List inProgressExecutionParamSet) { logger.info("Fetching the list of executions completed since last iteration"); List completedExecutions = new ArrayList(); try { @@ -50,7 +50,7 @@ protected Boolean analyzeCompletedJobsExecution(List } } catch (Exception e) { logger.error("Error in fetching list of completed executions", e); - e.printStackTrace(); + return false; } logger.info("Number of executions completed since last iteration: " + completedExecutions.size()); return true; diff --git a/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java similarity index 94% rename from app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java rename to app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java index 11d750f22..807929e4f 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/TuningTypeManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java @@ -1,24 +1,16 @@ package com.linkedin.drelephant.tuning.hbt; -import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; -import com.linkedin.drelephant.tuning.AbstractTuningTypeManager; +import com.linkedin.drelephant.tuning.AbstractParameterGenerateManager; import com.linkedin.drelephant.tuning.JobTuningInfo; import com.linkedin.drelephant.tuning.ExecutionEngine; -import com.linkedin.drelephant.tuning.Particle; import com.linkedin.drelephant.tuning.TuningHelper; -import controllers.AutoTuningMetricsController; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import models.AppHeuristicResult; import models.AppResult; import models.JobDefinition; import models.JobExecution; -import models.JobSavedState; import models.JobSuggestedParamSet; import models.JobSuggestedParamValue; import models.TuningAlgorithm; @@ -28,21 +20,18 @@ import play.libs.Json; import org.apache.commons.io.FileUtils; -import static java.lang.Math.*; - import com.avaje.ebean.Expr; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -public class TuningTypeManagerHBT extends AbstractTuningTypeManager { +public class ParameterGenerateManagerHBT extends AbstractParameterGenerateManager { private final Logger logger = Logger.getLogger(getClass()); private Map> usageDataGlobal = null; - public TuningTypeManagerHBT(ExecutionEngine executionEngine) { - tuningType = "HBT"; + public ParameterGenerateManagerHBT(ExecutionEngine executionEngine) { this._executionEngine = executionEngine; } @@ -146,7 +135,7 @@ private List getAppResults(JobExecution jobExecution) { * * @param jobTuningInfoList JobTuningInfo List */ - protected Boolean updateDatabase(List jobTuningInfoList) { + protected boolean updateDatabase(List jobTuningInfoList) { logger.info("Updating new parameter suggestion in database HBT"); if (jobTuningInfoList == null) { logger.info("No new parameter suggestion to update"); diff --git a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java index 06d01793c..68be3bbcb 100644 --- a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java +++ b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java @@ -1,10 +1,6 @@ package com.linkedin.drelephant.tuning.obt; import com.avaje.ebean.Expr; -import com.linkedin.drelephant.AutoTuner; -import com.linkedin.drelephant.ElephantContext; -import com.linkedin.drelephant.tuning.AbstractFitnessManager; -import com.linkedin.drelephant.util.Utils; import java.util.ArrayList; import java.util.List; import models.AppResult; @@ -77,7 +73,7 @@ protected void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExec */ private void parameterSpaceOptimization(JobSuggestedParamSet jobSuggestedParamSet, JobExecution jobExecution, List results) { - TuningTypeManagerOBT optimizeManager = + ParameterGenerateManagerOBT optimizeManager = OptimizationAlgoFactory.getOptimizationAlogrithm(jobSuggestedParamSet.tuningAlgorithm); if (optimizeManager != null) { optimizeManager.parameterOptimizer(results,jobExecution); diff --git a/app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java b/app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java index a2782aa7c..c17c69764 100644 --- a/app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java +++ b/app/com/linkedin/drelephant/tuning/obt/OptimizationAlgoFactory.java @@ -13,26 +13,26 @@ public class OptimizationAlgoFactory { private static final Logger logger = Logger.getLogger(OptimizationAlgoFactory.class); - public static TuningTypeManagerOBT getOptimizationAlogrithm(TuningAlgorithm tuningAlgorithm) { + public static ParameterGenerateManagerOBT getOptimizationAlogrithm(TuningAlgorithm tuningAlgorithm) { if (tuningAlgorithm.optimizationAlgo.name().equals(TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) && tuningAlgorithm.jobType.name().equals(TuningAlgorithm.JobType.PIG.name())) { logger.info("OPTIMIZATION ALGORITHM PSO_IPSO MR"); - return new TuningTypeManagerOBTAlgoIPSO(new MRExecutionEngine()); + return new ParameterGenerateManagerOBTAlgoPSOIPSOImpl(new MRExecutionEngine()); } if (tuningAlgorithm.optimizationAlgo.name().equals(TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) && tuningAlgorithm.jobType.name().equals(TuningAlgorithm.JobType.SPARK.name())) { logger.info("OPTIMIZATION ALGORITHM PSO_IPSO SPARK"); - return new TuningTypeManagerOBTAlgoIPSO(new SparkExecutionEngine()); + return new ParameterGenerateManagerOBTAlgoPSOIPSOImpl(new SparkExecutionEngine()); } if (tuningAlgorithm.optimizationAlgo.name().equals(TuningAlgorithm.OptimizationAlgo.PSO.name()) && tuningAlgorithm.jobType.name().equals(TuningAlgorithm.JobType.PIG.name())) { logger.info("OPTIMIZATION ALGORITHM PSO PIG"); - return new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); + return new ParameterGenerateManagerOBTAlgoPSOImpl(new MRExecutionEngine()); } if (tuningAlgorithm.optimizationAlgo.name().equals(TuningAlgorithm.OptimizationAlgo.PSO.name()) && tuningAlgorithm.jobType.name().equals(TuningAlgorithm.JobType.SPARK.name())) { logger.info("OPTIMIZATION ALGORITHM PSO SPARK"); - return new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); + return new ParameterGenerateManagerOBTAlgoPSOImpl(new MRExecutionEngine()); } logger.info("OPTIMIZATION ALGORITHM HBT"); return null; diff --git a/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBT.java new file mode 100644 index 000000000..4ef47cee0 --- /dev/null +++ b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBT.java @@ -0,0 +1,60 @@ +package com.linkedin.drelephant.tuning.obt; + +import com.avaje.ebean.Expr; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.linkedin.drelephant.ElephantContext; +import com.linkedin.drelephant.tuning.AbstractParameterGenerateManager; +import com.linkedin.drelephant.tuning.JobTuningInfo; +import com.linkedin.drelephant.tuning.Particle; +import controllers.AutoTuningMetricsController; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; +import models.AppResult; +import models.JobDefinition; +import models.JobExecution; +import models.JobSavedState; +import models.JobSuggestedParamSet; +import models.JobSuggestedParamValue; +import models.TuningAlgorithm; +import models.TuningJobDefinition; +import models.TuningParameter; +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; +import play.libs.Json; +import org.apache.hadoop.conf.Configuration; + + +public abstract class ParameterGenerateManagerOBT extends AbstractParameterGenerateManager { + + protected final String PARAMS_TO_TUNE_FIELD_NAME = "parametersToTune"; + + private final Logger logger = Logger.getLogger(getClass()); + + /* + Intialize any prequisite require for Optimizer + Calls once in lifetime of the flow + */ + public abstract void initializePrerequisite(TuningAlgorithm tuningAlgorithm, + JobSuggestedParamSet jobSuggestedParamSet); + + /* + Optimize search space + call after each execution of flow + */ + public abstract void parameterOptimizer(List appResults, JobExecution jobExecution); + + /* + Saved the Job State , so that it can be used in next execution. In PSO , it saves the state of previous parameter generated + */ + protected abstract void saveJobState(JobTuningInfo jobTuningInfo, JobDefinition job); + + @Override + public String getManagerName() { + return "ParameterGenerateManagerOBT"; + } +} diff --git a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSO.java similarity index 93% rename from app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java rename to app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSO.java index 1746ac068..19606f810 100644 --- a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBT.java +++ b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSO.java @@ -1,61 +1,38 @@ package com.linkedin.drelephant.tuning.obt; -import com.avaje.ebean.Expr; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.JsonNodeFactory; -import com.fasterxml.jackson.databind.node.ObjectNode; import com.linkedin.drelephant.ElephantContext; -import com.linkedin.drelephant.tuning.AbstractTuningTypeManager; +import com.linkedin.drelephant.tuning.AbstractParameterGenerateManager; import com.linkedin.drelephant.tuning.JobTuningInfo; import com.linkedin.drelephant.tuning.Particle; -import com.linkedin.drelephant.tuning.ExecutionEngine; -import com.linkedin.drelephant.tuning.TuningHelper; import controllers.AutoTuningMetricsController; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import models.AppResult; import models.JobDefinition; -import models.JobExecution; import models.JobSavedState; import models.JobSuggestedParamSet; import models.JobSuggestedParamValue; import models.TuningAlgorithm; import models.TuningJobDefinition; import models.TuningParameter; -import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import play.libs.Json; import org.apache.hadoop.conf.Configuration; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.commons.io.FileUtils; +public abstract class ParameterGenerateManagerOBTAlgoPSO extends ParameterGenerateManagerOBT { -public abstract class TuningTypeManagerOBT extends AbstractTuningTypeManager { - - private static final String PARAMS_TO_TUNE_FIELD_NAME = "parametersToTune"; + private final Logger logger = Logger.getLogger(getClass()); private static final String PYTHON_PATH_CONF = "python.path"; private static final String PSO_DIR_PATH_ENV_VARIABLE = "PSO_DIR_PATH"; private static final String PYTHON_PATH_ENV_VARIABLE = "PYTHONPATH"; private String PYTHON_PATH = null; private String TUNING_SCRIPT_PATH = null; - private final Logger logger = Logger.getLogger(getClass()); - - - /* - Intialize any prequisite require for Optimizer - Calls once in lifetime of the flow - */ - public abstract void initializePrerequisite(TuningAlgorithm tuningAlgorithm, - JobSuggestedParamSet jobSuggestedParamSet); - - /* - Optimize search space - call after each execution of flow - */ - public abstract void parameterOptimizer(List appResults, JobExecution jobExecution); /** * Swarm size specific to OBT @@ -63,8 +40,7 @@ public abstract void initializePrerequisite(TuningAlgorithm tuningAlgorithm, */ protected abstract int getSwarmSize(); - public TuningTypeManagerOBT() { - tuningType = "OBT"; + public ParameterGenerateManagerOBTAlgoPSO() { Configuration configuration = ElephantContext.instance().getAutoTuningConf(); PYTHON_PATH = configuration.get(PYTHON_PATH_CONF); @@ -84,6 +60,7 @@ public TuningTypeManagerOBT() { logger.info("Python path: " + PYTHON_PATH); } + @Override protected void saveJobState(JobTuningInfo jobTuningInfo, JobDefinition job) { boolean validSavedState = true; @@ -233,7 +210,7 @@ public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { * * @param jobTuningInfoList JobTuningInfo List */ - protected Boolean updateDatabase(List jobTuningInfoList) { + protected boolean updateDatabase(List jobTuningInfoList) { logger.info("Updating new parameter suggestion in database"); if (jobTuningInfoList == null) { logger.info("No new parameter suggestion to update"); @@ -402,7 +379,9 @@ private Long saveSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { @Override public String getManagerName() { - return "TuningTypeManagerOBT"; + return "ParameterGenerateManagerOBTAlgoAbstractPSO"; } + + } diff --git a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoIPSO.java b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java similarity index 52% rename from app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoIPSO.java rename to app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java index aca1bfa5c..2fdc1ce0a 100644 --- a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoIPSO.java +++ b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java @@ -1,9 +1,7 @@ package com.linkedin.drelephant.tuning.obt; import com.avaje.ebean.Expr; -import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; import com.linkedin.drelephant.tuning.ExecutionEngine; -import com.linkedin.drelephant.tuning.engine.MRExecutionEngine; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -19,19 +17,15 @@ import models.TuningParameterConstraint; import org.apache.log4j.Logger; -import static java.lang.Math.*; - -public class TuningTypeManagerOBTAlgoIPSO extends TuningTypeManagerOBT { +public class ParameterGenerateManagerOBTAlgoPSOIPSOImpl extends ParameterGenerateManagerOBTAlgoPSO { private final Logger logger = Logger.getLogger(getClass()); - private Map> usageDataGlobal = null; enum UsageCounterSchema {USED_PHYSICAL_MEMORY, USED_VIRTUAL_MEMORY, USED_HEAP_MEMORY} - public TuningTypeManagerOBTAlgoIPSO(ExecutionEngine executionEngine) { - tuningAlgorithm = TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name(); + public ParameterGenerateManagerOBTAlgoPSOIPSOImpl(ExecutionEngine executionEngine) { this._executionEngine = executionEngine; } @@ -45,13 +39,11 @@ public boolean isParamConstraintViolated(List jobSuggest return _executionEngine.isParamConstraintViolatedIPSO(jobSuggestedParamValues); } - - @Override protected List getPendingParamSets() { List pendingParamSetList = _executionEngine.getPendingJobs() - .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm - + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) + .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.optimizationAlgo, + TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) .findList(); return pendingParamSetList; @@ -60,64 +52,70 @@ protected List getPendingParamSets() { @Override protected List getTuningJobDefinitions() { return _executionEngine.getTuningJobDefinitionsForParameterSuggestion() - .eq(TuningJobDefinition.TABLE.tuningAlgorithm - + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) + .eq(TuningJobDefinition.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.optimizationAlgo, + TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) .findList(); } @Override public void initializePrerequisite(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { logger.info(" Intialize Prerequisite "); - setDefaultParameterValues(tuningAlgorithm, jobSuggestedParamSet); + try { + setDefaultParameterValues(tuningAlgorithm, jobSuggestedParamSet); + } catch (Exception e) { + logger.info("Cannot intialize parameter in IPSO " + e.getMessage()); + } } - private void setDefaultParameterValues(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { - List tuningParameters = - TuningParameter.find.where().eq(TuningParameter.TABLE.tuningAlgorithm, tuningAlgorithm).findList(); - for (TuningParameter tuningParameter : tuningParameters) { - TuningParameterConstraint tuningParameterConstraint = new TuningParameterConstraint(); - tuningParameterConstraint.jobDefinition = jobSuggestedParamSet.jobDefinition; - tuningParameterConstraint.tuningParameter = tuningParameter; - tuningParameterConstraint.lowerBound = tuningParameter.minValue; - tuningParameterConstraint.upperBound = tuningParameter.maxValue; - tuningParameterConstraint.constraintType = TuningParameterConstraint.ConstraintType.BOUNDARY; - tuningParameterConstraint.save(); + private void setDefaultParameterValues(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) + throws Exception { + try { + List tuningParameters = + TuningParameter.find.where().eq(TuningParameter.TABLE.tuningAlgorithm, tuningAlgorithm).findList(); + for (TuningParameter tuningParameter : tuningParameters) { + TuningParameterConstraint tuningParameterConstraint = new TuningParameterConstraint(); + tuningParameterConstraint.jobDefinition = jobSuggestedParamSet.jobDefinition; + tuningParameterConstraint.tuningParameter = tuningParameter; + tuningParameterConstraint.lowerBound = tuningParameter.minValue; + tuningParameterConstraint.upperBound = tuningParameter.maxValue; + tuningParameterConstraint.constraintType = TuningParameterConstraint.ConstraintType.BOUNDARY; + tuningParameterConstraint.save(); + } + } catch (Exception e) { + logger.info( + " Error in setting up intial parameter constraint . IPSO will not work in this case ." + e.getMessage()); + throw e; } } - - - - @Override public void parameterOptimizer(List appResults, JobExecution jobExecution) { logger.info(" IPSO Optimizer"); - _executionEngine.parameterOptimizerIPSO(appResults,jobExecution); + _executionEngine.parameterOptimizerIPSO(appResults, jobExecution); } - public void applyIntelligenceOnParameter(List tuningParameterList, JobDefinition job) { logger.info(" Apply Intelligence"); List tuningParameterConstraintList = new ArrayList(); - try { - tuningParameterConstraintList = TuningParameterConstraint.find.where() - .eq("job_definition_id", job.id) - .eq(TuningParameterConstraint.TABLE.constraintType, TuningParameterConstraint.ConstraintType.BOUNDARY) - .findList(); - } catch (NullPointerException e) { - logger.info("No boundary constraints found for job: " + job.jobName); - } + tuningParameterConstraintList = TuningParameterConstraint.find.where() + .eq("job_definition_id", job.id) + .eq(TuningParameterConstraint.TABLE.constraintType, TuningParameterConstraint.ConstraintType.BOUNDARY) + .findList(); - Map paramConstrainIndexMap = new HashMap(); + Map paramConstraintIndexMap = new HashMap(); int i = 0; - for (TuningParameterConstraint tuningParameterConstraint : tuningParameterConstraintList) { - paramConstrainIndexMap.put(tuningParameterConstraint.tuningParameter.id, i); - i += 1; + if (tuningParameterConstraintList != null && tuningParameterConstraintList.size() >= 1) { + for (TuningParameterConstraint tuningParameterConstraint : tuningParameterConstraintList) { + paramConstraintIndexMap.put(tuningParameterConstraint.tuningParameter.id, i); + i += 1; + } + } else { + logger.info("No boundary constraints found for job: " + job.jobName); } for (TuningParameter tuningParameter : tuningParameterList) { - if (paramConstrainIndexMap.containsKey(tuningParameter.id)) { - int index = paramConstrainIndexMap.get(tuningParameter.id); + if (paramConstraintIndexMap.containsKey(tuningParameter.id)) { + int index = paramConstraintIndexMap.get(tuningParameter.id); tuningParameter.minValue = tuningParameterConstraintList.get(index).lowerBound; tuningParameter.maxValue = tuningParameterConstraintList.get(index).upperBound; } diff --git a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoPSO.java b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOImpl.java similarity index 83% rename from app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoPSO.java rename to app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOImpl.java index bf6c54996..acbcfc7f2 100644 --- a/app/com/linkedin/drelephant/tuning/obt/TuningTypeManagerOBTAlgoPSO.java +++ b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOImpl.java @@ -3,7 +3,6 @@ import com.avaje.ebean.Expr; import com.linkedin.drelephant.tuning.ExecutionEngine; import java.util.List; -import java.util.Map; import models.AppResult; import models.JobDefinition; import models.JobExecution; @@ -12,11 +11,11 @@ import models.TuningAlgorithm; import models.TuningJobDefinition; import models.TuningParameter; +import org.apache.log4j.Logger; - -public class TuningTypeManagerOBTAlgoPSO extends TuningTypeManagerOBT{ - public TuningTypeManagerOBTAlgoPSO(ExecutionEngine executionEngine) { - tuningAlgorithm = TuningAlgorithm.OptimizationAlgo.PSO.name(); +public class ParameterGenerateManagerOBTAlgoPSOImpl extends ParameterGenerateManagerOBTAlgoPSO { + private final Logger logger = Logger.getLogger(getClass()); + public ParameterGenerateManagerOBTAlgoPSOImpl(ExecutionEngine executionEngine) { this._executionEngine=executionEngine; } @@ -66,6 +65,6 @@ public int getSwarmSize() { } public String getManagerName() { - return "TuningTypeManagerOBTAlgoPSO" + this._executionEngine.getClass().getSimpleName(); + return "ParameterGenerateManagerOBTAlgoPSOImpl" + this._executionEngine.getClass().getSimpleName(); } } diff --git a/test/com/linkedin/drelephant/tuning/FlowTestRunner.java b/test/com/linkedin/drelephant/tuning/FlowTestRunner.java index 2ef479644..5b1ba87bb 100644 --- a/test/com/linkedin/drelephant/tuning/FlowTestRunner.java +++ b/test/com/linkedin/drelephant/tuning/FlowTestRunner.java @@ -5,12 +5,13 @@ import com.linkedin.drelephant.tuning.engine.SparkExecutionEngine; import com.linkedin.drelephant.tuning.hbt.BaselineManagerHBT; import com.linkedin.drelephant.tuning.hbt.FitnessManagerHBT; -import com.linkedin.drelephant.tuning.hbt.TuningTypeManagerHBT; +import com.linkedin.drelephant.tuning.hbt.ParameterGenerateManagerHBT; import com.linkedin.drelephant.tuning.obt.BaselineManagerOBT; import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoIPSO; import com.linkedin.drelephant.tuning.obt.FitnessManagerOBTAlgoPSO; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoIPSO; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoPSO; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBTAlgoPSOIPSOImpl; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBTAlgoPSO; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBTAlgoPSOImpl; import java.util.List; import static org.junit.Assert.*; @@ -63,29 +64,29 @@ private void testCreateTuningTypeManagersPipeline(Flow flow) { List tuningTypeManagers = pipelines.get(3); assertTrue(" Total Number of tuningType Managers ", tuningTypeManagers.size() == 6); - assertTrue(" TuningTypeManagerHBT ", tuningTypeManagers.get(0) instanceof TuningTypeManagerHBT); + assertTrue(" TuningTypeManagerHBT ", tuningTypeManagers.get(0) instanceof ParameterGenerateManagerHBT); assertTrue(" TuningTypeManagerHBTMR ", - ((TuningTypeManagerHBT) tuningTypeManagers.get(0))._executionEngine instanceof MRExecutionEngine); + ((ParameterGenerateManagerHBT) tuningTypeManagers.get(0))._executionEngine instanceof MRExecutionEngine); - assertTrue(" TuningTypeManagerHBT ", tuningTypeManagers.get(1) instanceof TuningTypeManagerHBT); + assertTrue(" TuningTypeManagerHBT ", tuningTypeManagers.get(1) instanceof ParameterGenerateManagerHBT); assertTrue(" TuningTypeManagerHBTSpark ", - ((TuningTypeManagerHBT) tuningTypeManagers.get(1))._executionEngine instanceof SparkExecutionEngine); + ((ParameterGenerateManagerHBT) tuningTypeManagers.get(1))._executionEngine instanceof SparkExecutionEngine); - assertTrue(" TuningTypeManagerOBTPSO ", tuningTypeManagers.get(2) instanceof TuningTypeManagerOBTAlgoPSO); + assertTrue(" TuningTypeManagerOBTPSO ", tuningTypeManagers.get(2) instanceof ParameterGenerateManagerOBTAlgoPSO); assertTrue(" TuningTypeManagerOBTPSOMR ", - ((TuningTypeManagerOBTAlgoPSO) tuningTypeManagers.get(2))._executionEngine instanceof MRExecutionEngine); + ((ParameterGenerateManagerOBTAlgoPSOImpl) tuningTypeManagers.get(2))._executionEngine instanceof MRExecutionEngine); - assertTrue(" TuningTypeManagerOBTPSO ", tuningTypeManagers.get(3) instanceof TuningTypeManagerOBTAlgoPSO); + assertTrue(" TuningTypeManagerOBTPSO ", tuningTypeManagers.get(3) instanceof ParameterGenerateManagerOBTAlgoPSO); assertTrue(" TuningTypeManagerOBTPSOSpark ", - ((TuningTypeManagerOBTAlgoPSO) tuningTypeManagers.get(3))._executionEngine instanceof SparkExecutionEngine); + ((ParameterGenerateManagerOBTAlgoPSOImpl) tuningTypeManagers.get(3))._executionEngine instanceof SparkExecutionEngine); - assertTrue(" TuningTypeManagerOBTIPSO ", tuningTypeManagers.get(4) instanceof TuningTypeManagerOBTAlgoIPSO); + assertTrue(" TuningTypeManagerOBTIPSO ", tuningTypeManagers.get(4) instanceof ParameterGenerateManagerOBTAlgoPSOIPSOImpl); assertTrue(" TuningTypeManagerOBTIPSOMR ", - ((TuningTypeManagerOBTAlgoIPSO) tuningTypeManagers.get(4))._executionEngine instanceof MRExecutionEngine); + ((ParameterGenerateManagerOBTAlgoPSOIPSOImpl) tuningTypeManagers.get(4))._executionEngine instanceof MRExecutionEngine); - assertTrue(" TuningTypeManagerOBTIPSO ", tuningTypeManagers.get(5) instanceof TuningTypeManagerOBTAlgoIPSO); + assertTrue(" TuningTypeManagerOBTIPSO ", tuningTypeManagers.get(5) instanceof ParameterGenerateManagerOBTAlgoPSOIPSOImpl); assertTrue(" TuningTypeManagerOBTPSOSpark ", - ((TuningTypeManagerOBTAlgoIPSO) tuningTypeManagers.get(5))._executionEngine instanceof SparkExecutionEngine); + ((ParameterGenerateManagerOBTAlgoPSOIPSOImpl) tuningTypeManagers.get(5))._executionEngine instanceof SparkExecutionEngine); } } /* StringBuffer stringBuffer = new StringBuffer(); diff --git a/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java b/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java index 0ed363678..8a5bae802 100644 --- a/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java +++ b/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java @@ -2,11 +2,10 @@ import com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic; import com.linkedin.drelephant.tuning.obt.OptimizationAlgoFactory; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBT; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoIPSO; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBT; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBTAlgoPSOIPSOImpl; import java.util.ArrayList; import java.util.List; -import java.util.Map; import models.AppHeuristicResult; import models.AppResult; import models.JobDefinition; @@ -43,7 +42,7 @@ public void run() { JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.where().eq("fitness_job_execution_id", 1541).findUnique(); JobExecution jobExecution = JobExecution.find.byId(1541L); - TuningTypeManagerOBT optimizeManager = checkIPSOManager(tuningAlgorithm); + ParameterGenerateManagerOBT optimizeManager = checkIPSOManager(tuningAlgorithm); testIPSOInitializePrerequisite(optimizeManager, tuningAlgorithm, jobSuggestedParamSet); //testIPSOExtractParameterInformation(jobExecution, optimizeManager); testIPSOParameterOptimizer(jobExecution, optimizeManager); @@ -51,13 +50,13 @@ public void run() { testIPSONumberOfConstraintsViolated(optimizeManager); } - private TuningTypeManagerOBT checkIPSOManager(TuningAlgorithm tuningAlgorithm) { - TuningTypeManagerOBT optimizeManager = OptimizationAlgoFactory.getOptimizationAlogrithm(tuningAlgorithm); - assertTrue("Optimization Algorithm type ", optimizeManager instanceof TuningTypeManagerOBTAlgoIPSO); + private ParameterGenerateManagerOBT checkIPSOManager(TuningAlgorithm tuningAlgorithm) { + ParameterGenerateManagerOBT optimizeManager = OptimizationAlgoFactory.getOptimizationAlogrithm(tuningAlgorithm); + assertTrue("Optimization Algorithm type ", optimizeManager instanceof ParameterGenerateManagerOBTAlgoPSOIPSOImpl); return optimizeManager; } - private void testIPSOInitializePrerequisite(TuningTypeManagerOBT optimizeManager, TuningAlgorithm tuningAlgorithm, + private void testIPSOInitializePrerequisite(ParameterGenerateManagerOBT optimizeManager, TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { optimizeManager.initializePrerequisite(tuningAlgorithm, jobSuggestedParamSet); List tuningParameterConstraint = @@ -100,7 +99,7 @@ private void testReduceData(Map> usageData) { == 2100); }*/ - private void testIPSOParameterOptimizer(JobExecution jobExecution, TuningTypeManagerOBT optimizeManager) { + private void testIPSOParameterOptimizer(JobExecution jobExecution, ParameterGenerateManagerOBT optimizeManager) { List results = AppResult.find.select("*") .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS, "*") .fetch(AppResult.TABLE.APP_HEURISTIC_RESULTS + "." + AppHeuristicResult.TABLE.APP_HEURISTIC_RESULT_DETAILS, "*") @@ -145,7 +144,7 @@ private void testReduceParameterBoundries(TuningParameterConstraint parameterCon } private void testIPSOApplyIntelligenceOnParameter(TuningJobDefinition tuningJobDefinition, - JobDefinition jobDefinition, TuningTypeManagerOBT optimizeManager) { + JobDefinition jobDefinition, ParameterGenerateManagerOBT optimizeManager) { List tuningParameterList = TuningParameter.find.where() .eq(TuningParameter.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.id, tuningJobDefinition.tuningAlgorithm.id) @@ -189,7 +188,7 @@ private void testNonIPSOParameter(TuningParameter tuningParameter) { } } - private void testIPSONumberOfConstraintsViolated(TuningTypeManagerOBT optimizeManager) { + private void testIPSONumberOfConstraintsViolated(ParameterGenerateManagerOBT optimizeManager) { JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); jobSuggestedParamValue.tuningParameter = TuningParameter.find.byId(11); jobSuggestedParamValue.paramValue = 2048.0; diff --git a/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java b/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java index 32f10e1a5..8c45657d6 100644 --- a/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java +++ b/test/com/linkedin/drelephant/tuning/PSOParamGeneratorTest.java @@ -21,8 +21,9 @@ import com.linkedin.drelephant.DrElephant; import com.linkedin.drelephant.ElephantContext; import com.linkedin.drelephant.tuning.engine.MRExecutionEngine; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBT; -import com.linkedin.drelephant.tuning.obt.TuningTypeManagerOBTAlgoPSO; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBT; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBTAlgoPSO; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBTAlgoPSOImpl; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -111,7 +112,7 @@ public void run() { jobTuningInfo.setJobType(TuningAlgorithm.JobType.PIG); //PSOParamGenerator psoParamGenerator = new PSOParamGenerator(); - TuningTypeManagerOBT tuningTypeManagerOBT = new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); + ParameterGenerateManagerOBT tuningTypeManagerOBT = new ParameterGenerateManagerOBTAlgoPSOImpl(new MRExecutionEngine()); //algorithmManagerOBT.generateParamSet() JobTuningInfo updatedJobTuningInfo = tuningTypeManagerOBT.generateParamSet(jobTuningInfo); @@ -208,7 +209,7 @@ public void run() { /* PSOParamGenerator psoParamGenerator = new PSOParamGenerator(); psoParamGenerator.getParams();*/ - Manager manager = new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine()); + Manager manager = new ParameterGenerateManagerOBTAlgoPSOImpl(new MRExecutionEngine()); manager.execute(); List jobSuggestedParamSetList = JobSuggestedParamSet.find.where() From 909b24cadbd4ea7925631387cb455bb8a4197c03 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Mon, 1 Oct 2018 10:59:41 +0530 Subject: [PATCH 07/22] Added Unit Test cases , Changes specific to IPSO --- .../tuning/AbstractJobStatusManager.java | 40 ++-- .../tuning/AutoTuningAPIHelper.java | 17 +- .../Schduler/AzkabanJobStatusManager.java | 12 +- .../obt/ParameterGenerateManagerOBT.java | 2 +- ...eterGenerateManagerOBTAlgoPSOIPSOImpl.java | 35 ++-- ...arameterGenerateManagerOBTAlgoPSOImpl.java | 2 +- app/models/TuningParameterConstraint.java | 2 +- .../tuning/BaselineManagerTestRunner.java | 40 +++- .../tuning/IPSOManagerTestRunner.java | 10 +- .../tuning/JobStatusManagerTestRunner.java | 62 +++++++ .../drelephant/tuning/TuningManagerTest.java | 11 ++ test/common/DBTestUtil.java | 5 + test/common/TestConstants.java | 1 + test/resources/test-init-baseline.sql | 31 +++- test/resources/test-job-status-data.sql | 173 ++++++++++++++++++ 15 files changed, 384 insertions(+), 59 deletions(-) create mode 100644 test/com/linkedin/drelephant/tuning/JobStatusManagerTestRunner.java create mode 100644 test/resources/test-job-status-data.sql diff --git a/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java b/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java index 68f7007c1..4b24fa4a3 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java @@ -38,16 +38,21 @@ protected List detectJobsExecutionInProgress() { } protected boolean updateDataBase(List jobs) { - for (TuningJobExecutionParamSet job : jobs) { - JobSuggestedParamSet jobSuggestedParamSet = job.jobSuggestedParamSet; - JobExecution jobExecution = job.jobExecution; - if (isJobCompleted(jobExecution)) { - jobExecution.update(); - jobSuggestedParamSet.update(); - logger.info("Execution " + jobExecution.jobExecId + " is completed"); - } else { - logger.info("Execution " + jobExecution.jobExecId + " is still in running state"); + try { + for (TuningJobExecutionParamSet job : jobs) { + JobSuggestedParamSet jobSuggestedParamSet = job.jobSuggestedParamSet; + JobExecution jobExecution = job.jobExecution; + if (isJobCompleted(jobExecution)) { + jobExecution.update(); + jobSuggestedParamSet.update(); + logger.info("Execution " + jobExecution.jobExecId + " is completed"); + } else { + logger.info("Execution " + jobExecution.jobExecId + " is still in running state"); + } } + }catch(Exception e){ + logger.info ( "Exception occur while updating database " + e.getMessage()); + return false; } return true; } @@ -63,13 +68,18 @@ private Boolean isJobCompleted(JobExecution jobExecution) { } protected boolean updateMetrics(List completedJobs) { - for (TuningJobExecutionParamSet completedJob : completedJobs) { - JobExecution jobExecution = completedJob.jobExecution; - if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { - AutoTuningMetricsController.markSuccessfulJobs(); - } else if (jobExecution.executionState.equals(JobExecution.ExecutionState.FAILED)) { - AutoTuningMetricsController.markFailedJobs(); + try { + for (TuningJobExecutionParamSet completedJob : completedJobs) { + JobExecution jobExecution = completedJob.jobExecution; + if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { + AutoTuningMetricsController.markSuccessfulJobs(); + } else if (jobExecution.executionState.equals(JobExecution.ExecutionState.FAILED)) { + AutoTuningMetricsController.markFailedJobs(); + } } + }catch (Exception e){ + logger.info(" Exception while updating Metrics to ingraph " + e.getMessage()); + return false; } return true; } diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java index b01c64cd2..6709224aa 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java @@ -363,10 +363,12 @@ public Map getCurrentRunParameters(TuningInput tuningInput) thro if (tuningJobDefinition == null) { tuningInput.setTuningAlgorithm(getTuningAlgorithmForfirstTime(tuningInput)); logger.info(" Tuning Algorithm Type " + tuningInput.getTuningAlgorithm().optimizationAlgo.name()); + initializeOptimizationAlgoPrerequisite(tuningInput); return processForFirstExecution(tuningInput, jobExecution); } else { tuningInput.setTuningAlgorithm(tuningJobDefinition.tuningAlgorithm); logger.info(" Tuning Algorithm Type " + tuningInput.getTuningAlgorithm().optimizationAlgo.name()); + initializeOptimizationAlgoPrerequisite(tuningInput); return processForSubsequentExecutions(tuningJobDefinition, tuningInput, jobExecution); } } /*catch (Exception e) { @@ -610,7 +612,6 @@ private void insertParamSet(JobDefinition job, TuningAlgorithm tuningAlgorithm, jobSuggestedParamSet.isParamSetSuggested = false; jobSuggestedParamSet.save(); insertParameterValues(jobSuggestedParamSet, paramValueMap, tuningAlgorithm); - initializeOptimizationAlgoPrerequisite(tuningAlgorithm, jobSuggestedParamSet); logger.debug("Default parameter set inserted for job: " + job.jobName); } @@ -632,12 +633,20 @@ private void insertParameterValues(JobSuggestedParamSet jobSuggestedParamSet, Ma } } - private void initializeOptimizationAlgoPrerequisite(TuningAlgorithm tuningAlgorithm, - JobSuggestedParamSet jobSuggestedParamSet) { + private void initializeOptimizationAlgoPrerequisite(TuningInput tuningInput) { + String jobDefId = tuningInput.getJobDefId(); + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") + .fetch(TuningJobDefinition.TABLE.job, "*") + .where() + .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.jobDefId, jobDefId) + .setMaxRows(1) + .orderBy(TuningJobDefinition.TABLE.createdTs + " desc") + .findUnique(); + TuningAlgorithm tuningAlgorithm = tuningInput.getTuningAlgorithm(); logger.info("Inserting parameter constraint " + tuningAlgorithm.optimizationAlgo.name()); ParameterGenerateManagerOBT manager = OptimizationAlgoFactory.getOptimizationAlogrithm(tuningAlgorithm); if (manager != null) { - manager.initializePrerequisite(tuningAlgorithm, jobSuggestedParamSet); + manager.initializePrerequisite(tuningAlgorithm, tuningJobDefinition.job); } } diff --git a/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java b/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java index c4093753c..187fb503e 100644 --- a/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java +++ b/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java @@ -24,9 +24,10 @@ public enum AzkabanJobStatus { protected List detectJobsExecutionInProgress() { logger.info("Fetching the executions which are in progress"); List tuningJobExecutionParamSets = - TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobExecution) + TuningJobExecutionParamSet.find. + fetch(TuningJobExecutionParamSet.TABLE.jobExecution) .fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet) - .where() + .where() .eq(TuningJobExecutionParamSet.TABLE.jobExecution + '.' + JobExecution.TABLE.executionState, JobExecution.ExecutionState.IN_PROGRESS) .findList(); @@ -46,7 +47,7 @@ protected boolean analyzeCompletedJobsExecution(List JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; logger.info("Checking current status of started execution: " + jobExecution.jobExecId); assignAzkabanJobStatusUtil(); - analyzeJobExecution(jobExecution,jobSuggestedParamSet); + return analyzeJobExecution(jobExecution,jobSuggestedParamSet); } } catch (Exception e) { logger.error("Error in fetching list of completed executions", e); @@ -63,7 +64,7 @@ private void assignAzkabanJobStatusUtil() { } } - private void analyzeJobExecution(JobExecution jobExecution,JobSuggestedParamSet jobSuggestedParamSet){ + private boolean analyzeJobExecution(JobExecution jobExecution,JobSuggestedParamSet jobSuggestedParamSet){ try { Map jobStatus = _azkabanJobStatusUtil.getJobsFromFlow(jobExecution.flowExecution.flowExecId); if (jobStatus != null) { @@ -75,10 +76,13 @@ private void analyzeJobExecution(JobExecution jobExecution,JobSuggestedParamSet } } else { logger.info("No jobs found for flow execution: " + jobExecution.flowExecution.flowExecId); + return false; } } catch (Exception e) { logger.error("Error in checking status of execution: " + jobExecution.jobExecId, e); + return false; } + return true; } private void updateJobExecutionMetrics(Map.Entry job, JobSuggestedParamSet jobSuggestedParamSet, diff --git a/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBT.java index 4ef47cee0..6236d5b43 100644 --- a/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBT.java +++ b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBT.java @@ -40,7 +40,7 @@ public abstract class ParameterGenerateManagerOBT extends AbstractParameterGener Calls once in lifetime of the flow */ public abstract void initializePrerequisite(TuningAlgorithm tuningAlgorithm, - JobSuggestedParamSet jobSuggestedParamSet); + JobDefinition job); /* Optimize search space diff --git a/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java index 2fdc1ce0a..0e1afdbda 100644 --- a/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java +++ b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java @@ -22,7 +22,6 @@ public class ParameterGenerateManagerOBTAlgoPSOIPSOImpl extends ParameterGenerat private final Logger logger = Logger.getLogger(getClass()); - enum UsageCounterSchema {USED_PHYSICAL_MEMORY, USED_VIRTUAL_MEMORY, USED_HEAP_MEMORY} public ParameterGenerateManagerOBTAlgoPSOIPSOImpl(ExecutionEngine executionEngine) { @@ -58,28 +57,38 @@ protected List getTuningJobDefinitions() { } @Override - public void initializePrerequisite(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { + public void initializePrerequisite(TuningAlgorithm tuningAlgorithm, JobDefinition job) { logger.info(" Intialize Prerequisite "); try { - setDefaultParameterValues(tuningAlgorithm, jobSuggestedParamSet); + setDefaultParameterValues(tuningAlgorithm, job); } catch (Exception e) { logger.info("Cannot intialize parameter in IPSO " + e.getMessage()); } } - private void setDefaultParameterValues(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) - throws Exception { + private void setDefaultParameterValues(TuningAlgorithm tuningAlgorithm, JobDefinition job) throws Exception{ try { List tuningParameters = TuningParameter.find.where().eq(TuningParameter.TABLE.tuningAlgorithm, tuningAlgorithm).findList(); - for (TuningParameter tuningParameter : tuningParameters) { - TuningParameterConstraint tuningParameterConstraint = new TuningParameterConstraint(); - tuningParameterConstraint.jobDefinition = jobSuggestedParamSet.jobDefinition; - tuningParameterConstraint.tuningParameter = tuningParameter; - tuningParameterConstraint.lowerBound = tuningParameter.minValue; - tuningParameterConstraint.upperBound = tuningParameter.maxValue; - tuningParameterConstraint.constraintType = TuningParameterConstraint.ConstraintType.BOUNDARY; - tuningParameterConstraint.save(); + + List tuningParameterConstrains= TuningParameterConstraint.find.where() + .eq(TuningParameterConstraint.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, job.id) + .findList(); + + if(tuningParameterConstrains== null || tuningParameterConstrains.size()==0) { + logger.info("Parameter constraints not added . Hence adding parameter constraint. "); + for (TuningParameter tuningParameter : tuningParameters) { + TuningParameterConstraint tuningParameterConstraint = new TuningParameterConstraint(); + tuningParameterConstraint.jobDefinition = job; + tuningParameterConstraint.tuningParameter = tuningParameter; + tuningParameterConstraint.lowerBound = tuningParameter.minValue; + tuningParameterConstraint.upperBound = tuningParameter.maxValue; + tuningParameterConstraint.constraintType = TuningParameterConstraint.ConstraintType.BOUNDARY; + tuningParameterConstraint.save(); + } + } + else{ + logger.info(" Parameter constraints already added . Hence not adding them"); } } catch (Exception e) { logger.info( diff --git a/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOImpl.java b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOImpl.java index acbcfc7f2..ba3558444 100644 --- a/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOImpl.java +++ b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOImpl.java @@ -50,7 +50,7 @@ protected List getTuningJobDefinitions() { } @Override - public void initializePrerequisite(TuningAlgorithm tuningAlgorithm, JobSuggestedParamSet jobSuggestedParamSet) { + public void initializePrerequisite(TuningAlgorithm tuningAlgorithm, JobDefinition job) { } diff --git a/app/models/TuningParameterConstraint.java b/app/models/TuningParameterConstraint.java index 52a7d4832..08d34b6b8 100644 --- a/app/models/TuningParameterConstraint.java +++ b/app/models/TuningParameterConstraint.java @@ -49,7 +49,7 @@ public enum ConstraintType { public static class TABLE { public static final String TABLE_NAME = "tuning_parameter_constraint"; public static final String id = "id"; - public static final String jobDefinitionId = "jobDefinitionId"; + public static final String jobDefinition = "jobDefinition"; public static final String constraintType = "constraintType"; public static final String tuningParameterId = "tuningParameterId"; public static final String lowerBound = "lowerBound"; diff --git a/test/com/linkedin/drelephant/tuning/BaselineManagerTestRunner.java b/test/com/linkedin/drelephant/tuning/BaselineManagerTestRunner.java index 37bafe7cb..997bde8d8 100644 --- a/test/com/linkedin/drelephant/tuning/BaselineManagerTestRunner.java +++ b/test/com/linkedin/drelephant/tuning/BaselineManagerTestRunner.java @@ -1,8 +1,12 @@ package com.linkedin.drelephant.tuning; +import com.linkedin.drelephant.tuning.engine.MRExecutionEngine; import com.linkedin.drelephant.tuning.hbt.BaselineManagerHBT; +import com.linkedin.drelephant.tuning.hbt.ParameterGenerateManagerHBT; import com.linkedin.drelephant.tuning.obt.BaselineManagerOBT; +import com.linkedin.drelephant.tuning.obt.ParameterGenerateManagerOBTAlgoPSOIPSOImpl; import java.util.List; +import models.TuningJobDefinition; import static common.DBTestUtil.*; import static org.junit.Assert.*; @@ -21,17 +25,33 @@ private void populateTestData() { @Override public void run() { populateTestData(); - Flow flow = new Flow(); - flow.createBaseLineManagersPipeline(); - List> pipeline = flow.getPipeline(); - List baseLineManagers = pipeline.get(0); - testManagerCreations(baseLineManagers); + testBaselineManagerOBT(); + testBaselineManagerHBT(); } - private void testManagerCreations(List baseLineManagers){ - assertTrue("Base line Manager HBT ", baseLineManagers.get(0) instanceof BaselineManagerHBT); - assertTrue("Base line Manager OBT ", baseLineManagers.get(1) instanceof BaselineManagerOBT); - BaselineManagerHBT baselineManagerHBT = (BaselineManagerHBT)baseLineManagers.get(0); - BaselineManagerOBT baselineManagerOBT = (BaselineManagerOBT)baseLineManagers.get(1); + private void testBaselineManagerOBT(){ + AbstractBaselineManager abstractBaselineManager = new BaselineManagerOBT(); + List tuningJobDefinitions = abstractBaselineManager.detectJobsForBaseLineComputation(); + assertTrue(" Base line detection jobs for OBT "+tuningJobDefinitions.size(), tuningJobDefinitions.size()==1); + assertTrue (" Calculating Base line ",abstractBaselineManager.calculateBaseLine(tuningJobDefinitions)); + for(TuningJobDefinition tuningJobDefinition : tuningJobDefinitions){ + assertTrue(" Average Resource usage is not null "+tuningJobDefinition.averageResourceUsage!=null); + } + abstractBaselineManager.updateDataBase(tuningJobDefinitions); + tuningJobDefinitions = abstractBaselineManager.detectJobsForBaseLineComputation(); + assertTrue ("Base line Done . No jobs for Baseline ",(tuningJobDefinitions ==null ||tuningJobDefinitions.size()==0)); + } + + private void testBaselineManagerHBT(){ + AbstractBaselineManager abstractBaselineManager = new BaselineManagerHBT(); + List tuningJobDefinitions = abstractBaselineManager.detectJobsForBaseLineComputation(); + assertTrue(" Base line detection jobs for HBT "+tuningJobDefinitions.size(), tuningJobDefinitions.size()==1); + assertTrue (" Calculating Base line ",abstractBaselineManager.calculateBaseLine(tuningJobDefinitions)); + for(TuningJobDefinition tuningJobDefinition : tuningJobDefinitions){ + assertTrue(" Average Resource usage is not null "+tuningJobDefinition.averageResourceUsage!=null); + } + abstractBaselineManager.updateDataBase(tuningJobDefinitions); + tuningJobDefinitions = abstractBaselineManager.detectJobsForBaseLineComputation(); + assertTrue ("Base line Done . No jobs for Baseline ",(tuningJobDefinitions ==null ||tuningJobDefinitions.size()==0)); } } diff --git a/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java b/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java index 8a5bae802..12111e0dd 100644 --- a/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java +++ b/test/com/linkedin/drelephant/tuning/IPSOManagerTestRunner.java @@ -43,7 +43,7 @@ public void run() { JobSuggestedParamSet.find.where().eq("fitness_job_execution_id", 1541).findUnique(); JobExecution jobExecution = JobExecution.find.byId(1541L); ParameterGenerateManagerOBT optimizeManager = checkIPSOManager(tuningAlgorithm); - testIPSOInitializePrerequisite(optimizeManager, tuningAlgorithm, jobSuggestedParamSet); + testIPSOInitializePrerequisite(optimizeManager, tuningAlgorithm, tuningJobDefinition); //testIPSOExtractParameterInformation(jobExecution, optimizeManager); testIPSOParameterOptimizer(jobExecution, optimizeManager); testIPSOApplyIntelligenceOnParameter(tuningJobDefinition, jobDefinition, optimizeManager); @@ -57,11 +57,13 @@ private ParameterGenerateManagerOBT checkIPSOManager(TuningAlgorithm tuningAlgor } private void testIPSOInitializePrerequisite(ParameterGenerateManagerOBT optimizeManager, TuningAlgorithm tuningAlgorithm, - JobSuggestedParamSet jobSuggestedParamSet) { - optimizeManager.initializePrerequisite(tuningAlgorithm, jobSuggestedParamSet); + TuningJobDefinition tuningJobDefinition) { + + optimizeManager.initializePrerequisite(tuningAlgorithm, tuningJobDefinition.job); List tuningParameterConstraint = TuningParameterConstraint.find.where().eq("job_definition_id", 100003).findList(); - assertTrue(" Parameters Constraint Size ", tuningParameterConstraint.size() == 9); + + assertTrue(" Parameters Constraint Size " + tuningParameterConstraint.size(), tuningParameterConstraint.size() == 9); } /* private void testIPSOExtractParameterInformation(JobExecution jobExecution, TuningTypeManagerOBT optimizeManager) { diff --git a/test/com/linkedin/drelephant/tuning/JobStatusManagerTestRunner.java b/test/com/linkedin/drelephant/tuning/JobStatusManagerTestRunner.java new file mode 100644 index 000000000..289d23095 --- /dev/null +++ b/test/com/linkedin/drelephant/tuning/JobStatusManagerTestRunner.java @@ -0,0 +1,62 @@ +package com.linkedin.drelephant.tuning; + +import com.linkedin.drelephant.tuning.Schduler.AzkabanJobStatusManager; +import com.linkedin.drelephant.tuning.obt.BaselineManagerOBT; +import java.util.List; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.TuningJobDefinition; +import models.TuningJobExecutionParamSet; + +import static common.DBTestUtil.*; +import static org.junit.Assert.*; +import static play.test.Helpers.*; +import static common.DBTestUtil.*; + + +public class JobStatusManagerTestRunner implements Runnable { + + private void populateTestData() { + try { + initDBAzkabanJobStatus(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public void run() { + populateTestData(); + testJobStatusAzkaban(); + } + + private void testJobStatusAzkaban() { + AbstractJobStatusManager jobStatusManager = new AzkabanJobStatusManager(); + List tuningJobExecutionParamSets = jobStatusManager.detectJobsExecutionInProgress(); + /*List tuningJobExecutionParamSets = + TuningJobExecutionParamSet.find.select("*").fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*").findList();*/ + assertTrue(" Number of Jobs in progress " + tuningJobExecutionParamSets.size(), + tuningJobExecutionParamSets.size() == 1); + + boolean calculateCompletedJobExDone = jobStatusManager.analyzeCompletedJobsExecution(tuningJobExecutionParamSets); + + assertTrue(" Analyize completed job execution ", !calculateCompletedJobExDone); + + boolean updateDatabase = jobStatusManager.updateDataBase(tuningJobExecutionParamSets); + + + assertTrue(" Update database done ", updateDatabase); + + JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*").findUnique(); + assertTrue(" Job Suggested param set " + jobSuggestedParamSet.paramSetState.name(), + jobSuggestedParamSet.paramSetState.name().equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED.name())); + + JobExecution jobExecution = JobExecution.find.select("*").findUnique(); + assertTrue(" Job Execution Status " , jobExecution.executionState.name().equals(JobExecution.ExecutionState.IN_PROGRESS.name())); + + boolean updateMetrics = jobStatusManager.updateMetrics(tuningJobExecutionParamSets); + + assertTrue(" Update Metrics " , updateMetrics); + + } +} diff --git a/test/com/linkedin/drelephant/tuning/TuningManagerTest.java b/test/com/linkedin/drelephant/tuning/TuningManagerTest.java index f39e5975c..338d703a7 100644 --- a/test/com/linkedin/drelephant/tuning/TuningManagerTest.java +++ b/test/com/linkedin/drelephant/tuning/TuningManagerTest.java @@ -57,4 +57,15 @@ public void testIPSOManager() { public void testFlowTestRunner(){ running(testServer(TEST_SERVER_PORT, fakeApp), new FlowTestRunner()); } + + + @Test + public void testBaselineManagerTestRunner(){ + running(testServer(TEST_SERVER_PORT, fakeApp), new BaselineManagerTestRunner()); + } + + @Test + public void testJobStatusManagerTestRunner(){ + running(testServer(TEST_SERVER_PORT, fakeApp), new JobStatusManagerTestRunner()); + } } diff --git a/test/common/DBTestUtil.java b/test/common/DBTestUtil.java index 04e5f0193..c80690654 100644 --- a/test/common/DBTestUtil.java +++ b/test/common/DBTestUtil.java @@ -33,6 +33,11 @@ public static void initDBBaseline() throws IOException, SQLException { initDBUtil(TEST_BASELINE_DATA_FILE); } + public static void initDBAzkabanJobStatus() throws IOException, SQLException { + initDBUtil(TEST_JOB_STATUS_DATA_FILE); + } + + public static void initDBIPSO() throws IOException, SQLException { initDBUtil(TEST_IPSO_DATA_FILE); } diff --git a/test/common/TestConstants.java b/test/common/TestConstants.java index ab13f13b4..80646d75f 100644 --- a/test/common/TestConstants.java +++ b/test/common/TestConstants.java @@ -24,6 +24,7 @@ public class TestConstants { public static final String TEST_DATA_FILE = "test/resources/test-init.sql"; public static final String TEST_IPSO_DATA_FILE = "test/resources/test-init-ipso.sql"; public static final String TEST_BASELINE_DATA_FILE = "test/resources/test-init-baseline.sql"; + public static final String TEST_JOB_STATUS_DATA_FILE = "test/resources/test-job-status-data.sql"; public static final String TEST_AUTO_TUNING_DATA_FILE1 = "test/resources/tunein-test1.sql"; public static final int RESPONSE_TIMEOUT = 3000; // milliseconds diff --git a/test/resources/test-init-baseline.sql b/test/resources/test-init-baseline.sql index 47830484f..fc9c8c21e 100644 --- a/test/resources/test-init-baseline.sql +++ b/test/resources/test-init-baseline.sql @@ -45,6 +45,7 @@ insert into yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name (137594520,'Median task runtime','11 sec','NULL'), (137594520,'Median task speed','3 MB/s','NULL'), (137594520,'Number of tasks','2','NULL'), + (137594520,'Total input size in MB','58.65','NULL'), (137594523,'Avg output records per task','56687','NULL'), (137594523,'Avg spilled records per task','79913','NULL'), (137594523,'Number of tasks','2','NULL'), @@ -100,6 +101,7 @@ insert into yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name (137594620,'Median task runtime','11 sec','NULL'), (137594620,'Median task speed','3 MB/s','NULL'), (137594620,'Number of tasks','2','NULL'), + (137594620,'Total input size in MB','58.65','NULL'), (137594623,'Avg output records per task','56687','NULL'), (137594623,'Avg spilled records per task','79913','NULL'), (137594623,'Number of tasks','2','NULL'), @@ -139,16 +141,16 @@ insert into yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name (137594640,'Average sort time','(0.04x)','NULL'), (137594640,'Number of tasks','20','NULL'); -INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES (10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'); +INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES +(10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'); + +INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES +(100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','pkumar2','2018-02-12 08:40:42','2018-02-12 08:40:43'); -INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES (100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','mkumar1','2018-02-12 08:40:42','2018-02-12 08:40:43'); INSERT INTO tuning_job_definition(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason, number_of_iterations, auto_apply) VALUES -(100003,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true), -(100004,'azkaban',3,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true), -(100005,'azkaban',1,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true), -(100003,'azkaban',4,1,5.178423333333334,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true); +(100003,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true); @@ -160,4 +162,21 @@ INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow (1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1541,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'); +INSERT INTO yarn_app_result VALUES +('application_1529108724527_640141','PigLatin:data_simple.pig','pkumar2','sna_default',1535553409318,1535553457517,'http://ltx1-farojh01.grid.linkedin.com:19888/jobhistory/job/job_1529108724527_640141','Pig',1,0,0,'azkaban','dimDataSimpleFlow_dimDataSimple','https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1088630&job=dimDataSimpleFlow_dimDataSimple&attempt=0','https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1088630','https://ltx1-faroaz01.grid.linkedin.com:8443/manager?project=tuning_hbt&flow=dimDataSimpleFlow&job=dimDataSimpleFlow_dimDataSimple','https://ltx1-faroaz01.grid.linkedin.com:8443/manager?project=tuning_hbt&flow=dimDataSimpleFlow','https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1088630&job=dimDataSimpleFlow_dimDataSimple&attempt=0','https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1088630','https://ltx1-faroaz01.grid.linkedin.com:8443/manager?project=tuning_hbt&flow=dimDataSimpleFlow&job=dimDataSimpleFlow_dimDataSimple','https://ltx1-faroaz01.grid.linkedin.com:8443/manager?project=tuning_hbt&flow=dimDataSimpleFlow',104448,19227,9384); + +INSERT INTO yarn_app_heuristic_result VALUES (16093,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0),(16094,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0),(16095,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0),(16096,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.ConfigurationHeuristic','MapReduceConfiguration',1,0),(16097,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0),(16098,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0),(16099,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0),(16100,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0),(16101,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer GC',0,0),(16102,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer Time',0,0),(16103,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0),(16104,'application_1529108724527_640141','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0); + +INSERT INTO yarn_app_heuristic_result_details VALUES +(16097,'Median task input size','300 MB',NULL),(16097,'Median task runtime','25 sec',NULL),(16097,'Median task speed','9 MB/s',NULL),(16097,'Number of tasks','2',NULL),(16097,'Total input size in MB','601.4286088943481',NULL); + + +INSERT INTO flow_definition VALUES (10046,'https://ltx1-faroaz01.grid.linkedin.com:8443/manager?project=tuning_hbt&flow=dimDataSimpleFlow','https://ltx1-faroaz01.grid.linkedin.com:8443/manager?project=tuning_hbt&flow=dimDataSimpleFlow','2018-09-12 08:57:43','2018-09-11 20:27:44') ; + +INSERT INTO job_definition VALUES (100046,'https://ltx1-faroaz01.grid.linkedin.com:8443/manager?project=tuning_hbt&flow=dimDataSimpleFlow&job=dimDataSimpleFlow_dimDataSimple','https://ltx1-faroaz01.grid.linkedin.com:8443/manager?project=tuning_hbt&flow=dimDataSimpleFlow&job=dimDataSimpleFlow_dimDataSimple',10046,'dimDataSimpleFlow_dimDataSimple','azkaban','pkumar2','2018-09-12 08:57:43','2018-09-11 20:27:44'); + +INSERT INTO tuning_job_definition VALUES (100046,'azkaban',3,1,1,NULL,1.15030667,1912288051,150,150,NULL,10,'2018-09-12 08:57:43','2018-09-11 20:27:49'); + +INSERT INTO flow_execution VALUES (1084,'https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1122321','https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1122321',10046,'2018-09-12 08:57:43','2018-09-11 20:27:44'),(1085,'https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1122375','https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1122375',10046,'2018-09-12 09:20:16','2018-09-11 20:50:16'); +INSERT INTO job_execution VALUES (1084,'https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1122321&job=dimDataSimpleFlow_dimDataSimple&attempt=0','https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1122321&job=dimDataSimpleFlow_dimDataSimple&attempt=0',100046,1084,'SUCCEEDED',0.6155555555555555,1.2017666666666666,1918012391,'2018-09-12 08:57:43','2018-09-11 20:35:58'),(1085,'https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1122375&job=dimDataSimpleFlow_dimDataSimple&attempt=0','https://ltx1-faroaz01.grid.linkedin.com:8443/executor?execid=1122375&job=dimDataSimpleFlow_dimDataSimple&attempt=0',100046,1085,'SUCCEEDED',0.15722222222222224,1.2016666666666667,1918012391,'2018-09-12 09:20:16','2018-09-11 20:58:03'); \ No newline at end of file diff --git a/test/resources/test-job-status-data.sql b/test/resources/test-job-status-data.sql new file mode 100644 index 000000000..6e8363d70 --- /dev/null +++ b/test/resources/test-job-status-data.sql @@ -0,0 +1,173 @@ +insert into yarn_app_result + (id,name,username,queue_name,start_time,finish_time,tracking_url,job_type,severity,score,workflow_depth,scheduler,job_name,job_exec_id,flow_exec_id,job_def_id,flow_def_id,job_exec_url,flow_exec_url,job_def_url,flow_def_url,resource_used,resource_wasted,total_delay) values + ('application_1458194917883_1453361','Email Overwriter','growth','misc_default',1460980616502,1460980723925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453361','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654676&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654676','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 100, 30, 20), + ('application_1458194917883_1453362','Email Overwriter','metrics','misc_default',1460980823925,1460980923925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453362','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654677&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654677','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 200, 40, 10); + +insert into yarn_app_heuristic_result(id,yarn_app_result_id,heuristic_class,heuristic_name,severity,score) values +(137594512,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), +(137594513,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), +(137594516,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), +(137594520,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), +(137594523,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), +(137594525,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), +(137594530,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), +(137594531,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), +(137594534,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), +(137594537,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), +(137594540,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0), +(137594612,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), +(137594613,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), +(137594616,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), +(137594620,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), +(137594623,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), +(137594625,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), +(137594630,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), +(137594631,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), +(137594634,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), +(137594637,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), +(137594640,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0); + +insert into yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name,value,details) values +(137594512,'Group A','1 tasks @ 4 MB avg','NULL'), +(137594512,'Group B','1 tasks @ 79 MB avg','NULL'), +(137594512,'Number of tasks','2','NULL'), +(137594513,'Avg task CPU time (ms)','11510','NULL'), + (137594513,'Avg task GC time (ms)','76','NULL'), + (137594513,'Avg task runtime (ms)','11851','NULL'), + (137594513,'Number of tasks','2','NULL'), + (137594513,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), + (137594516,'Average task input size','42 MB','NULL'), + (137594516,'Average task runtime','11 sec','NULL'), + (137594516,'Max task runtime','12 sec','NULL'), + (137594516,'Min task runtime','11 sec','NULL'), + (137594516,'Number of tasks','2','NULL'), + (137594520,'Median task input size','42 MB','NULL'), + (137594520,'Median task runtime','11 sec','NULL'), + (137594520,'Median task speed','3 MB/s','NULL'), + (137594520,'Number of tasks','2','NULL'), + (137594520,'Total input size in MB','58.65','NULL'), + (137594523,'Avg output records per task','56687','NULL'), + (137594523,'Avg spilled records per task','79913','NULL'), + (137594523,'Number of tasks','2','NULL'), + (137594523,'Ratio of spilled records to output records','1.4097111356119074','NULL'), + (137594525,'Avg Physical Memory (MB)','522','NULL'), + (137594525,'Avg task runtime','11 sec','NULL'), + (137594525,'Avg Virtual Memory (MB)','3307','NULL'), + (137594525,'Max Physical Memory (MB)','595','NULL'), + (137594525,'Max Total Committed Heap Usage Memory (MB)','427','NULL'), + (137594525,'Max Virtual Memory (MB)','1426','NULL'), + (137594525,'Min Physical Memory (MB)','449','NULL'), + (137594525,'Number of tasks','2','NULL'), + (137594525,'Requested Container Memory','2 GB','NULL'), + (137594530,'Group A','11 tasks @ 868 KB avg','NULL'), + (137594530,'Group B','9 tasks @ 883 KB avg ','NULL'), + (137594530,'Number of tasks','20','NULL'), + (137594531,'Avg task CPU time (ms)','8912','NULL'), + (137594531,'Avg task GC time (ms)','73','NULL'), + (137594531,'Avg task runtime (ms)','11045','NULL'), + (137594531,'Number of tasks','20','NULL'), + (137594531,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), + (137594534,'Average task runtime','11 sec','NULL'), + (137594534,'Max task runtime','14 sec','NULL'), + (137594534,'Min task runtime','8 sec','NULL'), + (137594534,'Number of tasks','20','NULL'), + (137594537,'Avg Physical Memory (MB)','416','NULL'), + (137594537,'Avg task runtime','11 sec','NULL'), + (137594537,'Avg Virtual Memory (MB)','3326','NULL'), + (137594537,'Max Physical Memory (MB)','497','NULL'), + (137594537,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594537,'Max Virtual Memory (MB)','1350','NULL'), + (137594537,'Min Physical Memory (MB)','354','NULL'), + (137594537,'Number of tasks','20','NULL'), + (137594537,'Requested Container Memory','2 GB','NULL'), + (137594540,'Average code runtime','1 sec','NULL'), + (137594540,'Average shuffle time','9 sec (5.49x)','NULL'), + (137594540,'Average sort time','(0.04x)','NULL'), + (137594540,'Number of tasks','20','NULL'), + (137594612,'Group A','1 tasks @ 4 MB avg','NULL'), + (137594612,'Group B','1 tasks @ 79 MB avg','NULL'), + (137594612,'Number of tasks','2','NULL'), + (137594613,'Avg task CPU time (ms)','11510','NULL'), + (137594613,'Avg task GC time (ms)','76','NULL'), + (137594613,'Avg task runtime (ms)','11851','NULL'), + (137594613,'Number of tasks','2','NULL'), + (137594613,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), + (137594616,'Average task input size','42 MB','NULL'), + (137594616,'Average task runtime','11 sec','NULL'), + (137594616,'Max task runtime','12 sec','NULL'), + (137594616,'Min task runtime','11 sec','NULL'), + (137594616,'Number of tasks','2','NULL'), + (137594620,'Median task input size','42 MB','NULL'), + (137594620,'Median task runtime','11 sec','NULL'), + (137594620,'Median task speed','3 MB/s','NULL'), + (137594620,'Number of tasks','2','NULL'), + (137594620,'Total input size in MB','58.65','NULL'), + (137594623,'Avg output records per task','56687','NULL'), + (137594623,'Avg spilled records per task','79913','NULL'), + (137594623,'Number of tasks','2','NULL'), + (137594623,'Ratio of spilled records to output records','1.4097111356119074','NULL'), + (137594625,'Avg Physical Memory (MB)','522','NULL'), + (137594625,'Avg task runtime','11 sec','NULL'), + (137594625,'Avg Virtual Memory (MB)','3307','NULL'), + (137594625,'Max Physical Memory (MB)','595','NULL'), + (137594625,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594625,'Max Virtual Memory (MB)','2200','NULL'), + (137594625,'Min Physical Memory (MB)','449','NULL'), + (137594625,'Number of tasks','2','NULL'), + (137594625,'Requested Container Memory','2 GB','NULL'), + (137594630,'Group A','11 tasks @ 868 KB avg','NULL'), + (137594630,'Group B','9 tasks @ 883 KB avg ','NULL'), + (137594630,'Number of tasks','20','NULL'), + (137594631,'Avg task CPU time (ms)','8912','NULL'), + (137594631,'Avg task GC time (ms)','73','NULL'), + (137594631,'Avg task runtime (ms)','11045','NULL'), + (137594631,'Number of tasks','20','NULL'), + (137594631,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), + (137594634,'Average task runtime','11 sec','NULL'), + (137594634,'Max task runtime','14 sec','NULL'), + (137594634,'Min task runtime','8 sec','NULL'), + (137594634,'Number of tasks','20','NULL'), + (137594637,'Avg Physical Memory (MB)','416','NULL'), + (137594637,'Avg task runtime','11 sec','NULL'), + (137594637,'Avg Virtual Memory (MB)','3326','NULL'), + (137594637,'Max Physical Memory (MB)','497','NULL'), + (137594637,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594637,'Max Virtual Memory (MB)','2100','NULL'), + (137594637,'Min Physical Memory (MB)','354','NULL'), + (137594637,'Number of tasks','20','NULL'), + (137594637,'Requested Container Memory','2 GB','NULL'), + (137594640,'Average code runtime','1 sec','NULL'), + (137594640,'Average shuffle time','9 sec (5.49x)','NULL'), + (137594640,'Average sort time','(0.04x)','NULL'), + (137594640,'Number of tasks','20','NULL'); + +INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES +(10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'); + +INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES +(100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','pkumar2','2018-02-12 08:40:42','2018-02-12 08:40:43'); + + +INSERT INTO tuning_job_definition(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason, number_of_iterations, auto_apply) +VALUES +(100003,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true); + + + + +INSERT INTO flow_execution(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003); + +INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1541,'IN_PROGRESS',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'); + +INSERT INTO job_suggested_param_set VALUES +(1137,100003,4,'FITNESS_COMPUTED', 0 ,1 , 0 , 0 , 0 ,20 , 1086,'2018-09-17 23:22:31' ,'2018-09-17 11:09:31'); + +INSERT INTO tuning_job_execution_param_set (job_suggested_param_set_id,job_execution_id,tuning_enabled,created_ts,updated_ts) VALUES +(1137,1541,1,'2018-09-17 23:22:31','2018-09-17 10:52:31'); + + + + + From e2f0097a2b239dd49785771a53a3222098804319 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Wed, 3 Oct 2018 14:55:30 +0530 Subject: [PATCH 08/22] Unit test cases for Fitness Manager for HBT --- .../tuning/AbstractFitnessManager.java | 3 +- .../tuning/FitnessManagerTestRunner.java | 98 ++++++++++ .../drelephant/tuning/TuningManagerTest.java | 6 + test/common/DBTestUtil.java | 4 + test/common/TestConstants.java | 1 + test/resources/AutoTuningConf.xml | 4 +- .../test-fitness-calculation-data.sql | 173 ++++++++++++++++++ 7 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 test/com/linkedin/drelephant/tuning/FitnessManagerTestRunner.java create mode 100644 test/resources/test-fitness-calculation-data.sql diff --git a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java index 26c265bb5..25b58f2e7 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java @@ -78,10 +78,11 @@ protected boolean calculateFitness(List completedJob handleFitnessCalculation(jobExecution, results, tuningJobDefinition, jobSuggestedParamSet); } catch (Exception e) { logger.error("Error updating fitness of execution: " + jobExecution.id + "\n Stacktrace: ", e); + return false; } } logger.info("Execution metrics updated"); - return false; + return true; } private TuningJobDefinition getTuningJobDefinition(JobDefinition job) { diff --git a/test/com/linkedin/drelephant/tuning/FitnessManagerTestRunner.java b/test/com/linkedin/drelephant/tuning/FitnessManagerTestRunner.java new file mode 100644 index 000000000..770cc3111 --- /dev/null +++ b/test/com/linkedin/drelephant/tuning/FitnessManagerTestRunner.java @@ -0,0 +1,98 @@ +package com.linkedin.drelephant.tuning; + +import com.linkedin.drelephant.tuning.hbt.FitnessManagerHBT; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import models.AppHeuristicResult; +import models.AppResult; +import models.JobDefinition; +import models.JobExecution; +import models.JobSuggestedParamSet; +import models.TuningJobDefinition; +import models.TuningJobExecutionParamSet; + +import static common.DBTestUtil.*; +import static org.junit.Assert.*; +import static play.test.Helpers.*; +import static common.DBTestUtil.*; + + +public class FitnessManagerTestRunner implements Runnable { + + private void populateTestData() { + try { + initFitnessComputation(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public void run() { + populateTestData(); + testFitnessManagerHBT(); + } + + public void testFitnessManagerHBT() { + AbstractFitnessManager fitnessManager = new FitnessManagerHBT(); + List jobsForFitnessComputation = fitnessManager.detectJobsForFitnessComputation(); + assertTrue(" Jobs for fitness computation HBT " + jobsForFitnessComputation.size(), + jobsForFitnessComputation.size() == 1); + + + JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") + .where() + .eq(JobSuggestedParamSet.TABLE.id, 1137) + .findUnique(); + + assertTrue("Job Suggested Param Set Before " + jobSuggestedParamSet.paramSetState.name(), + jobSuggestedParamSet.paramSetState.name().equals(JobSuggestedParamSet.ParamSetStatus.EXECUTED.name())); + + assertTrue("Fitness Calculated " ,fitnessManager.calculateFitness(jobsForFitnessComputation)); + + JobSuggestedParamSet jobSuggestedParamSet1 = JobSuggestedParamSet.find.select("*") + .where() + .eq(JobSuggestedParamSet.TABLE.id, 1137) + .findUnique(); + + + assertTrue("Job Suggested Param Set After " + jobSuggestedParamSet1.paramSetState.name(), + jobSuggestedParamSet1.paramSetState.name().equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED.name())); + + + testTuningDisabled(jobsForFitnessComputation,fitnessManager); + testFitnessDelay(jobsForFitnessComputation,fitnessManager); + + } + + private void testTuningDisabled(List jobsForFitnessComputation,AbstractFitnessManager fitnessManager ){ + JobDefinition jobDefinition = jobsForFitnessComputation.get(0).jobSuggestedParamSet.jobDefinition; + assertTrue(" Number of executions reach to threshold " ,!fitnessManager.reachToNumberOfThresholdIterations(jobsForFitnessComputation,jobDefinition)); + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*").where() + .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, 100003) + .eq(TuningJobDefinition.TABLE.tuningEnabled, 1).findUnique(); + + tuningJobDefinition.numberOfIterations=1; + tuningJobDefinition.save(); + assertTrue(" Number of executions reach to threshold " ,fitnessManager.reachToNumberOfThresholdIterations(jobsForFitnessComputation,jobDefinition)); + assertTrue(" Tuning Enabled ", tuningJobDefinition.tuningEnabled); + fitnessManager.disableTuning(jobDefinition, "User Specified Iterations reached"); + TuningJobDefinition tuningJobDefinition1 = TuningJobDefinition.find.select("*").where() + .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, 100003).findUnique(); + assertTrue(" Tuning Disabled reason ", tuningJobDefinition1.tuningDisabledReason.equals("User Specified Iterations reached")); + assertTrue(" Tuning Disabled reason "+tuningJobDefinition1.tuningEnabled, !tuningJobDefinition1.tuningEnabled); + } + + private void testFitnessDelay(List jobsForFitnessComputation,AbstractFitnessManager fitnessManager){ + TuningJobExecutionParamSet tuningJobExecutionParamSet = jobsForFitnessComputation.get(0); + tuningJobExecutionParamSet.jobExecution.updatedTs.setTime(System.currentTimeMillis()); + List completedJobExecutionParamSet = new ArrayList(); + fitnessManager.getCompletedExecution(jobsForFitnessComputation, completedJobExecutionParamSet); + assertTrue("Current Time in millis: " + System.currentTimeMillis() + ", Job execution last updated time " + + tuningJobExecutionParamSet.jobExecution.updatedTs.getTime() + " FitnessComputeInterval " + + fitnessManager.fitnessComputeWaitInterval, completedJobExecutionParamSet.size() == 0); + } + +} diff --git a/test/com/linkedin/drelephant/tuning/TuningManagerTest.java b/test/com/linkedin/drelephant/tuning/TuningManagerTest.java index 338d703a7..4f0ead234 100644 --- a/test/com/linkedin/drelephant/tuning/TuningManagerTest.java +++ b/test/com/linkedin/drelephant/tuning/TuningManagerTest.java @@ -68,4 +68,10 @@ public void testBaselineManagerTestRunner(){ public void testJobStatusManagerTestRunner(){ running(testServer(TEST_SERVER_PORT, fakeApp), new JobStatusManagerTestRunner()); } + + + @Test + public void testFitnessManagerTestRunner(){ + running(testServer(TEST_SERVER_PORT, fakeApp), new FitnessManagerTestRunner()); + } } diff --git a/test/common/DBTestUtil.java b/test/common/DBTestUtil.java index c80690654..b90af2073 100644 --- a/test/common/DBTestUtil.java +++ b/test/common/DBTestUtil.java @@ -33,6 +33,10 @@ public static void initDBBaseline() throws IOException, SQLException { initDBUtil(TEST_BASELINE_DATA_FILE); } + public static void initFitnessComputation() throws IOException, SQLException { + initDBUtil(TEST_FITNESS_CALCULATION_DATA_FILE); + } + public static void initDBAzkabanJobStatus() throws IOException, SQLException { initDBUtil(TEST_JOB_STATUS_DATA_FILE); } diff --git a/test/common/TestConstants.java b/test/common/TestConstants.java index 80646d75f..e7a615cfc 100644 --- a/test/common/TestConstants.java +++ b/test/common/TestConstants.java @@ -26,6 +26,7 @@ public class TestConstants { public static final String TEST_BASELINE_DATA_FILE = "test/resources/test-init-baseline.sql"; public static final String TEST_JOB_STATUS_DATA_FILE = "test/resources/test-job-status-data.sql"; public static final String TEST_AUTO_TUNING_DATA_FILE1 = "test/resources/tunein-test1.sql"; + public static final String TEST_FITNESS_CALCULATION_DATA_FILE = "test/resources/test-fitness-calculation-data.sql"; public static final int RESPONSE_TIMEOUT = 3000; // milliseconds diff --git a/test/resources/AutoTuningConf.xml b/test/resources/AutoTuningConf.xml index e2e6cacea..0011e5c70 100644 --- a/test/resources/AutoTuningConf.xml +++ b/test/resources/AutoTuningConf.xml @@ -44,7 +44,7 @@ fitness.compute.wait_interval.ms - 0 + 1800000 Wait time after the job is completed till fitness is computer @@ -58,4 +58,4 @@ - \ No newline at end of file + diff --git a/test/resources/test-fitness-calculation-data.sql b/test/resources/test-fitness-calculation-data.sql new file mode 100644 index 000000000..99e872b69 --- /dev/null +++ b/test/resources/test-fitness-calculation-data.sql @@ -0,0 +1,173 @@ +insert into yarn_app_result + (id,name,username,queue_name,start_time,finish_time,tracking_url,job_type,severity,score,workflow_depth,scheduler,job_name,job_exec_id,flow_exec_id,job_def_id,flow_def_id,job_exec_url,flow_exec_url,job_def_url,flow_def_url,resource_used,resource_wasted,total_delay) values + ('application_1458194917883_1453361','Email Overwriter','growth','misc_default',1460980616502,1460980723925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453361','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654676&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654676','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 100, 30, 20), + ('application_1458194917883_1453362','Email Overwriter','metrics','misc_default',1460980823925,1460980923925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453362','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654677&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654677','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 200, 40, 10); + +insert into yarn_app_heuristic_result(id,yarn_app_result_id,heuristic_class,heuristic_name,severity,score) values +(137594512,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), +(137594513,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), +(137594516,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), +(137594520,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), +(137594523,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), +(137594525,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), +(137594530,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), +(137594531,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), +(137594534,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), +(137594537,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), +(137594540,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0), +(137594612,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), +(137594613,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), +(137594616,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), +(137594620,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), +(137594623,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), +(137594625,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), +(137594630,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), +(137594631,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), +(137594634,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), +(137594637,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), +(137594640,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0); + +insert into yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name,value,details) values +(137594512,'Group A','1 tasks @ 4 MB avg','NULL'), +(137594512,'Group B','1 tasks @ 79 MB avg','NULL'), +(137594512,'Number of tasks','2','NULL'), +(137594513,'Avg task CPU time (ms)','11510','NULL'), + (137594513,'Avg task GC time (ms)','76','NULL'), + (137594513,'Avg task runtime (ms)','11851','NULL'), + (137594513,'Number of tasks','2','NULL'), + (137594513,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), + (137594516,'Average task input size','42 MB','NULL'), + (137594516,'Average task runtime','11 sec','NULL'), + (137594516,'Max task runtime','12 sec','NULL'), + (137594516,'Min task runtime','11 sec','NULL'), + (137594516,'Number of tasks','2','NULL'), + (137594520,'Median task input size','42 MB','NULL'), + (137594520,'Median task runtime','11 sec','NULL'), + (137594520,'Median task speed','3 MB/s','NULL'), + (137594520,'Number of tasks','2','NULL'), + (137594520,'Total input size in MB','58.65','NULL'), + (137594523,'Avg output records per task','56687','NULL'), + (137594523,'Avg spilled records per task','79913','NULL'), + (137594523,'Number of tasks','2','NULL'), + (137594523,'Ratio of spilled records to output records','1.4097111356119074','NULL'), + (137594525,'Avg Physical Memory (MB)','522','NULL'), + (137594525,'Avg task runtime','11 sec','NULL'), + (137594525,'Avg Virtual Memory (MB)','3307','NULL'), + (137594525,'Max Physical Memory (MB)','595','NULL'), + (137594525,'Max Total Committed Heap Usage Memory (MB)','427','NULL'), + (137594525,'Max Virtual Memory (MB)','1426','NULL'), + (137594525,'Min Physical Memory (MB)','449','NULL'), + (137594525,'Number of tasks','2','NULL'), + (137594525,'Requested Container Memory','2 GB','NULL'), + (137594530,'Group A','11 tasks @ 868 KB avg','NULL'), + (137594530,'Group B','9 tasks @ 883 KB avg ','NULL'), + (137594530,'Number of tasks','20','NULL'), + (137594531,'Avg task CPU time (ms)','8912','NULL'), + (137594531,'Avg task GC time (ms)','73','NULL'), + (137594531,'Avg task runtime (ms)','11045','NULL'), + (137594531,'Number of tasks','20','NULL'), + (137594531,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), + (137594534,'Average task runtime','11 sec','NULL'), + (137594534,'Max task runtime','14 sec','NULL'), + (137594534,'Min task runtime','8 sec','NULL'), + (137594534,'Number of tasks','20','NULL'), + (137594537,'Avg Physical Memory (MB)','416','NULL'), + (137594537,'Avg task runtime','11 sec','NULL'), + (137594537,'Avg Virtual Memory (MB)','3326','NULL'), + (137594537,'Max Physical Memory (MB)','497','NULL'), + (137594537,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594537,'Max Virtual Memory (MB)','1350','NULL'), + (137594537,'Min Physical Memory (MB)','354','NULL'), + (137594537,'Number of tasks','20','NULL'), + (137594537,'Requested Container Memory','2 GB','NULL'), + (137594540,'Average code runtime','1 sec','NULL'), + (137594540,'Average shuffle time','9 sec (5.49x)','NULL'), + (137594540,'Average sort time','(0.04x)','NULL'), + (137594540,'Number of tasks','20','NULL'), + (137594612,'Group A','1 tasks @ 4 MB avg','NULL'), + (137594612,'Group B','1 tasks @ 79 MB avg','NULL'), + (137594612,'Number of tasks','2','NULL'), + (137594613,'Avg task CPU time (ms)','11510','NULL'), + (137594613,'Avg task GC time (ms)','76','NULL'), + (137594613,'Avg task runtime (ms)','11851','NULL'), + (137594613,'Number of tasks','2','NULL'), + (137594613,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), + (137594616,'Average task input size','42 MB','NULL'), + (137594616,'Average task runtime','11 sec','NULL'), + (137594616,'Max task runtime','12 sec','NULL'), + (137594616,'Min task runtime','11 sec','NULL'), + (137594616,'Number of tasks','2','NULL'), + (137594620,'Median task input size','42 MB','NULL'), + (137594620,'Median task runtime','11 sec','NULL'), + (137594620,'Median task speed','3 MB/s','NULL'), + (137594620,'Number of tasks','2','NULL'), + (137594620,'Total input size in MB','58.65','NULL'), + (137594623,'Avg output records per task','56687','NULL'), + (137594623,'Avg spilled records per task','79913','NULL'), + (137594623,'Number of tasks','2','NULL'), + (137594623,'Ratio of spilled records to output records','1.4097111356119074','NULL'), + (137594625,'Avg Physical Memory (MB)','522','NULL'), + (137594625,'Avg task runtime','11 sec','NULL'), + (137594625,'Avg Virtual Memory (MB)','3307','NULL'), + (137594625,'Max Physical Memory (MB)','595','NULL'), + (137594625,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594625,'Max Virtual Memory (MB)','2200','NULL'), + (137594625,'Min Physical Memory (MB)','449','NULL'), + (137594625,'Number of tasks','2','NULL'), + (137594625,'Requested Container Memory','2 GB','NULL'), + (137594630,'Group A','11 tasks @ 868 KB avg','NULL'), + (137594630,'Group B','9 tasks @ 883 KB avg ','NULL'), + (137594630,'Number of tasks','20','NULL'), + (137594631,'Avg task CPU time (ms)','8912','NULL'), + (137594631,'Avg task GC time (ms)','73','NULL'), + (137594631,'Avg task runtime (ms)','11045','NULL'), + (137594631,'Number of tasks','20','NULL'), + (137594631,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), + (137594634,'Average task runtime','11 sec','NULL'), + (137594634,'Max task runtime','14 sec','NULL'), + (137594634,'Min task runtime','8 sec','NULL'), + (137594634,'Number of tasks','20','NULL'), + (137594637,'Avg Physical Memory (MB)','416','NULL'), + (137594637,'Avg task runtime','11 sec','NULL'), + (137594637,'Avg Virtual Memory (MB)','3326','NULL'), + (137594637,'Max Physical Memory (MB)','497','NULL'), + (137594637,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594637,'Max Virtual Memory (MB)','2100','NULL'), + (137594637,'Min Physical Memory (MB)','354','NULL'), + (137594637,'Number of tasks','20','NULL'), + (137594637,'Requested Container Memory','2 GB','NULL'), + (137594640,'Average code runtime','1 sec','NULL'), + (137594640,'Average shuffle time','9 sec (5.49x)','NULL'), + (137594640,'Average sort time','(0.04x)','NULL'), + (137594640,'Number of tasks','20','NULL'); + +INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES +(10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'); + +INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES +(100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','pkumar2','2018-02-12 08:40:42','2018-02-12 08:40:43'); + + +INSERT INTO tuning_job_definition(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason, number_of_iterations, auto_apply) +VALUES +(100003,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true); + + + + +INSERT INTO flow_execution(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003); + +INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1541,'SUCCEEDED',NULL,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'); + +INSERT INTO job_suggested_param_set VALUES +(1137,100003,4,'EXECUTED', 0 ,1 , 0 , 0 , 0 ,0 , 1086,'2018-09-17 23:22:31' ,'2018-09-17 11:09:31'); + +INSERT INTO tuning_job_execution_param_set (job_suggested_param_set_id,job_execution_id,tuning_enabled,created_ts,updated_ts) VALUES +(1137,1541,1,'2018-09-17 23:22:31','2018-09-17 10:52:31'); + + + + + From 030b0369cb5cc8506ceb6179d73dd8084117d2d3 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Wed, 3 Oct 2018 17:51:41 +0530 Subject: [PATCH 09/22] Unit test cases for Paramgenerator HBT . Documentation added for entity --- .../tuning/engine/MRExecutionEngine.java | 4 +- app/models/FlowExecution.java | 3 + app/models/JobDefinition.java | 4 +- app/models/JobExecution.java | 3 + app/models/JobSavedState.java | 4 + app/models/JobSuggestedParamSet.java | 6 + app/models/JobSuggestedParamValue.java | 3 + app/models/TuningAlgorithm.java | 5 + app/models/TuningJobDefinition.java | 3 + app/models/TuningJobExecutionParamSet.java | 5 + app/models/TuningParameter.java | 4 + app/models/TuningParameterConstraint.java | 6 +- .../ParameterGenerateManagerTestRunner.java | 64 +++++++ .../drelephant/tuning/TuningManagerTest.java | 5 + test/common/DBTestUtil.java | 4 + test/common/TestConstants.java | 1 + test/resources/test-param-generate-data.sql | 173 ++++++++++++++++++ 17 files changed, 293 insertions(+), 4 deletions(-) create mode 100644 test/com/linkedin/drelephant/tuning/ParameterGenerateManagerTestRunner.java create mode 100644 test/resources/test-param-generate-data.sql diff --git a/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java b/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java index d054bd4ad..23467889d 100644 --- a/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java +++ b/app/com/linkedin/drelephant/tuning/engine/MRExecutionEngine.java @@ -34,7 +34,7 @@ public class MRExecutionEngine implements ExecutionEngine { private final Logger logger = Logger.getLogger(getClass()); boolean debugEnabled = logger.isDebugEnabled(); enum UsageCounterSchema {USED_PHYSICAL_MEMORY, USED_VIRTUAL_MEMORY, USED_HEAP_MEMORY} - + final static int MAX_MAP_MEM_SORT_MEM_DIFF = 768; private String functionTypes[] = {"map", "reduce"}; @Override @@ -126,7 +126,7 @@ public Boolean isParamConstraintViolatedIPSO(List jobSug } violations++; } - if (mrMapMemory - mrSortMemory < 768) { + if (mrMapMemory - mrSortMemory < MAX_MAP_MEM_SORT_MEM_DIFF) { if (debugEnabled) { logger.debug("Sort Memory " + mrSortMemory); logger.debug("Mapper Memory " + mrMapMemory); diff --git a/app/models/FlowExecution.java b/app/models/FlowExecution.java index 0bcee835f..87d9fb247 100644 --- a/app/models/FlowExecution.java +++ b/app/models/FlowExecution.java @@ -32,6 +32,9 @@ import play.db.ebean.Model; +/** + * Table have information about flow execution . + */ @Entity @Table(name = "flow_execution") public class FlowExecution extends Model { diff --git a/app/models/JobDefinition.java b/app/models/JobDefinition.java index 387b7ff12..b478eab58 100644 --- a/app/models/JobDefinition.java +++ b/app/models/JobDefinition.java @@ -33,7 +33,9 @@ import play.db.ebean.Model; - +/** + * This table have information about job definition.This also have information about username and schduler. + */ @Entity @Table(name = "job_definition") public class JobDefinition extends Model { diff --git a/app/models/JobExecution.java b/app/models/JobExecution.java index 0da6f3c2f..019a43a5b 100644 --- a/app/models/JobExecution.java +++ b/app/models/JobExecution.java @@ -34,6 +34,9 @@ import play.db.ebean.Model; +/** + * This table have information about the exeuction of the job. + */ @Entity @Table(name = "job_execution") public class JobExecution extends Model { diff --git a/app/models/JobSavedState.java b/app/models/JobSavedState.java index f7ca624ae..cc1f90b75 100644 --- a/app/models/JobSavedState.java +++ b/app/models/JobSavedState.java @@ -29,6 +29,10 @@ import com.avaje.ebean.annotation.UpdatedTimestamp; +/** + * This table contains information about previous state of the job . This is required for the algorithms + * which require previous information for e.g PSO. + */ @Entity @Table(name = "job_saved_state") public class JobSavedState extends Model { diff --git a/app/models/JobSuggestedParamSet.java b/app/models/JobSuggestedParamSet.java index b8e8b6fdb..b0c6739e7 100644 --- a/app/models/JobSuggestedParamSet.java +++ b/app/models/JobSuggestedParamSet.java @@ -35,6 +35,12 @@ import play.db.ebean.Model; +/** + * This table contains information about suggested param set . This also contains state of param set, + * whether the param set is created , sent , executed , fitness_computed , discarded. + * . Table also have information about which parameter is best so far , which is default , and which is manually + * overridden. + */ @Entity @Table(name = "job_suggested_param_set") public class JobSuggestedParamSet extends Model { diff --git a/app/models/JobSuggestedParamValue.java b/app/models/JobSuggestedParamValue.java index d78ba21ba..7f06afa74 100644 --- a/app/models/JobSuggestedParamValue.java +++ b/app/models/JobSuggestedParamValue.java @@ -33,6 +33,9 @@ import play.db.ebean.Model; +/** + * This table have mapping between job suggested param set and the param value. + */ @Entity @Table(name = "job_suggested_param_value") public class JobSuggestedParamValue extends Model { diff --git a/app/models/TuningAlgorithm.java b/app/models/TuningAlgorithm.java index 3258cd8b2..5f387a340 100644 --- a/app/models/TuningAlgorithm.java +++ b/app/models/TuningAlgorithm.java @@ -32,6 +32,11 @@ import play.db.ebean.Model; +/** + * This table have information about tuning algorithm . + * For e.g job type PIG and optimization algo PSO and optimization metric RESOURCE + * the algorithm id 1. + */ @Entity @Table(name = "tuning_algorithm") public class TuningAlgorithm extends Model { diff --git a/app/models/TuningJobDefinition.java b/app/models/TuningJobDefinition.java index c905b563c..7fd4b75eb 100644 --- a/app/models/TuningJobDefinition.java +++ b/app/models/TuningJobDefinition.java @@ -31,6 +31,9 @@ import play.db.ebean.Model; +/** + * Table describe about the job which is for tuning. It have one entry per job . + */ @Entity @Table(name = "tuning_job_definition") public class TuningJobDefinition extends Model { diff --git a/app/models/TuningJobExecutionParamSet.java b/app/models/TuningJobExecutionParamSet.java index 1be048e17..81ce36406 100644 --- a/app/models/TuningJobExecutionParamSet.java +++ b/app/models/TuningJobExecutionParamSet.java @@ -27,6 +27,11 @@ import javax.persistence.Table; import play.db.ebean.Model; + +/** + * This table tags / bridges between job_suggested_param and job execution. Basically it tags + * each execution with parameter set. + */ @Entity @Table(name = "tuning_job_execution_param_set") public class TuningJobExecutionParamSet extends Model { diff --git a/app/models/TuningParameter.java b/app/models/TuningParameter.java index 7ecce1acc..25a052e5c 100644 --- a/app/models/TuningParameter.java +++ b/app/models/TuningParameter.java @@ -34,6 +34,10 @@ import play.db.ebean.Model; +/** + * This table represent tuning parameters . Parameters which is going to be tune + * ,with their default values and min and max values + */ @Entity @Table(name = "tuning_parameter") public class TuningParameter extends Model { diff --git a/app/models/TuningParameterConstraint.java b/app/models/TuningParameterConstraint.java index 08d34b6b8..2b287fd48 100644 --- a/app/models/TuningParameterConstraint.java +++ b/app/models/TuningParameterConstraint.java @@ -35,7 +35,11 @@ import play.db.ebean.Model; - +/** + * TuningParameterConstraint refers to the table tuning_parameter_constraint. + * This table will contain boundry constraint for each parameter. This table will be specifically + * use in IPSO + */ @Entity @Table(name = "tuning_parameter_constraint") public class TuningParameterConstraint extends Model { diff --git a/test/com/linkedin/drelephant/tuning/ParameterGenerateManagerTestRunner.java b/test/com/linkedin/drelephant/tuning/ParameterGenerateManagerTestRunner.java new file mode 100644 index 000000000..858e6c810 --- /dev/null +++ b/test/com/linkedin/drelephant/tuning/ParameterGenerateManagerTestRunner.java @@ -0,0 +1,64 @@ +package com.linkedin.drelephant.tuning; + +import com.linkedin.drelephant.tuning.engine.MRExecutionEngine; +import com.linkedin.drelephant.tuning.hbt.FitnessManagerHBT; +import com.linkedin.drelephant.tuning.hbt.ParameterGenerateManagerHBT; +import java.util.List; +import models.JobSuggestedParamSet; + +import static org.junit.Assert.*; +import static play.test.Helpers.*; +import static common.DBTestUtil.*; +import static common.DBTestUtil.*; + + +public class ParameterGenerateManagerTestRunner implements Runnable { + + private void populateTestData() { + try { + initParamGenerater(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + @Override + public void run() { + populateTestData(); + testParamGeneraterHBT(); + } + + public void testParamGeneraterHBT() { + AbstractParameterGenerateManager parameterGenerateManager = + new ParameterGenerateManagerHBT(new MRExecutionEngine()); + List jobTuningInfos = parameterGenerateManager.detectJobsForParameterGeneration(); + assertTrue(" Job Needed for Param Generation " + jobTuningInfos.size(), jobTuningInfos.size() == 1); + assertTrue(" Job State " + jobTuningInfos.get(0).getTunerState().replaceAll("\\{\\}", ""), + jobTuningInfos.get(0).getTunerState().replaceAll("\\{\\}", "").length() == 0); + + List updatedJobTuningInfoList = parameterGenerateManager.generateParameters(jobTuningInfos); + assertTrue(" Job State After param generation " + updatedJobTuningInfoList.get(0).getTunerState(), + updatedJobTuningInfoList.get(0).getTunerState().replaceAll("\\{\\}", "").length() > 0); + + List jobSuggestedParamSets = JobSuggestedParamSet.find.select("*") + .where() + .eq(JobSuggestedParamSet.TABLE.paramSetState,JobSuggestedParamSet.ParamSetStatus.CREATED) + .findList(); + + assertTrue(" Parameter Created "+jobSuggestedParamSets.size(),jobSuggestedParamSets.size()==0); + + boolean isDataBaseUpdated = parameterGenerateManager.updateDatabase(updatedJobTuningInfoList); + + assertTrue(" Updated database with param generation "+isDataBaseUpdated,isDataBaseUpdated); + + List jobSuggestedParamSets1 = JobSuggestedParamSet.find.select("*") + .where() + .eq(JobSuggestedParamSet.TABLE.paramSetState,JobSuggestedParamSet.ParamSetStatus.CREATED) + .findList(); + + + assertTrue(" Parameter Created "+jobSuggestedParamSets1.size(),jobSuggestedParamSets1.size()==1); + + + } +} diff --git a/test/com/linkedin/drelephant/tuning/TuningManagerTest.java b/test/com/linkedin/drelephant/tuning/TuningManagerTest.java index 4f0ead234..96cc3e11c 100644 --- a/test/com/linkedin/drelephant/tuning/TuningManagerTest.java +++ b/test/com/linkedin/drelephant/tuning/TuningManagerTest.java @@ -74,4 +74,9 @@ public void testJobStatusManagerTestRunner(){ public void testFitnessManagerTestRunner(){ running(testServer(TEST_SERVER_PORT, fakeApp), new FitnessManagerTestRunner()); } + + @Test + public void testParamGenerterTestRunner(){ + running(testServer(TEST_SERVER_PORT, fakeApp), new ParameterGenerateManagerTestRunner()); + } } diff --git a/test/common/DBTestUtil.java b/test/common/DBTestUtil.java index b90af2073..55963eceb 100644 --- a/test/common/DBTestUtil.java +++ b/test/common/DBTestUtil.java @@ -37,6 +37,10 @@ public static void initFitnessComputation() throws IOException, SQLException { initDBUtil(TEST_FITNESS_CALCULATION_DATA_FILE); } + public static void initParamGenerater() throws IOException, SQLException { + initDBUtil(TEST_PARAM_GENERATE_DATA_FILE); + } + public static void initDBAzkabanJobStatus() throws IOException, SQLException { initDBUtil(TEST_JOB_STATUS_DATA_FILE); } diff --git a/test/common/TestConstants.java b/test/common/TestConstants.java index e7a615cfc..bc98933e5 100644 --- a/test/common/TestConstants.java +++ b/test/common/TestConstants.java @@ -27,6 +27,7 @@ public class TestConstants { public static final String TEST_JOB_STATUS_DATA_FILE = "test/resources/test-job-status-data.sql"; public static final String TEST_AUTO_TUNING_DATA_FILE1 = "test/resources/tunein-test1.sql"; public static final String TEST_FITNESS_CALCULATION_DATA_FILE = "test/resources/test-fitness-calculation-data.sql"; + public static final String TEST_PARAM_GENERATE_DATA_FILE = "test/resources/test-param-generate-data.sql"; public static final int RESPONSE_TIMEOUT = 3000; // milliseconds diff --git a/test/resources/test-param-generate-data.sql b/test/resources/test-param-generate-data.sql new file mode 100644 index 000000000..e881df44e --- /dev/null +++ b/test/resources/test-param-generate-data.sql @@ -0,0 +1,173 @@ +insert into yarn_app_result + (id,name,username,queue_name,start_time,finish_time,tracking_url,job_type,severity,score,workflow_depth,scheduler,job_name,job_exec_id,flow_exec_id,job_def_id,flow_def_id,job_exec_url,flow_exec_url,job_def_url,flow_def_url,resource_used,resource_wasted,total_delay) values + ('application_1458194917883_1453361','Email Overwriter','growth','misc_default',1460980616502,1460980723925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453361','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654676&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654676','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 100, 30, 20), + ('application_1458194917883_1453362','Email Overwriter','metrics','misc_default',1460980823925,1460980923925,'http://elephant.linkedin.com:19888/jobhistory/job/job_1458194917883_1453362','HadoopJava',0,0,0,'azkaban','overwriter-reminder2','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder','https://elephant.linkedin.com:8443/executor?execid=1654677&job=overwriter-reminder2&attempt=0','https://elephant.linkedin.com:8443/executor?execid=1654677','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder&job=overwriter-reminder2','https://elephant.linkedin.com:8443/manager?project=b2-confirm-email-reminder&flow=reminder', 200, 40, 10); + +insert into yarn_app_heuristic_result(id,yarn_app_result_id,heuristic_class,heuristic_name,severity,score) values +(137594512,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), +(137594513,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), +(137594516,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), +(137594520,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), +(137594523,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), +(137594525,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), +(137594530,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), +(137594531,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), +(137594534,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), +(137594537,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), +(137594540,'application_1458194917883_1453361','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0), +(137594612,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic','Mapper Skew',0,0), +(137594613,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperGCHeuristic','Mapper GC',0,0), +(137594616,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperTimeHeuristic','Mapper Time',0,0), +(137594620,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpeedHeuristic','Mapper Speed',0,0), +(137594623,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperSpillHeuristic','Mapper Spill',0,0), +(137594625,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.MapperMemoryHeuristic','Mapper Memory',0,0), +(137594630,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerSkewHeuristic','Reducer Skew',0,0), +(137594631,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerGCHeuristic','Reducer Time',0,0), +(137594634,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerTimeHeuristic','Reducer GC',0,0), +(137594637,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ReducerMemoryHeuristic','Reducer Memory',0,0), +(137594640,'application_1458194917883_1453362','com.linkedin.drelephant.mapreduce.heuristics.ShuffleSortHeuristic','Shuffle & Sort',0,0); + +insert into yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name,value,details) values +(137594512,'Group A','1 tasks @ 4 MB avg','NULL'), +(137594512,'Group B','1 tasks @ 79 MB avg','NULL'), +(137594512,'Number of tasks','2','NULL'), +(137594513,'Avg task CPU time (ms)','11510','NULL'), + (137594513,'Avg task GC time (ms)','76','NULL'), + (137594513,'Avg task runtime (ms)','11851','NULL'), + (137594513,'Number of tasks','2','NULL'), + (137594513,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), + (137594516,'Average task input size','42 MB','NULL'), + (137594516,'Average task runtime','11 sec','NULL'), + (137594516,'Max task runtime','12 sec','NULL'), + (137594516,'Min task runtime','11 sec','NULL'), + (137594516,'Number of tasks','2','NULL'), + (137594520,'Median task input size','42 MB','NULL'), + (137594520,'Median task runtime','11 sec','NULL'), + (137594520,'Median task speed','3 MB/s','NULL'), + (137594520,'Number of tasks','2','NULL'), + (137594520,'Total input size in MB','58.65','NULL'), + (137594523,'Avg output records per task','56687','NULL'), + (137594523,'Avg spilled records per task','79913','NULL'), + (137594523,'Number of tasks','2','NULL'), + (137594523,'Ratio of spilled records to output records','1.4097111356119074','NULL'), + (137594525,'Avg Physical Memory (MB)','522','NULL'), + (137594525,'Avg task runtime','11 sec','NULL'), + (137594525,'Avg Virtual Memory (MB)','3307','NULL'), + (137594525,'Max Physical Memory (MB)','595','NULL'), + (137594525,'Max Total Committed Heap Usage Memory (MB)','427','NULL'), + (137594525,'Max Virtual Memory (MB)','1426','NULL'), + (137594525,'Min Physical Memory (MB)','449','NULL'), + (137594525,'Number of tasks','2','NULL'), + (137594525,'Requested Container Memory','2 GB','NULL'), + (137594530,'Group A','11 tasks @ 868 KB avg','NULL'), + (137594530,'Group B','9 tasks @ 883 KB avg ','NULL'), + (137594530,'Number of tasks','20','NULL'), + (137594531,'Avg task CPU time (ms)','8912','NULL'), + (137594531,'Avg task GC time (ms)','73','NULL'), + (137594531,'Avg task runtime (ms)','11045','NULL'), + (137594531,'Number of tasks','20','NULL'), + (137594531,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), + (137594534,'Average task runtime','11 sec','NULL'), + (137594534,'Max task runtime','14 sec','NULL'), + (137594534,'Min task runtime','8 sec','NULL'), + (137594534,'Number of tasks','20','NULL'), + (137594537,'Avg Physical Memory (MB)','416','NULL'), + (137594537,'Avg task runtime','11 sec','NULL'), + (137594537,'Avg Virtual Memory (MB)','3326','NULL'), + (137594537,'Max Physical Memory (MB)','497','NULL'), + (137594537,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594537,'Max Virtual Memory (MB)','1350','NULL'), + (137594537,'Min Physical Memory (MB)','354','NULL'), + (137594537,'Number of tasks','20','NULL'), + (137594537,'Requested Container Memory','2 GB','NULL'), + (137594540,'Average code runtime','1 sec','NULL'), + (137594540,'Average shuffle time','9 sec (5.49x)','NULL'), + (137594540,'Average sort time','(0.04x)','NULL'), + (137594540,'Number of tasks','20','NULL'), + (137594612,'Group A','1 tasks @ 4 MB avg','NULL'), + (137594612,'Group B','1 tasks @ 79 MB avg','NULL'), + (137594612,'Number of tasks','2','NULL'), + (137594613,'Avg task CPU time (ms)','11510','NULL'), + (137594613,'Avg task GC time (ms)','76','NULL'), + (137594613,'Avg task runtime (ms)','11851','NULL'), + (137594613,'Number of tasks','2','NULL'), + (137594613,'Task GC/CPU ratio','0.006602953953084275 ','NULL'), + (137594616,'Average task input size','42 MB','NULL'), + (137594616,'Average task runtime','11 sec','NULL'), + (137594616,'Max task runtime','12 sec','NULL'), + (137594616,'Min task runtime','11 sec','NULL'), + (137594616,'Number of tasks','2','NULL'), + (137594620,'Median task input size','42 MB','NULL'), + (137594620,'Median task runtime','11 sec','NULL'), + (137594620,'Median task speed','3 MB/s','NULL'), + (137594620,'Number of tasks','2','NULL'), + (137594620,'Total input size in MB','58.65','NULL'), + (137594623,'Avg output records per task','56687','NULL'), + (137594623,'Avg spilled records per task','79913','NULL'), + (137594623,'Number of tasks','2','NULL'), + (137594623,'Ratio of spilled records to output records','1.4097111356119074','NULL'), + (137594625,'Avg Physical Memory (MB)','522','NULL'), + (137594625,'Avg task runtime','11 sec','NULL'), + (137594625,'Avg Virtual Memory (MB)','3307','NULL'), + (137594625,'Max Physical Memory (MB)','595','NULL'), + (137594625,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594625,'Max Virtual Memory (MB)','2200','NULL'), + (137594625,'Min Physical Memory (MB)','449','NULL'), + (137594625,'Number of tasks','2','NULL'), + (137594625,'Requested Container Memory','2 GB','NULL'), + (137594630,'Group A','11 tasks @ 868 KB avg','NULL'), + (137594630,'Group B','9 tasks @ 883 KB avg ','NULL'), + (137594630,'Number of tasks','20','NULL'), + (137594631,'Avg task CPU time (ms)','8912','NULL'), + (137594631,'Avg task GC time (ms)','73','NULL'), + (137594631,'Avg task runtime (ms)','11045','NULL'), + (137594631,'Number of tasks','20','NULL'), + (137594631,'Task GC/CPU ratio','0.008191202872531419 ','NULL'), + (137594634,'Average task runtime','11 sec','NULL'), + (137594634,'Max task runtime','14 sec','NULL'), + (137594634,'Min task runtime','8 sec','NULL'), + (137594634,'Number of tasks','20','NULL'), + (137594637,'Avg Physical Memory (MB)','416','NULL'), + (137594637,'Avg task runtime','11 sec','NULL'), + (137594637,'Avg Virtual Memory (MB)','3326','NULL'), + (137594637,'Max Physical Memory (MB)','497','NULL'), + (137594637,'Max Total Committed Heap Usage Memory (MB)','300','NULL'), + (137594637,'Max Virtual Memory (MB)','2100','NULL'), + (137594637,'Min Physical Memory (MB)','354','NULL'), + (137594637,'Number of tasks','20','NULL'), + (137594637,'Requested Container Memory','2 GB','NULL'), + (137594640,'Average code runtime','1 sec','NULL'), + (137594640,'Average shuffle time','9 sec (5.49x)','NULL'), + (137594640,'Average sort time','(0.04x)','NULL'), + (137594640,'Number of tasks','20','NULL'); + +INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES +(10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'); + +INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES +(100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','pkumar2','2018-02-12 08:40:42','2018-02-12 08:40:43'); + + +INSERT INTO tuning_job_definition(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason, number_of_iterations, auto_apply) +VALUES +(100003,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true); + + + + +INSERT INTO flow_execution(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003); + +INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1541,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'); + +INSERT INTO job_suggested_param_set VALUES +(1137,100003,4,'FITNESS_COMPUTED', 0 ,1 , 0 , 0 , 0 ,20 , 1086,'2018-09-17 23:22:31' ,'2018-09-17 11:09:31'); + +INSERT INTO tuning_job_execution_param_set (job_suggested_param_set_id,job_execution_id,tuning_enabled,created_ts,updated_ts) VALUES +(1137,1541,1,'2018-09-17 23:22:31','2018-09-17 10:52:31'); + + + + + From d93a2c961211de3701f6d211fdf85e7ba4c67fff Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Thu, 4 Oct 2018 14:51:39 +0530 Subject: [PATCH 10/22] Changed logging to debug from info , optimization and indenetation changes --- app/com/linkedin/drelephant/AutoTuner.java | 4 +- .../tuning/AbstractBaselineManager.java | 48 +++++++----- .../tuning/AbstractFitnessManager.java | 78 ++++++++++--------- .../tuning/{Flow.java => AutoTuningFlow.java} | 28 +++---- .../tuning/hbt/FitnessManagerHBT.java | 31 ++++---- .../hbt/ParameterGenerateManagerHBT.java | 61 ++++----------- .../tuning/obt/BaselineManagerOBT.java | 4 +- .../tuning/obt/FitnessManagerOBT.java | 10 +-- .../tuning/obt/FitnessManagerOBTAlgoIPSO.java | 8 +- .../tuning/obt/FitnessManagerOBTAlgoPSO.java | 8 +- .../ParameterGenerateManagerOBTAlgoPSO.java | 36 +++++---- ...eterGenerateManagerOBTAlgoPSOIPSOImpl.java | 16 ++-- .../drelephant/tuning/FlowTestRunner.java | 34 ++++---- 13 files changed, 171 insertions(+), 195 deletions(-) rename app/com/linkedin/drelephant/tuning/{Flow.java => AutoTuningFlow.java} (74%) diff --git a/app/com/linkedin/drelephant/AutoTuner.java b/app/com/linkedin/drelephant/AutoTuner.java index 4a1da8cbd..9442e8feb 100644 --- a/app/com/linkedin/drelephant/AutoTuner.java +++ b/app/com/linkedin/drelephant/AutoTuner.java @@ -16,7 +16,7 @@ package com.linkedin.drelephant; -import com.linkedin.drelephant.tuning.Flow; +import com.linkedin.drelephant.tuning.AutoTuningFlow; import org.apache.hadoop.conf.Configuration; import org.apache.log4j.Logger; @@ -50,7 +50,7 @@ public void run() { Utils.getNonNegativeLong(configuration, AUTO_TUNING_DAEMON_WAIT_INTERVAL, DEFAULT_METRICS_COMPUTATION_INTERVAL); try { AutoTuningMetricsController.init(); - Flow autoTuningFlow= new Flow(); + AutoTuningFlow autoTuningFlow= new AutoTuningFlow(); while (!Thread.currentThread().isInterrupted()) { try { autoTuningFlow.executeFlow(); diff --git a/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java b/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java index 1bf5e2a12..7277ad7ee 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java @@ -35,6 +35,7 @@ public abstract class AbstractBaselineManager implements Manager { + "AND yahrd.name='" + CommonConstantsHeuristic.TOTAL_INPUT_SIZE_IN_MB + "' " + "GROUP BY job_exec_id ORDER BY start_time DESC LIMIT :num ) temp"; private final Logger logger = Logger.getLogger(getClass()); + boolean debugEnabled = logger.isDebugEnabled(); protected Integer _numJobsForBaseline = null; protected Configuration configuration = null; @@ -51,23 +52,28 @@ public AbstractBaselineManager() { */ @Override public final boolean execute() { - logger.info("Executing BaseLine"); - boolean baseLineComputationDone = false, databaseUpdateDone = false, updateMetricsDone = false; - List tuningJobDefinitions = detectJobsForBaseLineComputation(); - if (tuningJobDefinitions != null && tuningJobDefinitions.size() >= 1) { - logger.info("Computing BaseLine"); - baseLineComputationDone = calculateBaseLine(tuningJobDefinitions); - } - if (baseLineComputationDone) { - logger.info("Updating Database"); - databaseUpdateDone = updateDataBase(tuningJobDefinitions); - } - if (databaseUpdateDone) { - logger.info("Updating Metrics"); - updateMetricsDone = updateMetrics(tuningJobDefinitions); + try { + logger.info("Executing BaseLine"); + boolean baseLineComputationDone = false, databaseUpdateDone = false, updateMetricsDone = false; + List tuningJobDefinitions = detectJobsForBaseLineComputation(); + if (tuningJobDefinitions != null && tuningJobDefinitions.size() >= 1) { + logger.info("Computing BaseLine"); + baseLineComputationDone = calculateBaseLine(tuningJobDefinitions); + } + if (baseLineComputationDone) { + logger.info("Updating Database"); + databaseUpdateDone = updateDataBase(tuningJobDefinitions); + } + if (databaseUpdateDone) { + logger.info("Updating Metrics"); + updateMetricsDone = updateMetrics(tuningJobDefinitions); + } + logger.info("Baseline Done "); + return updateMetricsDone; + } catch (Exception e) { + logger.error(" Execution of the base line manager is failed " + e.getMessage(),e); + return false; } - logger.info("Baseline Done "); - return updateMetricsDone; } /** @@ -86,8 +92,10 @@ public final boolean execute() { protected boolean calculateBaseLine(List tuningJobDefinitions) { for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitions) { try { - logger.info("Computing and updating baseline metric values for job: " + tuningJobDefinition.job.jobName); - logger.debug("Running query for baseline computation " + baseLineCalculationSQL); + if(debugEnabled) { + logger.debug("Computing and updating baseline metric values for job: " + tuningJobDefinition.job.jobName); + logger.debug("Running query for baseline computation " + baseLineCalculationSQL); + } SqlRow baseline = Ebean.createSqlQuery(baseLineCalculationSQL) .setParameter("jobDefId", tuningJobDefinition.job.jobDefId) .setParameter("num", _numJobsForBaseline) @@ -135,7 +143,7 @@ protected boolean updateDataBase(List tuningJobDefinitions) try { for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitions) { tuningJobDefinition.update(); - logger.info("Updated baseline metric value for job: " + tuningJobDefinition.job.jobName); + logger.debug("Updated baseline metric value for job: " + tuningJobDefinition.job.jobName); } } catch (Exception e) { logger.error(" Error Updating Database " + e.getMessage()); @@ -161,7 +169,7 @@ protected boolean updateMetrics(List tuningJobDefinitions) } AutoTuningMetricsController.setBaselineComputeWaitJobs(baselineComputeWaitJobs); } catch (Exception e) { - logger.error(" Error Updating Metrics in Ingraph" + e.getMessage()); + logger.error(" Error Updating Metrics in Ingraph" + e.getMessage(),e); return false; } return true; diff --git a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java index 25b58f2e7..0c524edf0 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java @@ -34,14 +34,19 @@ public abstract class AbstractFitnessManager implements Manager { private final Logger logger = Logger.getLogger(getClass()); + boolean debugEnabled = logger.isDebugEnabled(); protected final String FITNESS_COMPUTE_WAIT_INTERVAL = "fitness.compute.wait_interval.ms"; protected final String IGNORE_EXECUTION_WAIT_INTERVAL = "ignore.execution.wait.interval.ms"; protected final String MAX_TUNING_EXECUTIONS = "max.tuning.executions"; protected final String MIN_TUNING_EXECUTIONS = "min.tuning.executions"; + private static final int MIN_TO_MS = (1000 * 60); + private static final int HOUR_TO_SEC = 3600; + //private stati protected int maxTuningExecutions; protected int minTuningExecutions; protected Long fitnessComputeWaitInterval; protected Long ignoreExecutionWaitInterval; + private Map tuningDefintionCollection = new HashMap(); /** * Detects & return the jobs for which fitness computation have to be done. @@ -70,8 +75,7 @@ protected boolean calculateFitness(List completedJob JobExecution jobExecution = completedJobExecutionParamSet.jobExecution; JobSuggestedParamSet jobSuggestedParamSet = completedJobExecutionParamSet.jobSuggestedParamSet; JobDefinition job = jobExecution.job; - - logger.info("Updating execution metrics and fitness for execution: " + jobExecution.jobExecId); + logger.debug("Updating execution metrics and fitness for execution: " + jobExecution.jobExecId); try { TuningJobDefinition tuningJobDefinition = getTuningJobDefinition(job); List results = getAppResults(jobExecution); @@ -81,7 +85,8 @@ protected boolean calculateFitness(List completedJob return false; } } - logger.info("Execution metrics updated"); + + logger.debug("Execution metrics updated"); return true; } @@ -92,7 +97,9 @@ private TuningJobDefinition getTuningJobDefinition(JobDefinition job) { .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, job.id) .order() .desc(TuningJobDefinition.TABLE.createdTs) + .setMaxRows(1) .findUnique(); + tuningDefintionCollection.put(tuningJobDefinition.job.id, tuningJobDefinition); return tuningJobDefinition; } @@ -116,15 +123,13 @@ private void handleFitnessCalculation(JobExecution jobExecution, List } } - - protected void updateJobExecution(JobExecution jobExecution, Double totalResourceUsed, Double totalInputBytesInBytes, Long totalExecutionTime) { - jobExecution.executionTime = totalExecutionTime * 1.0 / (1000 * 60); - jobExecution.resourceUsage = totalResourceUsed * 1.0 / (1024 * 3600); + jobExecution.executionTime = totalExecutionTime * 1.0 / MIN_TO_MS; + jobExecution.resourceUsage = totalResourceUsed * 1.0 / (FileUtils.ONE_KB * HOUR_TO_SEC); jobExecution.inputSizeInBytes = totalInputBytesInBytes; jobExecution.update(); - logger.info("Metric Values for execution " + jobExecution.jobExecId + ": Execution time = " + totalExecutionTime + logger.debug("Metric Values for execution " + jobExecution.jobExecId + ": Execution time = " + totalExecutionTime + ", Resource usage = " + totalResourceUsed + " and total input size = " + totalInputBytesInBytes); } @@ -140,7 +145,7 @@ private void handleEmptyResultScenario(JobExecution jobExecution, JobSuggestedPa logger.debug("Current Time in millis: " + System.currentTimeMillis() + ", job execution last updated time " + jobExecution.updatedTs.getTime()); if (diff > ignoreExecutionWaitInterval) { - logger.info( + logger.debug( "Fitness of param set " + jobSuggestedParamSet.id + " corresponding to execution id: " + jobExecution.id + " not computed for more than the maximum duration specified to compute fitness. " + "Resetting the param set to CREATED state"); @@ -177,7 +182,7 @@ protected Long getTotalInputBytes(AppResult appResult) { */ protected void resetParamSetToCreated(JobSuggestedParamSet jobSuggestedParamSet) { if (!jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED)) { - logger.info("Resetting parameter set to created: " + jobSuggestedParamSet.id); + logger.debug("Resetting parameter set to created: " + jobSuggestedParamSet.id); jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; jobSuggestedParamSet.save(); } @@ -205,7 +210,7 @@ protected void updateJobSuggestedParamSetSucceededExecution(JobExecution jobExec if (resourceUsagePerGBInput > maxDesiredResourceUsagePerGBInput || executionTimePerGBInput > maxDesiredExecutionTimePerGBInput) { - logger.info("Execution " + jobExecution.jobExecId + " violates constraint on resource usage per GB input"); + logger.debug("Execution " + jobExecution.jobExecId + " violates constraint on resource usage per GB input"); jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; } else { jobSuggestedParamSet.fitness = resourceUsagePerGBInput; @@ -222,7 +227,7 @@ protected void updateJobSuggestedParamSetSucceededExecution(JobExecution jobExec * @param jobSuggestedParamSet JobSuggestedParamSet */ protected JobSuggestedParamSet updateBestJobSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { - logger.info("Checking if a new best param set is found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId); + logger.debug("Checking if a new best param set is found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId); JobSuggestedParamSet currentBestJobSuggestedParamSet = JobSuggestedParamSet.find.where() .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobSuggestedParamSet.jobDefinition.id) @@ -230,14 +235,14 @@ protected JobSuggestedParamSet updateBestJobSuggestedParamSet(JobSuggestedParamS .findUnique(); if (currentBestJobSuggestedParamSet != null) { if (currentBestJobSuggestedParamSet.fitness > jobSuggestedParamSet.fitness) { - logger.info("Param set: " + jobSuggestedParamSet.id + " is the new best param set for job: " + logger.debug("Param set: " + jobSuggestedParamSet.id + " is the new best param set for job: " + jobSuggestedParamSet.jobDefinition.jobDefId); currentBestJobSuggestedParamSet.isParamSetBest = false; jobSuggestedParamSet.isParamSetBest = true; currentBestJobSuggestedParamSet.save(); } } else { - logger.info("No best param set found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId + logger.debug("No best param set found for job: " + jobSuggestedParamSet.jobDefinition.jobDefId + ". Marking current param set " + jobSuggestedParamSet.id + " as best"); jobSuggestedParamSet.isParamSetBest = true; } @@ -270,25 +275,23 @@ private boolean updateMetrics(List completedJobExecu } private boolean isTuningEnabled(Integer jobDefinitionId) { - TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() - .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinitionId) - .order() - // There can be multiple entries in tuningJobDefinition if the job is switch on/off multiple times. - // The latest entry gives the information regarding whether tuning is enabled or not - .desc(TuningJobDefinition.TABLE.createdTs) - .setMaxRows(1) - .findUnique(); - - return tuningJobDefinition != null && tuningJobDefinition.tuningEnabled; + TuningJobDefinition tuningJobDefinition = tuningDefintionCollection.get(jobDefinitionId); + if (tuningJobDefinition == null) { + TuningJobDefinition tuningJobDefinitionNew = TuningJobDefinition.find.where() + .eq(TuningJobDefinition.TABLE.job + '.' + JobDefinition.TABLE.id, jobDefinitionId) + .order() + // There can be multiple entries in tuningJobDefinition if the job is switch on/off multiple times. + // The latest entry gives the information regarding whether tuning is enabled or not + .desc(TuningJobDefinition.TABLE.createdTs) + .setMaxRows(1) + .findUnique(); + + return tuningJobDefinitionNew != null && tuningJobDefinitionNew.tuningEnabled; + } else { + return tuningJobDefinition != null && tuningJobDefinition.tuningEnabled; + } } - - - - - - - public boolean reachToNumberOfThresholdIterations(List tuningJobExecutionParamSets, JobDefinition jobDefinition) { TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.where() @@ -316,8 +319,6 @@ public void disableTuning(JobDefinition jobDefinition, String reason) { } } - - public final boolean execute() { logger.info("Executing Fitness Manager"); boolean calculateFitnessDone = false, databaseUpdateDone = false, updateMetricsDone = false; @@ -337,7 +338,8 @@ public final boolean execute() { logger.info("Updating Metrics"); updateMetricsDone = updateMetrics(tuningJobExecutionParamSet); } - logger.info("Disable Tuning if Required"); + logger.info( + "Disable Tuning if Required . Disabled based on number of iteration reached or other terminating conditions for algorithms"); if (tuningJobExecutionParamSet != null && tuningJobExecutionParamSet.size() >= 1) { Set jobDefinitionSet = new HashSet(); for (TuningJobExecutionParamSet completedJobExecutionParamSet : tuningJobExecutionParamSet) { @@ -357,16 +359,16 @@ protected void getCompletedExecution(List tuningJobE for (TuningJobExecutionParamSet tuningJobExecutionParamSet : tuningJobExecutionParamSets) { JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; long diff = System.currentTimeMillis() - jobExecution.updatedTs.getTime(); - logger.info("Current Time in millis: " + System.currentTimeMillis() + ", Job execution last updated time " + logger.debug("Current Time in millis: " + System.currentTimeMillis() + ", Job execution last updated time " + jobExecution.updatedTs.getTime()); if (diff < fitnessComputeWaitInterval) { - logger.info("Delaying fitness compute for execution: " + jobExecution.jobExecId); + logger.debug("Delaying fitness compute for execution: " + jobExecution.jobExecId); } else { - logger.info("Adding execution " + jobExecution.jobExecId + " to fitness computation queue"); + logger.debug("Adding execution " + jobExecution.jobExecId + " to fitness computation queue"); completedJobExecutionParamSet.add(tuningJobExecutionParamSet); } } - logger.info( + logger.debug( "Number of completed execution fetched for fitness computation: " + completedJobExecutionParamSet.size()); } } diff --git a/app/com/linkedin/drelephant/tuning/Flow.java b/app/com/linkedin/drelephant/tuning/AutoTuningFlow.java similarity index 74% rename from app/com/linkedin/drelephant/tuning/Flow.java rename to app/com/linkedin/drelephant/tuning/AutoTuningFlow.java index cc272d5d0..0791b4110 100644 --- a/app/com/linkedin/drelephant/tuning/Flow.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningFlow.java @@ -14,11 +14,11 @@ * TuningTypeManager : This will have Combinations of TuningType , AlgorithmType and execution engine. for e.g * TuningTypeManagerOBTAlgoIPSO for Map Reduce and Spark ... */ -public class Flow { +public class AutoTuningFlow { List> pipelines = null; - private static final Logger logger = Logger.getLogger(Flow.class); + private static final Logger logger = Logger.getLogger(AutoTuningFlow.class); - public Flow() { + public AutoTuningFlow() { pipelines = new ArrayList>(); createBaseLineManagersPipeline(); createJobStatusManagersPipeline(); @@ -35,10 +35,10 @@ public void executeFlow() throws InterruptedException { Thread t1 = new Thread(new Runnable() { @Override public void run() { - for (Manager manager : pipelineType) { - logger.info(" Manager execution Status " + manager.getManagerName()); + for (Manager manager : pipelineType) { + logger.info(" Starting Manager " + manager.getManagerName()); Boolean execute = manager.execute(); - logger.info(" Manager execution Status " + execute + " " + manager.getManagerName()); + logger.info(" Ending Manager " + execute + " " + manager.getManagerName()); } } }); @@ -71,15 +71,12 @@ public void createFitnessManagersPipeline() { Manager manager = ManagerFactory.getManager(tuningType.name(), algotihmType.name(), null, AbstractFitnessManager.class.getSimpleName()); if (manager != null) { - logger.info(manager.getManagerName()); + logger.info("Loading " + manager.getManagerName()); fitnessManagers.add(manager); } } } - /*fitnessManagers.add(new FitnessManagerHBT()); - fitnessManagers.add(new FitnessManagerOBTAlgoIPSO()); - fitnessManagers.add(new FitnessManagerOBTAlgoPSO());*/ - //this.pipelines.add(new FitnessManagerHBT()) + this.pipelines.add(fitnessManagers); } @@ -92,19 +89,12 @@ public void createTuningTypeManagersPipeline() { ManagerFactory.getManager(tuningType.name(), algotihmType.name(), executionEngineTypes.name(), AbstractParameterGenerateManager.class.getSimpleName()); if (manager != null) { - logger.info(manager.getManagerName()); + logger.info("Loading " + manager.getManagerName()); algorithmManagers.add(manager); } } } } - /*algorithmManagers.add(new TuningTypeManagerHBT(new MRExecutionEngine())); - algorithmManagers.add(new TuningTypeManagerHBT(new SparkExecutionEngine())); - algorithmManagers.add(new TuningTypeManagerOBTAlgoPSO(new MRExecutionEngine())); - algorithmManagers.add(new TuningTypeManagerOBTAlgoPSO(new SparkExecutionEngine())); - algorithmManagers.add(new TuningTypeManagerOBTAlgoIPSO(new MRExecutionEngine())); - algorithmManagers.add(new TuningTypeManagerOBTAlgoIPSO(new SparkExecutionEngine())); -*/ this.pipelines.add(algorithmManagers); } diff --git a/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java index 642ab91c5..7b6a3c15b 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/FitnessManagerHBT.java @@ -44,7 +44,7 @@ public FitnessManagerHBT() { @Override protected List detectJobsForFitnessComputation() { - logger.info("Fetching completed executions whose fitness are yet to be computed"); + logger.debug("Fetching completed executions whose fitness are yet to be computed"); List completedJobExecutionParamSet = new ArrayList(); List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.select("*") @@ -62,7 +62,7 @@ protected List detectJobsForFitnessComputation() { + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.HBT.name()) .findList(); - logger.info("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); + logger.debug("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); getCompletedExecution(tuningJobExecutionParamSets, completedJobExecutionParamSet); @@ -72,7 +72,7 @@ protected List detectJobsForFitnessComputation() { @Override protected void calculateAndUpdateFitness(JobExecution jobExecution, List results, TuningJobDefinition tuningJobDefinition, JobSuggestedParamSet jobSuggestedParamSet) { - logger.info("calculateAndUpdateFitness"); + logger.debug("calculateAndUpdateFitness"); Double totalResourceUsed = 0D; Double totalInputBytesInBytes = 0D; Double score=0D; @@ -104,7 +104,7 @@ protected void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExec if (!jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED) || !jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.DISCARDED)) { if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { - logger.info("Execution id: " + jobExecution.id + " succeeded"); + logger.debug("Execution id: " + jobExecution.id + " succeeded"); updateJobSuggestedParamSetSucceededExecution(jobExecution, jobSuggestedParamSet, tuningJobDefinition); } else { // Resetting param set to created state because this case captures the scenarios when @@ -112,7 +112,7 @@ protected void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExec // In all the above scenarios, fitness cannot be computed for the param set correctly. // Note that the penalty on failures caused by auto tuning is applied when the job execution is retried // after failure. - logger.info("HBT Execution id: " + jobExecution.id + " was not successful for reason other than tuning." + logger.debug("HBT Execution id: " + jobExecution.id + " was not successful for reason other than tuning." + "Resetting param set: " + jobSuggestedParamSet.id + " to CREATED state"); resetParamSetToCreated(jobSuggestedParamSet); } @@ -140,7 +140,7 @@ protected void updateJobSuggestedParamSetSucceededExecution(JobExecution jobExec protected void resetParamSetToCreated(JobSuggestedParamSet jobSuggestedParamSet) { if (!jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED) && !jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.DISCARDED)) { - logger.info("Resetting parameter set to created: " + jobSuggestedParamSet.id); + logger.debug("Resetting parameter set to created: " + jobSuggestedParamSet.id); jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; jobSuggestedParamSet.save(); } @@ -148,6 +148,7 @@ protected void resetParamSetToCreated(JobSuggestedParamSet jobSuggestedParamSet) @Override protected void checkToDisableTuning(Set jobDefinitionSet) { + Long currentTimeBefore = System.currentTimeMillis(); for (JobDefinition jobDefinition : jobDefinitionSet) { List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.fetch(TuningJobExecutionParamSet.TABLE.jobSuggestedParamSet, "*") @@ -166,24 +167,26 @@ protected void checkToDisableTuning(Set jobDefinitionSet) { disableTuning(jobDefinition, "All Heuristics Passed1"); } } + Long currentTimeAfter= System.currentTimeMillis(); + logger.info(" Total time taken by disable tuning " + (currentTimeAfter-currentTimeBefore)); } private boolean areHeuristicsPassed(List tuningJobExecutionParamSets) { - logger.info(" Testing All Heuristics "); + logger.debug(" Testing All Heuristics "); if (tuningJobExecutionParamSets != null && tuningJobExecutionParamSets.size() >= 1) { - logger.info("tuningJobExecutionParamSets have some values"); + logger.debug("tuningJobExecutionParamSets have some values"); TuningJobExecutionParamSet tuningJobExecutionParamSet = tuningJobExecutionParamSets.get(0); JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; List results = getAppResult(jobExecution); if (results != null) { - logger.info(" Results are not null "); + logger.debug(" Results are not null "); return areAppResultsHaveSeverity(results); } else { - logger.info(" App Results are null "); + logger.debug(" App Results are null "); return false; } } else { - logger.info(" Tuning Job Execution Param Set is null "); + logger.debug(" Tuning Job Execution Param Set is null "); return false; } } @@ -200,7 +203,7 @@ private boolean areAppResultsHaveSeverity(List results) { } } } else { - logger.info(appResult.id + " " + appResult.jobDefId + " have yarn app result null "); + logger.debug(appResult.id + " " + appResult.jobDefId + " have yarn app result null "); return true; } } @@ -209,11 +212,11 @@ private boolean areAppResultsHaveSeverity(List results) { private boolean checkHeuriticsforSeverity(List heuristicsWithHighSeverity) { if (heuristicsWithHighSeverity.size() == 0) { - logger.info(" No sever heursitics "); + logger.debug(" No severe heursitics "); return true; } else { for (String failedHeuristics : heuristicsWithHighSeverity) { - logger.info(failedHeuristics); + logger.debug(failedHeuristics); } return false; } diff --git a/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java index 807929e4f..350fd12f0 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java @@ -42,7 +42,7 @@ protected List getPendingParamSets() { TuningAlgorithm.OptimizationAlgo.HBT.name()) // .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) .findList(); - logger.info( + logger.debug( " Number of Pending Jobs for parameter suggestion " + this._executionEngine + " " + pendingParamSetList.size()); return pendingParamSetList; } @@ -54,7 +54,7 @@ protected List getTuningJobDefinitions() { TuningAlgorithm.OptimizationAlgo.HBT.name()) .findList(); - logger.info(" Number of Total Jobs " + this._executionEngine + " " + totalJobs.size()); + logger.debug(" Number of Total Jobs " + this._executionEngine + " " + totalJobs.size()); return totalJobs; } @@ -65,7 +65,7 @@ protected void saveJobState(JobTuningInfo jobTuningInfo, JobDefinition job) { @Override public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { - logger.info("Generating param set for job: " + jobTuningInfo.getTuningJob().jobName); + logger.debug("Generating param set for job: " + jobTuningInfo.getTuningJob().jobName); String newTunedParameters = generateParamSet(jobTuningInfo.getParametersToTune(), jobTuningInfo.getTuningJob()); jobTuningInfo.setTunerState(newTunedParameters); return jobTuningInfo; @@ -79,16 +79,16 @@ private String generateParamSet(List tuningParameters, JobDefin .desc(JobExecution.TABLE.updatedTs) .setMaxRows(1) .findUnique(); - logger.info("Job Status " + jobExecution.executionState.name()); + logger.debug("Job Status " + jobExecution.executionState.name()); if (jobExecution.executionState.name().equals(JobExecution.ExecutionState.IN_PROGRESS.name()) || jobExecution.executionState.name().equals(JobExecution.ExecutionState.NOT_STARTED.name())) { - logger.info(" Job is still running , cannot use for param generation "); + logger.debug(" Job is still running , cannot use for param generation "); return ""; } List results = getAppResults(jobExecution); if (results == null) { - logger.info( + logger.debug( " Job is analyzing , cannot use for param generation " + jobExecution.id + " " + jobExecution.job.id); return ""; } @@ -136,14 +136,14 @@ private List getAppResults(JobExecution jobExecution) { * @param jobTuningInfoList JobTuningInfo List */ protected boolean updateDatabase(List jobTuningInfoList) { - logger.info("Updating new parameter suggestion in database HBT"); + logger.debug("Updating new parameter suggestion in database HBT"); if (jobTuningInfoList == null) { - logger.info("No new parameter suggestion to update"); + logger.debug("No new parameter suggestion to update"); return false; } for (JobTuningInfo jobTuningInfo : jobTuningInfoList) { - logger.info("Updating new parameter suggestion for job:" + jobTuningInfo.getTuningJob().jobDefId); + logger.debug("Updating new parameter suggestion for job:" + jobTuningInfo.getTuningJob().jobDefId); JobDefinition job = jobTuningInfo.getTuningJob(); List paramList = jobTuningInfo.getParametersToTune(); @@ -157,7 +157,7 @@ protected boolean updateDatabase(List jobTuningInfoList) { List derivedParameterList = TuningHelper.getDerivedParameterList(tuningJobDefinition); - logger.info("No. of derived tuning params for job " + tuningJobDefinition.job.jobName + ": " + logger.debug("No. of derived tuning params for job " + tuningJobDefinition.job.jobName + ": " + derivedParameterList.size()); List jobSuggestedParamValueList = getParamValueList(stringTunerState); @@ -171,7 +171,7 @@ protected boolean updateDatabase(List jobTuningInfoList) { if (isParamConstraintViolated(jobSuggestedParamValueList)) { penaltyApplication(jobSuggestedParamSet, tuningJobDefinition); } else { - logger.info(" Parameters constraints not violeted "); + logger.debug(" Parameters constraints not violeted "); jobSuggestedParamSet.areConstraintsViolated = false; jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; //processParamSetStatus(jobSuggestedParamSet); @@ -181,46 +181,17 @@ protected boolean updateDatabase(List jobTuningInfoList) { for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValueList) { jobSuggestedParamValue.jobSuggestedParamSet = jobSuggestedParamSet; } - logger.info(" Job Suggested list " + jobSuggestedParamValueList.size()); + logger.debug(" Job Suggested list " + jobSuggestedParamValueList.size()); saveSuggestedParams(jobSuggestedParamValueList); } return true; } - private void processParamSetStatus(JobSuggestedParamSet jobSuggestedParamSet) { - TuningJobDefinition tuningJobDefinition1 = TuningJobDefinition.find.select("*") - .where() - .eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, jobSuggestedParamSet.jobDefinition.id) - .setMaxRows(1) - .findUnique(); - //jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; - //handleDiscarding(tuningJobDefinition1, jobSuggestedParamSet); - } - - /* private void handleDiscarding(TuningJobDefinition tuningJobDefinition1, JobSuggestedParamSet jobSuggestedParamSet) { - if (tuningJobDefinition1.autoApply) { - jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.CREATED; - } else { - jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.DISCARDED; - } - List tempJobSuggestedParamSet = JobSuggestedParamSet.find.select("*") - .where() - .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, tuningJobDefinition1.job.id) - .findList(); - Boolean isManuallyOverriden = false; - for (JobSuggestedParamSet jobSuggestedParamSet1 : tempJobSuggestedParamSet) { - if (jobSuggestedParamSet1.isManuallyOverridenParameter) { - isManuallyOverriden = true; - } - } - if (isManuallyOverriden) { - jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.DISCARDED; - } - }*/ + private void penaltyApplication(JobSuggestedParamSet jobSuggestedParamSet, TuningJobDefinition tuningJobDefinition) { - logger.info("Parameter constraint violated. Applying penalty."); + logger.debug("Parameter constraint violated. Applying penalty."); int penaltyConstant = 4; Double averageResourceUsagePerGBInput = tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; @@ -255,7 +226,7 @@ private Long saveSuggestedParamSet(JobSuggestedParamSet jobSuggestedParamSet) { private List getParamValueList(String tunerState) { List jobSuggestedParamValueList = new ArrayList(); for (String parameter : tunerState.split("\n")) { - logger.info(" Parameter values " + parameter); + logger.debug(" Parameter values " + parameter); String paramIDValues[] = parameter.split("\t"); if (paramIDValues.length == 2) { JobSuggestedParamValue jobSuggestedParamValue = new JobSuggestedParamValue(); @@ -264,7 +235,7 @@ private List getParamValueList(String tunerState) { jobSuggestedParamValueList.add(jobSuggestedParamValue); } } - logger.info(" Job Suggested Values " + jobSuggestedParamValueList.size()); + logger.debug(" Job Suggested Values " + jobSuggestedParamValueList.size()); return jobSuggestedParamValueList; } diff --git a/app/com/linkedin/drelephant/tuning/obt/BaselineManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/BaselineManagerOBT.java index 8c9e23262..0e4ed27d7 100644 --- a/app/com/linkedin/drelephant/tuning/obt/BaselineManagerOBT.java +++ b/app/com/linkedin/drelephant/tuning/obt/BaselineManagerOBT.java @@ -22,7 +22,7 @@ public BaselineManagerOBT() { @Override protected List detectJobsForBaseLineComputation() { - logger.info("Fetching jobs for which baseline metrics need to be computed"); + logger.debug("Fetching jobs for which baseline metrics need to be computed"); List tuningJobDefinitions = TuningJobDefinition.find.where() .eq(TuningJobDefinition.TABLE.averageResourceUsage, null) .or(Expr.eq(TuningJobDefinition.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.optimizationAlgo, @@ -31,7 +31,7 @@ protected List detectJobsForBaseLineComputation() { TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name())) .findList(); if(tuningJobDefinitions!=null){ - logger.info("Total jobs for Baseline Computation in OBT"); + logger.debug("Total jobs for Baseline Computation in OBT"); } return tuningJobDefinitions; } diff --git a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java index 6e8cae0e8..d80c6c78f 100644 --- a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java +++ b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java @@ -118,13 +118,13 @@ protected void checkToDisableTuning(Set jobDefinitionSet) { } if (tuningJobExecutionParamSets.size() >= minTuningExecutions) { if (didParameterSetConverge(tuningJobExecutionParamSets)) { - logger.info("Parameters converged. Disabling tuning for job: " + jobDefinition.jobName); + logger.debug("Parameters converged. Disabling tuning for job: " + jobDefinition.jobName); disableTuning(jobDefinition, "Parameters converged"); } else if (isMedianGainNegative(tuningJobExecutionParamSets)) { - logger.info("Unable to get gain while tuning. Disabling tuning for job: " + jobDefinition.jobName); + logger.debug("Unable to get gain while tuning. Disabling tuning for job: " + jobDefinition.jobName); disableTuning(jobDefinition, "Unable to get gain"); } else if (tuningJobExecutionParamSets.size() >= maxTuningExecutions) { - logger.info("Maximum tuning executions limit reached. Disabling tuning for job: " + jobDefinition.jobName); + logger.debug("Maximum tuning executions limit reached. Disabling tuning for job: " + jobDefinition.jobName); disableTuning(jobDefinition, "Maximum executions reached"); } } @@ -193,7 +193,7 @@ private boolean didParameterSetConverge(List tuningJ } if (result) { - logger.info("Switching off tuning for job: " + tuningJobExecutionParamSets.get( + logger.debug("Switching off tuning for job: " + tuningJobExecutionParamSets.get( 0).jobSuggestedParamSet.jobDefinition.jobName + " Reason: parameter set converged"); } return result; @@ -243,7 +243,7 @@ private boolean isMedianGainNegative(List tuningJobE tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; if (medianFitness > baselineFitness) { - logger.info("Switching off tuning for job: " + jobDefinition.jobName + " Reason: unable to tune enough"); + logger.debug("Switching off tuning for job: " + jobDefinition.jobName + " Reason: unable to tune enough"); return true; } else { return false; diff --git a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java index 68be3bbcb..d0830ee78 100644 --- a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java +++ b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoIPSO.java @@ -23,7 +23,7 @@ public FitnessManagerOBTAlgoIPSO() { @Override protected List detectJobsForFitnessComputation() { - logger.info("Fetching completed executions whose fitness are yet to be computed"); + logger.debug("Fetching completed executions whose fitness are yet to be computed"); List completedJobExecutionParamSet = new ArrayList(); List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.select("*") @@ -41,7 +41,7 @@ protected List detectJobsForFitnessComputation() { + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name()) .findList(); - logger.info("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); + logger.debug("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); getCompletedExecution(tuningJobExecutionParamSets, completedJobExecutionParamSet); @@ -52,7 +52,7 @@ protected void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExec TuningJobDefinition tuningJobDefinition, List results) { if (!jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED)) { if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { - logger.info("Execution id: " + jobExecution.id + " succeeded"); + logger.debug("Execution id: " + jobExecution.id + " succeeded"); parameterSpaceOptimization(jobSuggestedParamSet, jobExecution, results); updateJobSuggestedParamSetSucceededExecution(jobExecution, jobSuggestedParamSet, tuningJobDefinition); } else { @@ -61,7 +61,7 @@ protected void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExec // In all the above scenarios, fitness cannot be computed for the param set correctly. // Note that the penalty on failures caused by auto tuning is applied when the job execution is retried // after failure. - logger.info("Execution id: " + jobExecution.id + " was not successful for reason other than tuning." + logger.debug("Execution id: " + jobExecution.id + " was not successful for reason other than tuning." + "Resetting param set: " + jobSuggestedParamSet.id + " to CREATED state"); resetParamSetToCreated(jobSuggestedParamSet); } diff --git a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoPSO.java b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoPSO.java index c5fdb1a54..0ed125ecd 100644 --- a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoPSO.java +++ b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBTAlgoPSO.java @@ -26,7 +26,7 @@ public FitnessManagerOBTAlgoPSO() { @Override protected List detectJobsForFitnessComputation() { - logger.info("Fetching completed executions whose fitness are yet to be computed"); + logger.debug("Fetching completed executions whose fitness are yet to be computed"); List completedJobExecutionParamSet = new ArrayList(); List tuningJobExecutionParamSets = TuningJobExecutionParamSet.find.select("*") @@ -44,7 +44,7 @@ protected List detectJobsForFitnessComputation() { + "." + TuningAlgorithm.TABLE.optimizationAlgo, TuningAlgorithm.OptimizationAlgo.PSO.name()) .findList(); - logger.info("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); + logger.debug("#completed executions whose metrics are not computed: " + tuningJobExecutionParamSets.size()); getCompletedExecution(tuningJobExecutionParamSets,completedJobExecutionParamSet); @@ -56,7 +56,7 @@ protected void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExec TuningJobDefinition tuningJobDefinition, List results) { if (!jobSuggestedParamSet.paramSetState.equals(JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED)) { if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED)) { - logger.info("Execution id: " + jobExecution.id + " succeeded"); + logger.debug("Execution id: " + jobExecution.id + " succeeded"); updateJobSuggestedParamSetSucceededExecution(jobExecution, jobSuggestedParamSet, tuningJobDefinition); } else { // Resetting param set to created state because this case captures the scenarios when @@ -64,7 +64,7 @@ protected void computeFitness(JobSuggestedParamSet jobSuggestedParamSet, JobExec // In all the above scenarios, fitness cannot be computed for the param set correctly. // Note that the penalty on failures caused by auto tuning is applied when the job execution is retried // after failure. - logger.info("Execution id: " + jobExecution.id + " was not successful for reason other than tuning." + logger.debug("Execution id: " + jobExecution.id + " was not successful for reason other than tuning." + "Resetting param set: " + jobSuggestedParamSet.id + " to CREATED state"); resetParamSetToCreated(jobSuggestedParamSet); } diff --git a/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSO.java b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSO.java index 19606f810..28076f7f0 100644 --- a/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSO.java +++ b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSO.java @@ -33,7 +33,7 @@ public abstract class ParameterGenerateManagerOBTAlgoPSO extends ParameterGenera private static final String PYTHON_PATH_ENV_VARIABLE = "PYTHONPATH"; private String PYTHON_PATH = null; private String TUNING_SCRIPT_PATH = null; - + boolean debugEnabled = logger.isDebugEnabled(); /** * Swarm size specific to OBT * @return @@ -56,8 +56,10 @@ public ParameterGenerateManagerOBTAlgoPSO() { PYTHON_PATH = "python"; } TUNING_SCRIPT_PATH = PSO_DIR_PATH + "/pso_param_generation.py"; - logger.info("Tuning script path: " + TUNING_SCRIPT_PATH); - logger.info("Python path: " + PYTHON_PATH); + if(debugEnabled) { + logger.debug("Tuning script path: " + TUNING_SCRIPT_PATH); + logger.debug("Python path: " + PYTHON_PATH); + } } @@ -73,7 +75,7 @@ protected void saveJobState(JobTuningInfo jobTuningInfo, JobDefinition job) { for (Particle particle : currentPopulation) { Long paramSetId = particle.getParamSetId(); - logger.info("Param set id: " + paramSetId.toString()); + logger.debug("Param set id: " + paramSetId.toString()); JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*").where().eq(JobSuggestedParamSet.TABLE.id, paramSetId).findUnique(); @@ -94,7 +96,7 @@ protected void saveJobState(JobTuningInfo jobTuningInfo, JobDefinition job) { jobTuningInfo.setTunerState(savedState); } } else { - logger.info("Saved state empty for job: " + job.jobDefId); + logger.debug("Saved state empty for job: " + job.jobDefId); validSavedState = false; } @@ -112,7 +114,7 @@ private List jsonToParticleList(JsonNode jsonParticleList) { List particleList = new ArrayList(); if (jsonParticleList == null) { - logger.info("Null json, empty particle list returned"); + logger.debug("Null json, empty particle list returned"); } else { for (JsonNode jsonParticle : jsonParticleList) { Particle particle; @@ -135,7 +137,7 @@ private JsonNode particleListToJson(List particleList) { if (particleList == null) { jsonNode = JsonNodeFactory.instance.objectNode(); - logger.info("Null particleList, returning empty json"); + logger.debug("Null particleList, returning empty json"); } else { jsonNode = Json.toJson(particleList); } @@ -150,7 +152,7 @@ private JsonNode particleListToJson(List particleList) { @Override public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { - logger.info("Generating param set for job: " + jobTuningInfo.getTuningJob().jobName); + logger.debug("Generating param set for job: " + jobTuningInfo.getTuningJob().jobName); JobTuningInfo newJobTuningInfo = new JobTuningInfo(); newJobTuningInfo.setTuningJob(jobTuningInfo.getTuningJob()); @@ -163,7 +165,7 @@ public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { stringTunerState = stringTunerState.replaceAll("\\s+", ""); String jobType = jobTuningInfo.getJobType().toString(); - logger.info(" ID " + jobTuningInfo.getTuningJob().id); + logger.debug(" ID " + jobTuningInfo.getTuningJob().id); int swarmSize = getSwarmSize(); @@ -181,7 +183,7 @@ public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { BufferedReader inputStream = new BufferedReader(new InputStreamReader(p.getInputStream())); BufferedReader errorStream = new BufferedReader(new InputStreamReader(p.getErrorStream())); String updatedStringTunerState = inputStream.readLine(); - logger.info("Param Generator Testing" + updatedStringTunerState); + logger.debug("Param Generator Testing" + updatedStringTunerState); newJobTuningInfo.setTunerState(updatedStringTunerState); String errorLine; while ((errorLine = errorStream.readLine()) != null) { @@ -211,16 +213,16 @@ public JobTuningInfo generateParamSet(JobTuningInfo jobTuningInfo) { * @param jobTuningInfoList JobTuningInfo List */ protected boolean updateDatabase(List jobTuningInfoList) { - logger.info("Updating new parameter suggestion in database"); + logger.debug("Updating new parameter suggestion in database"); if (jobTuningInfoList == null) { - logger.info("No new parameter suggestion to update"); + logger.debug("No new parameter suggestion to update"); return false; } int paramSetNotGeneratedJobs = jobTuningInfoList.size(); for (JobTuningInfo jobTuningInfo : jobTuningInfoList) { - logger.info("Updating new parameter suggestion for job:" + jobTuningInfo.getTuningJob().jobDefId); + logger.debug("Updating new parameter suggestion for job:" + jobTuningInfo.getTuningJob().jobDefId); JobDefinition job = jobTuningInfo.getTuningJob(); List paramList = jobTuningInfo.getParametersToTune(); @@ -244,7 +246,7 @@ protected boolean updateDatabase(List jobTuningInfoList) { .eq(TuningParameter.TABLE.isDerived, 1) .findList(); - logger.info("No. of derived tuning params for job " + tuningJobDefinition.job.jobName + ": " + logger.debug("No. of derived tuning params for job " + tuningJobDefinition.job.jobName + ": " + derivedParameterList.size()); JsonNode jsonTunerState = Json.parse(stringTunerState); @@ -321,7 +323,7 @@ private List getParamValueList(Particle particle, List getParamValueList(Particle particle, List getTuningJobDefinitions() { @Override public void initializePrerequisite(TuningAlgorithm tuningAlgorithm, JobDefinition job) { - logger.info(" Intialize Prerequisite "); + logger.debug(" Intialize Prerequisite "); try { setDefaultParameterValues(tuningAlgorithm, job); } catch (Exception e) { - logger.info("Cannot intialize parameter in IPSO " + e.getMessage()); + logger.debug("Cannot intialize parameter in IPSO " + e.getMessage()); } } @@ -76,7 +76,7 @@ private void setDefaultParameterValues(TuningAlgorithm tuningAlgorithm, JobDefin .findList(); if(tuningParameterConstrains== null || tuningParameterConstrains.size()==0) { - logger.info("Parameter constraints not added . Hence adding parameter constraint. "); + logger.debug("Parameter constraints not added . Hence adding parameter constraint. "); for (TuningParameter tuningParameter : tuningParameters) { TuningParameterConstraint tuningParameterConstraint = new TuningParameterConstraint(); tuningParameterConstraint.jobDefinition = job; @@ -88,11 +88,11 @@ private void setDefaultParameterValues(TuningAlgorithm tuningAlgorithm, JobDefin } } else{ - logger.info(" Parameter constraints already added . Hence not adding them"); + logger.debug(" Parameter constraints already added . Hence not adding them"); } } catch (Exception e) { - logger.info( - " Error in setting up intial parameter constraint . IPSO will not work in this case ." + e.getMessage()); + logger.debug( + " Error in setting up intial parameter constraint . IPSO will not work in this case ." + e.getMessage(),e); throw e; } } @@ -104,7 +104,7 @@ public void parameterOptimizer(List appResults, JobExecution jobExecu } public void applyIntelligenceOnParameter(List tuningParameterList, JobDefinition job) { - logger.info(" Apply Intelligence"); + logger.debug(" Apply Intelligence"); List tuningParameterConstraintList = new ArrayList(); tuningParameterConstraintList = TuningParameterConstraint.find.where() .eq("job_definition_id", job.id) @@ -119,7 +119,7 @@ public void applyIntelligenceOnParameter(List tuningParameterLi i += 1; } } else { - logger.info("No boundary constraints found for job: " + job.jobName); + logger.debug("No boundary constraints found for job: " + job.jobName); } for (TuningParameter tuningParameter : tuningParameterList) { diff --git a/test/com/linkedin/drelephant/tuning/FlowTestRunner.java b/test/com/linkedin/drelephant/tuning/FlowTestRunner.java index 5b1ba87bb..0596ff2dc 100644 --- a/test/com/linkedin/drelephant/tuning/FlowTestRunner.java +++ b/test/com/linkedin/drelephant/tuning/FlowTestRunner.java @@ -21,37 +21,37 @@ public class FlowTestRunner implements Runnable { @Override public void run() { - Flow flow = new Flow(); - testPipeline(flow); - testCreateBaseLineManagersPipeline(flow); - testCreateJobStatusManagersPipeline(flow); - testCreateFitnessManagersPipeline(flow); - testCreateTuningTypeManagersPipeline(flow); - testCreateTuningTypeManagersPipeline(flow); + AutoTuningFlow autoTuningFlow = new AutoTuningFlow(); + testPipeline(autoTuningFlow); + testCreateBaseLineManagersPipeline(autoTuningFlow); + testCreateJobStatusManagersPipeline(autoTuningFlow); + testCreateFitnessManagersPipeline(autoTuningFlow); + testCreateTuningTypeManagersPipeline(autoTuningFlow); + testCreateTuningTypeManagersPipeline(autoTuningFlow); } - private void testPipeline(Flow flow) { - List> pipelines = flow.getPipeline(); + private void testPipeline(AutoTuningFlow autoTuningFlow) { + List> pipelines = autoTuningFlow.getPipeline(); assertTrue(" Total Number of pipeline ", pipelines.size() == 4); } - private void testCreateBaseLineManagersPipeline(Flow flow) { - List> pipelines = flow.getPipeline(); + private void testCreateBaseLineManagersPipeline(AutoTuningFlow autoTuningFlow) { + List> pipelines = autoTuningFlow.getPipeline(); List baseLineManagers = pipelines.get(0); assertTrue(" Total Number of Base line Managers ", baseLineManagers.size() == 2); assertTrue(" HBT Baseline Manager ", baseLineManagers.get(0) instanceof BaselineManagerHBT); assertTrue(" OBT Baseline Manager ", baseLineManagers.get(1) instanceof BaselineManagerOBT); } - private void testCreateJobStatusManagersPipeline(Flow flow) { - List> pipelines = flow.getPipeline(); + private void testCreateJobStatusManagersPipeline(AutoTuningFlow autoTuningFlow) { + List> pipelines = autoTuningFlow.getPipeline(); List jobStatusManagers = pipelines.get(1); assertTrue(" Total Number of Base line Managers ", jobStatusManagers.size() == 1); assertTrue(" Azkaban Job Status Manager ", jobStatusManagers.get(0) instanceof AzkabanJobStatusManager); } - private void testCreateFitnessManagersPipeline(Flow flow) { - List> pipelines = flow.getPipeline(); + private void testCreateFitnessManagersPipeline(AutoTuningFlow autoTuningFlow) { + List> pipelines = autoTuningFlow.getPipeline(); List fitnessManagers = pipelines.get(2); assertTrue(" Total Number of fitness Managers ", fitnessManagers.size() == 3); assertTrue(" FitnessManagerHBT ", fitnessManagers.get(0) instanceof FitnessManagerHBT); @@ -59,8 +59,8 @@ private void testCreateFitnessManagersPipeline(Flow flow) { assertTrue(" FitnessManagerOBTIPSO ", fitnessManagers.get(2) instanceof FitnessManagerOBTAlgoIPSO); } - private void testCreateTuningTypeManagersPipeline(Flow flow) { - List> pipelines = flow.getPipeline(); + private void testCreateTuningTypeManagersPipeline(AutoTuningFlow autoTuningFlow) { + List> pipelines = autoTuningFlow.getPipeline(); List tuningTypeManagers = pipelines.get(3); assertTrue(" Total Number of tuningType Managers ", tuningTypeManagers.size() == 6); From 21ea0ef94931dfd984c293439bb5f3c37c214fa5 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Mon, 8 Oct 2018 11:15:17 +0530 Subject: [PATCH 11/22] Changed logic to have algorithm selection based on version --- .../tuning/AbstractParameterGenerateManager.java | 2 +- .../drelephant/tuning/AutoTuningAPIHelper.java | 5 +++-- .../tuning/hbt/ParameterGenerateManagerHBT.java | 4 ++-- ...ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java | 2 +- app/controllers/Application.java | 14 ++++++++++++++ .../drelephant/tuning/TuningManagerTest.java | 13 +++++++++++++ 6 files changed, 34 insertions(+), 6 deletions(-) diff --git a/app/com/linkedin/drelephant/tuning/AbstractParameterGenerateManager.java b/app/com/linkedin/drelephant/tuning/AbstractParameterGenerateManager.java index 1591d43b8..ee3aca1fd 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractParameterGenerateManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractParameterGenerateManager.java @@ -153,8 +153,8 @@ public final boolean execute() { return databaseUpdateDone; } catch (Exception e) { logger.info("Exception in generating parameters ", e); + return false; } - return false; } /** diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java index 6709224aa..dd1f17544 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java @@ -258,12 +258,13 @@ private TuningJobDefinition addNewJobForTuning(TuningInput tuningInput) { } private TuningAlgorithm getTuningAlgorithmForfirstTime(TuningInput tuningInput) { - logger.info(" Since its new job . Algorithm type is HBT "); + logger.info("Version "+tuningInput.getVersion() + "\t" + tuningInput.getOptimizationAlgo()); TuningAlgorithm tuningAlgorithm = TuningAlgorithm.find.select("*") .where() .eq(TuningAlgorithm.TABLE.jobType, tuningInput.getJobType()) .eq(TuningAlgorithm.TABLE.optimizationMetric, "RESOURCE") - .eq(TuningAlgorithm.TABLE.optimizationAlgo, "HBT") + // .eq(TuningAlgorithm.TABLE.optimizationAlgo, "HBT") + .eq(TuningAlgorithm.TABLE.optimizationAlgo,tuningInput.getOptimizationAlgo()) .findUnique(); if (tuningAlgorithm == null) { throw new IllegalArgumentException("Wrong job type " + tuningInput.getJobType()); diff --git a/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java index 350fd12f0..5e0bf73d8 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java @@ -188,7 +188,7 @@ protected boolean updateDatabase(List jobTuningInfoList) { return true; } - + private void penaltyApplication(JobSuggestedParamSet jobSuggestedParamSet, TuningJobDefinition tuningJobDefinition) { logger.debug("Parameter constraint violated. Applying penalty."); @@ -252,6 +252,6 @@ public boolean isParamConstraintViolated(List jobSuggest @Override public String getManagerName() { - return "TuningTypeManagerHBT"; + return "ParameterGenerateManagerHBT"; } } diff --git a/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java index d7bff0f32..12d4006f1 100644 --- a/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java +++ b/app/com/linkedin/drelephant/tuning/obt/ParameterGenerateManagerOBTAlgoPSOIPSOImpl.java @@ -137,6 +137,6 @@ public int getSwarmSize() { } public String getManagerName() { - return "TuningTypeManagerOBTAlgoIPSO" + this._executionEngine.getClass().getSimpleName(); + return "ParameterGenerateManagerOBTAlgoPSOIPSOImpl" + this._executionEngine.getClass().getSimpleName(); } } diff --git a/app/controllers/Application.java b/app/controllers/Application.java index b18931c6c..728e08ad5 100644 --- a/app/controllers/Application.java +++ b/app/controllers/Application.java @@ -943,6 +943,11 @@ public static Result getCurrentRunParameters() { if (paramValueMap.containsKey("allowedMaxExecutionTimePercent")) { allowedMaxExecutionTimePercent = Double.parseDouble(paramValueMap.get("allowedMaxExecutionTimePercent")); } + /* + This logic is for backward compatailbity. Ideally we should remove this logic in next release + because HBT should be the default optimization algorithm for all the jobs. + */ + optimizationAlgo = getAlgoBasedOnVersion(version); tuningInput.setFlowDefId(flowDefId); tuningInput.setJobDefId(jobDefId); tuningInput.setFlowDefUrl(flowDefUrl); @@ -977,6 +982,15 @@ public static Result getCurrentRunParameters() { } } + public static String getAlgoBasedOnVersion(int version){ + if(version == 1){ + return TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name(); + } + else{ + return TuningAlgorithm.OptimizationAlgo.HBT.name(); + } + } + private static JsonNode formatGetCurrentRunParametersOutput(Map outputParams, Integer version) { if (version == 1) { return Json.toJson(outputParams); diff --git a/test/com/linkedin/drelephant/tuning/TuningManagerTest.java b/test/com/linkedin/drelephant/tuning/TuningManagerTest.java index 96cc3e11c..b62fee36e 100644 --- a/test/com/linkedin/drelephant/tuning/TuningManagerTest.java +++ b/test/com/linkedin/drelephant/tuning/TuningManagerTest.java @@ -1,5 +1,6 @@ package com.linkedin.drelephant.tuning; +import com.google.common.annotations.VisibleForTesting; import com.linkedin.drelephant.DrElephant; import com.linkedin.drelephant.ElephantContext; import java.util.HashMap; @@ -8,9 +9,11 @@ import static common.DBTestUtil.*; import static common.TestConstants.*; +import models.TuningAlgorithm; import org.slf4j.LoggerFactory; import play.Application; import play.GlobalSettings; +import controllers.*; import play.test.FakeApplication; import org.apache.hadoop.conf.Configuration; @@ -79,4 +82,14 @@ public void testFitnessManagerTestRunner(){ public void testParamGenerterTestRunner(){ running(testServer(TEST_SERVER_PORT, fakeApp), new ParameterGenerateManagerTestRunner()); } + + @Test + public void testAlgoBasedOnVersion(){ + assertTrue("Alorithm Based on Version Test", controllers.Application.getAlgoBasedOnVersion(1).equals( + TuningAlgorithm.OptimizationAlgo.PSO_IPSO.name())); + assertTrue("Alorithm Based on Version Test", controllers.Application.getAlgoBasedOnVersion(2).equals( + TuningAlgorithm.OptimizationAlgo.HBT.name())); + assertTrue("Alorithm Based on Version Test", controllers.Application.getAlgoBasedOnVersion(3).equals( + TuningAlgorithm.OptimizationAlgo.HBT.name())); + } } From c29f969053f03ed030541d0d272e2a1f0f006c22 Mon Sep 17 00:00:00 2001 From: "Manoj Kumar(Bangalore Engineering)" Date: Mon, 8 Oct 2018 12:10:34 +0530 Subject: [PATCH 12/22] Adding Spark Job Specific Configuration Input Output --- .../drelephant/tuning/TuningInput.java | 59 ++++++++++++++- .../engine/SparkConfigurationConstants.java | 3 + .../tuning/engine/SparkExecutionEngine.java | 2 +- app/controllers/Application.java | 74 ++++++++++++++----- 4 files changed, 116 insertions(+), 22 deletions(-) diff --git a/app/com/linkedin/drelephant/tuning/TuningInput.java b/app/com/linkedin/drelephant/tuning/TuningInput.java index 7ccf80a23..296295f47 100644 --- a/app/com/linkedin/drelephant/tuning/TuningInput.java +++ b/app/com/linkedin/drelephant/tuning/TuningInput.java @@ -19,10 +19,17 @@ import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.drelephant.tuning.engine.SparkConfigurationConstants; + import java.io.IOException; import java.util.HashMap; import java.util.Map; + +import org.apache.spark.network.util.ByteUnit; +import org.apache.spark.network.util.JavaUtils; + import models.TuningAlgorithm; +import models.TuningAlgorithm.JobType; /** @@ -286,6 +293,15 @@ public void setScheduler(String scheduler) { this._scheduler = scheduler; } + public Map getDefaultParams() throws JsonParseException, JsonMappingException, IOException { + if(getJobType().equals(JobType.SPARK.name())) + { + return getSparkDefaultParams(); + }else + { + return getMRDefaultParams(); + } + } /** * Returns the default parameters * @return default parameters @@ -294,7 +310,7 @@ public void setScheduler(String scheduler) { * @throws JsonParseException */ @SuppressWarnings("unchecked") - public Map getDefaultParams() throws JsonParseException, JsonMappingException, IOException { + public Map getMRDefaultParams() throws JsonParseException, JsonMappingException, IOException { ObjectMapper mapper = new ObjectMapper(); Map paramValueMap; if (version == 1) { @@ -321,6 +337,47 @@ public Map getDefaultParams() throws JsonParseException, JsonMap return paramValueMap; } + public Map getSparkDefaultParams() throws JsonParseException, JsonMappingException, IOException{ + ObjectMapper mapper = new ObjectMapper(); + Map paramValueMap = new HashMap(); + Map paramsStringMap = (Map) mapper.readValue(this._defaultParams, Map.class); + for (Map.Entry entry : paramsStringMap.entrySet()) { + String confKey = entry.getKey(); + String confVal = entry.getValue(); + Double confValDouble = null; + try { + if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY)) { + paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY, Double.parseDouble(confVal)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_DRIVER_CORES)) { + paramValueMap.put(SparkConfigurationConstants.SPARK_DRIVER_CORES, Double.parseDouble(confVal)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_INSTANCES_KEY)) { + paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_INSTANCES_KEY, Double.parseDouble(confVal)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY)) { + long memoryMb = JavaUtils.byteStringAsMb(confVal); + paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY, new Double(memoryMb)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_OVERHEAD)) { + long memoryMb = JavaUtils.byteStringAsMb(confVal); + paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_OVERHEAD, new Double(memoryMb)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD)) { + long memoryMb = JavaUtils.byteStringAsMb(confVal); + paramValueMap.put(SparkConfigurationConstants.SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD, new Double(memoryMb)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY)) { + long memoryMb = JavaUtils.byteStringAsMb(confVal); + paramValueMap.put(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY, new Double(memoryMb)); + } else if (confVal != null) { + confValDouble = Double.parseDouble(confVal); + } + } catch (NumberFormatException nfe) { + //Do Nothing + } + if (confValDouble != null) { + paramValueMap.put(confKey, confValDouble); + } + } + return paramValueMap; + } + + /** * Sets the default parameters * @param defaultParams default parameters diff --git a/app/com/linkedin/drelephant/tuning/engine/SparkConfigurationConstants.java b/app/com/linkedin/drelephant/tuning/engine/SparkConfigurationConstants.java index abe3ca392..2c865b991 100644 --- a/app/com/linkedin/drelephant/tuning/engine/SparkConfigurationConstants.java +++ b/app/com/linkedin/drelephant/tuning/engine/SparkConfigurationConstants.java @@ -14,4 +14,7 @@ public class SparkConfigurationConstants { public final static String SPARK_DYNAMIC_ALLOCATION_MAX_EXECUTORS = "spark.dynamicAllocation.maxExecutors"; public final static String SPARK_YARN_JARS = "spark.yarn.secondary.jars"; public final static String SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD = "spark.yarn.executor.memoryOverhead"; + public final static String SPARK_EXECUTOR_MEMORY_OVERHEAD = "spark.executor.memoryOverhead"; + public final static String SPARK_DRIVER_CORES = "spark.driver.cores"; + } diff --git a/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java b/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java index 890d4304e..aa7c9d16c 100644 --- a/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java +++ b/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java @@ -84,8 +84,8 @@ public String parameterGenerationsHBT(List results, List outputParams, Integer version) { - if (version == 1) { + private static JsonNode formatGetCurrentRunParametersOutput(Map outputParams, TuningInput tuningInput) { + if (tuningInput.getVersion() == 1) { return Json.toJson(outputParams); + } else if (tuningInput.getJobType().equals(JobType.SPARK.name())) { + return formatSparkOutput(outputParams); } else { - Map outputParamFormatted = new HashMap(); - - //Temporarily removing input split parameters - outputParams.remove("pig.maxCombinedSplitSize"); - outputParams.remove("mapreduce.input.fileinputformat.split.maxsize"); - - for (Map.Entry param : outputParams.entrySet()) { - if (param.getKey().equals("mapreduce.map.sort.spill.percent")) { - outputParamFormatted.put(param.getKey(), String.valueOf(param.getValue())); - } else if (param.getKey().equals("mapreduce.map.java.opts") - || param.getKey().equals("mapreduce.reduce.java.opts")) { - outputParamFormatted.put(param.getKey(), "-Xmx" + Math.round(param.getValue()) + "m"); - } else { - outputParamFormatted.put(param.getKey(), String.valueOf(Math.round(param.getValue()))); - } + return formatMROutput(outputParams); + } + } + + private static JsonNode formatMROutput(Map outputParams) { + Map outputParamFormatted = new HashMap(); + + //Temporarily removing input split parameters + outputParams.remove("pig.maxCombinedSplitSize"); + outputParams.remove("mapreduce.input.fileinputformat.split.maxsize"); + + for (Map.Entry param : outputParams.entrySet()) { + if (param.getKey().equals("mapreduce.map.sort.spill.percent")) { + outputParamFormatted.put(param.getKey(), String.valueOf(param.getValue())); + } else if (param.getKey().equals("mapreduce.map.java.opts") + || param.getKey().equals("mapreduce.reduce.java.opts")) { + outputParamFormatted.put(param.getKey(), "-Xmx" + Math.round(param.getValue()) + "m"); + } else { + outputParamFormatted.put(param.getKey(), String.valueOf(Math.round(param.getValue()))); + } + } + return Json.toJson(outputParamFormatted); + } + + private static JsonNode formatSparkOutput(Map outputParams) { + Map outputParamFormatted = new HashMap(); + for (Map.Entry param : outputParams.entrySet()) { + if (param.getKey().equals(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY)) { + outputParamFormatted.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY, param.getValue().intValue() + + "m"); + } else if (param.getKey().equals(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY)) { + outputParamFormatted.put(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY, + String.valueOf(param.getValue().intValue())); + } else if (param.getKey().equals(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_OVERHEAD)) { + outputParamFormatted.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_OVERHEAD, param.getValue() + .intValue() + "m"); + } else if (param.getKey().equals(SparkConfigurationConstants.SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD)) { + outputParamFormatted.put(SparkConfigurationConstants.SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD, param.getValue() + .intValue() + "m"); + } else if (param.getKey().equals(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY)) { + outputParamFormatted + .put(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY, param.getValue().intValue() + "m"); + } else { + outputParamFormatted.put(param.getKey(), param.getValue().toString()); } - return Json.toJson(outputParamFormatted); } + return Json.toJson(outputParamFormatted); } private static Result getCurrentRunParameters(TuningInput tuningInput) throws Exception { @@ -1020,7 +1054,7 @@ private static Result getCurrentRunParameters(TuningInput tuningInput) throws Ex Map outputParams = autoTuningAPIHelper.getCurrentRunParameters(tuningInput); if (outputParams != null) { logger.info("Output params " + outputParams); - return ok(formatGetCurrentRunParametersOutput(outputParams, tuningInput.getVersion())); + return ok(formatGetCurrentRunParametersOutput(outputParams, tuningInput)); } else { AutoTuningMetricsController.markGetCurrentRunParametersFailures(); return notFound("Unable to find parameters. Job id: " + tuningInput.getJobDefId() + " Flow id: " From d3cc99f3884190b9a45021a0e5bf14afd80a1308 Mon Sep 17 00:00:00 2001 From: "Manoj Kumar(Bangalore Engineering)" Date: Mon, 8 Oct 2018 13:02:19 +0530 Subject: [PATCH 13/22] Changing spark input configuration for version 1 api --- .../drelephant/tuning/TuningInput.java | 69 ++++++++++--------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/app/com/linkedin/drelephant/tuning/TuningInput.java b/app/com/linkedin/drelephant/tuning/TuningInput.java index 296295f47..fa70ef101 100644 --- a/app/com/linkedin/drelephant/tuning/TuningInput.java +++ b/app/com/linkedin/drelephant/tuning/TuningInput.java @@ -25,7 +25,6 @@ import java.util.HashMap; import java.util.Map; -import org.apache.spark.network.util.ByteUnit; import org.apache.spark.network.util.JavaUtils; import models.TuningAlgorithm; @@ -337,41 +336,47 @@ public Map getMRDefaultParams() throws JsonParseException, JsonM return paramValueMap; } + @SuppressWarnings("unchecked") public Map getSparkDefaultParams() throws JsonParseException, JsonMappingException, IOException{ + ObjectMapper mapper = new ObjectMapper(); Map paramValueMap = new HashMap(); - Map paramsStringMap = (Map) mapper.readValue(this._defaultParams, Map.class); - for (Map.Entry entry : paramsStringMap.entrySet()) { - String confKey = entry.getKey(); - String confVal = entry.getValue(); - Double confValDouble = null; - try { - if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY)) { - paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY, Double.parseDouble(confVal)); - } else if (confKey.equals(SparkConfigurationConstants.SPARK_DRIVER_CORES)) { - paramValueMap.put(SparkConfigurationConstants.SPARK_DRIVER_CORES, Double.parseDouble(confVal)); - } else if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_INSTANCES_KEY)) { - paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_INSTANCES_KEY, Double.parseDouble(confVal)); - } else if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY)) { - long memoryMb = JavaUtils.byteStringAsMb(confVal); - paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY, new Double(memoryMb)); - } else if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_OVERHEAD)) { - long memoryMb = JavaUtils.byteStringAsMb(confVal); - paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_OVERHEAD, new Double(memoryMb)); - } else if (confKey.equals(SparkConfigurationConstants.SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD)) { - long memoryMb = JavaUtils.byteStringAsMb(confVal); - paramValueMap.put(SparkConfigurationConstants.SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD, new Double(memoryMb)); - } else if (confKey.equals(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY)) { - long memoryMb = JavaUtils.byteStringAsMb(confVal); - paramValueMap.put(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY, new Double(memoryMb)); - } else if (confVal != null) { - confValDouble = Double.parseDouble(confVal); + if (version == 1) { + paramValueMap = (Map) mapper.readValue(this._defaultParams, Map.class); + } else { + Map paramsStringMap = (Map) mapper.readValue(this._defaultParams, Map.class); + for (Map.Entry entry : paramsStringMap.entrySet()) { + String confKey = entry.getKey(); + String confVal = entry.getValue(); + Double confValDouble = null; + try { + if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY)) { + paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY, Double.parseDouble(confVal)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_DRIVER_CORES)) { + paramValueMap.put(SparkConfigurationConstants.SPARK_DRIVER_CORES, Double.parseDouble(confVal)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_INSTANCES_KEY)) { + paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_INSTANCES_KEY, Double.parseDouble(confVal)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY)) { + long memoryMb = JavaUtils.byteStringAsMb(confVal); + paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY, new Double(memoryMb)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_OVERHEAD)) { + long memoryMb = JavaUtils.byteStringAsMb(confVal); + paramValueMap.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_OVERHEAD, new Double(memoryMb)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD)) { + long memoryMb = JavaUtils.byteStringAsMb(confVal); + paramValueMap.put(SparkConfigurationConstants.SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD, new Double(memoryMb)); + } else if (confKey.equals(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY)) { + long memoryMb = JavaUtils.byteStringAsMb(confVal); + paramValueMap.put(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY, new Double(memoryMb)); + } else if (confVal != null) { + confValDouble = Double.parseDouble(confVal); + } + } catch (NumberFormatException nfe) { + //Do Nothing + } + if (confValDouble != null) { + paramValueMap.put(confKey, confValDouble); } - } catch (NumberFormatException nfe) { - //Do Nothing - } - if (confValDouble != null) { - paramValueMap.put(confKey, confValDouble); } } return paramValueMap; From 1905c2a0d3540c641001286d0a6e41bd8f8ddd1b Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Thu, 11 Oct 2018 18:21:51 +0530 Subject: [PATCH 14/22] Code changes to handle multiple jobs in Spark --- .../tuning/Schduler/AzkabanJobStatusManager.java | 9 +++++++-- .../drelephant/tuning/engine/SparkExecutionEngine.java | 2 +- .../tuning/hbt/ParameterGenerateManagerHBT.java | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java b/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java index 187fb503e..7d6fea6fc 100644 --- a/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java +++ b/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java @@ -41,20 +41,24 @@ protected List detectJobsExecutionInProgress() { protected boolean analyzeCompletedJobsExecution(List inProgressExecutionParamSet) { logger.info("Fetching the list of executions completed since last iteration"); List completedExecutions = new ArrayList(); + boolean isAnalyzeDone = true; try { for (TuningJobExecutionParamSet tuningJobExecutionParamSet : inProgressExecutionParamSet) { JobSuggestedParamSet jobSuggestedParamSet = tuningJobExecutionParamSet.jobSuggestedParamSet; JobExecution jobExecution = tuningJobExecutionParamSet.jobExecution; logger.info("Checking current status of started execution: " + jobExecution.jobExecId); assignAzkabanJobStatusUtil(); - return analyzeJobExecution(jobExecution,jobSuggestedParamSet); + isAnalyzeDone = analyzeJobExecution(jobExecution,jobSuggestedParamSet); + if(isAnalyzeDone) { + completedExecutions.add(jobExecution); + } } } catch (Exception e) { logger.error("Error in fetching list of completed executions", e); return false; } logger.info("Number of executions completed since last iteration: " + completedExecutions.size()); - return true; + return inProgressExecutionParamSet.size()==completedExecutions.size(); } private void assignAzkabanJobStatusUtil() { @@ -71,6 +75,7 @@ private boolean analyzeJobExecution(JobExecution jobExecution,JobSuggestedParamS for (Map.Entry job : jobStatus.entrySet()) { logger.info("Job Found:" + job.getKey() + ". Status: " + job.getValue()); if (job.getKey().equals(jobExecution.job.jobName)) { + logger.info(" Job Updated " + jobExecution.job.jobName); updateJobExecutionMetrics(job, jobSuggestedParamSet, jobExecution); } } diff --git a/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java b/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java index aa7c9d16c..2d6c44c54 100644 --- a/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java +++ b/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java @@ -41,7 +41,7 @@ public ExpressionList getPendingJobs() { .or(Expr.or(Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.CREATED), Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.SENT)), Expr.eq(JobSuggestedParamSet.TABLE.paramSetState, JobSuggestedParamSet.ParamSetStatus.EXECUTED)) - .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) + // .eq(JobSuggestedParamSet.TABLE.isParamSetDefault, 0) .eq(JobSuggestedParamSet.TABLE.tuningAlgorithm + "." + TuningAlgorithm.TABLE.jobType, TuningAlgorithm.JobType.SPARK.name()).eq(JobSuggestedParamSet.TABLE.isParamSetBest, 0); } diff --git a/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java index 5e0bf73d8..b509849a1 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java @@ -87,7 +87,7 @@ private String generateParamSet(List tuningParameters, JobDefin } List results = getAppResults(jobExecution); - if (results == null) { + if (results == null || results.size()==0 ) { logger.debug( " Job is analyzing , cannot use for param generation " + jobExecution.id + " " + jobExecution.job.id); return ""; From c519f89e2681ae4a0d282f39dc89f02c015fdb85 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Fri, 12 Oct 2018 11:18:02 +0530 Subject: [PATCH 15/22] Changing log status to debug --- .../drelephant/tuning/Schduler/AzkabanJobStatusManager.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java b/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java index 7d6fea6fc..bcb158d05 100644 --- a/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java +++ b/app/com/linkedin/drelephant/tuning/Schduler/AzkabanJobStatusManager.java @@ -73,14 +73,14 @@ private boolean analyzeJobExecution(JobExecution jobExecution,JobSuggestedParamS Map jobStatus = _azkabanJobStatusUtil.getJobsFromFlow(jobExecution.flowExecution.flowExecId); if (jobStatus != null) { for (Map.Entry job : jobStatus.entrySet()) { - logger.info("Job Found:" + job.getKey() + ". Status: " + job.getValue()); + logger.debug("Job Found:" + job.getKey() + ". Status: " + job.getValue()); if (job.getKey().equals(jobExecution.job.jobName)) { - logger.info(" Job Updated " + jobExecution.job.jobName); + logger.debug(" Job Updated " + jobExecution.job.jobName); updateJobExecutionMetrics(job, jobSuggestedParamSet, jobExecution); } } } else { - logger.info("No jobs found for flow execution: " + jobExecution.flowExecution.flowExecId); + logger.debug("No jobs found for flow execution: " + jobExecution.flowExecution.flowExecId); return false; } } catch (Exception e) { From b8f2ccc1eee9bdfbc624f7c6a4c3ee76cd88ea70 Mon Sep 17 00:00:00 2001 From: "Manoj Kumar(Bangalore Engineering)" Date: Mon, 15 Oct 2018 11:17:49 +0530 Subject: [PATCH 16/22] Updated HBT parameter generation for Spark --- .../heuristics/ExecutorGcHeuristic.scala | 3 +- .../engine/SparkHBTParamRecommender.java | 77 ++++++++++++++----- 2 files changed, 58 insertions(+), 22 deletions(-) diff --git a/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala index da48c351b..c544ac152 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala @@ -48,7 +48,7 @@ class ExecutorGcHeuristic(private val heuristicConfigurationData: HeuristicConfi override def apply(data: SparkApplicationData): HeuristicResult = { val evaluator = new Evaluator(this, data) var resultDetails = Seq( - new HeuristicResultDetails("GC time to Executor Run time ratio", evaluator.ratio.toString), + new HeuristicResultDetails(GC_RUN_TIME_RATIO, evaluator.ratio.toString), new HeuristicResultDetails("Total GC time", evaluator.msecToString(evaluator.jvmTime)), new HeuristicResultDetails("Total Executor Runtime", evaluator.msecToString(evaluator.executorRunTimeTotal)) ) @@ -77,6 +77,7 @@ object ExecutorGcHeuristic { val SPARK_EXECUTOR_MEMORY = "spark.executor.memory" val SPARK_EXECUTOR_CORES = "spark.executor.cores" val EXECUTOR_RUNTIME_THRESHOLD_IN_MINUTES = 5 + val GC_RUN_TIME_RATIO="GC time to Executor Run time ratio"; /** The ascending severity thresholds for the ratio of JVM GC Time and executor Run Time (checking whether ratio is above normal) * These thresholds are experimental and are likely to change */ diff --git a/app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java b/app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java index fe61d5248..eb42cb322 100644 --- a/app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java +++ b/app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java @@ -2,11 +2,15 @@ import java.util.HashMap; import java.util.Map; + import models.AppResult; + import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; + import com.linkedin.drelephant.spark.heuristics.ConfigurationHeuristic; import com.linkedin.drelephant.spark.heuristics.DriverHeuristic; +import com.linkedin.drelephant.spark.heuristics.ExecutorGcHeuristic; import com.linkedin.drelephant.spark.heuristics.JvmUsedMemoryHeuristic; import com.linkedin.drelephant.spark.heuristics.UnifiedMemoryHeuristic; import com.linkedin.drelephant.util.MemoryFormatUtils; @@ -21,14 +25,20 @@ public class SparkHBTParamRecommender { AppResult appResult; // TODos Move these to configuration - public static final int MAX_EXECUTOR_CORE = 4; + public static final int MAX_EXECUTOR_CORE = 3; public static final long MAX_EXECUTOR_MEMORY = 10 * FileUtils.ONE_GB; + public static final long MIN_EXECUTOR_MEMORY = 900 * FileUtils.ONE_MB; + public static final long MIN_DRIVER_MEMORY = 900 * FileUtils.ONE_MB; + public static final int CLUSTER_DEFAULT_EXECUTOR_CORE = 1; public static final long RESERVED_MEMORY = 300 * FileUtils.ONE_MB; - private static final long EXECUTOR_MEMORY_BUFFER_PER_CORE = 5; - private static final long EXECUTOR_MEMORY_BUFFER_OVERALL = 5; - private static final long DRIVER_MEMORY_BUFFER = 20; + private static final long EXECUTOR_MEMORY_BUFFER_PER_CORE = 0; + private static final long EXECUTOR_MEMORY_BUFFER_OVERALL = 0; + private static final long DRIVER_MEMORY_BUFFER = 0; + + private static final int GC_MEMORY_INCREASE = 5; + private static final int GC_MEMORY_DECREASE = -5; private long maxPeakUnifiedMemory; private long maxPeakJVMUsedMemory; @@ -39,7 +49,7 @@ public class SparkHBTParamRecommender { private int lastRunExecutorCore; private long lastRunExecutorMemoryOverhead; private long lastRunDriverMemoryOverhead; - + private Float gcRunTimeRatio; private Long suggestedExecutorMemory; private Integer suggestedCore; private Double suggestedMemoryFactor; @@ -71,6 +81,9 @@ public SparkHBTParamRecommender(AppResult appResult) { } catch (NumberFormatException e) { // Do Nothing } + gcRunTimeRatio = + Float.parseFloat(appHeuristicsResultDetailsMap.get(ExecutorGcHeuristic.class.getCanonicalName() + "_" + + ExecutorGcHeuristic.GC_RUN_TIME_RATIO())); driverMaxPeakJVMUsedMemory = MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(DriverHeuristic.class.getCanonicalName() @@ -189,23 +202,27 @@ public void setSuggestedSparkDriverMemory(long suggestedSparkDriverMemory) { */ public HashMap getHBTSuggestion() { HashMap suggestedParameters = new HashMap(); - suggestExecutorMemoryCore(); - suggestMemoryFactor(); - suggestDriverMemory(); - suggestedParameters.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY, (double) suggestedExecutorMemory - / FileUtils.ONE_MB); - suggestedParameters.put(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY, suggestedCore.doubleValue()); - if (suggestedMemoryFactor < UnifiedMemoryHeuristic.SPARK_MEMORY_FRACTION_THRESHOLD()) { - suggestedMemoryFactor = UnifiedMemoryHeuristic.SPARK_MEMORY_FRACTION_THRESHOLD(); + try { + suggestExecutorMemoryCore(); + suggestMemoryFactor(); + suggestDriverMemory(); + suggestedParameters.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY, (double) suggestedExecutorMemory + / FileUtils.ONE_MB); + suggestedParameters.put(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY, suggestedCore.doubleValue()); + if (suggestedMemoryFactor < UnifiedMemoryHeuristic.SPARK_MEMORY_FRACTION_THRESHOLD()) { + suggestedMemoryFactor = UnifiedMemoryHeuristic.SPARK_MEMORY_FRACTION_THRESHOLD(); + } + suggestedParameters.put(SparkConfigurationConstants.SPARK_MEMORY_FRACTION_KEY, suggestedMemoryFactor); + suggestedParameters.put(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY, (double) suggestedDriverMemory + / FileUtils.ONE_MB); + logger.info("Following are the suggestions for spark parameters for app id : " + appResult.flowExecId); + logger.info("suggestedExecutorMemory " + suggestedExecutorMemory); + logger.info("suggestedCore " + suggestedCore); + logger.info("suggestedMemoryFactor " + suggestedMemoryFactor); + logger.info("suggestedDriverMemory " + suggestedDriverMemory); + } catch (Exception e) { + logger.error("Error in generating parameters ", e); } - suggestedParameters.put(SparkConfigurationConstants.SPARK_MEMORY_FRACTION_KEY, suggestedMemoryFactor); - suggestedParameters.put(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY, (double) suggestedDriverMemory - / FileUtils.ONE_MB); - logger.info("Following are the suggestions for spark parameters for app id : " + appResult.flowExecId); - logger.info("suggestedExecutorMemory " + suggestedExecutorMemory); - logger.info("suggestedCore " + suggestedCore); - logger.info("suggestedMemoryFactor " + suggestedMemoryFactor); - logger.info("suggestedDriverMemory " + suggestedDriverMemory); return suggestedParameters; } @@ -239,6 +256,12 @@ private void suggestExecutorMemoryCore() { suggestedExecutorMemory = lastRunExecutorMemory; suggestedCore = lastRunExecutorCore; } + if (getMemoryIncreaseForGC() != 0) { + suggestedExecutorMemory = suggestedExecutorMemory * (100 + getMemoryIncreaseForGC()) / 100; + } + if (suggestedExecutorMemory < MIN_EXECUTOR_MEMORY) { + suggestedExecutorMemory = MIN_EXECUTOR_MEMORY; + } suggestedExecutorMemory = getRoundedExecutorMemory(); } @@ -310,6 +333,9 @@ private void suggestMemoryFactor() { */ private void suggestDriverMemory() { suggestedDriverMemory = driverMaxPeakJVMUsedMemory * (100 + DRIVER_MEMORY_BUFFER) / 100; + if (suggestedDriverMemory < MIN_DRIVER_MEMORY) { + suggestedDriverMemory = MIN_DRIVER_MEMORY; + } suggestedDriverMemory = getRoundedDriverMemory(); } @@ -323,4 +349,13 @@ private long getRoundedContainerSize(long memory) { return ((long) Math.ceil(memory * 1.0 / FileUtils.ONE_GB)) * FileUtils.ONE_GB; } + private int getMemoryIncreaseForGC() { + if (gcRunTimeRatio > ExecutorGcHeuristic.DEFAULT_GC_SEVERITY_A_THRESHOLDS().moderate().floatValue()) { + return GC_MEMORY_INCREASE; + } else if (gcRunTimeRatio < ExecutorGcHeuristic.DEFAULT_GC_SEVERITY_D_THRESHOLDS().moderate().floatValue()) { + return GC_MEMORY_DECREASE; + } else { + return 0; + } + } } From f126f9a47820be27149892f0758c0cc1ebec6191 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Tue, 16 Oct 2018 11:33:42 +0530 Subject: [PATCH 17/22] Documentation/Javadoc added --- app/com/linkedin/drelephant/AutoTuner.java | 8 +- .../heuristics/ConfigurationHeuristic.java | 8 +- .../tuning/AbstractBaselineManager.java | 15 +-- .../tuning/AbstractJobStatusManager.java | 33 +++++- .../AbstractParameterGenerateManager.java | 46 ++++---- .../tuning/AutoTuningAPIHelper.java | 106 ++++++++++++------ .../drelephant/tuning/AutoTuningFlow.java | 8 +- .../linkedin/drelephant/tuning/Constant.java | 5 + .../linkedin/drelephant/tuning/Manager.java | 18 ++- 9 files changed, 167 insertions(+), 80 deletions(-) diff --git a/app/com/linkedin/drelephant/AutoTuner.java b/app/com/linkedin/drelephant/AutoTuner.java index 9442e8feb..3d4b2f6b8 100644 --- a/app/com/linkedin/drelephant/AutoTuner.java +++ b/app/com/linkedin/drelephant/AutoTuner.java @@ -39,10 +39,12 @@ public class AutoTuner implements Runnable { private static final Logger logger = Logger.getLogger(AutoTuner.class); private static final long DEFAULT_METRICS_COMPUTATION_INTERVAL = ONE_MIN / 5; public static final String AUTO_TUNING_DAEMON_WAIT_INTERVAL = "autotuning.daemon.wait.interval.ms"; - private static final String tuningTypes[] = {"OBT"}; - public void run() { + /** + * It will start auto tuning thread . + */ + public void run() { logger.info("Starting Auto Tuning thread"); HDFSContext.load(); Configuration configuration = ElephantContext.instance().getAutoTuningConf(); @@ -64,6 +66,4 @@ public void run() { } logger.info("Auto tuning thread shutting down"); } - - } diff --git a/app/com/linkedin/drelephant/mapreduce/heuristics/ConfigurationHeuristic.java b/app/com/linkedin/drelephant/mapreduce/heuristics/ConfigurationHeuristic.java index 5a10a8dd7..866cc6a88 100644 --- a/app/com/linkedin/drelephant/mapreduce/heuristics/ConfigurationHeuristic.java +++ b/app/com/linkedin/drelephant/mapreduce/heuristics/ConfigurationHeuristic.java @@ -21,14 +21,12 @@ import com.linkedin.drelephant.analysis.Severity; import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData; import com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData; - import static com.linkedin.drelephant.mapreduce.heuristics.CommonConstantsHeuristic.ParameterKeys.*; - import org.apache.log4j.Logger; - -/* -Heuristics to collect memory data/counter values about application previous exeution. +/** + * This heuristics is to collect memory data / properties / counter values about jobs + * previous execution */ public class ConfigurationHeuristic implements Heuristic { private static final Logger logger = Logger.getLogger(ConfigurationHeuristic.class); diff --git a/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java b/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java index 7277ad7ee..264312fe5 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractBaselineManager.java @@ -43,12 +43,12 @@ public AbstractBaselineManager() { configuration = ElephantContext.instance().getAutoTuningConf(); } - /* - Execute the whole logic for Base line Computing - 1) Get the jobs for which Baseline have to be calculated - 2) Calculate the baseline - 3) Update the base line - 4) Update the metrics . + /** + * 1) Get the jobs for which Baseline have to be calculated + * 2) Calculate the baseline + * 3) Update the base line + * 4) Update the metrics + * @return true if method executes successfully . Otherwise false */ @Override public final boolean execute() { @@ -81,14 +81,12 @@ public final boolean execute() { * This is done by returning the jobs with null average resource usage * @return List of jobs whose baseline needs to be added */ - protected abstract List detectJobsForBaseLineComputation(); /** * Adds baseline metric values for a job * @param tuningJobDefinitions Job for which baseline is to be computed */ - protected boolean calculateBaseLine(List tuningJobDefinitions) { for (TuningJobDefinition tuningJobDefinition : tuningJobDefinitions) { try { @@ -156,7 +154,6 @@ protected boolean updateDataBase(List tuningJobDefinitions) * This method update metrics for auto tuning monitoring for baseline computation * @param tuningJobDefinitions */ - protected boolean updateMetrics(List tuningJobDefinitions) { try { int baselineComputeWaitJobs = 0; diff --git a/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java b/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java index 4b24fa4a3..57a77d30a 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractJobStatusManager.java @@ -23,6 +23,10 @@ public abstract class AbstractJobStatusManager implements Manager { protected abstract boolean analyzeCompletedJobsExecution( List inProgressExecutionParamSet); + /** + * This method detect jobs whose execution state in job execution table is in in progress + * @return List of tuning jobs whose status is in progress. + */ protected List detectJobsExecutionInProgress() { logger.info("Fetching the executions which are in progress"); List tuningJobExecutionParamSets = @@ -37,6 +41,12 @@ protected List detectJobsExecutionInProgress() { return tuningJobExecutionParamSets; } + /** + * + * @param jobs : It takes jobs whose state is changes from execution to completed and also paramset w + * whose status is changed to executed. + * @return : If the database update successfully then it return true other false + */ protected boolean updateDataBase(List jobs) { try { for (TuningJobExecutionParamSet job : jobs) { @@ -50,13 +60,18 @@ protected boolean updateDataBase(List jobs) { logger.info("Execution " + jobExecution.jobExecId + " is still in running state"); } } - }catch(Exception e){ - logger.info ( "Exception occur while updating database " + e.getMessage()); + } catch (Exception e) { + logger.info("Exception occur while updating database " + e.getMessage()); return false; } return true; } + /** + * + * @param jobExecution : Take jobExecution as input + * @return true if the job execution is not in progress. + */ private Boolean isJobCompleted(JobExecution jobExecution) { if (jobExecution.executionState.equals(JobExecution.ExecutionState.SUCCEEDED) || jobExecution.executionState.equals( JobExecution.ExecutionState.FAILED) || jobExecution.executionState.equals( @@ -67,6 +82,11 @@ private Boolean isJobCompleted(JobExecution jobExecution) { } } + /** + * + * @param completedJobs : Take jobs whose execution is completed. Update ingraph metrics + * @return true if update is successfull else false. + */ protected boolean updateMetrics(List completedJobs) { try { for (TuningJobExecutionParamSet completedJob : completedJobs) { @@ -77,13 +97,20 @@ protected boolean updateMetrics(List completedJobs) AutoTuningMetricsController.markFailedJobs(); } } - }catch (Exception e){ + } catch (Exception e) { logger.info(" Exception while updating Metrics to ingraph " + e.getMessage()); return false; } return true; } + /** + * 1) Get the jobs for which Status need to be queried from Azkaban + * 2) Queried the Azkaban and get current status + * 3) Update the data base if the execution is completed + * 4) Update the metrics + * @return true if method executes successfully . Otherwise false + */ @Override public final boolean execute() { logger.info("Executing Job Status Manager"); diff --git a/app/com/linkedin/drelephant/tuning/AbstractParameterGenerateManager.java b/app/com/linkedin/drelephant/tuning/AbstractParameterGenerateManager.java index ee3aca1fd..1cf185578 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractParameterGenerateManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractParameterGenerateManager.java @@ -38,7 +38,6 @@ public abstract class AbstractParameterGenerateManager implements Manager { * @param tuningJobDefinitions * @return */ - protected abstract boolean updateDatabase(List tuningJobDefinitions); /** @@ -52,29 +51,20 @@ public abstract class AbstractParameterGenerateManager implements Manager { * generation have to done * @return */ - protected abstract List getTuningJobDefinitions(); /** - * If the algorithm requires to save previous state ,then this method should be implemented + * If the algorithm requires to save previous state ,then this method should be implemented. * @param jobTuningInfo * @param job */ - protected abstract void saveJobState(JobTuningInfo jobTuningInfo, JobDefinition job); /** - * - * @param tuningParameterList - * @param job - */ - - /** - * Update the boundry constraint based on the previous executions + * Update the boundry constraint based on the previous executions. Used in IPSO. * @param tuningParameterList * @param job */ - protected abstract void updateBoundryConstraint(List tuningParameterList, JobDefinition job); /** @@ -88,7 +78,6 @@ public abstract class AbstractParameterGenerateManager implements Manager { * Fetches the list to job which need new parameter suggestion * @return Job list */ - protected List detectJobsForParameterGeneration() { List jobsForSwarmSuggestion = getJobsForParamSuggestion(); List jobTuningInfoList = getJobsTuningInfo(jobsForSwarmSuggestion); @@ -129,6 +118,11 @@ private List getJobsForParamSuggestion() { return jobsForParamSuggestion; } + /** + * Generate parameters for the job + * @param jobsForParameterSuggestion + * @return Update jobTuningInfo with the suggested parameters . + */ protected List generateParameters(List jobsForParameterSuggestion) { List updatedJobTuningInfoList = new ArrayList(); for (JobTuningInfo jobTuningInfo : jobsForParameterSuggestion) { @@ -138,6 +132,13 @@ protected List generateParameters(List jobsForPara return updatedJobTuningInfoList; } + /** + * Execute the whole logic for parameter generations + * 1) Get the jobs for which parameters need to be generated + * 2) Generate the parameters + * 3) Update the database + * @return true if the parameters are generated successfully else false. + */ public final boolean execute() { try { logger.info("Executing Tuning Algorithm"); @@ -152,7 +153,7 @@ public final boolean execute() { logger.info("Param Generation Done"); return databaseUpdateDone; } catch (Exception e) { - logger.info("Exception in generating parameters ", e); + logger.warn("Exception in generating parameters ", e); return false; } } @@ -163,27 +164,29 @@ public final boolean execute() { * @return Tuning information list */ protected List getJobsTuningInfo(List tuningJobs) { - List jobTuningInfoList = + List jobTuningInfoList = new ArrayList(); for (TuningJobDefinition tuningJobDefinition : tuningJobs) { JobDefinition job = tuningJobDefinition.job; List tuningParameterList = generateTuningParameterListWithDefaultValues(tuningJobDefinition); // updating boundary constraints for the job updateBoundryConstraint(tuningParameterList, job); - - com.linkedin.drelephant.tuning.JobTuningInfo jobTuningInfo = new com.linkedin.drelephant.tuning.JobTuningInfo(); + JobTuningInfo jobTuningInfo = new JobTuningInfo(); jobTuningInfo.setTuningJob(job); jobTuningInfo.setJobType(tuningJobDefinition.tuningAlgorithm.jobType); jobTuningInfo.setParametersToTune(tuningParameterList); - saveJobState(jobTuningInfo, job); - logger.info("Adding JobTuningInfo " + Json.toJson(jobTuningInfo)); jobTuningInfoList.add(jobTuningInfo); } return jobTuningInfoList; } + /** + * Assign default values to tuning parameters . Then later on the boundaries are updated + * @param tuningJobDefinition + * @return + */ protected List generateTuningParameterListWithDefaultValues(TuningJobDefinition tuningJobDefinition) { JobDefinition job = tuningJobDefinition.job; logger.info("Getting tuning information for job: " + job.jobDefId); @@ -195,6 +198,11 @@ protected List generateTuningParameterListWithDefaultValues(Tun return tuningParameterList; } + /** + * Assign default values to the parameters + * @param defaultJobParamSet + * @param tuningParameterList + */ protected void assignDefaultValues(JobSuggestedParamSet defaultJobParamSet, List tuningParameterList) { if (defaultJobParamSet != null) { logger.info("Fetching default parameter values for job "); diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java index dd1f17544..b1d55041f 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java @@ -128,15 +128,6 @@ private void setMaxAllowedMetricIncreasePercentage(TuningInput tuningInput) { } } - /** - * Sets the tuning algorithm based on the job type and optimization metric - * @param tuningInput TuningInput for which tuning algorithm is to be set - *//* - private void setTuningAlgorithm(TuningInput tuningInput,TuningJobDefinition tuningJobDefinition) throws IllegalArgumentException { - TuningAlgorithm tuningAlgorithm = tuningJobDefinition.tuningAlgorithm; - tuningInput.setTuningAlgorithm(tuningAlgorithm); - }*/ - /** * Applies penalty to the param set corresponding to the given execution * @param jobExecId String job execution id/url of the execution whose parameter set has to be penalized @@ -167,7 +158,6 @@ private void applyPenalty(String jobExecId) { jobSuggestedParamSet.paramSetState = ParamSetStatus.FITNESS_COMPUTED; jobSuggestedParamSet.fitnessJobExecution = jobExecution; jobSuggestedParamSet.update(); - jobExecution.resourceUsage = 0D; jobExecution.executionTime = 0D; jobExecution.inputSizeInBytes = 1D; @@ -233,7 +223,6 @@ private TuningJobDefinition addNewJobForTuning(TuningInput tuningInput) { job.flowDefinition = flowDefinition; job.save(); } - String client = tuningInput.getClient(); Map defaultParams = null; try { @@ -250,21 +239,28 @@ private TuningJobDefinition addNewJobForTuning(TuningInput tuningInput) { tuningJobDefinition.allowedMaxExecutionTimePercent = tuningInput.getAllowedMaxExecutionTimePercent(); tuningJobDefinition.allowedMaxResourceUsagePercent = tuningInput.getAllowedMaxResourceUsagePercent(); tuningJobDefinition.save(); - insertParamSet(job, tuningInput.getTuningAlgorithm(), defaultParams); - logger.info("Added job: " + tuningInput.getJobDefId() + " for tuning"); return tuningJobDefinition; } + /** + * This method will be used to get tuning algorithm for the first execution of the job. + * Currently for backward compatibility the tuning algorithm is Azkaban version depended. + * (See Version in Application.java) + * . + * But in future HBT ,will be the by default tuning algorithm for first time. + * .eq(TuningAlgorithm.TABLE.optimizationAlgo, "HBT") + * @param tuningInput + * @return + */ private TuningAlgorithm getTuningAlgorithmForfirstTime(TuningInput tuningInput) { - logger.info("Version "+tuningInput.getVersion() + "\t" + tuningInput.getOptimizationAlgo()); + logger.info("Version " + tuningInput.getVersion() + "\t" + tuningInput.getOptimizationAlgo()); TuningAlgorithm tuningAlgorithm = TuningAlgorithm.find.select("*") .where() .eq(TuningAlgorithm.TABLE.jobType, tuningInput.getJobType()) .eq(TuningAlgorithm.TABLE.optimizationMetric, "RESOURCE") - // .eq(TuningAlgorithm.TABLE.optimizationAlgo, "HBT") - .eq(TuningAlgorithm.TABLE.optimizationAlgo,tuningInput.getOptimizationAlgo()) + .eq(TuningAlgorithm.TABLE.optimizationAlgo, tuningInput.getOptimizationAlgo()) .findUnique(); if (tuningAlgorithm == null) { throw new IllegalArgumentException("Wrong job type " + tuningInput.getJobType()); @@ -307,7 +303,6 @@ private JobDefinition getJobDefinition(TuningInput tuningInput) { private JobExecution addNewExecution(TuningInput tuningInput) { JobDefinition jobDefinition = getJobDefinition(tuningInput); FlowExecution flowExecution = getFlowExecution(tuningInput); - JobExecution jobExecution = new JobExecution(); jobExecution.jobExecId = tuningInput.getJobExecId(); jobExecution.jobExecUrl = tuningInput.getJobExecUrl(); @@ -324,7 +319,6 @@ private JobExecution addNewExecution(TuningInput tuningInput) { * @return JobExecution corresponding to the given tuning input */ private JobExecution getJobExecution(TuningInput tuningInput) { - JobExecution jobExecution = JobExecution.find.select("*") .fetch(JobExecution.TABLE.job, "*") .where() @@ -343,13 +337,11 @@ private JobExecution getJobExecution(TuningInput tuningInput) { * @return Parameter Suggestion */ public Map getCurrentRunParameters(TuningInput tuningInput) throws Exception { - /* try {*/ logger.info("Parameter set request received from execution: " + tuningInput.getJobExecId()); if (tuningInput.getAllowedMaxExecutionTimePercent() == null || tuningInput.getAllowedMaxResourceUsagePercent() == null) { setMaxAllowedMetricIncreasePercentage(tuningInput); } - String jobDefId = tuningInput.getJobDefId(); TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") .fetch(TuningJobDefinition.TABLE.job, "*") @@ -358,9 +350,7 @@ public Map getCurrentRunParameters(TuningInput tuningInput) thro .setMaxRows(1) .orderBy(TuningJobDefinition.TABLE.createdTs + " desc") .findUnique(); - JobExecution jobExecution = getJobExecution(tuningInput); - //setTuningAlgorithm(tuningInput,tuningJobDefinition); if (tuningJobDefinition == null) { tuningInput.setTuningAlgorithm(getTuningAlgorithmForfirstTime(tuningInput)); logger.info(" Tuning Algorithm Type " + tuningInput.getTuningAlgorithm().optimizationAlgo.name()); @@ -372,14 +362,14 @@ public Map getCurrentRunParameters(TuningInput tuningInput) thro initializeOptimizationAlgoPrerequisite(tuningInput); return processForSubsequentExecutions(tuningJobDefinition, tuningInput, jobExecution); } - } /*catch (Exception e) { - logger.info(" Exception occured " + e.getMessage()); - logger.info(" Running Default parameters "); - return new HashMap(); - }*/ - -//} + } + /** + * This method will be called for the first execution of the job + * @param tuningInput + * @param jobExecution + * @return Map of property name and value + */ private Map processForFirstExecution(TuningInput tuningInput, JobExecution jobExecution) { logger.info( " New Job. Hence Not checking for AutoTuning . Running with default Parameters " + tuningInput.getJobExecId()); @@ -393,6 +383,16 @@ private Map processForFirstExecution(TuningInput tuningInput, Jo return jobSuggestedParamValueListToMap(jobSuggestedParamValues); } + /** + * This method will be called for all the next execution of the job (except the first one). Execution path would + * be different for first execution and subsequent execution . Since user cannot enable auto apply before first execution + * . If user doesn't enable after seeing in Dr elephant , then suggested parameters will not be applied instead default + * parameters will be applied. + * @param tuningJobDefinition + * @param tuningInput + * @param jobExecution + * @return + */ private Map processForSubsequentExecutions(TuningJobDefinition tuningJobDefinition, TuningInput tuningInput, JobExecution jobExecution) { logger.info(" Not a New Job . Hence check for autoTuning" + tuningInput.getJobExecId()); @@ -406,6 +406,12 @@ private Map processForSubsequentExecutions(TuningJobDefinition t } } + /** + * If auto apply is enabled , then get the suggested parameters . Retry if the job is failed . + * @param tuningInput + * @param jobExecution + * @return + */ private List processParameterTuningEnabled(TuningInput tuningInput, JobExecution jobExecution) { JobSuggestedParamSet jobSuggestedParamSet; @@ -427,6 +433,12 @@ private List processParameterTuningEnabled(TuningInput t return jobSuggestedParamValues; } + /** + * Since Auto Tuning disabled , no parameter suggestion . + * @param tuningInput + * @param jobExecution + * @return + */ private Map sendDefaultParameters(TuningInput tuningInput, JobExecution jobExecution) { JobSuggestedParamSet jobSuggestedParamSet; logger.info(" Auto Tuning Disabled . Hence no parameter suggestion. Tagging execution with default values"); @@ -438,6 +450,10 @@ private Map sendDefaultParameters(TuningInput tuningInput, JobEx return new HashMap(); } + /** + * Update the data base with default values of the parameters + * @param tuningInput + */ public void updateDataBaseWithDefaultValues(TuningInput tuningInput) { JobDefinition job = JobDefinition.find.select("*").where().eq(JobDefinition.TABLE.jobDefId, tuningInput.getJobDefId()).findUnique(); @@ -451,6 +467,11 @@ public void updateDataBaseWithDefaultValues(TuningInput tuningInput) { insertParamSetForDefault(job, tuningInput.getTuningAlgorithm(), defaultParams); } + /** + * Get the default parameters which will be send . + * @param jobDefinition + * @return + */ public JobSuggestedParamSet getDefaultParameters(JobDefinition jobDefinition) { JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") @@ -464,6 +485,11 @@ public JobSuggestedParamSet getDefaultParameters(JobDefinition jobDefinition) { return jobSuggestedParamSet; } + /** + * If auto apply is disabled then , we have to discard all the generated parameters + * and send default values to the execution. + * @param jobDefinition + */ public void discardGeneratedParameter(JobDefinition jobDefinition) { List jobSuggestedParamSets = JobSuggestedParamSet.find.select("*") .fetch(JobSuggestedParamSet.TABLE.jobDefinition, "*") @@ -471,7 +497,6 @@ public void discardGeneratedParameter(JobDefinition jobDefinition) { .eq(JobSuggestedParamSet.TABLE.jobDefinition + "." + JobDefinition.TABLE.id, jobDefinition.id) .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED) .findList(); - if (jobSuggestedParamSets != null && jobSuggestedParamSets.size() >= 1) { logger.info(" Discarding Generated Parameters , as auto tuning off." + jobSuggestedParamSets.size()); for (JobSuggestedParamSet jobSuggestedParamSet : jobSuggestedParamSets) { @@ -481,6 +506,12 @@ public void discardGeneratedParameter(JobDefinition jobDefinition) { } } + /** + * Inserted data in JobSuggestedParamSet . + * @param job + * @param tuningAlgorithm + * @param paramValueMap + */ private void insertParamSetForDefault(JobDefinition job, TuningAlgorithm tuningAlgorithm, Map paramValueMap) { logger.debug("Inserting default parameter set for job: " + job.jobName); @@ -549,6 +580,12 @@ private JobSuggestedParamSet getNewSuggestedParamSet(JobDefinition jobDefinition return jobSuggestedParamSet; } + /** + * If the algorithm type is changed from HBT to OBT or vice versa , + * then discard all the parameters which was generated by previous algorithm type. + * @param jobDefinition + * @param tuningAlgorithm + */ private void makeOtherAlgoParamGeneratedDiscarded(JobDefinition jobDefinition, TuningAlgorithm tuningAlgorithm) { logger.info("Making other tuning type suggested as discarded , if tuning type is changed"); List jobSuggestedParamSets = JobSuggestedParamSet.find.select("*") @@ -558,9 +595,9 @@ private void makeOtherAlgoParamGeneratedDiscarded(JobDefinition jobDefinition, T .eq(JobSuggestedParamSet.TABLE.paramSetState, ParamSetStatus.CREATED) .not(Expr.eq(JobSuggestedParamSet.TABLE.tuningAlgorithm, tuningAlgorithm)) .findList(); - if(jobSuggestedParamSets != null){ + if (jobSuggestedParamSets != null) { logger.info(" other algorithm parameter created " + jobSuggestedParamSets.size()); - for(JobSuggestedParamSet jobSuggestedParamSet : jobSuggestedParamSets){ + for (JobSuggestedParamSet jobSuggestedParamSet : jobSuggestedParamSets) { jobSuggestedParamSet.paramSetState = ParamSetStatus.DISCARDED; jobSuggestedParamSet.save(); } @@ -634,6 +671,11 @@ private void insertParameterValues(JobSuggestedParamSet jobSuggestedParamSet, Ma } } + /** + * This is specifically used in IPSO ,where tuning parameter constraint + * table is populated with default boundaries of the parameters . + * @param tuningInput + */ private void initializeOptimizationAlgoPrerequisite(TuningInput tuningInput) { String jobDefId = tuningInput.getJobDefId(); TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*") diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningFlow.java b/app/com/linkedin/drelephant/tuning/AutoTuningFlow.java index 0791b4110..669c52047 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningFlow.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningFlow.java @@ -47,7 +47,7 @@ public void run() { } } - public void createBaseLineManagersPipeline() { + private void createBaseLineManagersPipeline() { List baselineManagers = new ArrayList(); for (Constant.TuningType tuningType : Constant.TuningType.values()) { @@ -57,14 +57,14 @@ public void createBaseLineManagersPipeline() { this.pipelines.add(baselineManagers); } - public void createJobStatusManagersPipeline() { + private void createJobStatusManagersPipeline() { List jobStatusManagers = new ArrayList(); jobStatusManagers.add(ManagerFactory.getManager(null, null, null, AbstractJobStatusManager.class.getSimpleName())); //jobStatusManagers.add(new JobStatusManagerOBT()); this.pipelines.add(jobStatusManagers); } - public void createFitnessManagersPipeline() { + private void createFitnessManagersPipeline() { List fitnessManagers = new ArrayList(); for (Constant.TuningType tuningType : Constant.TuningType.values()) { for (Constant.AlgotihmType algotihmType : Constant.AlgotihmType.values()) { @@ -80,7 +80,7 @@ public void createFitnessManagersPipeline() { this.pipelines.add(fitnessManagers); } - public void createTuningTypeManagersPipeline() { + private void createTuningTypeManagersPipeline() { List algorithmManagers = new ArrayList(); for (Constant.TuningType tuningType : Constant.TuningType.values()) { for (Constant.AlgotihmType algotihmType : Constant.AlgotihmType.values()) { diff --git a/app/com/linkedin/drelephant/tuning/Constant.java b/app/com/linkedin/drelephant/tuning/Constant.java index 72e267142..48bd64110 100644 --- a/app/com/linkedin/drelephant/tuning/Constant.java +++ b/app/com/linkedin/drelephant/tuning/Constant.java @@ -4,6 +4,11 @@ import java.util.Map; +/** + * Constants describe different TuningType , Algorithm Type + * Execution Engine and managers . + * These are used in AutoTuningFlow . + */ public class Constant { public enum TuningType {HBT,OBT} public enum AlgotihmType{PSO,PSO_IPSO,HBT} diff --git a/app/com/linkedin/drelephant/tuning/Manager.java b/app/com/linkedin/drelephant/tuning/Manager.java index c910965fe..c8ce2adde 100644 --- a/app/com/linkedin/drelephant/tuning/Manager.java +++ b/app/com/linkedin/drelephant/tuning/Manager.java @@ -1,12 +1,22 @@ package com.linkedin.drelephant.tuning; +/** + * This is the top most managers . All the implementation + * of managers should implement this interface and expose execute method. + * Execute method will be the core of the manager and will do the bulk of the work . + * Execute will be exposed as public method and will be used to execute the functionality + * of the respective manager + */ public interface Manager { - /* - Use to execute the logic of all the managers . + /** + * Use to execute the logic of all the managers . + * @return */ boolean execute(); - /* - Manager Name + + /** + * Used to get the manager name + * @return */ String getManagerName(); From 4ae3c0a57318587aaa0269ab6f98e87e041c14f1 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Tue, 16 Oct 2018 16:35:04 +0530 Subject: [PATCH 18/22] Handling scenario where no param generator implementation for algorith type --- .../tuning/hbt/ParameterGenerateManagerHBT.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java b/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java index b509849a1..a4616c6cd 100644 --- a/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java +++ b/app/com/linkedin/drelephant/tuning/hbt/ParameterGenerateManagerHBT.java @@ -87,17 +87,19 @@ private String generateParamSet(List tuningParameters, JobDefin } List results = getAppResults(jobExecution); - if (results == null || results.size()==0 ) { + if (results == null || results.size() == 0) { logger.debug( " Job is analyzing , cannot use for param generation " + jobExecution.id + " " + jobExecution.job.id); return ""; } String idParameters = null; - try - { - idParameters=this._executionEngine.parameterGenerationsHBT(results, tuningParameters); - }catch(Exception e) - { + try { + idParameters = this._executionEngine.parameterGenerationsHBT(results, tuningParameters); + if (idParameters == null) { + logger.error(" id parameters are null "); + idParameters = ""; + } + } catch (Exception e) { logger.error("Exception in getting specific parameters ", e); } return idParameters.toString(); @@ -188,8 +190,6 @@ protected boolean updateDatabase(List jobTuningInfoList) { return true; } - - private void penaltyApplication(JobSuggestedParamSet jobSuggestedParamSet, TuningJobDefinition tuningJobDefinition) { logger.debug("Parameter constraint violated. Applying penalty."); int penaltyConstant = 4; From 62c87348f7507af104aa17cd6c69158ed25ec302 Mon Sep 17 00:00:00 2001 From: "Manoj Kumar(Bangalore Engineering)" Date: Tue, 16 Oct 2018 16:59:56 +0530 Subject: [PATCH 19/22] Removing spark heuristics specific changes to separate commit --- .../spark/heuristics/DriverHeuristic.scala | 9 +- .../heuristics/ExecutorGcHeuristic.scala | 3 +- .../heuristics/JvmUsedMemoryHeuristic.scala | 4 +- .../heuristics/UnifiedMemoryHeuristic.scala | 3 +- .../tuning/engine/SparkExecutionEngine.java | 13 - .../engine/SparkHBTParamRecommender.java | 361 ------------------ 6 files changed, 6 insertions(+), 387 deletions(-) delete mode 100644 app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java diff --git a/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala index 0510dcef8..8a1b71cc5 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/DriverHeuristic.scala @@ -23,9 +23,8 @@ import scala.util.Try import com.linkedin.drelephant.analysis._ import com.linkedin.drelephant.configurations.heuristic.HeuristicConfigurationData import com.linkedin.drelephant.spark.data.SparkApplicationData -import com.linkedin.drelephant.util.MemoryFormatUtils +import com.linkedin.drelephant.util.{MemoryFormatUtils, Utils} import com.linkedin.drelephant.spark.fetchers.statusapiv1.ExecutorSummary -import com.linkedin.drelephant.util.Utils /** * A heuristic based the driver's configurations and memory used. @@ -74,11 +73,11 @@ class DriverHeuristic(private val heuristicConfigurationData: HeuristicConfigura SPARK_YARN_DRIVER_MEMORY_OVERHEAD, evaluator.sparkYarnDriverMemoryOverhead ), - new HeuristicResultDetails(DRIVER_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME, MemoryFormatUtils.bytesToString(evaluator.maxDriverPeakJvmUsedMemory)) + new HeuristicResultDetails("Max driver peak JVM used memory", MemoryFormatUtils.bytesToString(evaluator.maxDriverPeakJvmUsedMemory)) ) if(evaluator.severityJvmUsedMemory != Severity.NONE) { resultDetails = resultDetails :+ new HeuristicResultDetails("Driver Peak JVM used Memory", "The allocated memory for the driver (in " + SPARK_DRIVER_MEMORY_KEY + ") is much more than the peak JVM used memory by the driver.") - resultDetails = resultDetails :+ new HeuristicResultDetails(SUGGESTED_SPARK_DRIVER_MEMORY_HEURISTIC_NAME, MemoryFormatUtils.roundOffMemoryStringToNextInteger(MemoryFormatUtils.bytesToString(((1 + BUFFER_FRACTION) * (evaluator.maxDriverPeakJvmUsedMemory + reservedMemory)).toLong))) + resultDetails = resultDetails :+ new HeuristicResultDetails("Suggested spark.driver.memory", MemoryFormatUtils.roundOffMemoryStringToNextInteger(MemoryFormatUtils.bytesToString(((1 + BUFFER_FRACTION) * (evaluator.maxDriverPeakJvmUsedMemory + reservedMemory)).toLong))) } if (evaluator.severityGc != Severity.NONE) { resultDetails = resultDetails :+ new HeuristicResultDetails("Gc ratio high", "The driver is spending too much time on GC. We recommend increasing the driver memory.") @@ -116,8 +115,6 @@ object DriverHeuristic { val EXECUTION_MEMORY = "executionMemory" val STORAGE_MEMORY = "storageMemory" val JVM_USED_MEMORY = "jvmUsedMemory" - val DRIVER_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME="Max driver peak JVM used memory" - val SUGGESTED_SPARK_DRIVER_MEMORY_HEURISTIC_NAME="Suggested spark.driver.memory" val BUFFER_FRACTION = 0.2 // 300 * FileUtils.ONE_MB (300 * 1024 * 1024) diff --git a/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala index c544ac152..da48c351b 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/ExecutorGcHeuristic.scala @@ -48,7 +48,7 @@ class ExecutorGcHeuristic(private val heuristicConfigurationData: HeuristicConfi override def apply(data: SparkApplicationData): HeuristicResult = { val evaluator = new Evaluator(this, data) var resultDetails = Seq( - new HeuristicResultDetails(GC_RUN_TIME_RATIO, evaluator.ratio.toString), + new HeuristicResultDetails("GC time to Executor Run time ratio", evaluator.ratio.toString), new HeuristicResultDetails("Total GC time", evaluator.msecToString(evaluator.jvmTime)), new HeuristicResultDetails("Total Executor Runtime", evaluator.msecToString(evaluator.executorRunTimeTotal)) ) @@ -77,7 +77,6 @@ object ExecutorGcHeuristic { val SPARK_EXECUTOR_MEMORY = "spark.executor.memory" val SPARK_EXECUTOR_CORES = "spark.executor.cores" val EXECUTOR_RUNTIME_THRESHOLD_IN_MINUTES = 5 - val GC_RUN_TIME_RATIO="GC time to Executor Run time ratio"; /** The ascending severity thresholds for the ratio of JVM GC Time and executor Run Time (checking whether ratio is above normal) * These thresholds are experimental and are likely to change */ diff --git a/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala index 3c9202838..929f8af50 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/JvmUsedMemoryHeuristic.scala @@ -44,7 +44,7 @@ class JvmUsedMemoryHeuristic(private val heuristicConfigurationData: HeuristicCo val evaluator = new Evaluator(this, data) var resultDetails = Seq( - new HeuristicResultDetails(MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME, MemoryFormatUtils.bytesToString(evaluator.maxExecutorPeakJvmUsedMemory)), + new HeuristicResultDetails("Max executor peak JVM used memory", MemoryFormatUtils.bytesToString(evaluator.maxExecutorPeakJvmUsedMemory)), new HeuristicResultDetails("spark.executor.memory", MemoryFormatUtils.bytesToString(evaluator.sparkExecutorMemory)) ) @@ -73,8 +73,6 @@ object JvmUsedMemoryHeuristic { val reservedMemory: Long = 314572800 val BUFFER_FRACTION: Double = 0.2 val MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_THRESHOLD_KEY = "executor_peak_jvm_memory_threshold" - val MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME = "Max executor peak JVM used memory" - lazy val DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD = "2G" class Evaluator(jvmUsedMemoryHeuristic: JvmUsedMemoryHeuristic, data: SparkApplicationData) { diff --git a/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala b/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala index 9a16d8bf4..80dfba54f 100644 --- a/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala +++ b/app/com/linkedin/drelephant/spark/heuristics/UnifiedMemoryHeuristic.scala @@ -47,7 +47,7 @@ class UnifiedMemoryHeuristic(private val heuristicConfigurationData: HeuristicCo var resultDetails = Seq( new HeuristicResultDetails("Unified Memory Space Allocated", MemoryFormatUtils.bytesToString(evaluator.maxMemory)), new HeuristicResultDetails("Mean peak unified memory", MemoryFormatUtils.bytesToString(evaluator.meanUnifiedMemory)), - new HeuristicResultDetails(MAX_PEAK_UNIFIED_MEMORY_HEURISTIC_NAME, MemoryFormatUtils.bytesToString(evaluator.maxUnifiedMemory)), + new HeuristicResultDetails("Max peak unified memory", MemoryFormatUtils.bytesToString(evaluator.maxUnifiedMemory)), new HeuristicResultDetails("spark.executor.memory", MemoryFormatUtils.bytesToString(evaluator.sparkExecutorMemory)), new HeuristicResultDetails("spark.memory.fraction", evaluator.sparkMemoryFraction.toString) ) @@ -74,7 +74,6 @@ object UnifiedMemoryHeuristic { val DEFAULT_SPARK_EXECUTOR_MEMORY_THRESHOLD = "2G" val UNIFIED_MEMORY_ALLOCATED_THRESHOLD = "256M" val SPARK_MEMORY_FRACTION_THRESHOLD : Double = 0.05 - val MAX_PEAK_UNIFIED_MEMORY_HEURISTIC_NAME="Max peak unified memory"; class Evaluator(unifiedMemoryHeuristic: UnifiedMemoryHeuristic, data: SparkApplicationData) { lazy val appConfigurationProperties: Map[String, String] = diff --git a/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java b/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java index 2d6c44c54..f607d1cbb 100644 --- a/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java +++ b/app/com/linkedin/drelephant/tuning/engine/SparkExecutionEngine.java @@ -74,19 +74,6 @@ public void parameterOptimizerIPSO(List results, JobExecution jobExec @Override public String parameterGenerationsHBT(List results, List tuningParameters) { - if (results != null && results.size() > 0) { - SparkHBTParamRecommender sparkHBTParamRecommender = new SparkHBTParamRecommender(results.get(0)); - HashMap suggestedParameters = sparkHBTParamRecommender.getHBTSuggestion(); - StringBuffer idParameters = new StringBuffer(); - for (TuningParameter tuningParameter : tuningParameters) { - if (suggestedParameters.containsKey(tuningParameter.paramName)) { - idParameters.append(tuningParameter.id).append("\t") - .append(suggestedParameters.get(tuningParameter.paramName)); - idParameters.append("\n"); - } - } - return idParameters.toString(); - } return null; } diff --git a/app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java b/app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java deleted file mode 100644 index eb42cb322..000000000 --- a/app/com/linkedin/drelephant/tuning/engine/SparkHBTParamRecommender.java +++ /dev/null @@ -1,361 +0,0 @@ -package com.linkedin.drelephant.tuning.engine; - -import java.util.HashMap; -import java.util.Map; - -import models.AppResult; - -import org.apache.commons.io.FileUtils; -import org.apache.log4j.Logger; - -import com.linkedin.drelephant.spark.heuristics.ConfigurationHeuristic; -import com.linkedin.drelephant.spark.heuristics.DriverHeuristic; -import com.linkedin.drelephant.spark.heuristics.ExecutorGcHeuristic; -import com.linkedin.drelephant.spark.heuristics.JvmUsedMemoryHeuristic; -import com.linkedin.drelephant.spark.heuristics.UnifiedMemoryHeuristic; -import com.linkedin.drelephant.util.MemoryFormatUtils; - - -/** - * This class recommends spark parameters based on previous run heuristics. For spark it expects one - */ -public class SparkHBTParamRecommender { - private final Logger logger = Logger.getLogger(SparkHBTParamRecommender.class); - - AppResult appResult; - - // TODos Move these to configuration - public static final int MAX_EXECUTOR_CORE = 3; - public static final long MAX_EXECUTOR_MEMORY = 10 * FileUtils.ONE_GB; - public static final long MIN_EXECUTOR_MEMORY = 900 * FileUtils.ONE_MB; - public static final long MIN_DRIVER_MEMORY = 900 * FileUtils.ONE_MB; - - public static final int CLUSTER_DEFAULT_EXECUTOR_CORE = 1; - public static final long RESERVED_MEMORY = 300 * FileUtils.ONE_MB; - - private static final long EXECUTOR_MEMORY_BUFFER_PER_CORE = 0; - private static final long EXECUTOR_MEMORY_BUFFER_OVERALL = 0; - private static final long DRIVER_MEMORY_BUFFER = 0; - - private static final int GC_MEMORY_INCREASE = 5; - private static final int GC_MEMORY_DECREASE = -5; - - private long maxPeakUnifiedMemory; - private long maxPeakJVMUsedMemory; - private long driverMaxPeakJVMUsedMemory; - private long suggestedSparkDriverMemory; - - private long lastRunExecutorMemory; - private int lastRunExecutorCore; - private long lastRunExecutorMemoryOverhead; - private long lastRunDriverMemoryOverhead; - private Float gcRunTimeRatio; - private Long suggestedExecutorMemory; - private Integer suggestedCore; - private Double suggestedMemoryFactor; - private Long suggestedDriverMemory; - - public SparkHBTParamRecommender(AppResult appResult) { - this.appResult = appResult; - Map appHeuristicsResultDetailsMap = appResult.getHeuristicsResultDetailsMap(); - - maxPeakUnifiedMemory = - MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(UnifiedMemoryHeuristic.class - .getCanonicalName() + "_" + UnifiedMemoryHeuristic.MAX_PEAK_UNIFIED_MEMORY_HEURISTIC_NAME())); - - maxPeakJVMUsedMemory = - MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(JvmUsedMemoryHeuristic.class - .getCanonicalName() + "_" + JvmUsedMemoryHeuristic.MAX_EXECUTOR_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME())); - - lastRunExecutorMemoryOverhead = - MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(ConfigurationHeuristic.class - .getCanonicalName() + "_" + ConfigurationHeuristic.SPARK_YARN_EXECUTOR_MEMORY_OVERHEAD())); - - String coreConfigStr = - appHeuristicsResultDetailsMap.get(ConfigurationHeuristic.class.getCanonicalName() + "_" - + ConfigurationHeuristic.SPARK_EXECUTOR_CORES_KEY()); - - lastRunExecutorCore = CLUSTER_DEFAULT_EXECUTOR_CORE; - try { - lastRunExecutorCore = Integer.parseInt(coreConfigStr); - } catch (NumberFormatException e) { - // Do Nothing - } - gcRunTimeRatio = - Float.parseFloat(appHeuristicsResultDetailsMap.get(ExecutorGcHeuristic.class.getCanonicalName() + "_" - + ExecutorGcHeuristic.GC_RUN_TIME_RATIO())); - - driverMaxPeakJVMUsedMemory = - MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(DriverHeuristic.class.getCanonicalName() - + "_" + DriverHeuristic.DRIVER_PEAK_JVM_USED_MEMORY_HEURISTIC_NAME())); - - lastRunDriverMemoryOverhead = - MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(DriverHeuristic.class.getCanonicalName() - + "_" + DriverHeuristic.SPARK_YARN_DRIVER_MEMORY_OVERHEAD())); - - suggestedSparkDriverMemory = - MemoryFormatUtils.stringToBytes(appHeuristicsResultDetailsMap.get(DriverHeuristic.class.getCanonicalName() - + "_" + DriverHeuristic.SUGGESTED_SPARK_DRIVER_MEMORY_HEURISTIC_NAME())); - - logger.info("Following are the heuristics values for last run : "); - logger.info("maxPeakUnifiedMemory: " + maxPeakUnifiedMemory); - logger.info("maxPeakJVMUsedMemory: " + maxPeakJVMUsedMemory); - logger.info("lastRunExecutorCore: " + lastRunExecutorCore); - logger.info("driverMaxPeakJVMUsedMemory: " + driverMaxPeakJVMUsedMemory); - logger.info("suggestedSparkDriverMemory: " + suggestedSparkDriverMemory); - - } - - public long getLastRunExecutorMemoryOverhead() { - return lastRunExecutorMemoryOverhead; - } - - public void setLastRunExecutorMemoryOverhead(long lastRunExecutorMemoryOverhead) { - this.lastRunExecutorMemoryOverhead = lastRunExecutorMemoryOverhead; - } - - public Long getSuggestedExecutorMemory() { - return suggestedExecutorMemory; - } - - public void setSuggestedExecutorMemory(Long suggestedExecutorMemory) { - this.suggestedExecutorMemory = suggestedExecutorMemory; - } - - public Integer getSuggestedCore() { - return suggestedCore; - } - - public void setSuggestedCore(Integer suggestedCore) { - this.suggestedCore = suggestedCore; - } - - public Double getSuggestedMemoryFactor() { - return suggestedMemoryFactor; - } - - public void setSuggestedMemoryFactor(Double suggestedMemoryFactor) { - this.suggestedMemoryFactor = suggestedMemoryFactor; - } - - public Long getSuggestedDriverMemory() { - return suggestedDriverMemory; - } - - public void setSuggestedDriverMemory(Long suggestedDriverMemory) { - this.suggestedDriverMemory = suggestedDriverMemory; - } - - public long getMaxPeakUnifiedMemory() { - return maxPeakUnifiedMemory; - } - - public void setMaxPeakUnifiedMemory(long maxPeakUnifiedMemory) { - this.maxPeakUnifiedMemory = maxPeakUnifiedMemory; - } - - public long getMaxPeakJVMUsedMemory() { - return maxPeakJVMUsedMemory; - } - - public void setMaxPeakJVMUsedMemory(long maxPeakJVMUsedMemory) { - this.maxPeakJVMUsedMemory = maxPeakJVMUsedMemory; - } - - public long getLastRunExecutorMemory() { - return lastRunExecutorMemory; - } - - public void setLastRunExecutorMemory(long lastRunExecutorMemory) { - this.lastRunExecutorMemory = lastRunExecutorMemory; - } - - public int getLastRunExecutorCore() { - return lastRunExecutorCore; - } - - public void setLastRunExecutorCore(int lastRunExecutorCore) { - this.lastRunExecutorCore = lastRunExecutorCore; - } - - public long getDriverMaxPeakJVMUsedMemory() { - return driverMaxPeakJVMUsedMemory; - } - - public void setDriverMaxPeakJVMUsedMemory(long driverMaxPeakJVMUsedMemory) { - this.driverMaxPeakJVMUsedMemory = driverMaxPeakJVMUsedMemory; - } - - public long getSuggestedSparkDriverMemory() { - return suggestedSparkDriverMemory; - } - - public void setSuggestedSparkDriverMemory(long suggestedSparkDriverMemory) { - this.suggestedSparkDriverMemory = suggestedSparkDriverMemory; - } - - /** - * This method returns suggested parameter for the job. Currently it runs following parameters: - * executor memory, core, driver memory and memory factor - * - * @return HashMap with property name as key and double value - */ - public HashMap getHBTSuggestion() { - HashMap suggestedParameters = new HashMap(); - try { - suggestExecutorMemoryCore(); - suggestMemoryFactor(); - suggestDriverMemory(); - suggestedParameters.put(SparkConfigurationConstants.SPARK_EXECUTOR_MEMORY_KEY, (double) suggestedExecutorMemory - / FileUtils.ONE_MB); - suggestedParameters.put(SparkConfigurationConstants.SPARK_EXECUTOR_CORES_KEY, suggestedCore.doubleValue()); - if (suggestedMemoryFactor < UnifiedMemoryHeuristic.SPARK_MEMORY_FRACTION_THRESHOLD()) { - suggestedMemoryFactor = UnifiedMemoryHeuristic.SPARK_MEMORY_FRACTION_THRESHOLD(); - } - suggestedParameters.put(SparkConfigurationConstants.SPARK_MEMORY_FRACTION_KEY, suggestedMemoryFactor); - suggestedParameters.put(SparkConfigurationConstants.SPARK_DRIVER_MEMORY_KEY, (double) suggestedDriverMemory - / FileUtils.ONE_MB); - logger.info("Following are the suggestions for spark parameters for app id : " + appResult.flowExecId); - logger.info("suggestedExecutorMemory " + suggestedExecutorMemory); - logger.info("suggestedCore " + suggestedCore); - logger.info("suggestedMemoryFactor " + suggestedMemoryFactor); - logger.info("suggestedDriverMemory " + suggestedDriverMemory); - } catch (Exception e) { - logger.error("Error in generating parameters ", e); - } - return suggestedParameters; - } - - private Long getMaxPeakJVMUsedMemoryPerCore() { - return maxPeakJVMUsedMemory / lastRunExecutorCore; - } - - private Long getMaxPeakUnifiedMemoryPerCore() { - return maxPeakUnifiedMemory / lastRunExecutorCore; - } - - /** - * Suggest executor memory and core We calculate per core peak JVM used memory. Then we start from - * max executor core and compute memory requirement for max executor core If this is below - * threshold of max executor core then good, otherwise reduce the core and check memory - * requirement. - */ - private void suggestExecutorMemoryCore() { - boolean validSuggestion = false; - long maxPeakJVMUsedMemoryPerCore = getMaxPeakJVMUsedMemoryPerCore(); - for (int core = MAX_EXECUTOR_CORE; core > 0; core--) { - long currSuggestedMemory = suggestExecutorMemory(maxPeakJVMUsedMemoryPerCore, core); - if (currSuggestedMemory < MAX_EXECUTOR_MEMORY || core == 1) { - suggestedExecutorMemory = currSuggestedMemory; - suggestedCore = core; - validSuggestion = true; - break; - } - } - if (!validSuggestion) { - suggestedExecutorMemory = lastRunExecutorMemory; - suggestedCore = lastRunExecutorCore; - } - if (getMemoryIncreaseForGC() != 0) { - suggestedExecutorMemory = suggestedExecutorMemory * (100 + getMemoryIncreaseForGC()) / 100; - } - if (suggestedExecutorMemory < MIN_EXECUTOR_MEMORY) { - suggestedExecutorMemory = MIN_EXECUTOR_MEMORY; - } - suggestedExecutorMemory = getRoundedExecutorMemory(); - } - - /** - * This method is to get the rounded executor memory. We don't want to round executor memory to - * multiple of 1GB as container size is decided by executor memory + memoryOverhead. So round the - * executor memory + memoryOverhead and then distribute extra memory to executor memory - * - * @return rounded executor memory - */ - private long getRoundedExecutorMemory() { - long memoryOverhead = lastRunExecutorMemoryOverhead; - if (lastRunExecutorMemoryOverhead == 0) { - memoryOverhead = Math.max(suggestedExecutorMemory / 10, 384); - } - - long roundedContainerSize = getRoundedContainerSize(suggestedExecutorMemory + memoryOverhead + FileUtils.ONE_MB); - long executorMemory = - (long) (suggestedExecutorMemory + (roundedContainerSize - suggestedExecutorMemory - memoryOverhead - FileUtils.ONE_MB) * 0.9); - - return executorMemory; - } - - /** - * This method is to get the rounded executor memory. We don't want to round executor memory to - * multiple of 1GB as container size is decided by executor memory + memoryOverhead. So round the - * executor memory + memoryOverhead and then distribute extra memory to executor memory - * - * @return rounded driver memory - */ - private long getRoundedDriverMemory() { - long memoryOverhead = lastRunDriverMemoryOverhead; - if (lastRunDriverMemoryOverhead == 0) { - memoryOverhead = Math.max(suggestedDriverMemory / 10, 384); - } - - long roundedContainerSize = getRoundedContainerSize(suggestedDriverMemory + memoryOverhead + FileUtils.ONE_MB); - long driverMemory = - (long) (suggestedDriverMemory + (roundedContainerSize - suggestedDriverMemory - memoryOverhead - FileUtils.ONE_MB) * 0.9); - - return driverMemory; - } - - /** - * Suggest executor memory based on peak JVM used memory per core and number of core after adding - * buffer - * - * @param maxPeakJVMUsedMemoryPerCore - * @param core - * @return suggested executor memory - */ - private long suggestExecutorMemory(long maxPeakJVMUsedMemoryPerCore, int core) { - return (maxPeakJVMUsedMemoryPerCore * (100 + EXECUTOR_MEMORY_BUFFER_PER_CORE) / 100) * core - * (100 + EXECUTOR_MEMORY_BUFFER_OVERALL) / 100; - } - - /** - * Suggest memory factor using peak unified memory per core and number of core after adding buffer - */ - private void suggestMemoryFactor() { - long unifiedMemoryRequirement = - (getMaxPeakUnifiedMemoryPerCore() * (100 + EXECUTOR_MEMORY_BUFFER_PER_CORE) / 100) * suggestedCore - * (100 + EXECUTOR_MEMORY_BUFFER_OVERALL) / 100; - suggestedMemoryFactor = unifiedMemoryRequirement * 1.0 / suggestedExecutorMemory; - } - - /** - * Suggest driver memory based on driver peak JVM used memory - */ - private void suggestDriverMemory() { - suggestedDriverMemory = driverMaxPeakJVMUsedMemory * (100 + DRIVER_MEMORY_BUFFER) / 100; - if (suggestedDriverMemory < MIN_DRIVER_MEMORY) { - suggestedDriverMemory = MIN_DRIVER_MEMORY; - } - suggestedDriverMemory = getRoundedDriverMemory(); - } - - /** - * Rounded container size in multiple of 1GB - * - * @param memory - * @return rounded container size - */ - private long getRoundedContainerSize(long memory) { - return ((long) Math.ceil(memory * 1.0 / FileUtils.ONE_GB)) * FileUtils.ONE_GB; - } - - private int getMemoryIncreaseForGC() { - if (gcRunTimeRatio > ExecutorGcHeuristic.DEFAULT_GC_SEVERITY_A_THRESHOLDS().moderate().floatValue()) { - return GC_MEMORY_INCREASE; - } else if (gcRunTimeRatio < ExecutorGcHeuristic.DEFAULT_GC_SEVERITY_D_THRESHOLDS().moderate().floatValue()) { - return GC_MEMORY_DECREASE; - } else { - return 0; - } - } -} From 2eaee219749a6c509a9411c2f0b39c9e16113e5e Mon Sep 17 00:00:00 2001 From: "Manoj Kumar(Bangalore Engineering)" Date: Tue, 16 Oct 2018 17:09:23 +0530 Subject: [PATCH 20/22] Removing spark heuristics specific changes to separate commit --- app/controllers/Application.java | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/Application.java b/app/controllers/Application.java index 1fe3a8910..507888c07 100644 --- a/app/controllers/Application.java +++ b/app/controllers/Application.java @@ -81,7 +81,6 @@ import com.linkedin.drelephant.tuning.AutoTuningAPIHelper; import com.linkedin.drelephant.tuning.TuningInput; import com.linkedin.drelephant.tuning.engine.SparkConfigurationConstants; -import com.linkedin.drelephant.tuning.engine.SparkHBTParamRecommender; public class Application extends Controller { From e523afe4289b45ccdf20b43d677a26858b1ce0b0 Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Tue, 16 Oct 2018 17:30:29 +0530 Subject: [PATCH 21/22] Changed Boolean to boolean --- app/com/linkedin/drelephant/tuning/AutoTuningFlow.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningFlow.java b/app/com/linkedin/drelephant/tuning/AutoTuningFlow.java index 669c52047..cb30b0fae 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningFlow.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningFlow.java @@ -37,7 +37,7 @@ public void executeFlow() throws InterruptedException { public void run() { for (Manager manager : pipelineType) { logger.info(" Starting Manager " + manager.getManagerName()); - Boolean execute = manager.execute(); + boolean execute = manager.execute(); logger.info(" Ending Manager " + execute + " " + manager.getManagerName()); } } From 2eb5e7eefff697f3ea6761375c1265716560cf2d Mon Sep 17 00:00:00 2001 From: Pralabh Kumar Date: Wed, 17 Oct 2018 10:56:34 +0530 Subject: [PATCH 22/22] Made fitness calculation, tuning type specific --- .../tuning/AbstractFitnessManager.java | 28 +------------- .../tuning/obt/FitnessManagerOBT.java | 38 ++++++++++++++++--- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java index 0c524edf0..1c5201712 100644 --- a/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java +++ b/app/com/linkedin/drelephant/tuning/AbstractFitnessManager.java @@ -194,32 +194,8 @@ protected void resetParamSetToCreated(JobSuggestedParamSet jobSuggestedParamSet) * @param jobSuggestedParamSet param set which is to be updated * @param tuningJobDefinition TuningJobDefinition of the job to which param set corresponds */ - protected void updateJobSuggestedParamSetSucceededExecution(JobExecution jobExecution, - JobSuggestedParamSet jobSuggestedParamSet, TuningJobDefinition tuningJobDefinition) { - int penaltyConstant = 3; - Double averageResourceUsagePerGBInput = - tuningJobDefinition.averageResourceUsage * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; - Double maxDesiredResourceUsagePerGBInput = - averageResourceUsagePerGBInput * tuningJobDefinition.allowedMaxResourceUsagePercent / 100.0; - Double averageExecutionTimePerGBInput = - tuningJobDefinition.averageExecutionTime * FileUtils.ONE_GB / tuningJobDefinition.averageInputSizeInBytes; - Double maxDesiredExecutionTimePerGBInput = - averageExecutionTimePerGBInput * tuningJobDefinition.allowedMaxExecutionTimePercent / 100.0; - Double resourceUsagePerGBInput = jobExecution.resourceUsage * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; - Double executionTimePerGBInput = jobExecution.executionTime * FileUtils.ONE_GB / jobExecution.inputSizeInBytes; - - if (resourceUsagePerGBInput > maxDesiredResourceUsagePerGBInput - || executionTimePerGBInput > maxDesiredExecutionTimePerGBInput) { - logger.debug("Execution " + jobExecution.jobExecId + " violates constraint on resource usage per GB input"); - jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; - } else { - jobSuggestedParamSet.fitness = resourceUsagePerGBInput; - } - jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED; - jobSuggestedParamSet.fitnessJobExecution = jobExecution; - jobSuggestedParamSet = updateBestJobSuggestedParamSet(jobSuggestedParamSet); - jobSuggestedParamSet.update(); - } + protected abstract void updateJobSuggestedParamSetSucceededExecution(JobExecution jobExecution, + JobSuggestedParamSet jobSuggestedParamSet, TuningJobDefinition tuningJobDefinition); /** * Updates the given job suggested param set to be the best param set if its fitness is less than the current best param set diff --git a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java index d80c6c78f..06ec060cf 100644 --- a/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java +++ b/app/com/linkedin/drelephant/tuning/obt/FitnessManagerOBT.java @@ -86,11 +86,39 @@ protected void calculateAndUpdateFitness(JobExecution jobExecution, List maxDesiredResourceUsagePerGBInput + || executionTimePerGBInput > maxDesiredExecutionTimePerGBInput) { + logger.debug("Execution " + jobExecution.jobExecId + " violates constraint on resource usage per GB input"); + jobSuggestedParamSet.fitness = penaltyConstant * maxDesiredResourceUsagePerGBInput; + } else { + jobSuggestedParamSet.fitness = resourceUsagePerGBInput; + } + jobSuggestedParamSet.paramSetState = JobSuggestedParamSet.ParamSetStatus.FITNESS_COMPUTED; + jobSuggestedParamSet.fitnessJobExecution = jobExecution; + jobSuggestedParamSet = updateBestJobSuggestedParamSet(jobSuggestedParamSet); + jobSuggestedParamSet.update(); + } /** * Checks and disables tuning for the given job definitions.