diff --git a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java index 16319645b..1ad96c6b1 100644 --- a/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java +++ b/app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java @@ -451,9 +451,19 @@ private List processParameterTuningEnabled(TuningInput t markParameterSetSent(jobSuggestedParamSet); } addNewTuningJobExecutionParamSet(jobSuggestedParamSet, jobExecution); - List jobSuggestedParamValues = getParamSetValues(jobSuggestedParamSet.id); - logger.debug("Number of output parameters for execution " + tuningInput.getJobExecId() + " = " - + jobSuggestedParamValues.size()); + List jobSuggestedParamValues = null; + if (jobSuggestedParamSet.isParamSetSuggested) { + jobSuggestedParamValues = getParamSetValues(jobSuggestedParamSet.id); + if(debugEnabled){ + logger.debug("Number of output parameters for execution " + tuningInput.getJobExecId() + " = " + + jobSuggestedParamValues.size()); + } + } else { + if(debugEnabled){ + logger.debug("Sending empty parameter hashmap for execution " + tuningInput.getJobExecId() + + " as parameters to be sent are default parameters"); + } + } logger.info("Finishing getCurrentRunParameters"); return jobSuggestedParamValues; } @@ -634,22 +644,9 @@ private void makeOtherAlgoParamGeneratedDiscarded(JobDefinition jobDefinition, T * @param jobSuggestedParamValues List of JobSuggestedParamValue * @return Map of string to double containing the parameter name and corresponding value */ - @VisibleForTesting - Map jobSuggestedParamValueListToMap(List jobSuggestedParamValues) { + private Map jobSuggestedParamValueListToMap(List jobSuggestedParamValues) { Map paramValues = new HashMap(); - if ((jobSuggestedParamValues != null) && jobSuggestedParamValues.size() > 0) { - /** - * If param values are not suggested values then - * return empty hashmap - */ - JobSuggestedParamValue jobSuggestedParamValueFirst = jobSuggestedParamValues.get(0); - JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*") - .where() - .eq(JobSuggestedParamSet.TABLE.id, jobSuggestedParamValueFirst.jobSuggestedParamSet.id) - .setMaxRows(1) - .findUnique(); - - if(jobSuggestedParamSet.isParamSetSuggested){ + if (jobSuggestedParamValues != null) { for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) { if (debugEnabled) { logger.debug("Param Name is " + jobSuggestedParamValue.tuningParameter.paramName + " And value is " + jobSuggestedParamValue.paramValue); @@ -657,7 +654,6 @@ Map jobSuggestedParamValueListToMap(List paramValues.put(jobSuggestedParamValue.tuningParameter.paramName, jobSuggestedParamValue.paramValue); } } - } return paramValues; } diff --git a/test/com/linkedin/drelephant/tuning/AlertingTest.java b/test/com/linkedin/drelephant/tuning/AlertingTest.java index 985fcd0f8..6ea3d73ae 100644 --- a/test/com/linkedin/drelephant/tuning/AlertingTest.java +++ b/test/com/linkedin/drelephant/tuning/AlertingTest.java @@ -12,6 +12,7 @@ import static common.DBTestUtil.*; import models.TuningJobDefinition; +import models.JobDefinition; import org.apache.hadoop.conf.Configuration; import static com.linkedin.drelephant.tuning.alerting.Constant.*; @@ -47,7 +48,7 @@ private void testDeveloperAlerting() { - JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*").where().findUnique(); + JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*").where().eq(JobSuggestedParamSet.TABLE.id, "1137").findUnique(); jobSuggestedParamSet.updatedTs = new Timestamp(startTime + 100); jobSuggestedParamSet.createdTs = new Timestamp(endTime+1-259200000); jobSuggestedParamSet.update(); @@ -85,7 +86,7 @@ private void testSKAlerting(){ long startTime = System.currentTimeMillis(); long endTime = System.currentTimeMillis() + 1000; - TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*").where().findUnique(); + TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*").where().eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, 100003).findUnique(); tuningJobDefinition.updatedTs = new Timestamp(startTime + 100); tuningJobDefinition.tuningEnabled=false; tuningJobDefinition.autoApply=true; diff --git a/test/com/linkedin/drelephant/tuning/AutoTunerApiTestRunner.java b/test/com/linkedin/drelephant/tuning/AutoTunerApiTestRunner.java index 5aaeef911..ed7e0bd05 100644 --- a/test/com/linkedin/drelephant/tuning/AutoTunerApiTestRunner.java +++ b/test/com/linkedin/drelephant/tuning/AutoTunerApiTestRunner.java @@ -25,6 +25,7 @@ import models.FlowExecution; import models.JobSuggestedParamSet; import models.JobSuggestedParamValue; +import models.TuningAlgorithm; import org.junit.Test; import static org.junit.Assert.*; @@ -69,16 +70,44 @@ public void run() { List flowExecution = FlowExecution.find.where().eq(FlowExecution.TABLE.flowExecId, 1).findList(); assertTrue(" Flow Execution ", flowExecution.size()==1); populateTestData(); - testJobSuggestedParamValueListToMap(); + testGetcurrentRunParameter(); } - private void testJobSuggestedParamValueListToMap(){ - List jobSuggestedParamValues = JobSuggestedParamValue.find.where() - .eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, 1137) - .findList(); + private void testGetcurrentRunParameter(){ + TuningInput tuningInput = new TuningInput(); + tuningInput.setFlowDefId("https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow"); + tuningInput.setJobDefId("https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry"); + tuningInput.setFlowDefUrl("https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow"); + tuningInput.setJobDefUrl("https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry"); + tuningInput.setFlowExecId("https://elephant.linkedin.com:8443/executor?execid=5416293"); + tuningInput.setJobExecId("https://elephant.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0"); + tuningInput.setFlowExecUrl("https://elephant.linkedin.com:8443/executor?execid=5416293"); + tuningInput.setJobExecUrl("https://elephant.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0"); + tuningInput.setJobName("countByCountryFlow_countByCountry"); +// tuningInput.setUserName("dukumar"); + tuningInput.setClient("azkaban"); + tuningInput.setScheduler("azkaban"); +// tuningInput.setDefaultParams(defaultParams); + tuningInput.setVersion(1); + tuningInput.setRetry(false); +// tuningInput.setSkipExecutionForOptimization(skipExecutionForOptimization); + tuningInput.setJobType("PIG"); + tuningInput.setOptimizationAlgo("HBT"); + tuningInput.setOptimizationAlgoVersion("4"); + tuningInput.setOptimizationMetric("RESOURCE"); + tuningInput.setAllowedMaxExecutionTimePercent(null); + tuningInput.setAllowedMaxResourceUsagePercent(null); AutoTuningAPIHelper autoTuningAPIHelper = new AutoTuningAPIHelper(); - Map paramValues = autoTuningAPIHelper.jobSuggestedParamValueListToMap(jobSuggestedParamValues); + /** + * Testing for empty hashMap if parameters to be sent are default parameters + */ + Map paramValues = null; + try { + paramValues = autoTuningAPIHelper.getCurrentRunParameters(tuningInput); + } catch (Exception e) { + e.printStackTrace(); + } assertTrue("Param values : " + paramValues, paramValues.isEmpty()); } } diff --git a/test/resources/test-param-generate-data.sql b/test/resources/test-param-generate-data.sql index a83287e1f..e6658a4da 100644 --- a/test/resources/test-param-generate-data.sql +++ b/test/resources/test-param-generate-data.sql @@ -142,27 +142,29 @@ insert into yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name (137594640,'Number of tasks','20','NULL'); INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES -(10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'); +(10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'), +(10004,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'); INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES -(100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','pkumar2','2018-02-12 08:40:42','2018-02-12 08:40:43'); - +(100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','pkumar2','2018-02-12 08:40:42','2018-02-12 08:40:43'), +(100004,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10004,'countByCountryFlow_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','dukumar','2018-02-12 08:40:42','2018-02-12 08:40:43'); INSERT INTO tuning_job_definition(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason, number_of_iterations, auto_apply) VALUES -(100003,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true); - - - +(100003,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true), +(100004,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true); INSERT INTO flow_execution(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES -(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003); +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003), +(1542,'https://elephant.linkedin.com:8443/executor?execid=5416293','https://elephant.linkedin.com:8443/executor?execid=5416293',10004); INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES -(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1541,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'); +(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100004,1541,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'), +(1542,'https://elephant.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100004,1542,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'); INSERT INTO job_suggested_param_set VALUES -(1137,100003,4,'FITNESS_COMPUTED', 0 ,1 , 0 , 1 , 0 ,10000 , 1086,'2018-09-17 23:22:31' ,'2018-09-17 11:09:31'); +(1137,100003,4,'FITNESS_COMPUTED', 0 ,1 , 0 , 1 , 0 ,10000 , 1086,'2018-09-17 23:22:31' ,'2018-09-17 11:09:31'), +(1138,100004,4,'FITNESS_COMPUTED', 0 ,1 , 0 , 1 , 0 ,10000 , 1542,'2018-09-17 23:22:31' ,'2018-09-17 11:09:31'); INSERT INTO tuning_job_execution_param_set (job_suggested_param_set_id,job_execution_id,tuning_enabled,created_ts,updated_ts) VALUES (1137,1541,1,'2018-09-17 23:22:31','2018-09-17 10:52:31'); @@ -171,8 +173,8 @@ INSERT INTO job_suggested_param_value VALUES (366044, 1137, 21, 6112.888799667358, '2018-12-21 14:55:36', '2018-12-21 14:55:37'), (366045, 1137, 22, 1854.7078742980957, '2018-12-21 14:55:36', '2018-12-21 14:55:37'), (366046, 1137, 23, 3, '2018-12-21 14:55:36', '2018-12-21 14:55:37'), -(366047, 1137, 24, 0.27054230043558763, '2018-12-21 14:55:36', '2018-12-21 14:55:37'); - - - - +(366047, 1137, 24, 0.27054230043558763, '2018-12-21 14:55:36', '2018-12-21 14:55:37'), +(366048, 1138, 21, 6112.888799667358, '2018-12-21 14:55:36', '2018-12-21 14:55:37'), +(366049, 1138, 22, 1854.7078742980957, '2018-12-21 14:55:36', '2018-12-21 14:55:37'), +(366050, 1138, 23, 3, '2018-12-21 14:55:36', '2018-12-21 14:55:37'), +(366051, 1138, 24, 0.27054230043558763, '2018-12-21 14:55:36', '2018-12-21 14:55:37'); \ No newline at end of file