Skip to content

Return empty hashmap when parameters to be returned are default parameters. #628

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dushyantk1509 wants to merge 2 commits into
base: tuning_20190221
Choose a base branch
from
Open

Return empty hashmap when parameters to be returned are default parameters. #628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c6563f4
Sending empty parameter hashmap if parameters to be set are default p…
dushyantk1509 Aug 30, 2019
f1f0192
Some minor changes in test sql file
dushyantk1509 Sep 3, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 15 additions & 19 deletions app/com/linkedin/drelephant/tuning/AutoTuningAPIHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,9 +451,19 @@ private List<JobSuggestedParamValue> processParameterTuningEnabled(TuningInput t
markParameterSetSent(jobSuggestedParamSet);
}
addNewTuningJobExecutionParamSet(jobSuggestedParamSet, jobExecution);
List<JobSuggestedParamValue> jobSuggestedParamValues = getParamSetValues(jobSuggestedParamSet.id);
logger.debug("Number of output parameters for execution " + tuningInput.getJobExecId() + " = "
+ jobSuggestedParamValues.size());
List<JobSuggestedParamValue> jobSuggestedParamValues = null;
if (jobSuggestedParamSet.isParamSetSuggested) {
jobSuggestedParamValues = getParamSetValues(jobSuggestedParamSet.id);
if(debugEnabled){
logger.debug("Number of output parameters for execution " + tuningInput.getJobExecId() + " = "
+ jobSuggestedParamValues.size());
}
} else {
if(debugEnabled){
logger.debug("Sending empty parameter hashmap for execution " + tuningInput.getJobExecId() +
" as parameters to be sent are default parameters");
}
}
logger.info("Finishing getCurrentRunParameters");
return jobSuggestedParamValues;
}
Expand Down Expand Up @@ -634,30 +644,16 @@ private void makeOtherAlgoParamGeneratedDiscarded(JobDefinition jobDefinition, T
* @param jobSuggestedParamValues List of JobSuggestedParamValue
* @return Map of string to double containing the parameter name and corresponding value
*/
@VisibleForTesting
Map<String, Double> jobSuggestedParamValueListToMap(List<JobSuggestedParamValue> jobSuggestedParamValues) {
private Map<String, Double> jobSuggestedParamValueListToMap(List<JobSuggestedParamValue> jobSuggestedParamValues) {
Map<String, Double> paramValues = new HashMap<String, Double>();
if ((jobSuggestedParamValues != null) && jobSuggestedParamValues.size() > 0) {
/**
* If param values are not suggested values then
* return empty hashmap
*/
JobSuggestedParamValue jobSuggestedParamValueFirst = jobSuggestedParamValues.get(0);
JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*")
.where()
.eq(JobSuggestedParamSet.TABLE.id, jobSuggestedParamValueFirst.jobSuggestedParamSet.id)
.setMaxRows(1)
.findUnique();

if(jobSuggestedParamSet.isParamSetSuggested){
if (jobSuggestedParamValues != null) {
for (JobSuggestedParamValue jobSuggestedParamValue : jobSuggestedParamValues) {
if (debugEnabled) {
logger.debug("Param Name is " + jobSuggestedParamValue.tuningParameter.paramName + " And value is " + jobSuggestedParamValue.paramValue);
}
paramValues.put(jobSuggestedParamValue.tuningParameter.paramName, jobSuggestedParamValue.paramValue);
}
}
}
return paramValues;
}

Expand Down
5 changes: 3 additions & 2 deletions test/com/linkedin/drelephant/tuning/AlertingTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import static common.DBTestUtil.*;

import models.TuningJobDefinition;
import models.JobDefinition;
import org.apache.hadoop.conf.Configuration;
import static com.linkedin.drelephant.tuning.alerting.Constant.*;

Expand Down Expand Up @@ -47,7 +48,7 @@ private void testDeveloperAlerting() {



JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*").where().findUnique();
JobSuggestedParamSet jobSuggestedParamSet = JobSuggestedParamSet.find.select("*").where().eq(JobSuggestedParamSet.TABLE.id, "1137").findUnique();
jobSuggestedParamSet.updatedTs = new Timestamp(startTime + 100);
jobSuggestedParamSet.createdTs = new Timestamp(endTime+1-259200000);
jobSuggestedParamSet.update();
Expand Down Expand Up @@ -85,7 +86,7 @@ private void testSKAlerting(){

long startTime = System.currentTimeMillis();
long endTime = System.currentTimeMillis() + 1000;
TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*").where().findUnique();
TuningJobDefinition tuningJobDefinition = TuningJobDefinition.find.select("*").where().eq(TuningJobDefinition.TABLE.job + "." + JobDefinition.TABLE.id, 100003).findUnique();
tuningJobDefinition.updatedTs = new Timestamp(startTime + 100);
tuningJobDefinition.tuningEnabled=false;
tuningJobDefinition.autoApply=true;
Expand Down
41 changes: 35 additions & 6 deletions test/com/linkedin/drelephant/tuning/AutoTunerApiTestRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import models.FlowExecution;
import models.JobSuggestedParamSet;
import models.JobSuggestedParamValue;
import models.TuningAlgorithm;
import org.junit.Test;

import static org.junit.Assert.*;
Expand Down Expand Up @@ -69,16 +70,44 @@ public void run() {
List<FlowExecution> flowExecution = FlowExecution.find.where().eq(FlowExecution.TABLE.flowExecId, 1).findList();
assertTrue(" Flow Execution ", flowExecution.size()==1);
populateTestData();
testJobSuggestedParamValueListToMap();
testGetcurrentRunParameter();
}

private void testJobSuggestedParamValueListToMap(){
List<JobSuggestedParamValue> jobSuggestedParamValues = JobSuggestedParamValue.find.where()
.eq(JobSuggestedParamValue.TABLE.jobSuggestedParamSet + '.' + JobSuggestedParamSet.TABLE.id, 1137)
.findList();
private void testGetcurrentRunParameter(){
TuningInput tuningInput = new TuningInput();
tuningInput.setFlowDefId("https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow");
tuningInput.setJobDefId("https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry");
tuningInput.setFlowDefUrl("https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow");
tuningInput.setJobDefUrl("https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry");
tuningInput.setFlowExecId("https://elephant.linkedin.com:8443/executor?execid=5416293");
tuningInput.setJobExecId("https://elephant.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0");
tuningInput.setFlowExecUrl("https://elephant.linkedin.com:8443/executor?execid=5416293");
tuningInput.setJobExecUrl("https://elephant.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0");
tuningInput.setJobName("countByCountryFlow_countByCountry");
// tuningInput.setUserName("dukumar");
tuningInput.setClient("azkaban");
tuningInput.setScheduler("azkaban");
// tuningInput.setDefaultParams(defaultParams);
tuningInput.setVersion(1);
tuningInput.setRetry(false);
// tuningInput.setSkipExecutionForOptimization(skipExecutionForOptimization);
tuningInput.setJobType("PIG");
tuningInput.setOptimizationAlgo("HBT");
tuningInput.setOptimizationAlgoVersion("4");
tuningInput.setOptimizationMetric("RESOURCE");
tuningInput.setAllowedMaxExecutionTimePercent(null);
tuningInput.setAllowedMaxResourceUsagePercent(null);

AutoTuningAPIHelper autoTuningAPIHelper = new AutoTuningAPIHelper();
Map<String, Double> paramValues = autoTuningAPIHelper.jobSuggestedParamValueListToMap(jobSuggestedParamValues);
/**
* Testing for empty hashMap if parameters to be sent are default parameters
*/
Map<String, Double> paramValues = null;
try {
paramValues = autoTuningAPIHelper.getCurrentRunParameters(tuningInput);
} catch (Exception e) {
e.printStackTrace();
}
assertTrue("Param values : " + paramValues, paramValues.isEmpty());
}
}
32 changes: 17 additions & 15 deletions test/resources/test-param-generate-data.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,27 +142,29 @@ insert into yarn_app_heuristic_result_details (yarn_app_heuristic_result_id,name
(137594640,'Number of tasks','20','NULL');

INSERT INTO flow_definition(id, flow_def_id, flow_def_url) VALUES
(10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow');
(10003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow'),
(10004,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow');

INSERT INTO job_definition(id, job_def_id, flow_definition_id, job_name, job_def_url, scheduler, username, created_ts, updated_ts) VALUES
(100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','pkumar2','2018-02-12 08:40:42','2018-02-12 08:40:43');

(100003,'https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10003,'countByCountryFlow_countByCountry','https://ltx1-holdemaz01.grid.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','pkumar2','2018-02-12 08:40:42','2018-02-12 08:40:43'),
(100004,'https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry',10004,'countByCountryFlow_countByCountry','https://elephant.linkedin.com:8443/manager?project=AzkabanHelloPigTest&flow=countByCountryFlow&job=countByCountryFlow_countByCountry','azkaban','dukumar','2018-02-12 08:40:42','2018-02-12 08:40:43');

INSERT INTO tuning_job_definition(job_definition_id, client, tuning_algorithm_id, tuning_enabled, average_resource_usage, average_execution_time, average_input_size_in_bytes, allowed_max_resource_usage_percent, allowed_max_execution_time_percent, created_ts, updated_ts, tuning_disabled_reason, number_of_iterations, auto_apply)
VALUES
(100003,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true);



(100003,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true),
(100004,'azkaban',4,1,null,5.178423333333334,324168876088,150,150,'2018-02-12 08:40:42','2018-02-12 08:40:43', NULL, 5, true);

INSERT INTO flow_execution(id, flow_exec_id, flow_exec_url, flow_definition_id) VALUES
(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003);
(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293',10003),
(1542,'https://elephant.linkedin.com:8443/executor?execid=5416293','https://elephant.linkedin.com:8443/executor?execid=5416293',10004);

INSERT INTO job_execution(id, job_exec_id, job_exec_url, job_definition_id, flow_execution_id, execution_state, resource_usage, execution_time, input_size_in_bytes, created_ts, updated_ts) VALUES
(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100003,1541,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42');
(1541,'https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://ltx1-holdemaz01.grid.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100004,1541,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42'),
(1542,'https://elephant.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0','https://elephant.linkedin.com:8443/executor?execid=5416293&job=countByCountryFlow_countByCountry&attempt=0',100004,1542,'SUCCEEDED',21.132545572916666,3.2694833333333335,324713861757,'2018-02-14 05:30:42','2018-02-14 05:30:42');

INSERT INTO job_suggested_param_set VALUES
(1137,100003,4,'FITNESS_COMPUTED', 0 ,1 , 0 , 1 , 0 ,10000 , 1086,'2018-09-17 23:22:31' ,'2018-09-17 11:09:31');
(1137,100003,4,'FITNESS_COMPUTED', 0 ,1 , 0 , 1 , 0 ,10000 , 1086,'2018-09-17 23:22:31' ,'2018-09-17 11:09:31'),
(1138,100004,4,'FITNESS_COMPUTED', 0 ,1 , 0 , 1 , 0 ,10000 , 1542,'2018-09-17 23:22:31' ,'2018-09-17 11:09:31');

INSERT INTO tuning_job_execution_param_set (job_suggested_param_set_id,job_execution_id,tuning_enabled,created_ts,updated_ts) VALUES
(1137,1541,1,'2018-09-17 23:22:31','2018-09-17 10:52:31');
Expand All @@ -171,8 +173,8 @@ INSERT INTO job_suggested_param_value VALUES
(366044, 1137, 21, 6112.888799667358, '2018-12-21 14:55:36', '2018-12-21 14:55:37'),
(366045, 1137, 22, 1854.7078742980957, '2018-12-21 14:55:36', '2018-12-21 14:55:37'),
(366046, 1137, 23, 3, '2018-12-21 14:55:36', '2018-12-21 14:55:37'),
(366047, 1137, 24, 0.27054230043558763, '2018-12-21 14:55:36', '2018-12-21 14:55:37');




(366047, 1137, 24, 0.27054230043558763, '2018-12-21 14:55:36', '2018-12-21 14:55:37'),
(366048, 1138, 21, 6112.888799667358, '2018-12-21 14:55:36', '2018-12-21 14:55:37'),
(366049, 1138, 22, 1854.7078742980957, '2018-12-21 14:55:36', '2018-12-21 14:55:37'),
(366050, 1138, 23, 3, '2018-12-21 14:55:36', '2018-12-21 14:55:37'),
(366051, 1138, 24, 0.27054230043558763, '2018-12-21 14:55:36', '2018-12-21 14:55:37');