diff --git a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/mutationweight/CompareAvgNumberToMutate.kt b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/mutationweight/CompareAvgNumberToMutate.kt index 4846c1cb5f..d74dac72c1 100644 --- a/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/mutationweight/CompareAvgNumberToMutate.kt +++ b/core-tests/integration-tests/core-it/src/test/kotlin/org/evomaster/core/mutationweight/CompareAvgNumberToMutate.kt @@ -11,7 +11,7 @@ import org.evomaster.core.search.action.ActionFilter import org.evomaster.core.search.gene.Gene import org.evomaster.core.search.mutationweight.GeneWeightTestSchema import org.evomaster.core.search.service.AdaptiveParameterControl -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.search.service.mutator.MutationWeightControl /** diff --git a/core/src/main/kotlin/org/evomaster/core/BaseModule.kt b/core/src/main/kotlin/org/evomaster/core/BaseModule.kt index 90041731c3..372164494c 100644 --- a/core/src/main/kotlin/org/evomaster/core/BaseModule.kt +++ b/core/src/main/kotlin/org/evomaster/core/BaseModule.kt @@ -9,6 +9,9 @@ import org.evomaster.core.search.service.* import org.evomaster.core.search.service.monitor.SearchProcessMonitor import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.ArchiveGeneMutator +import org.evomaster.core.search.service.time.ExecutionPhaseController +import org.evomaster.core.search.service.time.SearchStatusUpdater +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.search.tracer.ArchiveMutationTrackService import org.evomaster.core.search.tracer.TrackService import org.evomaster.core.solver.SMTLibZ3DbConstraintSolver diff --git a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt index f3c4ac0581..382b612f21 100644 --- a/core/src/main/kotlin/org/evomaster/core/EMConfig.kt +++ b/core/src/main/kotlin/org/evomaster/core/EMConfig.kt @@ -2616,6 +2616,17 @@ class EMConfig { var minimize: Boolean = true + @Cfg("When the main fuzzing session is over, there are several following phases in which test cases can be created" + + " and evaluated (e.g., for minimization, flakiness and security checks)." + + " Each of these follow up phases takes some time, which is not included in 'maxTime'." + + " All those phases are time-bounded, based on the main search budget." + + " For example, with a default of 10% and main budget of 1 hour, then each phase will take" + + " at most 6 minutes each after the 1 hour search." + + " Phases will be preemptively stopped if they reach their timeouts.") + @PercentageAsProbability + var extraPhaseBudgetPercentage: Double = 0.10 + + @Deprecated("No longer in use, replaced by 'extraPhaseBudgetPercentage'") @Cfg("Maximum number of minutes that will be dedicated to the minimization phase." + " A negative number mean no timeout is considered." + " A value of 0 means minimization will be skipped, even if minimize=true.") diff --git a/core/src/main/kotlin/org/evomaster/core/Main.kt b/core/src/main/kotlin/org/evomaster/core/Main.kt index 2151e16e11..44dada2f39 100644 --- a/core/src/main/kotlin/org/evomaster/core/Main.kt +++ b/core/src/main/kotlin/org/evomaster/core/Main.kt @@ -43,6 +43,8 @@ import org.evomaster.core.search.algorithms.* import org.evomaster.core.search.service.* import org.evomaster.core.search.service.monitor.SearchProcessMonitor import org.evomaster.core.search.service.mutator.genemutation.ArchiveImpactSelector +import org.evomaster.core.search.service.time.ExecutionPhaseController +import org.evomaster.core.search.service.time.SearchTimeController import java.lang.reflect.InvocationTargetException import java.util.Locale import kotlin.system.exitProcess @@ -269,7 +271,7 @@ class Main { solution = phaseHttpOracle(injector, config, epc, solution) solution = phaseFlaky(injector, config, epc, solution) - epc.startWriteOutput() + epc.markStartingWriteOutput() val suites = writeTests(injector, solution, controllerInfo) writeWFCReport(injector, solution, suites) @@ -298,7 +300,7 @@ class Main { solution.statistics = data.toMutableList() - epc.finishSearch() + epc.markFinishedSession() return solution } @@ -423,7 +425,7 @@ class Main { return when (config.problemType) { EMConfig.ProblemType.REST -> { LoggingUtil.getInfoLogger().info("Starting to apply flaky detection") - epc.startFlakiness() + epc.markStartingFlakiness() val flakinessDetector = injector.getInstance(Key.get(object : TypeLiteral>() {})) flakinessDetector.reexecuteToDetectFlakiness() @@ -450,7 +452,7 @@ class Main { } //apply security testing phase LoggingUtil.getInfoLogger().info("Starting to apply security testing") - epc.startSecurity() + epc.markStartingSecurity() //TODO might need to reset stc, and print some updated info again @@ -477,7 +479,7 @@ class Main { return if (config.httpOracles && config.problemType == EMConfig.ProblemType.REST) { LoggingUtil.getInfoLogger().info("Starting to apply HTTP") - epc.startHttpOracles() + epc.markStartingAdditionalOracles() val httpSemanticsService = injector.getInstance(HttpSemanticsService::class.java) httpSemanticsService.applyHttpSemanticsPhase() @@ -830,7 +832,7 @@ class Main { val config = injector.getInstance(EMConfig::class.java) val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() if (!config.blackBox || config.bbExperiments) { val rc = injector.getInstance(RemoteController::class.java) diff --git a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt index 8fb86c49d6..3d80d3d5c7 100644 --- a/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt +++ b/core/src/main/kotlin/org/evomaster/core/output/service/TestSuiteWriter.kt @@ -23,7 +23,7 @@ import org.evomaster.core.problem.security.service.HttpCallbackVerifier import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.Solution import org.evomaster.core.search.service.Sampler -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.sql.schema.TableId import org.evomaster.test.utils.EMTestUtils import org.evomaster.test.utils.SeleniumEMUtils diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt index afd6baab20..b9713424e9 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/service/EnterpriseFitness.kt @@ -25,7 +25,7 @@ import org.evomaster.core.search.gene.sql.SqlForeignKeyGene import org.evomaster.core.search.gene.sql.SqlPrimaryKeyGene import org.evomaster.core.search.service.ExtraHeuristicsLogger import org.evomaster.core.search.service.FitnessFunction -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.sql.* import org.evomaster.core.taint.TaintAnalysis import org.slf4j.Logger diff --git a/core/src/main/kotlin/org/evomaster/core/problem/graphql/IntrospectiveQuery.kt b/core/src/main/kotlin/org/evomaster/core/problem/graphql/IntrospectiveQuery.kt index e67296179e..23445621bf 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/graphql/IntrospectiveQuery.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/graphql/IntrospectiveQuery.kt @@ -7,14 +7,9 @@ import com.fasterxml.jackson.databind.ObjectMapper import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.remote.HttpClientFactory import org.evomaster.core.remote.SutProblemException -import org.evomaster.core.search.service.SearchTimeController -import org.glassfish.jersey.client.ClientConfig -import org.glassfish.jersey.client.ClientProperties -import org.glassfish.jersey.client.HttpUrlConnectorProvider +import org.evomaster.core.utils.TimeUtils import org.slf4j.LoggerFactory -import javax.net.ssl.SSLContext import javax.ws.rs.client.Client -import javax.ws.rs.client.ClientBuilder import javax.ws.rs.client.Entity import javax.ws.rs.core.MediaType @@ -58,8 +53,8 @@ class IntrospectiveQuery { """.trimIndent(), MediaType.APPLICATION_JSON_TYPE) //TODO check if TCP problems - val response = SearchTimeController.measureTimeMillis({ ms, res -> - LoggingUtil.getInfoLogger().info("Fetched GraphQL schema in ${ms}ms") + val response = TimeUtils.measureTimeMillis({ ms, res -> + LoggingUtil.getInfoLogger().info("Fetched GraphQL schema in ${ms}ms") }, { try { var request = client.target(graphQlEndpoint) @@ -72,8 +67,10 @@ class IntrospectiveQuery { .invoke() } catch (e: Exception) { log.error("Failed query to '$graphQlEndpoint' : $query") - throw SutProblemException("Failed introspection query to '$graphQlEndpoint'." + - " Please check connection and URL format. Error: ${e.message}") + throw SutProblemException( + "Failed introspection query to '$graphQlEndpoint'." + + " Please check connection and URL format. Error: ${e.message}" + ) } }) @@ -109,4 +106,4 @@ class IntrospectiveQuery { return body } -} \ No newline at end of file +} diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceSampleMethodController.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceSampleMethodController.kt index cbc00eb631..8e4ca8b528 100755 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceSampleMethodController.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/ResourceSampleMethodController.kt @@ -6,7 +6,7 @@ import org.evomaster.core.problem.rest.resource.RestResourceNode import org.evomaster.core.problem.rest.resource.SamplerSpecification import org.evomaster.core.problem.rest.service.ResourceSamplingMethod.* import org.evomaster.core.search.service.Randomness -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.slf4j.Logger import org.slf4j.LoggerFactory diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt index 48b10e6e5e..54aaf7d465 100644 --- a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt +++ b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/fitness/AbstractRestFitness.kt @@ -56,8 +56,8 @@ import org.evomaster.core.search.gene.string.StringGene import org.evomaster.core.search.gene.utils.GeneUtils import org.evomaster.core.search.service.DataPool import org.evomaster.core.search.service.ExecutionStats -import org.evomaster.core.search.service.SearchTimeController import org.evomaster.core.taint.TaintAnalysis +import org.evomaster.core.utils.TimeUtils import org.slf4j.Logger import org.slf4j.LoggerFactory import java.net.URI @@ -625,7 +625,7 @@ abstract class AbstractRestFitness : HttpWsFitness() { val response = try { val call = createInvocation(a, chainState, cookies, tokens) - SearchTimeController.measureTimeMillis( + TimeUtils.measureTimeMillis( { t, res -> rcr.setResponseTimeMs(t) executionStats.record(a.id, t) diff --git a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt index e97b9faf4a..99555c765f 100644 --- a/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt +++ b/core/src/main/kotlin/org/evomaster/core/remote/service/RemoteControllerImplementation.kt @@ -14,8 +14,8 @@ import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.remote.NoRemoteConnectionException import org.evomaster.core.remote.SutProblemException import org.evomaster.core.remote.TcpUtils -import org.evomaster.core.search.service.ExecutionPhaseController -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.ExecutionPhaseController +import org.evomaster.core.search.service.time.SearchTimeController import org.slf4j.Logger import org.slf4j.LoggerFactory import javax.annotation.PostConstruct diff --git a/core/src/main/kotlin/org/evomaster/core/search/algorithms/LipsBudget.kt b/core/src/main/kotlin/org/evomaster/core/search/algorithms/LipsBudget.kt index 3e0af2bc47..be132d41ac 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/algorithms/LipsBudget.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/algorithms/LipsBudget.kt @@ -1,7 +1,7 @@ package org.evomaster.core.search.algorithms import org.evomaster.core.EMConfig -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController /** * Encapsulates per-target budget accounting for LIPS. diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/AdaptiveParameterControl.kt b/core/src/main/kotlin/org/evomaster/core/search/service/AdaptiveParameterControl.kt index fb1379c6ed..ee8e0f8d4d 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/AdaptiveParameterControl.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/AdaptiveParameterControl.kt @@ -2,6 +2,7 @@ package org.evomaster.core.search.service import com.google.inject.Inject import org.evomaster.core.EMConfig +import org.evomaster.core.search.service.time.SearchTimeController import kotlin.math.roundToInt /** diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Archive.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Archive.kt index ac796618f4..f3eccdbaf6 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Archive.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Archive.kt @@ -13,6 +13,7 @@ import org.evomaster.core.search.impact.impactinfocollection.ImpactsOfIndividual import org.evomaster.core.search.service.IdMapper.Companion.LOCAL_OBJECTIVE_KEY import org.evomaster.core.search.service.monitor.SearchProcessMonitor import org.evomaster.core.search.service.mutator.EvaluatedMutation +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.search.tracer.ArchiveMutationTrackService import org.slf4j.LoggerFactory diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/ExecutionPhaseController.kt b/core/src/main/kotlin/org/evomaster/core/search/service/ExecutionPhaseController.kt deleted file mode 100644 index 9cada47c1d..0000000000 --- a/core/src/main/kotlin/org/evomaster/core/search/service/ExecutionPhaseController.kt +++ /dev/null @@ -1,103 +0,0 @@ -package org.evomaster.core.search.service - -import org.evomaster.core.EMConfig -import org.evomaster.core.logging.LoggingUtil -import javax.inject.Inject - -class ExecutionPhaseController { - - enum class Phase{ - NOT_STARTED, - SEARCH, - MINIMIZATION, - SECURITY, - HTTP_ORACLES, - FLAKINESS, - WRITE_OUTPUT, - FINISHED - } - - @Inject - private lateinit var config: EMConfig - - private var phase: Phase = Phase.NOT_STARTED - - private var lastPhaseStartMs : Long = 0 - - private val durationInSeconds : MutableMap = mutableMapOf() - - - fun getPhaseDurationInSeconds(target: Phase) : Long { - return durationInSeconds.getOrDefault(target, -1L) - } - - fun isInSearch() = phase == Phase.SEARCH - - fun startSearch() { - if(isRunning()){ - throw IllegalStateException("Illegal state to start new search: $phase") - } - startPhase(Phase.SEARCH) - } - - fun finishSearch() { - startPhase(Phase.FINISHED) - } - - fun startMinimization() { - if(!isRunning()) { - throw IllegalStateException("Illegal state to start minimization: $phase") - } - startPhase(Phase.MINIMIZATION) - } - - fun startSecurity(){ - if(!isRunning()) { - throw IllegalStateException("Illegal state to start security: $phase") - } - startPhase(Phase.SECURITY) - } - - fun startFlakiness() { - if (!isRunning()) { - throw IllegalStateException("Illegal state to start flakiness detection: $phase") - } - startPhase(Phase.FLAKINESS) - } - - fun startHttpOracles(){ - if(!isRunning()) { - throw IllegalStateException("Illegal state to start http oracles: $phase") - } - startPhase(Phase.HTTP_ORACLES) - } - - fun startWriteOutput (){ - if(!isRunning()) { - throw IllegalStateException("Illegal state to start write output: $phase") - } - startPhase(Phase.WRITE_OUTPUT) - } - - private fun isRunning() : Boolean{ - return phase != Phase.NOT_STARTED && phase != Phase.FINISHED - } - - private fun startPhase(newPhase: Phase){ - - if(phase != Phase.NOT_STARTED){ - //starting a new phase will end the current one - val elapsed = System.currentTimeMillis() - lastPhaseStartMs - val seconds = elapsed / 1000 - durationInSeconds[phase] = seconds - - if(!config.avoidNonDeterministicLogs) { - val time = SearchTimeController.getElapsedTime(seconds) - LoggingUtil.getInfoLogger().info("Phase $phase lasted: $time") - } - } - - phase = newPhase - lastPhaseStartMs = System.currentTimeMillis() - } -} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/ExtraHeuristicsLogger.kt b/core/src/main/kotlin/org/evomaster/core/search/service/ExtraHeuristicsLogger.kt index cda62a1f51..0dfd10ce12 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/ExtraHeuristicsLogger.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/ExtraHeuristicsLogger.kt @@ -7,6 +7,7 @@ import net.sf.jsqlparser.util.deparser.SelectDeParser import net.sf.jsqlparser.util.deparser.StatementDeParser import org.evomaster.client.java.controller.api.dto.ExtraHeuristicEntryDto import org.evomaster.core.EMConfig +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.sql.ReplaceValuesDeParser import java.nio.file.Files import java.nio.file.Path diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/FitnessFunction.kt b/core/src/main/kotlin/org/evomaster/core/search/service/FitnessFunction.kt index 08bb753189..f990a27a16 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/FitnessFunction.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/FitnessFunction.kt @@ -1,7 +1,6 @@ package org.evomaster.core.search.service import com.google.inject.Inject -import com.webfuzzing.commons.faults.FaultCategory import org.evomaster.core.EMConfig import org.evomaster.core.Lazy import org.evomaster.core.problem.enterprise.DetectedFaultUtils @@ -9,6 +8,9 @@ import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.Individual import org.evomaster.core.search.service.monitor.SearchProcessMonitor import org.evomaster.core.search.service.mutator.MutatedGeneSpecification +import org.evomaster.core.search.service.time.ExecutionPhaseController +import org.evomaster.core.search.service.time.SearchTimeController +import org.evomaster.core.utils.TimeUtils import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -59,8 +61,6 @@ abstract class FitnessFunction where T : Individual { val a = individual.seeMainExecutableActions().count() -// val calculatedBefore = individual.copy() - if(time.averageOverheadMsBetweenTests.isRecordingTimer()){ val computation = time.averageOverheadMsBetweenTests.addElapsedTime() executionInfoReporter.addLatestComputationOverhead(computation, time.evaluatedIndividuals) @@ -147,12 +147,20 @@ abstract class FitnessFunction where T : Individual { // By default, we optimize for performance in collecting coverage values, but for special cases, we want to collect full info val allTargetsWithDescriptive = config.processFormat == EMConfig.ProcessDataFormat.TARGET_HEURISTIC - val ei = SearchTimeController.measureTimeMillis( - { t, ind -> - time.reportExecutedIndividualTime(t, actionsSize) - ind?.executionTimeMs = t - }, - {doCalculateCoverage(individual, targets, allTargets = allTargetsWithDescriptive, fullyCovered = false, descriptiveIds = allTargetsWithDescriptive)} + val ei = TimeUtils.measureTimeMillis( + { t, ind -> + time.reportExecutedIndividualTime(t, actionsSize) + ind?.executionTimeMs = t + }, + { + doCalculateCoverage( + individual, + targets, + allTargets = allTargetsWithDescriptive, + fullyCovered = false, + descriptiveIds = allTargetsWithDescriptive + ) + } ) // plugin execution info reporter here, to avoid the time spent by execution reporter handleExecutionInfo(ei) @@ -177,4 +185,4 @@ abstract class FitnessFunction where T : Individual { executionInfoReporter.sqlExecutionInfo(ei.individual.seeAllActions(), ei.fitness.databaseExecutions) executionInfoReporter.actionExecutionInfo(ei.individual, ei.executionTimeMs, time.evaluatedIndividuals) } -} \ No newline at end of file +} diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt index 16a814f7b5..eae02a50cf 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Minimizer.kt @@ -11,6 +11,9 @@ import org.evomaster.core.problem.rest.data.RestIndividual import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.GroupsOfChildren import org.evomaster.core.search.Individual +import org.evomaster.core.search.service.time.ExecutionPhaseController +import org.evomaster.core.search.service.time.SearchStatusUpdater +import org.evomaster.core.search.service.time.TimeBoxedPhase import org.evomaster.core.sql.SqlAction import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -20,7 +23,7 @@ import org.slf4j.LoggerFactory * * WARN: currently minimization loses all history info from EvaluatedIndividual, eg impact of genes */ -class Minimizer { +class Minimizer : TimeBoxedPhase { companion object{ private val log : Logger = LoggerFactory.getLogger(Minimizer::class.java) @@ -37,45 +40,31 @@ class Minimizer { @Inject private lateinit var idMapper: IdMapper - - private var startTimer : Long = -1 + @Inject + private lateinit var epc: ExecutionPhaseController - fun doStartTheTimer(){ - startTimer = System.currentTimeMillis() - } + override fun applyPhase() { + minimizeMainActionsPerCoveredTargetInArchive() + pruneNonNeededDatabaseActions() + simplifyActions() - fun passedTimeInSecond() : Int { - if(startTimer < 0){ - throw IllegalStateException("Timer was not started") - } - return ((System.currentTimeMillis() - startTimer) / 1000).toInt() + //TODO there could be more to do here } - private fun checkHasTimedout() : Boolean{ - if(startTimer < 0){ - throw IllegalStateException("Timer was not started") - } - if(config.minimizeTimeout < 0){ - return false - } - if(config.minimizeTimeout == 0){ - return true - } - val current = System.currentTimeMillis() - val passed = (current - startTimer) / (1000 * 60.0) - return passed > config.minimizeTimeout + override fun hasPhaseTimedOut() : Boolean{ + return epc.hasPhaseTimedOut(ExecutionPhaseController.Phase.MINIMIZATION) } - fun pruneNonNeededDatabaseActions(){ + private fun pruneNonNeededDatabaseActions(){ //TODO } /** * eg, removed un-needed optional parameters */ - fun simplifyActions(){ + private fun simplifyActions(){ //TODO } @@ -87,10 +76,9 @@ class Minimizer { * * "EvoSuite: On The Challenges of Test Case Generation in the Real World" */ - fun minimizeMainActionsPerCoveredTargetInArchive() { + private fun minimizeMainActionsPerCoveredTargetInArchive() { - if(checkHasTimedout()){ - LoggingUtil.getInfoLogger().warn("Minimization phase has timed-out. You can use --minimizeTimeout to increase it.") + if(hasPhaseTimedOut()){ return } @@ -120,8 +108,7 @@ class Minimizer { current.forEach{ - if(checkHasTimedout()){ - LoggingUtil.getInfoLogger().warn("Minimization phase has timed-out. You can use --minimizeTimeout to increase it.") + if(hasPhaseTimedOut()){ return } diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt index 22f50c1c82..f1e3500690 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt @@ -8,6 +8,7 @@ import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.Individual import org.evomaster.core.search.action.MainAction import org.evomaster.core.search.gene.wrapper.OptionalGene +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.search.tracer.TrackOperator import org.slf4j.Logger import org.slf4j.LoggerFactory diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt b/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt index 695dbd693a..b60a9686c2 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/SearchAlgorithm.kt @@ -6,6 +6,9 @@ import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.search.Individual import org.evomaster.core.search.Solution import org.evomaster.core.search.service.mutator.Mutator +import org.evomaster.core.search.service.time.ExecutionPhaseController +import org.evomaster.core.search.service.time.SearchStatusUpdater +import org.evomaster.core.search.service.time.SearchTimeController abstract class SearchAlgorithm where T : Individual { @@ -100,17 +103,9 @@ abstract class SearchAlgorithm where T : Individual { time.doStopRecording() - ssu.enabled = false - if(config.minimize){ - epc.startMinimization() - - minimizer.doStartTheTimer() - minimizer.minimizeMainActionsPerCoveredTargetInArchive() - minimizer.pruneNonNeededDatabaseActions() - minimizer.simplifyActions() - val seconds = minimizer.passedTimeInSecond() - LoggingUtil.getInfoLogger().info("Minimization phase took $seconds seconds") + epc.markStartingMinimization() + minimizer.applyPhase() } if(config.addPreDefinedTests) { @@ -130,4 +125,4 @@ abstract class SearchAlgorithm where T : Individual { } abstract fun getType() : EMConfig.Algorithm -} \ No newline at end of file +} diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/SearchGlobalState.kt b/core/src/main/kotlin/org/evomaster/core/search/service/SearchGlobalState.kt index 6a77220331..5c63f9d331 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/SearchGlobalState.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/SearchGlobalState.kt @@ -4,6 +4,8 @@ import com.google.inject.Inject import org.evomaster.core.EMConfig import org.evomaster.core.problem.externalservice.httpws.service.HttpWsExternalServiceHandler import org.evomaster.core.problem.webfrontend.service.BrowserController +import org.evomaster.core.search.service.time.ExecutionPhaseController +import org.evomaster.core.search.service.time.SearchTimeController /** * Global state used in the search. diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/SearchListener.kt b/core/src/main/kotlin/org/evomaster/core/search/service/SearchListener.kt deleted file mode 100644 index 610446002c..0000000000 --- a/core/src/main/kotlin/org/evomaster/core/search/service/SearchListener.kt +++ /dev/null @@ -1,7 +0,0 @@ -package org.evomaster.core.search.service - - -interface SearchListener { - - fun newActionEvaluated() -} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt index fba7f78893..77466ee9bc 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/Statistics.kt @@ -4,22 +4,22 @@ import com.google.inject.Inject import org.evomaster.client.java.controller.api.dto.SutInfoDto import org.evomaster.client.java.instrumentation.shared.ObjectiveNaming import org.evomaster.core.EMConfig -import org.evomaster.core.output.service.PartialOracles import org.evomaster.core.problem.httpws.HttpWsCallResult import org.evomaster.core.problem.rest.data.RestCallAction import org.evomaster.core.problem.rest.service.AIResponseClassifier import org.evomaster.core.problem.rest.service.CallGraphService import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.Solution +import org.evomaster.core.search.service.time.ExecutionPhaseController +import org.evomaster.core.search.service.time.SearchListener +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.utils.IncrementalAverage import org.slf4j.Logger import org.slf4j.LoggerFactory import java.nio.file.Files import java.nio.file.Paths import javax.annotation.PostConstruct -import kotlin.reflect.full.isSubtypeOf -import kotlin.reflect.javaType -import kotlin.reflect.typeOf + class Statistics : SearchListener { @@ -61,9 +61,6 @@ class Statistics : SearchListener { @Inject(optional = true) private var remoteController: RemoteController? = null - @Inject - private lateinit var oracles: PartialOracles - @Inject(optional = true) private lateinit var aiResponseClassifier: AIResponseClassifier @@ -237,7 +234,13 @@ class Statistics : SearchListener { fun averageNumberOfEvaluatedDocumentsForRedisHeuristics(): Double = redisDocumentsAverageCalculator.mean - override fun newActionEvaluated() { + override fun newActionsEvaluated(n: Int) { + + if(!epc.isInSearch()){ + //we are only taking snapshots during the search + return + } + if (snapshotThreshold <= 0) { //not collecting snapshot data return @@ -245,7 +248,7 @@ class Statistics : SearchListener { val elapsed = 100 * time.percentageUsedBudget() - if (elapsed > snapshotThreshold) { + if (elapsed >= snapshotThreshold) { takeSnapshot() } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/monitor/SearchProcessMonitor.kt b/core/src/main/kotlin/org/evomaster/core/search/service/monitor/SearchProcessMonitor.kt index b27ab94f47..e394e847f1 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/monitor/SearchProcessMonitor.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/monitor/SearchProcessMonitor.kt @@ -14,6 +14,8 @@ import org.evomaster.core.remote.service.RemoteController import org.evomaster.core.search.* import org.evomaster.core.search.gene.Gene import org.evomaster.core.search.service.* +import org.evomaster.core.search.service.time.SearchListener +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.utils.ReportWriter.writeByChannel import org.slf4j.Logger import org.slf4j.LoggerFactory @@ -122,7 +124,7 @@ class SearchProcessMonitor: SearchListener { } } - override fun newActionEvaluated() { + override fun newActionsEvaluated(n: Int) { if(config.enableProcessMonitor && config.processFormat == EMConfig.ProcessDataFormat.JSON_ALL){ step = StepOfSearchProcess(archive, time.evaluatedIndividuals, eval!!.individual, eval!!, System.currentTimeMillis(),isMutated) } diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/mutator/Mutator.kt b/core/src/main/kotlin/org/evomaster/core/search/service/mutator/Mutator.kt index 66d926f9e2..2bc97a117e 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/mutator/Mutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/mutator/Mutator.kt @@ -11,6 +11,7 @@ import org.evomaster.core.search.gene.Gene import org.evomaster.core.search.service.* import org.evomaster.core.search.service.mutator.genemutation.ArchiveGeneMutator import org.evomaster.core.search.service.mutator.genemutation.ArchiveImpactSelector +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.search.tracer.ArchiveMutationTrackService import org.evomaster.core.search.tracer.TraceableElementCopyFilter import org.evomaster.core.search.tracer.TrackOperator diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/mutator/StructureMutator.kt b/core/src/main/kotlin/org/evomaster/core/search/service/mutator/StructureMutator.kt index b5992e71a4..e805a276f9 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/mutator/StructureMutator.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/mutator/StructureMutator.kt @@ -7,7 +7,7 @@ import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.Individual import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Randomness -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.search.tracer.TrackOperator /** diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/mutator/genemutation/ArchiveImpactSelector.kt b/core/src/main/kotlin/org/evomaster/core/search/service/mutator/genemutation/ArchiveImpactSelector.kt index a0488f7910..243274257a 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/mutator/genemutation/ArchiveImpactSelector.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/mutator/genemutation/ArchiveImpactSelector.kt @@ -10,7 +10,7 @@ import org.evomaster.core.search.impact.impactinfocollection.* import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Archive import org.evomaster.core.search.service.Randomness -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.search.service.mutator.EvaluatedMutation import java.nio.file.Files import java.nio.file.Paths diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/time/ExecutionPhaseController.kt b/core/src/main/kotlin/org/evomaster/core/search/service/time/ExecutionPhaseController.kt new file mode 100644 index 0000000000..d7cccbfadc --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/search/service/time/ExecutionPhaseController.kt @@ -0,0 +1,166 @@ +package org.evomaster.core.search.service.time + +import org.evomaster.core.EMConfig +import org.evomaster.core.logging.LoggingUtil +import org.evomaster.core.utils.TimeUtils +import javax.inject.Inject + +/** + * Service used to keep track of which phase of the fuzzing we are currently in, + * and how long we have spent in them. + * + * The actual starting/stopping of these phases is done elsewhere (as it depends + * on the problem type) + */ +class ExecutionPhaseController { + + enum class Phase( + val timeBoxed: Boolean + ){ + NOT_STARTED(false), + SEARCH(false), + MINIMIZATION(true), + SECURITY(true), + ADDITIONAL_ORACLES(true), + FLAKINESS(true), + WRITE_OUTPUT(false), + FINISHED(false) + } + + @Inject + private lateinit var config: EMConfig + + private var phase: Phase = Phase.NOT_STARTED + + private var lastPhaseStartMs : Long = 0 + + private val durationInSeconds : MutableMap = mutableMapOf() + + private val hasTimedOut : MutableSet = mutableSetOf() + + fun getCurrentPhase() : Phase{ + return phase + } + + fun hasPhaseTimedOut(target: Phase) : Boolean { + + if(!target.timeBoxed){ + throw IllegalArgumentException("Cannot query timeout for a non time-boxed phase: $target") + } + + if(config.stoppingCriterion != EMConfig.StoppingCriterion.TIME){ + //timeouts are defined only when we run base search on time, which is the default + return false + } + + if(hasTimedOut.contains(target)){ + return true + } + + val budget = (config.timeLimitInSeconds() * config.extraPhaseBudgetPercentage).toLong() + + val passed = elapsedSeconds() + val timeout = passed > budget + + if(timeout){ + LoggingUtil.getInfoLogger().warn("Phase '${phase.name}' has timed out after $budget seconds." + + " If you can run the fuzzing for longer, next time you might want to either increase " + + "'maxTime' (${config.maxTime})" + + "or 'extraPhaseBudgetPercentage' (${config.extraPhaseBudgetPercentage})") + hasTimedOut.add(phase) + } + + return timeout + } + + fun getPhaseDurationInSeconds(target: Phase) : Long { + return durationInSeconds.getOrDefault(target, -1L) + } + + fun isInSearch() = phase == Phase.SEARCH + + fun markStartingSearch() { + if(isRunning()){ + throw IllegalStateException("Illegal state to start new search: $phase") + } + startPhase(Phase.SEARCH) + } + + fun markFinishedSession() { + startPhase(Phase.FINISHED) + } + + fun markStartingMinimization() { + if(!isRunning()) { + throw IllegalStateException("Illegal state to start minimization: $phase") + } + startPhase(Phase.MINIMIZATION) + } + + fun markStartingSecurity(){ + if(!isRunning()) { + throw IllegalStateException("Illegal state to start security: $phase") + } + startPhase(Phase.SECURITY) + } + + fun markStartingFlakiness() { + if (!isRunning()) { + throw IllegalStateException("Illegal state to start flakiness detection: $phase") + } + startPhase(Phase.FLAKINESS) + } + + fun markStartingAdditionalOracles(){ + if(!isRunning()) { + throw IllegalStateException("Illegal state to start additional oracles: $phase") + } + startPhase(Phase.ADDITIONAL_ORACLES) + } + + fun markStartingWriteOutput (){ + if(!isRunning()) { + throw IllegalStateException("Illegal state to start write output: $phase") + } + startPhase(Phase.WRITE_OUTPUT) + } + + private fun isRunning() : Boolean{ + return phase != Phase.NOT_STARTED && phase != Phase.FINISHED + } + + private fun elapsedSeconds() : Long { + + if(phase == Phase.NOT_STARTED){ + throw IllegalStateException("Fuzzing session has not started yet") + } + + val elapsed = System.currentTimeMillis() - lastPhaseStartMs + val seconds = elapsed / 1000 + + return seconds + } + + private fun startPhase(newPhase: Phase){ + + if(newPhase == Phase.NOT_STARTED){ + throw IllegalStateException("Cannot start a 'not-started' phase") + } + + if(phase != Phase.NOT_STARTED){ + //starting a new phase will end the current one + val seconds = elapsedSeconds() + durationInSeconds[phase] = seconds + + if(!config.avoidNonDeterministicLogs) { + val time = TimeUtils.getElapsedTime(seconds) + LoggingUtil.getInfoLogger().info("Phase $phase lasted: $time") + } + } + + LoggingUtil.getInfoLogger().info("Starting phase $newPhase") + + phase = newPhase + lastPhaseStartMs = System.currentTimeMillis() + } +} diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/time/SearchListener.kt b/core/src/main/kotlin/org/evomaster/core/search/service/time/SearchListener.kt new file mode 100644 index 0000000000..e199832746 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/search/service/time/SearchListener.kt @@ -0,0 +1,6 @@ +package org.evomaster.core.search.service.time + +interface SearchListener { + + fun newActionsEvaluated(n: Int) +} diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/SearchStatusUpdater.kt b/core/src/main/kotlin/org/evomaster/core/search/service/time/SearchStatusUpdater.kt similarity index 75% rename from core/src/main/kotlin/org/evomaster/core/search/service/SearchStatusUpdater.kt rename to core/src/main/kotlin/org/evomaster/core/search/service/time/SearchStatusUpdater.kt index dbeea553c8..22d0e38ea2 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/SearchStatusUpdater.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/time/SearchStatusUpdater.kt @@ -1,26 +1,27 @@ -package org.evomaster.core.search.service +package org.evomaster.core.search.service.time import com.google.inject.Inject import org.evomaster.core.EMConfig +import org.evomaster.core.search.service.Archive +import org.evomaster.core.search.service.time.SearchListener +import java.io.PrintStream import java.nio.charset.Charset import javax.annotation.PostConstruct -import java.io.PrintStream - - - -class SearchStatusUpdater : SearchListener{ +class SearchStatusUpdater : SearchListener { @Inject private lateinit var time: SearchTimeController + @Inject + private lateinit var epc: ExecutionPhaseController + @Inject private lateinit var config: EMConfig @Inject private lateinit var archive: Archive<*> - var enabled = true companion object{ fun eraseLine(){ @@ -69,19 +70,23 @@ class SearchStatusUpdater : SearchListener{ } } - override fun newActionEvaluated() { + override fun newActionsEvaluated(n: Int) { - if(!enabled){ - return + when { + epc.isInSearch() -> printForSearch() + //TODO else for other phases + //TODO could save n in epc, and show progress bar with its value } + } + private fun printForSearch() { val percentageInt = (time.percentageUsedBudget() * 100).toInt() val current = String.format("%.3f", time.percentageUsedBudget() * 100) - if(first){ + if (first) { println() println() - if(config.e_u1f984){ + if (config.e_u1f984) { println() } first = false @@ -90,35 +95,36 @@ class SearchStatusUpdater : SearchListener{ val delta = System.currentTimeMillis() - lastUpdateMS //writing on console is I/O, which is expensive. So, can't do it too often - if(current != passed && delta > 500){ + if (current != passed && delta > 500) { lastUpdateMS += delta passed = current - if(percentageInt - lastCoverageComputation > 0){ + if (percentageInt - lastCoverageComputation > 0) { lastCoverageComputation = percentageInt //this is not too expensive, but still computation. so we do it only at each 1% coverage = archive.numberOfCoveredTargets() } - if(config.e_u1f984){ + if (config.e_u1f984) { upLineAndErase() } val avgTimeAndSize = time.computeExecutedIndividualTimeStatistics() val avgTime = String.format("%.1f", avgTimeAndSize.first) - val avgSize = String.format("%.1f",avgTimeAndSize.second) + val avgSize = String.format("%.1f", avgTimeAndSize.second) val sinceLast = time.getSecondsSinceLastImprovement() upLineAndErase() upLineAndErase() println("* Consumed search budget: $passed%") - println("* Covered targets: $coverage;" + - " time per test: ${avgTime}ms ($avgSize actions);" + - " since last improvement: ${sinceLast}s" + println( + "* Covered targets: $coverage;" + + " time per test: ${avgTime}ms ($avgSize actions);" + + " since last improvement: ${sinceLast}s" ) - if(config.e_u1f984){ + if (config.e_u1f984) { updateExtra() out.println(extra) } @@ -145,4 +151,4 @@ class SearchStatusUpdater : SearchListener{ */ -} \ No newline at end of file +} diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/SearchTimeController.kt b/core/src/main/kotlin/org/evomaster/core/search/service/time/SearchTimeController.kt similarity index 84% rename from core/src/main/kotlin/org/evomaster/core/search/service/SearchTimeController.kt rename to core/src/main/kotlin/org/evomaster/core/search/service/time/SearchTimeController.kt index cd590757ae..a8be6438d2 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/service/SearchTimeController.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/service/time/SearchTimeController.kt @@ -1,11 +1,13 @@ -package org.evomaster.core.search.service +package org.evomaster.core.search.service.time import com.google.inject.Inject import org.evomaster.core.EMConfig import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.utils.IncrementalAverage +import org.evomaster.core.utils.TimeUtils import org.slf4j.LoggerFactory -import java.util.* +import java.util.ArrayDeque +import java.util.Queue import kotlin.math.ceil /** @@ -21,36 +23,7 @@ class SearchTimeController { companion object{ private val log = LoggerFactory.getLogger(SearchTimeController::class.java) - /** - * Invoke the [function] lambda, which will return some result of generic type [T]. - * Once this is completed, the [loggingFunction] will be automatically called with, - * as input, the execution time expressed in milliseconds, as well as the [function]'s result - * of type [T]. - * - * From https://proandroiddev.com/measuring-execution-times-in-kotlin-460a0285e5ea - */ - inline fun measureTimeMillis(loggingFunction: (Long, T) -> Unit, - function: () -> T): T { - - val startTime = System.currentTimeMillis() - val result: T = function.invoke() - loggingFunction.invoke(System.currentTimeMillis() - startTime, result) - - return result - } - - fun getElapsedTime(totalInSeconds: Long) : String{ - - val seconds = totalInSeconds - val minutes = seconds / 60.0 - val hours = minutes / 60.0 - val ps = "%d".format(seconds % 60) - val pm = "%d".format(minutes.toInt() % 60) - val ph = "%d".format(hours.toInt()) - - return "${ph}h ${pm}m ${ps}s" - } } @@ -102,7 +75,7 @@ class SearchTimeController { * Time expressed in ms (Long). * Also keeping track of number of actions (Int) */ - private val executedIndividualTime : Queue> = ArrayDeque(100) + private val executedIndividualTime : Queue> = ArrayDeque(100) private val listeners = mutableListOf() @@ -229,9 +202,13 @@ class SearchTimeController { } fun newActionEvaluation(n: Int = 1) { - if(!recording) return + + listeners.forEach{it.newActionsEvaluated(n)} + + if(!recording){ + return + } evaluatedActions += n - listeners.forEach{it.newActionEvaluated()} } fun newCoveredTarget(){ @@ -256,7 +233,7 @@ class SearchTimeController { } fun getElapsedTime() : String{ - return getElapsedTime(getElapsedSeconds().toLong()) + return TimeUtils.getElapsedTime(getElapsedSeconds().toLong()) } fun shouldContinueSearch(): Boolean{ @@ -321,4 +298,4 @@ class SearchTimeController { return "$percentage%" } } -} \ No newline at end of file +} diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/time/TimeBoxedPhase.kt b/core/src/main/kotlin/org/evomaster/core/search/service/time/TimeBoxedPhase.kt new file mode 100644 index 0000000000..1ccceea21c --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/search/service/time/TimeBoxedPhase.kt @@ -0,0 +1,17 @@ +package org.evomaster.core.search.service.time + +/** + * A service representing one phase of fuzzing that is time-boxed. + * This means that the phase should not run more than a certain amount of time. + * If such limit is reached, the phase is prematurely stopped. + * Ideally, most of the time we would expect the phase to finish way earlier than the timeout. + * + * Note: this is quite different from typical "search" phase, where we usually run for all + * the available budget + */ +interface TimeBoxedPhase { + + fun applyPhase() + + fun hasPhaseTimedOut(): Boolean +} diff --git a/core/src/main/kotlin/org/evomaster/core/utils/TimeUtils.kt b/core/src/main/kotlin/org/evomaster/core/utils/TimeUtils.kt new file mode 100644 index 0000000000..a33d90be16 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/utils/TimeUtils.kt @@ -0,0 +1,38 @@ +package org.evomaster.core.utils + +object TimeUtils { + + + /** + * Invoke the [function] lambda, which will return some result of generic type [T]. + * Once this is completed, the [loggingFunction] will be automatically called with, + * as input, the execution time expressed in milliseconds, as well as the [function]'s result + * of type [T]. + * + * From https://proandroiddev.com/measuring-execution-times-in-kotlin-460a0285e5ea + */ + inline fun measureTimeMillis( + loggingFunction: (Long, T) -> Unit, + function: () -> T + ): T { + + val startTime = System.currentTimeMillis() + val result: T = function.invoke() + loggingFunction.invoke(System.currentTimeMillis() - startTime, result) + + return result + } + + fun getElapsedTime(totalInSeconds: Long) : String{ + + val seconds = totalInSeconds + val minutes = seconds / 60.0 + val hours = minutes / 60.0 + + val ps = "%d".format(seconds % 60) + val pm = "%d".format(minutes.toInt() % 60) + val ph = "%d".format(hours.toInt()) + + return "${ph}h ${pm}m ${ps}s" + } +} diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt index e94b49d7c8..f043ba2f61 100644 --- a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt +++ b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTestBase.kt @@ -42,7 +42,7 @@ import org.evomaster.core.search.EvaluatedIndividual import org.evomaster.core.search.impact.impactinfocollection.ImpactsOfIndividual import org.evomaster.core.search.service.Archive import org.evomaster.core.search.service.Randomness -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.search.service.mutator.StandardMutator import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.AfterEach diff --git a/core/src/test/kotlin/org/evomaster/core/search/StoppingCriterionTest.kt b/core/src/test/kotlin/org/evomaster/core/search/StoppingCriterionTest.kt index 6f682032cb..ea1fea3ccb 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/StoppingCriterionTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/StoppingCriterionTest.kt @@ -6,7 +6,7 @@ import com.netflix.governator.guice.LifecycleInjector import org.evomaster.core.BaseModule import org.evomaster.core.EMConfig import org.evomaster.core.search.algorithms.onemax.OneMaxModule -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/BreederGeneticAlgorithmTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/BreederGeneticAlgorithmTest.kt index 08bfec0c87..24a941d662 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/BreederGeneticAlgorithmTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/BreederGeneticAlgorithmTest.kt @@ -12,7 +12,7 @@ import org.evomaster.core.search.algorithms.onemax.OneMaxIndividual import org.evomaster.core.search.algorithms.onemax.OneMaxModule import org.evomaster.core.search.algorithms.onemax.OneMaxSampler import org.evomaster.core.search.algorithms.observer.GARecorder -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.evomaster.core.search.service.Randomness import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -44,9 +44,9 @@ class BreederGeneticAlgorithmTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = breederGA.search() - epc.finishSearch() + epc.markFinishedSession() assertTrue(solution.individuals.size == 1) assertEquals(OneMaxSampler.DEFAULT_N.toDouble(), solution.overall.computeFitnessScore(), 0.001) @@ -167,7 +167,7 @@ class BreederGeneticAlgorithmTest { val nextPop = breederGA.getViewOfPopulation() assertEquals(config.populationSize, nextPop.size) - // crossovers happen once per iteration (mutation probability = 1) + // crossovers happen once per iteration (mutation probability = 1) assertEquals(config.populationSize, rec.xoCalls.size) // mutations disabled @@ -175,5 +175,5 @@ class BreederGeneticAlgorithmTest { } } - + } diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/CellularGeneticAlgorithmTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/CellularGeneticAlgorithmTest.kt index 7540db685d..4130c82877 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/CellularGeneticAlgorithmTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/CellularGeneticAlgorithmTest.kt @@ -12,7 +12,7 @@ import org.evomaster.core.search.algorithms.observer.GARecorder import org.evomaster.core.search.algorithms.onemax.OneMaxIndividual import org.evomaster.core.search.algorithms.onemax.OneMaxModule import org.evomaster.core.search.algorithms.onemax.OneMaxSampler -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.evomaster.core.search.service.Randomness import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach @@ -88,15 +88,15 @@ class CellularGeneticAlgorithmTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = cga.search() - epc.finishSearch() + epc.markFinishedSession() assertTrue(solution.individuals.size == 1) assertEquals(OneMaxSampler.DEFAULT_N.toDouble(), solution.overall.computeFitnessScore(), 0.001) } } - + // Edge Case: CrossoverProbability=0 and MutationProbability=1 on CGA @Test fun testNoCrossoverWhenProbabilityZero_CGA() { @@ -167,7 +167,7 @@ class CellularGeneticAlgorithmTest { // two selections per iteration assertEquals(2 * n, rec.selections.size) // one crossover per iteration (crossover probability = 1) - assertEquals(n, rec.xoCalls.size) + assertEquals(n, rec.xoCalls.size) // mutation disabled assertEquals(0, rec.mutated.size) } diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/CroAlgorithmTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/CroAlgorithmTest.kt index 96195583af..48dd8b8caa 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/CroAlgorithmTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/CroAlgorithmTest.kt @@ -12,7 +12,7 @@ import org.evomaster.core.search.Individual import org.evomaster.core.search.algorithms.onemax.OneMaxIndividual import org.evomaster.core.search.algorithms.onemax.OneMaxModule import org.evomaster.core.search.algorithms.onemax.OneMaxSampler -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.algorithms.wts.WtsEvalIndividual import org.junit.jupiter.api.Assertions.assertEquals @@ -56,9 +56,9 @@ class CroAlgorithmTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = cro.search() - epc.finishSearch() + epc.markFinishedSession() assertTrue(solution.individuals.size == 1) assertEquals(OneMaxSampler.DEFAULT_N.toDouble(), solution.overall.computeFitnessScore(), 0.001) @@ -102,7 +102,7 @@ class CroAlgorithmTest { } } - + @Test fun testUniMolecular_DecompositionPath_rejected_NoReplacementAndParentCollisionsIncrement() { diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/LIPSAlgorithmTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/LIPSAlgorithmTest.kt index 8649c17b92..1f78fcfd1a 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/LIPSAlgorithmTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/LIPSAlgorithmTest.kt @@ -11,7 +11,7 @@ import org.evomaster.core.TestUtils import org.evomaster.core.search.algorithms.onemax.OneMaxIndividual import org.evomaster.core.search.algorithms.onemax.OneMaxModule import org.evomaster.core.search.algorithms.onemax.OneMaxSampler -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.evomaster.core.search.algorithms.observer.GARecorder import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue @@ -32,7 +32,7 @@ class LIPSAlgorithmTest { // Verifies that the LIPS algorithm can find the optimal solution for the OneMax problem @Test fun testLipsFindsOptimumOnOneMax() { - + val lips = injector.getInstance( Key.get(object : TypeLiteral>() {}) ) @@ -42,9 +42,9 @@ class LIPSAlgorithmTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = lips.search() - epc.finishSearch() + epc.markFinishedSession() assertTrue(solution.individuals.size == 1) assertEquals( @@ -53,7 +53,7 @@ class LIPSAlgorithmTest { 0.001 ) } - + // Edge Case: CrossoverProbability=0 on LIPS @Test fun testNoCrossoverWhenProbabilityZero_LIPS() { @@ -169,7 +169,7 @@ class LIPSAlgorithmTest { assertEquals(neededFromOffspring, rec.mutated.size) } } - + } diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/LipsBudgetTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/LipsBudgetTest.kt index 56f31002fc..9ea7dc0264 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/LipsBudgetTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/LipsBudgetTest.kt @@ -1,7 +1,7 @@ package org.evomaster.core.search.algorithms import org.evomaster.core.EMConfig -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnConstantTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnConstantTest.kt index 3bb7c146c3..08d099241d 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnConstantTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnConstantTest.kt @@ -9,7 +9,7 @@ import org.evomaster.core.BaseModule import org.evomaster.core.EMConfig import org.evomaster.core.search.algorithms.constant.ConstantIndividual import org.evomaster.core.search.algorithms.constant.ConstantModule -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.evomaster.core.search.service.Randomness import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test @@ -36,11 +36,11 @@ class MioAlgorithmOnConstantTest { config.maxEvaluations = 200 config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = mio.search() Assertions.assertEquals(1.0, solution.overall.computeFitnessScore(), 0.001); Assertions.assertEquals(1, solution.individuals.size) } -} \ No newline at end of file +} diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnOneMaxTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnOneMaxTest.kt index eef7164a43..a171b3008a 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnOneMaxTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MioAlgorithmOnOneMaxTest.kt @@ -10,7 +10,7 @@ import org.evomaster.core.EMConfig import org.evomaster.core.search.algorithms.onemax.OneMaxIndividual import org.evomaster.core.search.algorithms.onemax.OneMaxModule import org.evomaster.core.search.algorithms.onemax.OneMaxSampler -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.evomaster.core.search.service.Randomness import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Test @@ -37,7 +37,7 @@ class MioAlgorithmOnOneMaxTest { config.maxEvaluations = 30000 config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val n = 20 sampler.n = n @@ -47,4 +47,4 @@ class MioAlgorithmOnOneMaxTest { Assertions.assertEquals(n.toDouble(), solution.overall.computeFitnessScore(), 0.001); Assertions.assertEquals(1, solution.individuals.size) } -} \ No newline at end of file +} diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MonotonicGeneticAlgorithmTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MonotonicGeneticAlgorithmTest.kt index 0089e9d4fe..21f9f29c1b 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MonotonicGeneticAlgorithmTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MonotonicGeneticAlgorithmTest.kt @@ -13,7 +13,7 @@ import org.evomaster.core.search.algorithms.onemax.OneMaxIndividual import org.evomaster.core.search.algorithms.onemax.OneMaxModule import org.evomaster.core.search.algorithms.onemax.OneMaxSampler import org.evomaster.core.search.algorithms.strategy.FixedSelectionStrategy -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.evomaster.core.search.service.Randomness import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test @@ -44,15 +44,15 @@ class MonotonicGeneticAlgorithmTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = monoGA.search() - epc.finishSearch() + epc.markFinishedSession() assertTrue(solution.individuals.size == 1) assertEquals(OneMaxSampler.DEFAULT_N.toDouble(), solution.overall.computeFitnessScore(), 0.001) } } - + // Tests Edge Case: CrossoverProbability=0 on Monotonic GA @Test fun testNoCrossoverWhenProbabilityZero_Monotonic() { @@ -97,7 +97,7 @@ class MonotonicGeneticAlgorithmTest { assertEquals(2, rec.mutated.size) } } - + // Tests Edge Case: MutationProbability=0 on Monotonic GA @Test fun testNoMutationWhenProbabilityZero_Monotonic() { @@ -118,7 +118,7 @@ class MonotonicGeneticAlgorithmTest { config.gaSolutionSource = EMConfig.GASolutionSource.POPULATION config.maxEvaluations = 100_000 config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS - + ga.setupBeforeSearch() val pop = ga.getViewOfPopulation() @@ -139,7 +139,7 @@ class MonotonicGeneticAlgorithmTest { // crossover forced assertEquals(1, rec.xoCalls.size) // mutation disabled - assertEquals(0, rec.mutated.size) + assertEquals(0, rec.mutated.size) } } // Verifies that one generation is formed by elites plus monotonic replacement outcome @@ -163,7 +163,7 @@ class MonotonicGeneticAlgorithmTest { config.gaSolutionSource = EMConfig.GASolutionSource.POPULATION config.maxEvaluations = 100_000 config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS - + ga.setupBeforeSearch() val pop = ga.getViewOfPopulation() @@ -213,7 +213,7 @@ class MonotonicGeneticAlgorithmTest { assertTrue(rec.mutated.any { it === o2 }) } } - + // Ensures that maximum fitness never decreases across generations when running full search @Test fun testMonotonicReplacementRule() { @@ -232,9 +232,9 @@ class MonotonicGeneticAlgorithmTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = monoGA.search() - epc.finishSearch() + epc.markFinishedSession() // Check monotonicity across recorded generations: best score (selection metric) is non-decreasing val bestScores = rec.bestFitnessPerGeneration for (k in 1 until bestScores.size) { diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MuLambdaEvolutionaryAlgorithmTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MuLambdaEvolutionaryAlgorithmTest.kt index bcd3053716..dd415a4d08 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MuLambdaEvolutionaryAlgorithmTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MuLambdaEvolutionaryAlgorithmTest.kt @@ -12,7 +12,7 @@ import org.evomaster.core.search.algorithms.observer.GARecorder import org.evomaster.core.search.algorithms.onemax.OneMaxIndividual import org.evomaster.core.search.algorithms.onemax.OneMaxModule import org.evomaster.core.search.algorithms.onemax.OneMaxSampler -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -43,9 +43,9 @@ class MuLambdaEvolutionaryAlgorithmTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = ea.search() - epc.finishSearch() + epc.markFinishedSession() assertEquals(1, solution.individuals.size) assertEquals(OneMaxSampler.DEFAULT_N.toDouble(), solution.overall.computeFitnessScore(), 0.001) diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MuPlusLambdaEvolutionaryAlgorithmTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MuPlusLambdaEvolutionaryAlgorithmTest.kt index 54fb5f7a17..222f48252c 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/MuPlusLambdaEvolutionaryAlgorithmTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/MuPlusLambdaEvolutionaryAlgorithmTest.kt @@ -12,7 +12,7 @@ import org.evomaster.core.search.algorithms.observer.GARecorder import org.evomaster.core.search.algorithms.onemax.OneMaxIndividual import org.evomaster.core.search.algorithms.onemax.OneMaxModule import org.evomaster.core.search.algorithms.onemax.OneMaxSampler -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -43,9 +43,9 @@ class MuPlusLambdaEvolutionaryAlgorithmTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = ea.search() - epc.finishSearch() + epc.markFinishedSession() assertEquals(1, solution.individuals.size) assertEquals(OneMaxSampler.DEFAULT_N.toDouble(), solution.overall.computeFitnessScore(), 0.001) diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/OnePlusLambdaLambdaGeneticAlgorithmTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/OnePlusLambdaLambdaGeneticAlgorithmTest.kt index 3c7c20f493..d9f8c78fb0 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/OnePlusLambdaLambdaGeneticAlgorithmTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/OnePlusLambdaLambdaGeneticAlgorithmTest.kt @@ -12,7 +12,7 @@ import org.evomaster.core.search.algorithms.observer.GARecorder import org.evomaster.core.search.algorithms.onemax.OneMaxIndividual import org.evomaster.core.search.algorithms.onemax.OneMaxModule import org.evomaster.core.search.algorithms.onemax.OneMaxSampler -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.evomaster.core.search.service.Randomness import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach @@ -42,9 +42,9 @@ class OnePlusLambdaLambdaGeneticAlgorithmTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = ga.search() - epc.finishSearch() + epc.markFinishedSession() assertEquals(1, solution.individuals.size) assertEquals(OneMaxSampler.DEFAULT_N.toDouble(), solution.overall.computeFitnessScore(), 0.001) @@ -166,7 +166,7 @@ class OnePlusLambdaLambdaGeneticAlgorithmTest { } else { assertTrue(p1 === parent0) } - + } } diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomSearchTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomSearchTest.kt index c7eae41eae..a4327944a9 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomSearchTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomSearchTest.kt @@ -10,7 +10,7 @@ import org.evomaster.core.EMConfig import org.evomaster.core.TestUtils import org.evomaster.core.search.algorithms.onemax.OneMaxIndividual import org.evomaster.core.search.algorithms.onemax.OneMaxModule -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -33,7 +33,7 @@ class RandomSearchTest { config.maxEvaluations = 3000 config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = rs.search() @@ -41,4 +41,4 @@ class RandomSearchTest { assertTrue(solution.individuals.size <= 2) } } -} \ No newline at end of file +} diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomWalkSearchTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomWalkSearchTest.kt index 852b0b73fd..614cefe079 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomWalkSearchTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/RandomWalkSearchTest.kt @@ -11,10 +11,8 @@ import org.evomaster.core.TestUtils import org.evomaster.core.search.algorithms.onemax.OneMaxIndividual import org.evomaster.core.search.algorithms.onemax.OneMaxModule import org.evomaster.core.search.algorithms.onemax.OneMaxSampler -import org.evomaster.core.search.service.ExecutionPhaseController -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test @@ -43,11 +41,11 @@ class RandomWalkSearchTest { sampler.n = n val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = rw.search() assertEquals(n.toDouble(), solution.overall.computeFitnessScore(), 0.001) } } -} \ No newline at end of file +} diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/StandardGeneticAlgorithmTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/StandardGeneticAlgorithmTest.kt index 61daf3e74b..d999305514 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/StandardGeneticAlgorithmTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/StandardGeneticAlgorithmTest.kt @@ -14,7 +14,7 @@ import org.evomaster.core.search.algorithms.onemax.OneMaxSampler import org.evomaster.core.search.service.Randomness import org.junit.jupiter.api.Test import org.junit.jupiter.api.BeforeEach -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.junit.jupiter.api.Assertions.* import org.evomaster.core.search.algorithms.strategy.FixedSelectionStrategy @@ -45,9 +45,9 @@ class StandardGeneticAlgorithmTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = standardGeneticAlgorithm.search() - epc.finishSearch() + epc.markFinishedSession() assertTrue(solution.individuals.size == 1) assertEquals(OneMaxSampler.DEFAULT_N.toDouble(), solution.overall.computeFitnessScore(), 0.001) } @@ -118,7 +118,7 @@ class StandardGeneticAlgorithmTest { assertEquals(2, rec.mutated.size) assertTrue(rec.mutated.any { it === o1 }) assertTrue(rec.mutated.any { it === o2 }) - + } } @@ -165,7 +165,7 @@ class StandardGeneticAlgorithmTest { assertEquals(0, rec.xoCalls.size) // mutation still applied twice assertEquals(2, rec.mutated.size) - + } } @@ -210,7 +210,7 @@ class StandardGeneticAlgorithmTest { // crossover forced assertEquals(1, rec.xoCalls.size) // mutation disabled - assertEquals(0, rec.mutated.size) + assertEquals(0, rec.mutated.size) } } } @@ -230,4 +230,4 @@ private fun createGAWithSelection( // Override selection strategy directly on the GA instance (no DI here) ga.useSelectionStrategy(fixedSel) return ga to injector -} \ No newline at end of file +} diff --git a/core/src/test/kotlin/org/evomaster/core/search/algorithms/SteadyStateGeneticAlgorithmTest.kt b/core/src/test/kotlin/org/evomaster/core/search/algorithms/SteadyStateGeneticAlgorithmTest.kt index e352d7d32e..1d2c4e5469 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/algorithms/SteadyStateGeneticAlgorithmTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/algorithms/SteadyStateGeneticAlgorithmTest.kt @@ -13,7 +13,7 @@ import org.evomaster.core.search.algorithms.onemax.OneMaxModule import org.evomaster.core.search.algorithms.onemax.OneMaxSampler import org.evomaster.core.search.algorithms.observer.GARecorder import org.evomaster.core.search.algorithms.strategy.FixedSelectionStrategy -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.evomaster.core.search.service.Randomness import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach @@ -44,9 +44,9 @@ class SteadyStateGeneticAlgorithmTest { config.stoppingCriterion = EMConfig.StoppingCriterion.ACTION_EVALUATIONS val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val solution = steadyStateAlgorithm.search() - epc.finishSearch() + epc.markFinishedSession() assertTrue(solution.individuals.size == 1) assertEquals(OneMaxSampler.DEFAULT_N.toDouble(), solution.overall.computeFitnessScore(), 0.001) } diff --git a/core/src/test/kotlin/org/evomaster/core/search/gene/GeneUtilsGetDeltaTest.kt b/core/src/test/kotlin/org/evomaster/core/search/gene/GeneUtilsGetDeltaTest.kt index e89172b30d..4e244fe15f 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/gene/GeneUtilsGetDeltaTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/gene/GeneUtilsGetDeltaTest.kt @@ -8,7 +8,7 @@ import org.evomaster.core.EMConfig import org.evomaster.core.search.gene.utils.GeneUtils import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Randomness -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.assertThrows diff --git a/core/src/test/kotlin/org/evomaster/core/search/impact/weightcalculation/ImpactMutationWeightControlTest.kt b/core/src/test/kotlin/org/evomaster/core/search/impact/weightcalculation/ImpactMutationWeightControlTest.kt index 6ac4ee3aea..ad2b75f6bb 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/impact/weightcalculation/ImpactMutationWeightControlTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/impact/weightcalculation/ImpactMutationWeightControlTest.kt @@ -9,7 +9,7 @@ import org.evomaster.core.search.impact.impactinfocollection.GeneMutationSelecti import org.evomaster.core.search.impact.impactinfocollection.Impact import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Randomness -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.ArchiveImpactSelector import org.junit.jupiter.api.BeforeEach diff --git a/core/src/test/kotlin/org/evomaster/core/search/mutationweight/MutationWeightControlTest.kt b/core/src/test/kotlin/org/evomaster/core/search/mutationweight/MutationWeightControlTest.kt index cf697f90ae..5c1fe0571d 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/mutationweight/MutationWeightControlTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/mutationweight/MutationWeightControlTest.kt @@ -12,7 +12,7 @@ import org.evomaster.core.search.gene.ObjectGene import org.evomaster.core.search.gene.wrapper.OptionalGene import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Randomness -import org.evomaster.core.search.service.SearchTimeController +import org.evomaster.core.search.service.time.SearchTimeController import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy import org.junit.jupiter.api.BeforeEach diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/AdaptiveParameterControlTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/AdaptiveParameterControlTest.kt index 176d44a396..a999123716 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/AdaptiveParameterControlTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/AdaptiveParameterControlTest.kt @@ -5,6 +5,7 @@ import com.google.inject.Module import com.netflix.governator.guice.LifecycleInjector import org.evomaster.core.BaseModule import org.evomaster.core.EMConfig +import org.evomaster.core.search.service.time.SearchTimeController import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeEach diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/ProcessMonitorTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/ProcessMonitorTest.kt index abd5f20f1d..7e5e2bc0a5 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/ProcessMonitorTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/ProcessMonitorTest.kt @@ -18,6 +18,7 @@ import org.evomaster.core.search.algorithms.onemax.OneMaxSampler import org.evomaster.core.search.service.monitor.SearchOverall import org.evomaster.core.search.service.monitor.SearchProcessMonitor import org.evomaster.core.search.service.monitor.StepOfSearchProcess +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.hamcrest.CoreMatchers.containsString import org.hamcrest.CoreMatchers.not import org.hamcrest.MatcherAssert.assertThat @@ -84,7 +85,7 @@ class ProcessMonitorTest{ val eval = ff.calculateCoverage(a, modifiedSpec = null)!! processMonitor.eval = eval - processMonitor.newActionEvaluated() + processMonitor.newActionsEvaluated(1) val added = archive.addIfNeeded(eval) processMonitor.record(added, true, eval) @@ -109,7 +110,7 @@ class ProcessMonitorTest{ val eval = ff.calculateCoverage(a, modifiedSpec = null)!! processMonitor.eval = eval - processMonitor.newActionEvaluated() + processMonitor.newActionsEvaluated(1) val added = archive.addIfNeeded(eval) @@ -138,7 +139,7 @@ class ProcessMonitorTest{ val eval = ff.calculateCoverage(individual, modifiedSpec = null)!! processMonitor.eval = eval - processMonitor.newActionEvaluated() + processMonitor.newActionsEvaluated(1) val added = archive.addIfNeeded(eval) @@ -186,7 +187,7 @@ class ProcessMonitorTest{ a.setValue(0, 1.0) val evalA = ff.calculateCoverage(a, modifiedSpec = null)!! processMonitor.eval = evalA - processMonitor.newActionEvaluated() + processMonitor.newActionsEvaluated(1) val addedA = archive.addIfNeeded(evalA) assert(addedA) @@ -198,7 +199,7 @@ class ProcessMonitorTest{ b.setValue(1, 1.0) val evalB = ff.calculateCoverage(b, modifiedSpec = null)!! processMonitor.eval = evalB - processMonitor.newActionEvaluated() + processMonitor.newActionsEvaluated(1) val addedB = archive.addIfNeeded(evalB) @@ -249,7 +250,7 @@ class ProcessMonitorTest{ assertFalse(Files.exists(Paths.get(config.processFiles))) assertFalse(Files.exists(Paths.get(processMonitor.getStepDirAsPath()))) - epc.startSearch() + epc.markStartingSearch() mio.search() @@ -276,7 +277,7 @@ class ProcessMonitorTest{ val evalA = ff.calculateCoverage(a, modifiedSpec = null)!! processMonitor.eval = evalA - processMonitor.newActionEvaluated() + processMonitor.newActionsEvaluated(1) val addedA = archive.addIfNeeded(evalA) @@ -310,7 +311,7 @@ class ProcessMonitorTest{ val evalA = ff.calculateCoverage(a, modifiedSpec = null)!! processMonitor.eval = evalA - processMonitor.newActionEvaluated() + processMonitor.newActionsEvaluated(1) val addedA = archive.addIfNeeded(evalA) diff --git a/core/src/test/kotlin/org/evomaster/core/search/service/mutator/MutatorWithOneMaxTest.kt b/core/src/test/kotlin/org/evomaster/core/search/service/mutator/MutatorWithOneMaxTest.kt index 81e00eec52..e6a3acc637 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/service/mutator/MutatorWithOneMaxTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/service/mutator/MutatorWithOneMaxTest.kt @@ -9,11 +9,10 @@ import org.evomaster.core.BaseModule import org.evomaster.core.EMConfig import org.evomaster.core.search.algorithms.MioAlgorithm import org.evomaster.core.search.algorithms.onemax.* -import org.evomaster.core.search.service.ExecutionPhaseController +import org.evomaster.core.search.service.time.ExecutionPhaseController import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.junit.jupiter.api.Assertions.* -import org.junit.jupiter.api.Disabled import java.nio.file.Files import java.nio.file.Paths @@ -63,7 +62,7 @@ class MutatorWithOneMaxTest { Files.deleteIfExists(Paths.get(config.mutatedGeneFile)) val epc = injector.getInstance(ExecutionPhaseController::class.java) - epc.startSearch() + epc.markStartingSearch() val mio = injector.getInstance(Key.get( @@ -195,4 +194,4 @@ class MutatorWithOneMaxTest { assertTrue(result < 0.1, "less than 0.1 is expected, but $result") } -} \ No newline at end of file +} diff --git a/docs/options.md b/docs/options.md index ab7f4bdd1a..4d6a04ef2c 100644 --- a/docs/options.md +++ b/docs/options.md @@ -122,6 +122,7 @@ There are 3 types of options: |`exportImpacts`| __Boolean__. Specify whether to export derived impacts among genes. *DEBUG option*. *Default value*: `false`.| |`extraHeader`| __Boolean__. Add an extra HTTP header, to analyze how it is used/read by the SUT. Needed to discover new headers that were not specified in the schema. *Default value*: `true`.| |`extraHeuristicsFile`| __String__. Where the extra heuristics file (if any) is going to be written (in CSV format). *Default value*: `extra_heuristics.csv`.| +|`extraPhaseBudgetPercentage`| __Double__. When the main fuzzing session is over, there are several following phases in which test cases can be created and evaluated (e.g., for minimization, flakiness and security checks). Each of these follow up phases takes some time, which is not included in 'maxTime'. All those phases are time-bounded, based on the main search budget. For example, with a default of 10% and main budget of 1 hour, then each phase will take at most 6 minutes each after the 1 hour search. Phases will be preemptively stopped if they reach their timeouts. *Constraints*: `probability 0.0-1.0`. *Default value*: `0.1`.| |`extraQueryParam`| __Boolean__. Add an extra query param, to analyze how it is used/read by the SUT. Needed to discover new query params that were not specified in the schema. *Default value*: `true`.| |`extractMongoExecutionInfo`| __Boolean__. Enable extracting Mongo execution info. *Depends on*: `blackBox=false`. *Default value*: `true`.| |`extractSqlExecutionInfo`| __Boolean__. Enable extracting SQL execution info. *Depends on*: `blackBox=false`. *Default value*: `true`.|