diff --git a/app/com/linkedin/drelephant/exceptions/core/ExceptionFingerprintingRunner.java b/app/com/linkedin/drelephant/exceptions/core/ExceptionFingerprintingRunner.java index 49b96194d..5ca90a99d 100644 --- a/app/com/linkedin/drelephant/exceptions/core/ExceptionFingerprintingRunner.java +++ b/app/com/linkedin/drelephant/exceptions/core/ExceptionFingerprintingRunner.java @@ -27,6 +27,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import javax.persistence.PersistenceException; @@ -79,8 +80,9 @@ public void run() { if (SHOULD_PROCESS_AZKABAN_LOG.getValue() && exceptionInfos.size() <= 1) { exceptionInfos.addAll(getAzkabanExceptionInfoResults()); } - saveDriverExceptionLogForExceptionFingerPrinting(exceptionInfos, exceptionFingerprinting.getExceptionLogSourceInformation()); - LogClass logClass = exceptionFingerprinting.classifyException(exceptionInfos); + List deDupedExceptionInfoList = eliminateDuplicateLogs(exceptionInfos); + saveDriverExceptionLogForExceptionFingerPrinting(deDupedExceptionInfoList, exceptionFingerprinting.getExceptionLogSourceInformation()); + LogClass logClass = exceptionFingerprinting.classifyException(deDupedExceptionInfoList); boolean isAutoTuningFault = false; if (logClass != null && logClass.equals(LogClass.AUTOTUNING_ENABLED)) { isAutoTuningFault = true; @@ -238,4 +240,26 @@ private String processLogSourceInformation(Map logSourceInformat return logSource.toString(); } -} + private List eliminateDuplicateLogs(List exceptionInfos) { + HashSet exceptionInfoSet = new HashSet<>(); + List deDuplicatedExceptionInfo = new ArrayList<>(); + logger.info("Size of original exception info list " + exceptionInfos.size()); + for (ExceptionInfo exceptionInfo : exceptionInfos) { + boolean isDuplicate = false; + if (exceptionInfoSet.contains(exceptionInfo.getExceptionStackTrace())) { + for (String uniqueExceptions : exceptionInfoSet) { + if (uniqueExceptions.equals(exceptionInfo.getExceptionStackTrace())) { + isDuplicate = true; + break; + } + } + } + if (!isDuplicate) { + deDuplicatedExceptionInfo.add(exceptionInfo); + exceptionInfoSet.add(exceptionInfo.getExceptionStackTrace()); + } + } + logger.info("Size of exception info list after deduplication" + deDuplicatedExceptionInfo.size()); + return deDuplicatedExceptionInfo; + } +} \ No newline at end of file