Skip to content

Improve parallelism of Lock operations #1480

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
VictorCavichioli wants to merge 6 commits into
base: agent/master
Choose a base branch
from
Open

Improve parallelism of Lock operations #1480

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6ef9bb8
Improve parallelism of Lock operations
VictorCavichioli Apr 10, 2026
5e5fb10
Merge branch 'agent/master' into fix/jobs-late
VictorCavichioli Apr 22, 2026
9732a52
Merge branch 'agent/master' into fix/jobs-late
VictorCavichioli Apr 22, 2026
d23821f
Optimize ScheduleQueue to avoid Zombie threads
VictorCavichioli Apr 22, 2026
5f524cc
Merge branch 'agent/master' into fix/jobs-late
VictorCavichioli Apr 23, 2026
c802d79
Make JolokiaNotificationController with multiple threads
VictorCavichioli Apr 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions ...tion/src/main/java/com/ericsson/bss/cassandra/ecchronos/application/spring/ECChronos.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ public ECChronos(//NOPMD long parameter list
.withTableStorageStates(myECChronosInternals.getTableStorageStates())
.withRepairLockType(configuration.getRepairConfig().getRepairLockType())
.withTimeBasedRunPolicy(myTimeBasedRunPolicy)
.withThreadPoolSize(nativeConnectionProvider.getNodes().size())
.build();

AbstractRepairConfigurationProvider repairConfigurationProvider = new FileBasedRepairConfiguration(applicationContext);
Expand Down
7 changes: 5 additions & 2 deletions ...ava/com/ericsson/bss/cassandra/ecchronos/core/impl/jmx/JolokiaNotificationController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,7 @@ public class JolokiaNotificationController

private final Map<NotificationListener, String> myJolokiaRelationshipListeners = new HashMap<>();

private final ScheduledExecutorService myNotificationExecutor = Executors.newSingleThreadScheduledExecutor(
new ThreadFactoryBuilder().setNameFormat("NotificationRefresher-%d").build());
private final ScheduledExecutorService myNotificationExecutor;

private final Map<UUID, Map<String, String>> myClientIdMap = new ConcurrentHashMap<>();
private final Map<UUID, Map<String, NotificationListener>> myNodeListenersMap = new ConcurrentHashMap<>();
Expand All @@ -91,6 +90,10 @@ public JolokiaNotificationController(final Builder builder)
myRunDelay = builder.myRunDelay;
myIpTranslator = builder.myIpTranslator;
myCertificateHandler = builder.myCertificateHandler;
int nodeCount = myNativeConnectionProvider.getNodes().size();
myNotificationExecutor = Executors.newScheduledThreadPool(
Math.max(nodeCount, 1),
new ThreadFactoryBuilder().setNameFormat("NotificationRefresher-%d").setDaemon(true).build());
}

private HttpClient buildHttpClient()
Expand Down
7 changes: 5 additions & 2 deletions ...pl/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/impl/locks/CASLockFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.ericsson.bss.cassandra.ecchronos.connection.DistributedNativeConnectionProvider;
import com.ericsson.bss.cassandra.ecchronos.core.locks.LockFactory;
import com.ericsson.bss.cassandra.ecchronos.core.state.HostStates;
import com.ericsson.bss.cassandra.ecchronos.utils.exceptions.LockContentionException;
import com.ericsson.bss.cassandra.ecchronos.utils.exceptions.LockException;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
Expand Down Expand Up @@ -88,10 +89,12 @@ public final class CASLockFactory implements LockFactory, Closeable

CASLockFactory(final CASLockFactoryBuilder builder)
{
Map<UUID, Node> nodes = builder.getNativeConnectionProvider().getNodes();
int nodeCount = nodes != null ? Math.max(1, nodes.size()) : 1;
myCasLockProperties = new CASLockProperties(
builder.getNativeConnectionProvider().getConnectionType(),
builder.getKeyspaceName(),
Executors.newSingleThreadScheduledExecutor(
Executors.newScheduledThreadPool(nodeCount,
new ThreadFactoryBuilder().setNameFormat("LockRefresher-%d").build()),
builder.getConsistencyType(),
builder.getNativeConnectionProvider().getCqlSession());
Expand Down Expand Up @@ -293,7 +296,7 @@ private DistributedLock doTryLock(final String dataCenter,
}
else
{
throw new LockException(String.format("Unable to lock resource %s in datacenter %s", resource, dataCenter));
throw new LockContentionException(String.format("Unable to lock resource %s in datacenter %s", resource, dataCenter));
}
}

Expand Down
7 changes: 6 additions & 1 deletion core.impl/src/main/java/com/ericsson/bss/cassandra/ecchronos/core/impl/locks/LockCache.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package com.ericsson.bss.cassandra.ecchronos.core.impl.locks;

import com.ericsson.bss.cassandra.ecchronos.core.locks.LockFactory.DistributedLock;
import com.ericsson.bss.cassandra.ecchronos.utils.exceptions.LockContentionException;
import com.ericsson.bss.cassandra.ecchronos.utils.exceptions.LockException;
import com.github.benmanes.caffeine.cache.Cache;
import com.github.benmanes.caffeine.cache.Caffeine;
Expand Down Expand Up @@ -76,6 +77,10 @@ public DistributedLock getLock(final String dataCenter,
{
return myLockSupplier.getLock(dataCenter, resource, priority, metadata);
}
catch (LockContentionException e)
{
throw e;
}
catch (LockException e)
{
myFailureCache.put(lockKey, e);
Expand All @@ -85,7 +90,7 @@ public DistributedLock getLock(final String dataCenter,

private void throwCachedLockException(final LockException e) throws LockException
{
LOG.debug("Encountered cached locking failure, throwing exception", e);
LOG.info("[DIAG] Lock blocked by cached failure: {}", e.getMessage());
Copy link
Copy Markdown
Collaborator

@jwaeab jwaeab Apr 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the idea with these [DIAG] log entries? Are they temporary for this PR? I mean, going from DEBUG to INFO and attaching a DIAG tag...

Copy link
Copy Markdown
Collaborator

@tommystendahl tommystendahl Apr 23, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please, don't do e.getMessage(). It will only print the message, it dosen't tell you what the exception was. Remove to {} and do just e the way it was before, that will give you the expetion, the message and the stack trace.

throw e;
}

Expand Down
14 changes: 7 additions & 7 deletions ...src/main/java/com/ericsson/bss/cassandra/ecchronos/core/impl/multithreads/NodeWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,13 +208,13 @@ private void allTableOperation(
final String keyspaceName,
final BiConsumer<TableReference, TableMetadata> consumer)
{
for (TableMetadata tableMetadata : Metadata.getKeyspace(mySession, keyspaceName).get().getTables().values())
{
String tableName = tableMetadata.getName().asInternal();
TableReference tableReference = myTableReferenceFactory.forTable(keyspaceName, tableName);

consumer.accept(tableReference, tableMetadata);
}
Metadata.getKeyspace(mySession, keyspaceName).get().getTables().values().parallelStream()
.forEach(tableMetadata ->
{
String tableName = tableMetadata.getName().asInternal();
TableReference tableReference = myTableReferenceFactory.forTable(keyspaceName, tableName);
consumer.accept(tableReference, tableMetadata);
});
}

/**
Expand Down
86 changes: 53 additions & 33 deletions .../com/ericsson/bss/cassandra/ecchronos/core/impl/repair/scheduler/RepairSchedulerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,6 @@
import java.util.HashMap;
import java.util.List;
import java.util.ArrayList;
import java.util.AbstractMap;
import java.util.Collection;

import java.util.concurrent.ConcurrentHashMap;
Expand All @@ -68,11 +67,12 @@
public final class RepairSchedulerImpl implements RepairScheduler, Closeable
{
private static final int DEFAULT_TERMINATION_WAIT_IN_SECONDS = 10;
private static final int DEFAULT_THREAD_POOL_SIZE = 4;

private static final Logger LOG = LoggerFactory.getLogger(RepairSchedulerImpl.class);

private final Map<UUID, Map<TableReference, Set<ScheduledRepairJob>>> myScheduledJobs = new ConcurrentHashMap<>();
private final Object myLock = new Object();
private final ConcurrentHashMap<UUID, Object> myNodeLocks = new ConcurrentHashMap<>();

private final ExecutorService myExecutor;
private final TableRepairMetrics myTableRepairMetrics;
Expand Down Expand Up @@ -107,7 +107,8 @@ private Set<ScheduledRepairJob> validateScheduleMap(final UUID nodeID, final Tab

private RepairSchedulerImpl(final Builder builder)
{
myExecutor = Executors.newSingleThreadScheduledExecutor(
int poolSize = Math.max(DEFAULT_THREAD_POOL_SIZE, builder.myThreadPoolSize);
myExecutor = Executors.newFixedThreadPool(poolSize,
new ThreadFactoryBuilder().setNameFormat("RepairScheduler-%d").build());
myFaultReporter = builder.myFaultReporter;
myTableRepairMetrics = builder.myTableRepairMetrics;
Expand All @@ -129,6 +130,11 @@ public String getCurrentJobStatus()
return myScheduleManager.getCurrentJobStatus();
}

private Object getNodeLock(final UUID nodeID)
{
return myNodeLocks.computeIfAbsent(nodeID, k -> new Object());
}

@Override
public void close()
{
Expand All @@ -146,18 +152,16 @@ public void close()
Thread.currentThread().interrupt();
}

synchronized (myLock)
myScheduledJobs.forEach((nodeID, tableJobs) ->
{
myScheduledJobs.entrySet().stream()
.flatMap(nodeEntry -> nodeEntry.getValue().entrySet().stream()
.flatMap(tableEntry -> tableEntry.getValue().stream()
.map(job -> new AbstractMap.SimpleEntry<>(nodeEntry.getKey(), job))
)
)
.forEach(entry -> descheduleTableJob(entry.getKey(), entry.getValue()));

myScheduledJobs.clear();
}
synchronized (getNodeLock(nodeID))
{
tableJobs.values().stream()
.flatMap(Set::stream)
.forEach(job -> descheduleTableJob(nodeID, job));
}
});
myScheduledJobs.clear();
}

@Override
Expand All @@ -178,28 +182,31 @@ public void removeConfiguration(final Node node, final TableReference tableRefer
@Override
public List<ScheduledRepairJobView> getCurrentRepairJobs()
{
synchronized (myLock)
{
return myScheduledJobs.values().stream()
.flatMap(tableJobs -> tableJobs.values().stream())
.flatMap(Set::stream)
.map(ScheduledRepairJob::getView)
.collect(Collectors.toList());
}
return myScheduledJobs.entrySet().stream()
.flatMap(nodeEntry ->
{
synchronized (getNodeLock(nodeEntry.getKey()))
{
return nodeEntry.getValue().values().stream()
.flatMap(Set::stream)
.map(ScheduledRepairJob::getView)
.collect(Collectors.toList())
.stream();
}
})
.collect(Collectors.toList());
}

@Override
public List<ScheduledRepairJobView> getCurrentRepairJobsByNode(final UUID nodeId)
{
synchronized (myLock)
Map<TableReference, Set<ScheduledRepairJob>> tableJobs = myScheduledJobs.get(nodeId);
if (tableJobs == null)
{
return Collections.emptyList();
}
synchronized (getNodeLock(nodeId))
{
Map<TableReference, Set<ScheduledRepairJob>> tableJobs = myScheduledJobs.get(nodeId);

if (tableJobs == null)
{
return Collections.emptyList();
}

return tableJobs.values().stream()
.flatMap(Set::stream)
.map(ScheduledRepairJob::getView)
Expand All @@ -213,7 +220,7 @@ private void handleTableConfigurationChange(
final TableReference tableReference,
final Set<RepairConfiguration> repairConfigurations)
{
synchronized (myLock)
synchronized (getNodeLock(node.getHostId()))
{
try
{
Expand Down Expand Up @@ -311,7 +318,7 @@ private void createTableSchedule(

private void tableConfigurationRemoved(final Node node, final TableReference tableReference)
{
synchronized (myLock)
synchronized (getNodeLock(node.getHostId()))
{
try
{
Expand All @@ -336,7 +343,7 @@ public void removeAllConfigurationsForNode(final UUID nodeId)

private void nodeConfigurationRemoved(final UUID nodeId)
{
synchronized (myLock)
synchronized (getNodeLock(nodeId))
{
try
{
Expand Down Expand Up @@ -448,6 +455,7 @@ public static class Builder
private TableStorageStates myTableStorageStates;
private RepairLockType myRepairLockType;
private TimeBasedRunPolicy myTimeBasedRunPolicy;
private int myThreadPoolSize = DEFAULT_THREAD_POOL_SIZE;

/**
* RepairSchedulerImpl build with repair lock type.
Expand Down Expand Up @@ -593,6 +601,18 @@ public Builder withTimeBasedRunPolicy(final TimeBasedRunPolicy timeBasedRunPolic
return this;
}

/**
* RepairSchedulerImpl build with thread pool size.
*
* @param threadPoolSize Thread pool size for parallel configuration processing.
* @return Builder
*/
public Builder withThreadPoolSize(final int threadPoolSize)
{
myThreadPoolSize = threadPoolSize;
return this;
}

/**
* RepairSchedulerImpl build.
*
Expand Down
Loading
Loading