Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DeletedBlocksTransactionSummary;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionScmResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMListContainerRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerResponseProto;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
import org.apache.hadoop.hdds.scm.container.ContainerID;
Expand Down Expand Up @@ -115,6 +116,15 @@ void deleteContainer(long containerId, Pipeline pipeline, boolean force)
*/
void deleteContainer(long containerId, boolean force) throws IOException;

/**
* Lists containers using a single SCM list container request.
*
* @param request start containerID, count and optional filters
* @return a list of containers matching the filters.
* @throws IOException
*/
ContainerListResult listContainer(SCMListContainerRequestProto request) throws IOException;

/**
* Lists a range of containers and get their info.
*
Expand All @@ -125,6 +135,7 @@ void deleteContainer(long containerId, Pipeline pipeline, boolean force)
* in "ozone.scm.container.list.max.count" and total number of containers.
* @throws IOException
*/
@Deprecated
ContainerListResult listContainer(long startContainerID,
int count) throws IOException;

Expand All @@ -139,6 +150,7 @@ ContainerListResult listContainer(long startContainerID,
* in "ozone.scm.container.list.max.count" and total number of containers.
* @throws IOException
*/
@Deprecated
ContainerListResult listContainer(long startContainerID, int count,
HddsProtos.LifeCycleState state,
HddsProtos.ReplicationType replicationType,
Expand All @@ -157,6 +169,7 @@ ContainerListResult listContainer(long startContainerID, int count,
* in "ozone.scm.container.list.max.count" and total number of containers.
* @throws IOException
*/
@Deprecated
ContainerListResult listContainer(long startContainerID, int count,
HddsProtos.LifeCycleState state,
HddsProtos.ReplicationType replicationType,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.hadoop.hdds.scm.protocol;

import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType.EC;

import jakarta.annotation.Nullable;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicatedReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMListContainerRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMListContainerRequestProto.Builder;

/**
* Codec between {@link SCMListContainerRequestProto} and {@link ListContainerQuery} for list containers.
*/
public final class ScmListContainerRequestCodec {

private ScmListContainerRequestCodec() {
}

/**
* Immutable view of list container request fields after {@link #fromProto}.
*/
public static final class ListContainerQuery {
private final long startContainerID;
private final int count;
@Nullable
private final HddsProtos.LifeCycleState state;
@Nullable
private final HddsProtos.ReplicationFactor factor;
@Nullable
private final HddsProtos.ReplicationType replicationType;
@Nullable
private final ReplicationConfig replicationConfig;
@Nullable
private final Boolean suppressed;

private ListContainerQuery(long startContainerID, int count,
@Nullable HddsProtos.LifeCycleState state,
@Nullable HddsProtos.ReplicationFactor factor,
@Nullable HddsProtos.ReplicationType replicationType,
@Nullable ReplicationConfig replicationConfig,
@Nullable Boolean suppressed) {
this.startContainerID = startContainerID;
this.count = count;
this.state = state;
this.factor = factor;
this.replicationType = replicationType;
this.replicationConfig = replicationConfig;
this.suppressed = suppressed;
}

public long getStartContainerID() {
return startContainerID;
}

public int getCount() {
return count;
}

@Nullable
public HddsProtos.LifeCycleState getState() {
return state;
}

@Nullable
public HddsProtos.ReplicationFactor getFactor() {
return factor;
}

@Nullable
public HddsProtos.ReplicationType getReplicationType() {
return replicationType;
}

@Nullable
public ReplicationConfig getReplicationConfig() {
return replicationConfig;
}

@Nullable
public Boolean getSuppressed() {
return suppressed;
}
}

/**
* Builds {@link SCMListContainerRequestProto} from Java fields.
*/
@SuppressWarnings("checkstyle:ParameterNumber")
public static SCMListContainerRequestProto toProto(
long startContainerID,
int count,
@Nullable HddsProtos.LifeCycleState state,
@Nullable HddsProtos.ReplicationFactor factor,
@Nullable HddsProtos.ReplicationType replicationType,
@Nullable ReplicationConfig repConfig,
@Nullable Boolean suppressed,
@Nullable String traceId) {
Builder builder = SCMListContainerRequestProto.newBuilder()
.setCount(count)
.setStartContainerID(startContainerID);
if (traceId != null) {
builder.setTraceID(traceId);
}
if (suppressed != null) {
builder.setSuppressed(suppressed);
}
if (state != null) {
builder.setState(state);
}
if (repConfig != null) {
HddsProtos.ReplicationType repType = repConfig.getReplicationType();
if (repType == EC) {
builder.setType(EC);
builder.setEcReplicationConfig(((ECReplicationConfig) repConfig).toProto());
} else {
builder.setType(repType);
builder.setFactor(((ReplicatedReplicationConfig) repConfig)
.getReplicationFactor());
Comment thread
sarvekshayr marked this conversation as resolved.
}
} else if (replicationType != null) {
builder.setType(replicationType);
} else if (factor != null) {
builder.setFactor(factor);
}
return builder.build();
}

/**
* Decodes {@link SCMListContainerRequestProto} into {@link ListContainerQuery}.
*/
public static ListContainerQuery fromProto(SCMListContainerRequestProto request) {
long startContainerID = 0L;
if (request.hasStartContainerID()) {
startContainerID = request.getStartContainerID();
}
int count = request.getCount();
HddsProtos.LifeCycleState state = null;
HddsProtos.ReplicationFactor factor = null;
HddsProtos.ReplicationType replicationType = null;
ReplicationConfig repConfig = null;
if (request.hasState()) {
state = request.getState();
}
if (request.hasType()) {
replicationType = request.getType();
}
if (replicationType != null) {
if (replicationType == EC) {
if (request.hasEcReplicationConfig()) {
repConfig = new ECReplicationConfig(request.getEcReplicationConfig());
}
} else {
if (request.hasFactor()) {
repConfig = ReplicationConfig
.fromProtoTypeAndFactor(replicationType, request.getFactor());
}
}
} else if (request.hasFactor()) {
factor = request.getFactor();
}
Boolean suppressed = request.hasSuppressed() ? request.getSuppressed() : null;
return new ListContainerQuery(startContainerID, count, state, factor,
replicationType, repConfig, suppressed);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.hadoop.hdds.protocol.proto.HddsProtos.DeletedBlocksTransactionSummary;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.DecommissionScmResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SCMListContainerRequestProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.StartContainerBalancerResponseProto;
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.Type;
import org.apache.hadoop.hdds.scm.DatanodeAdminError;
Expand Down Expand Up @@ -147,110 +148,13 @@ List<ContainerWithPipeline> getExistContainerWithPipelinesInBatch(
List<Long> containerIDs);

/**
* Ask SCM a list of containers with a range of container names
* and the limit of count.
* Search container names between start name(exclusive), and
* use prefix name to filter the result. the max size of the
* searching range cannot exceed the value of count.
* Lists containers using a single SCM list container request.
*
* @param startContainerID start container ID.
* @param count count, if count {@literal <} 0, the max size is unlimited.(
* Usually the count will be replace with a very big
* value instead of being unlimited in case the db is very big)
*
* @return a list of containers capped by max count allowed
* in "ozone.scm.container.list.max.count" and total number of containers.
* @throws IOException
*/
ContainerListResult listContainer(long startContainerID,
int count) throws IOException;

/**
* Ask SCM a list of containers with a range of container names
* and the limit of count.
* Search container names between start name(exclusive), and
* use prefix name to filter the result. the max size of the
* searching range cannot exceed the value of count.
*
* @param startContainerID start container ID.
* @param count count, if count {@literal <} 0, the max size is unlimited.(
* Usually the count will be replace with a very big
* value instead of being unlimited in case the db is very big)
* @param state Container with this state will be returned.
*
* @return a list of containers capped by max count allowed
* in "ozone.scm.container.list.max.count" and total number of containers.
* @throws IOException
*/
ContainerListResult listContainer(long startContainerID,
int count, HddsProtos.LifeCycleState state) throws IOException;

/**
* Ask SCM a list of containers with a range of container names
* and the limit of count.
* Search container names between start name(exclusive), and
* use prefix name to filter the result. the max size of the
* searching range cannot exceed the value of count.
*
* @param startContainerID start container ID.
* @param count count, if count {@literal <} 0, the max size is unlimited.(
* Usually the count will be replace with a very big
* value instead of being unlimited in case the db is very big)
* @param state Container with this state will be returned.
* @param factor Container factor
* @return a list of containers capped by max count allowed
* in "ozone.scm.container.list.max.count" and total number of containers.
* @param request start containerID, count and optional filters
* @return a list of containers matching the filters.
* @throws IOException
*/
ContainerListResult listContainer(long startContainerID,
int count, HddsProtos.LifeCycleState state,
HddsProtos.ReplicationFactor factor) throws IOException;

/**
* Ask SCM for a list of containers with a range of container ID, state
* and replication config, and the limit of count.
* The containers are returned from startID (exclusive), and
* filtered by state and replication config. The returned list is limited to
* count entries.
*
* @param startContainerID start container ID.
* @param count count, if count {@literal <} 0, the max size is unlimited.(
* Usually the count will be replace with a very big
* value instead of being unlimited in case the db is very big)
* @param state Container with this state will be returned.
* @param replicationConfig Replication config for the containers
* @return a list of containers capped by max count allowed
* in "ozone.scm.container.list.max.count" and total number of containers.
* @throws IOException
*/
ContainerListResult listContainer(long startContainerID,
int count, HddsProtos.LifeCycleState state,
HddsProtos.ReplicationType replicationType,
ReplicationConfig replicationConfig) throws IOException;

/**
* Ask SCM for a list of containers with a range of container ID, state
* and replication config, and the limit of count.
* The containers are returned from startID (exclusive), and
* filtered by state and replication config. The returned list is limited to
* count entries.
*
* @param startContainerID start container ID.
* @param count count, if count {@literal <} 0, the max size is unlimited.(
* Usually the count will be replace with a very big
* value instead of being unlimited in case the db is very big)
* @param state Container with this state will be returned.
* @param replicationConfig Replication config for the containers
* @param suppressed container to be suppressed/unsuppressed from report
* @return a list of containers capped by max count allowed
* in "ozone.scm.container.list.max.count" and total number of containers.
* @throws IOException
*/
ContainerListResult listContainer(long startContainerID,
int count, HddsProtos.LifeCycleState state,
HddsProtos.ReplicationType replicationType,
ReplicationConfig replicationConfig,
Boolean suppressed) throws IOException;
ContainerListResult listContainer(SCMListContainerRequestProto request) throws IOException;

/**
* Deletes a container in SCM.
Expand Down
Loading