diff --git a/pom.xml b/pom.xml index 4ffed3fdc2..32c55768e0 100644 --- a/pom.xml +++ b/pom.xml @@ -184,6 +184,7 @@ 1.60.1 1.40.0 + 2.26.1 true _ @@ -414,6 +415,11 @@ opentelemetry-context ${opentelemetry.version} + + io.opentelemetry.javaagent + opentelemetry-javaagent + ${opentelemetry-javaagent.version} + diff --git a/ratis-assembly/pom.xml b/ratis-assembly/pom.xml index 0558693039..9987d91a15 100644 --- a/ratis-assembly/pom.xml +++ b/ratis-assembly/pom.xml @@ -289,6 +289,11 @@ org.apache.ratis ratis-shell + + + io.opentelemetry.javaagent + opentelemetry-javaagent + diff --git a/ratis-assembly/src/main/assembly/bin.xml b/ratis-assembly/src/main/assembly/bin.xml index 12161b00a8..6a4f3a62b1 100644 --- a/ratis-assembly/src/main/assembly/bin.xml +++ b/ratis-assembly/src/main/assembly/bin.xml @@ -33,6 +33,12 @@ examples/lib + + lib/trace + + io.opentelemetry.javaagent:* + + diff --git a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java index 27ae2e6bab..79a35385ee 100644 --- a/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java +++ b/ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java @@ -26,6 +26,7 @@ import org.apache.ratis.client.retry.ClientRetryEvent; import org.apache.ratis.conf.Parameters; import org.apache.ratis.conf.RaftProperties; +import io.opentelemetry.context.Context; import org.apache.ratis.proto.RaftProtos.SlidingWindowEntry; import org.apache.ratis.protocol.ClientId; import org.apache.ratis.protocol.Message; @@ -291,6 +292,9 @@ RaftClientRequest newRaftClientRequest( b.setLeaderId(getLeaderId()) .setRepliedCallIds(repliedCallIds.get(callId)); } + if (TraceUtils.getGlobalTracer() != null) { + b.setSpanContext(TraceUtils.injectContextToProto(Context.current())); + } return b.setClientId(clientId) .setGroupId(groupId) .setCallId(callId) diff --git a/ratis-common/src/main/java/org/apache/ratis/trace/TraceConfigKeys.java b/ratis-common/src/main/java/org/apache/ratis/trace/TraceConfigKeys.java index b0a1cbd9b1..90b77ce8e5 100644 --- a/ratis-common/src/main/java/org/apache/ratis/trace/TraceConfigKeys.java +++ b/ratis-common/src/main/java/org/apache/ratis/trace/TraceConfigKeys.java @@ -18,6 +18,7 @@ package org.apache.ratis.trace; import org.apache.ratis.conf.RaftProperties; +import org.apache.ratis.util.StringUtils; import java.util.function.Consumer; @@ -30,8 +31,20 @@ public interface TraceConfigKeys { String ENABLED_KEY = PREFIX + ".enabled"; boolean ENABLED_DEFAULT = false; + /** + * Whether Ratis should emit OpenTelemetry spans. When the key is absent from + * {@link RaftProperties}, the JVM system property {@value #ENABLED_KEY} is consulted so + * {@code -Draft.otel.tracing.enabled=true} works with empty example configs. + */ static boolean enabled(RaftProperties properties, Consumer logger) { - return getBoolean(properties::getBoolean, ENABLED_KEY, ENABLED_DEFAULT, logger); + if (properties.getRaw(ENABLED_KEY) != null) { + return getBoolean(properties::getBoolean, ENABLED_KEY, ENABLED_DEFAULT, logger); + } + final String fromSystem = System.getProperty(ENABLED_KEY); + if (fromSystem != null) { + return StringUtils.string2boolean(fromSystem.trim(), ENABLED_DEFAULT); + } + return ENABLED_DEFAULT; } static boolean enabled(RaftProperties properties) { diff --git a/ratis-examples/README.md b/ratis-examples/README.md index 1e50058dc1..b09ed94874 100644 --- a/ratis-examples/README.md +++ b/ratis-examples/README.md @@ -146,3 +146,47 @@ by using the `run_all_tests.sh` script found in [dev-support/vagrant/](../dev-su See the [dev-support/vagrant/README.md](../dev-support/vagrant/README.md) for more on dependencies and what is setup. This will allow one to try a fully setup three server Ratis cluster on a single VM image, preventing resource contention with your development host and allowing failure injection too. + + +## Enable Tracing when testing examples +Ratis uses OpenTracing to provide distributed tracing of requests across the cluster. +To enable tracing, you need to set the `-Draft.otel.tracing.enabled=true` as part of +the JVM options when running the server and client. +For example, to enable tracing for the FileStore server, you can run the following command: + +```bash +# build the assembly jar first +mvn clean package -DskipTests + +# extract the assembly jar, mainly we need the dependencies for tracing +pushd ratis-assembly/target +file=$(ls -1t ratis-assembly-*.tar.gz | head -n1) +tar -zxvf "$file" +popd + +# run the server, the tracing will be enabled because it found the +# opentelemetry agent jar +BIN=ratis-examples/src/main/bin +PEERS=n0:127.0.0.1:6000,n1:127.0.0.1:6001,n2:127.0.0.1:6002 + +ID=n0; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS} +ID=n1; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS} +ID=n2; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS} + +# some message should be shown +[otel.javaagent 2026-04-15 15:55:11:320 -0700] [main] INFO io.opentelemetry.javaagent.tooling.VersionLogger - opentelemetry-javaagent - version: 2.26.1 +``` + +If you want to configure the tracing further, you can set the following system properties + +```bash +# below is an example to configure the OTLP exporter to send the traces to a local collector, +# you can change the endpoint and protocol as needed, and please make sure +# raft.otel.tracing.enabled is set to true to enable tracing in Ratis +export OTEL_EXPORTER_OPTS="-Dotel.traces.exporter=otlp \ +-Dotel.exporter.otlp.protocol=grpc \ +-Dotel.exporter.otlp.endpoint=http://localhost:4317 \ +-Dotel.metrics.exporter=none \ +-Dotel.logs.exporter=none \ +-Draft.otel.tracing.enabled=true" +``` \ No newline at end of file diff --git a/ratis-examples/src/main/bin/client.sh b/ratis-examples/src/main/bin/client.sh index e42786fa3c..d5e00268ea 100755 --- a/ratis-examples/src/main/bin/client.sh +++ b/ratis-examples/src/main/bin/client.sh @@ -23,7 +23,8 @@ if [ "$#" -lt 2 ]; then exit 1 fi -source $DIR/common.sh +OTEL_SERVICE_NAME=ratis.client +source "${DIR}/common.sh" # One of the examples, e.g. "filestore" or "arithmetic" example="$1" @@ -32,4 +33,4 @@ shift subcommand="$1" shift -java ${LOGGER_OPTS} -jar $ARTIFACT "$example" "$subcommand" "$@" +java ${OTEL_OPTS} ${LOGGER_OPTS} -jar $ARTIFACT "$example" "$subcommand" "$@" diff --git a/ratis-examples/src/main/bin/common.sh b/ratis-examples/src/main/bin/common.sh index b05bf4dbf0..8fa3b3ba7d 100755 --- a/ratis-examples/src/main/bin/common.sh +++ b/ratis-examples/src/main/bin/common.sh @@ -46,3 +46,43 @@ if [[ -d "${CONF_DIR}" ]]; then else LOGGER_OPTS="-Dlog4j.configuration=file:${DIR}/../resources/log4j.properties" fi + + +# for opentelemetry: release tarball uses lib/trace next to examples/; +# dev tree uses ratis-assembly/target/apache-ratis-*-bin/lib/trace after package. +if [[ -n "${OTEL_JAR:-}" && -f "${OTEL_JAR}" ]]; then + : # use OTEL_JAR from environment +else + otel_jar_candidates=() + for _jar in "${SCRIPT_DIR}"/../../lib/trace/opentelemetry-javaagent*.jar; do + [[ -f "${_jar}" ]] && otel_jar_candidates+=("${_jar}") + done + for _otel_trace_dir in "${SCRIPT_DIR}"/../../../../ratis-assembly/target/apache-ratis-*-bin/lib/trace; do + [[ -d "${_otel_trace_dir}" ]] || continue + for _jar in "${_otel_trace_dir}"/opentelemetry-javaagent*.jar; do + [[ -f "${_jar}" ]] && otel_jar_candidates+=("${_jar}") + done + done + if ((${#otel_jar_candidates[@]} > 0)); then + OTEL_JAR="$(printf '%s\n' "${otel_jar_candidates[@]}" | sort -V | tail -n 1)" + else + echo "Warning: OpenTelemetry agent jar not found; OTEL disabled" + OTEL_JAR="" + fi +fi + +OTEL_SERVICE_NAME="${OTEL_SERVICE_NAME:-ratis.server}" +OTEL_DEFAULT_EXPORTER_OPTS="-Dotel.traces.exporter=logging \ +-Dotel.metrics.exporter=none \ +-Dotel.logs.exporter=logging" +# Unset or empty → default exporter flags (empty is not a supported override). +OTEL_EXPORTER_OPTS="${OTEL_EXPORTER_OPTS:-${OTEL_DEFAULT_EXPORTER_OPTS}}" + +# Enable javaagent when OTEL_JAR resolves to a file (env or discovery above). +if [[ -n "${OTEL_JAR}" && -f "${OTEL_JAR}" ]]; then + OTEL_OPTS="-javaagent:${OTEL_JAR} -Dotel.resource.attributes=service.name=${OTEL_SERVICE_NAME} ${OTEL_EXPORTER_OPTS} -Draft.otel.tracing.enabled=true" +else + OTEL_OPTS="" +fi + +echo "Using OTEL_OPTS: ${OTEL_OPTS}" \ No newline at end of file diff --git a/ratis-examples/src/main/bin/server.sh b/ratis-examples/src/main/bin/server.sh index 3cc069edf1..c0468c15bf 100755 --- a/ratis-examples/src/main/bin/server.sh +++ b/ratis-examples/src/main/bin/server.sh @@ -16,5 +16,4 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )" source $DIR/common.sh - -java ${LOGGER_OPTS} -jar $ARTIFACT "$@" +java ${OTEL_OPTS} ${LOGGER_OPTS} -jar $ARTIFACT "$@"