RATIS-2398. Add opentelemetry-javaagent to ratis-examples and assembly

pom.xml

@@ -184,6 +184,7 @@
     <!-- OpenTelemetry versions -->
     <opentelemetry.version>1.60.1</opentelemetry.version>
     <opentelemetry-semconv.version>1.40.0</opentelemetry-semconv.version>
+    <opentelemetry-javaagent.version>2.26.1</opentelemetry-javaagent.version>
     <!-- Test properties -->
     <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
     <test.exclude.pattern>_</test.exclude.pattern>
@@ -414,6 +415,11 @@
         <artifactId>opentelemetry-context</artifactId>
         <version>${opentelemetry.version}</version>
       </dependency>
+      <dependency>
+        <groupId>io.opentelemetry.javaagent</groupId>
+        <artifactId>opentelemetry-javaagent</artifactId>
+        <version>${opentelemetry-javaagent.version}</version>
+      </dependency>
     </dependencies>
 
   </dependencyManagement>

ratis-assembly/pom.xml

@@ -289,6 +289,11 @@
       <groupId>org.apache.ratis</groupId>
       <artifactId>ratis-shell</artifactId>
     </dependency>
+    <!-- Include OpenTelemetry agent -->
+    <dependency>
+      <groupId>io.opentelemetry.javaagent</groupId>
+      <artifactId>opentelemetry-javaagent</artifactId>
+    </dependency>
   </dependencies>
 
   <profiles>

ratis-assembly/src/main/assembly/bin.xml

@@ -33,6 +33,12 @@
       </includes>
       <outputDirectory>examples/lib</outputDirectory>
     </dependencySet>
+    <dependencySet>
+      <outputDirectory>lib/trace</outputDirectory>
+      <includes>
+        <include>io.opentelemetry.javaagent:*</include>
+      </includes>
+    </dependencySet>
   </dependencySets>
   <moduleSets>
     <moduleSet>

ratis-client/src/main/java/org/apache/ratis/client/impl/RaftClientImpl.java

@@ -26,6 +26,7 @@
 import org.apache.ratis.client.retry.ClientRetryEvent;
 import org.apache.ratis.conf.Parameters;
 import org.apache.ratis.conf.RaftProperties;
+import io.opentelemetry.context.Context;
 import org.apache.ratis.proto.RaftProtos.SlidingWindowEntry;
 import org.apache.ratis.protocol.ClientId;
 import org.apache.ratis.protocol.Message;
@@ -291,6 +292,9 @@ RaftClientRequest newRaftClientRequest(
       b.setLeaderId(getLeaderId())
        .setRepliedCallIds(repliedCallIds.get(callId));
     }
+    if (TraceUtils.getGlobalTracer() != null) {
+      b.setSpanContext(TraceUtils.injectContextToProto(Context.current()));
+    }
     return b.setClientId(clientId)
         .setGroupId(groupId)
         .setCallId(callId)

ratis-common/src/main/java/org/apache/ratis/trace/TraceConfigKeys.java

@@ -18,6 +18,7 @@
 package org.apache.ratis.trace;
 
 import org.apache.ratis.conf.RaftProperties;
+import org.apache.ratis.util.StringUtils;
 
 import java.util.function.Consumer;
 
@@ -30,8 +31,20 @@ public interface TraceConfigKeys {
   String ENABLED_KEY = PREFIX + ".enabled";
   boolean ENABLED_DEFAULT = false;
 
+  /**
+   * Whether Ratis should emit OpenTelemetry spans. When the key is absent from
+   * {@link RaftProperties}, the JVM system property {@value #ENABLED_KEY} is consulted so
+   * {@code -Draft.otel.tracing.enabled=true} works with empty example configs.
+   */
   static boolean enabled(RaftProperties properties, Consumer<String> logger) {
-    return getBoolean(properties::getBoolean, ENABLED_KEY, ENABLED_DEFAULT, logger);
+    if (properties.getRaw(ENABLED_KEY) != null) {
+      return getBoolean(properties::getBoolean, ENABLED_KEY, ENABLED_DEFAULT, logger);
+    }
+    final String fromSystem = System.getProperty(ENABLED_KEY);
+    if (fromSystem != null) {
+      return StringUtils.string2boolean(fromSystem.trim(), ENABLED_DEFAULT);
+    }
+    return ENABLED_DEFAULT;
   }
 
   static boolean enabled(RaftProperties properties) {

ratis-examples/README.md

@@ -146,3 +146,47 @@ by using the `run_all_tests.sh` script found in [dev-support/vagrant/](../dev-su
 See the [dev-support/vagrant/README.md](../dev-support/vagrant/README.md) for more on dependencies and what is setup.
 This will allow one to try a fully setup three server Ratis cluster on a single VM image,
 preventing resource contention with your development host and allowing failure injection too.
+
+
+## Enable Tracing when testing examples
+Ratis uses OpenTracing to provide distributed tracing of requests across the cluster. 
+To enable tracing, you need to set the `-Draft.otel.tracing.enabled=true` as part of 
+the JVM options when running the server and client.
+For example, to enable tracing for the FileStore server, you can run the following command:
+
+```bash
+# build the assembly jar first
+mvn clean package -DskipTests
+
+# extract the assembly jar, mainly we need the dependencies for tracing
+pushd ratis-assembly/target
+file=$(ls -1t ratis-assembly-*.tar.gz | head -n1)
+tar -zxvf "$file"
+popd
+
+# run the server, the tracing will be enabled because it found the 
+# opentelemetry agent jar
+BIN=ratis-examples/src/main/bin
+PEERS=n0:127.0.0.1:6000,n1:127.0.0.1:6001,n2:127.0.0.1:6002
+
+ID=n0; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS}
+ID=n1; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS}
+ID=n2; ${BIN}/server.sh filestore server --id ${ID} --storage /tmp/ratis/${ID} --peers ${PEERS}
+
+# some message should be shown
+[otel.javaagent 2026-04-15 15:55:11:320 -0700] [main] INFO io.opentelemetry.javaagent.tooling.VersionLogger - opentelemetry-javaagent - version: 2.26.1
+```
+
+If you want to configure the tracing further, you can set the following system properties
+
+```bash
+# below is an example to configure the OTLP exporter to send the traces to a local collector, 
+# you can change the endpoint and protocol as needed, and please make sure 
+# raft.otel.tracing.enabled is set to true to enable tracing in Ratis
+export OTEL_EXPORTER_OPTS="-Dotel.traces.exporter=otlp \
+-Dotel.exporter.otlp.protocol=grpc \
+-Dotel.exporter.otlp.endpoint=http://localhost:4317 \
+-Dotel.metrics.exporter=none \
+-Dotel.logs.exporter=none \
+-Draft.otel.tracing.enabled=true"
+```

ratis-examples/src/main/bin/client.sh

@@ -23,7 +23,8 @@ if [ "$#" -lt 2 ]; then
   exit 1
 fi
 
-source $DIR/common.sh
+OTEL_SERVICE_NAME=ratis.client
+source "${DIR}/common.sh"
 
 # One of the examples, e.g. "filestore" or "arithmetic"
 example="$1"
@@ -32,4 +33,4 @@ shift
 subcommand="$1"
 shift
 
-java ${LOGGER_OPTS} -jar $ARTIFACT "$example" "$subcommand" "$@"
+java ${OTEL_OPTS} ${LOGGER_OPTS} -jar $ARTIFACT "$example" "$subcommand" "$@"

ratis-examples/src/main/bin/common.sh

@@ -46,3 +46,43 @@ if [[ -d "${CONF_DIR}" ]]; then
 else
   LOGGER_OPTS="-Dlog4j.configuration=file:${DIR}/../resources/log4j.properties"
 fi
+
+
+# for opentelemetry: release tarball uses lib/trace next to examples/;
+# dev tree uses ratis-assembly/target/apache-ratis-*-bin/lib/trace after package.
+if [[ -n "${OTEL_JAR:-}" && -f "${OTEL_JAR}" ]]; then
+  : # use OTEL_JAR from environment
+else
+  otel_jar_candidates=()
+  for _jar in "${SCRIPT_DIR}"/../../lib/trace/opentelemetry-javaagent*.jar; do
+    [[ -f "${_jar}" ]] && otel_jar_candidates+=("${_jar}")
+  done
+  for _otel_trace_dir in "${SCRIPT_DIR}"/../../../../ratis-assembly/target/apache-ratis-*-bin/lib/trace; do
+    [[ -d "${_otel_trace_dir}" ]] || continue
+    for _jar in "${_otel_trace_dir}"/opentelemetry-javaagent*.jar; do
+      [[ -f "${_jar}" ]] && otel_jar_candidates+=("${_jar}")
+    done
+  done
+  if ((${#otel_jar_candidates[@]} > 0)); then
+    OTEL_JAR="$(printf '%s\n' "${otel_jar_candidates[@]}" | sort -V | tail -n 1)"
+  else
+    echo "Warning: OpenTelemetry agent jar not found; OTEL disabled"
+    OTEL_JAR=""
+  fi
+fi
+
+OTEL_SERVICE_NAME="${OTEL_SERVICE_NAME:-ratis.server}"
+OTEL_DEFAULT_EXPORTER_OPTS="-Dotel.traces.exporter=logging \
+-Dotel.metrics.exporter=none \
+-Dotel.logs.exporter=logging"
+# Unset or empty → default exporter flags (empty is not a supported override).
+OTEL_EXPORTER_OPTS="${OTEL_EXPORTER_OPTS:-${OTEL_DEFAULT_EXPORTER_OPTS}}"
+
+# Enable javaagent when OTEL_JAR resolves to a file (env or discovery above).
+if [[ -n "${OTEL_JAR}" && -f "${OTEL_JAR}" ]]; then
+  OTEL_OPTS="-javaagent:${OTEL_JAR} -Dotel.resource.attributes=service.name=${OTEL_SERVICE_NAME} ${OTEL_EXPORTER_OPTS} -Draft.otel.tracing.enabled=true"
+else
+  OTEL_OPTS=""
+fi
+
+echo "Using OTEL_OPTS: ${OTEL_OPTS}"

ratis-examples/src/main/bin/server.sh

@@ -16,5 +16,4 @@
 
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )"
 source $DIR/common.sh
-
-java ${LOGGER_OPTS} -jar $ARTIFACT "$@"
+java ${OTEL_OPTS} ${LOGGER_OPTS} -jar $ARTIFACT "$@"