diff --git a/appserver/concurrent/concurrent-impl/pom.xml b/appserver/concurrent/concurrent-impl/pom.xml index 73866a3b50d..14997f65c65 100644 --- a/appserver/concurrent/concurrent-impl/pom.xml +++ b/appserver/concurrent/concurrent-impl/pom.xml @@ -150,6 +150,11 @@ opentelemetry-tracing-starter ${project.version} + + io.opentelemetry + opentelemetry-api + provided + diff --git a/appserver/concurrent/concurrent-impl/src/main/java/org/glassfish/concurrent/runtime/ContextSetupProviderImpl.java b/appserver/concurrent/concurrent-impl/src/main/java/org/glassfish/concurrent/runtime/ContextSetupProviderImpl.java index 0e2fd2e54b0..aea7ff8dbbd 100644 --- a/appserver/concurrent/concurrent-impl/src/main/java/org/glassfish/concurrent/runtime/ContextSetupProviderImpl.java +++ b/appserver/concurrent/concurrent-impl/src/main/java/org/glassfish/concurrent/runtime/ContextSetupProviderImpl.java @@ -49,6 +49,12 @@ import com.sun.enterprise.security.SecurityContext; import com.sun.enterprise.transaction.api.JavaEETransactionManager; import com.sun.enterprise.util.Utility; +import io.opentelemetry.api.baggage.Baggage; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanBuilder; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; import org.glassfish.api.invocation.ComponentInvocation; import org.glassfish.api.invocation.InvocationManager; import org.glassfish.concurrent.LogFacade; @@ -107,6 +113,9 @@ public class ContextSetupProviderImpl implements ContextSetupProvider { public static final String CONTEXT_TYPE_NAMING = "NAMING"; // Concurrency 3.0: APPLICATION public static final String CONTEXT_TYPE_WORKAREA = "WORKAREA"; // Concurrency 3.0: TRANSACTION + private static final ThreadLocal currentConcurrentSpan = new ThreadLocal<>(); + private static final ThreadLocal currentConcurrentScope = new ThreadLocal<>(); + // TODO: do we need these booleans if we have sets? private boolean classloading, security, naming, workArea; private final Set contextPropagate; @@ -306,6 +315,10 @@ public ContextHandle setup(ContextHandle contextHandle) throws IllegalStateExcep transactionManager.clearThreadTx(); } + if (requestTracing != null && requestTracing.isRequestTracingEnabled()) { + startConcurrentContextSpan(invocation, handle); + } + if (stuckThreads != null) { stuckThreads.registerThread(Thread.currentThread().threadId()); } @@ -322,6 +335,50 @@ public ContextHandle setup(ContextHandle contextHandle) throws IllegalStateExcep Collections.EMPTY_LIST, restorers); } + private void startConcurrentContextSpan(ComponentInvocation invocation, InvocationContext handle) { + Tracer tracer = openTracing.getTracer(openTracing.getApplicationName( + Globals.getDefaultBaseServiceLocator().getService(InvocationManager.class))); + SpanBuilder builder = tracer.spanBuilder("executeConcurrentContext"); + Context parentContext = handle.getParentTraceContext(); + if (parentContext != null) { + builder.setParent(parentContext); + + String parentOperationName = Baggage.fromContext(parentContext).getEntryValue("operation.name"); + if (parentOperationName != null) { + builder.setAttribute("Parent Operation Name", parentOperationName); + } + } + + if (invocation != null) { + builder.setAttribute("App Name", invocation.getAppName()) + .setAttribute("Component ID", invocation.getComponentId()) + .setAttribute("Module Name", invocation.getModuleName()); + + Object instance = invocation.getInstance(); + if (instance != null) { + builder.setAttribute("Class Name", instance.getClass().getName()); + } + } + + builder.setAttribute("Thread Name", Thread.currentThread().getName()); + Span span = builder.startSpan(); + currentConcurrentSpan.set(span); + currentConcurrentScope.set(span.makeCurrent()); + } + + private void stopConcurrentContextSpan() { + Scope scope = currentConcurrentScope.get(); + Span span = currentConcurrentSpan.get(); + if (scope != null) { + scope.close(); + currentConcurrentScope.remove(); + } + if (span != null) { + span.end(); + currentConcurrentSpan.remove(); + } + } + @Override public void reset(ContextHandle contextHandle) { if (! (contextHandle instanceof InvocationContext)) { @@ -362,6 +419,7 @@ public void reset(ContextHandle contextHandle) { } if (requestTracing != null && requestTracing.isRequestTracingEnabled()) { + stopConcurrentContextSpan(); requestTracing.endTrace(); } if (stuckThreads != null) { diff --git a/appserver/concurrent/concurrent-impl/src/main/java/org/glassfish/concurrent/runtime/InvocationContext.java b/appserver/concurrent/concurrent-impl/src/main/java/org/glassfish/concurrent/runtime/InvocationContext.java index c3bb12fc9ac..55c52bbe6d1 100644 --- a/appserver/concurrent/concurrent-impl/src/main/java/org/glassfish/concurrent/runtime/InvocationContext.java +++ b/appserver/concurrent/concurrent-impl/src/main/java/org/glassfish/concurrent/runtime/InvocationContext.java @@ -42,10 +42,16 @@ package org.glassfish.concurrent.runtime; import com.sun.enterprise.security.SecurityContext; +import fish.payara.nucleus.requesttracing.RequestTracingService; +import fish.payara.opentracing.OpenTracingService; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; import jakarta.enterprise.concurrent.spi.ThreadContextRestorer; import jakarta.enterprise.concurrent.spi.ThreadContextSnapshot; import org.glassfish.api.invocation.ComponentInvocation; import org.glassfish.concurro.spi.ContextHandle; +import org.glassfish.hk2.api.ServiceLocator; +import org.glassfish.internal.api.Globals; import org.glassfish.internal.data.ApplicationInfo; import org.glassfish.internal.data.ApplicationRegistry; @@ -63,7 +69,8 @@ public class InvocationContext implements ContextHandle { private List threadContextSnapshots; private List threadContextRestorers; - + + private Context parentTraceContext; public InvocationContext(ComponentInvocation invocation, ClassLoader contextClassLoader, SecurityContext securityContext, boolean useTransactionOfExecutionThread, List threadContextSnapshots, @@ -74,8 +81,25 @@ public InvocationContext(ComponentInvocation invocation, ClassLoader contextClas this.useTransactionOfExecutionThread = useTransactionOfExecutionThread; this.threadContextSnapshots = threadContextSnapshots; this.threadContextRestorers = threadContextRestorers; + saveTracingContext(); + } + + private void saveTracingContext() { + ServiceLocator serviceLocator = Globals.getDefaultBaseServiceLocator(); + + if (serviceLocator != null) { + RequestTracingService requestTracing = serviceLocator.getService(RequestTracingService.class); + OpenTracingService openTracing = serviceLocator.getService(OpenTracingService.class); + + if (requestTracing != null && requestTracing.isRequestTracingEnabled() + && requestTracing.isTraceInProgress() && openTracing != null) { + Span currentSpan = Span.current(); + if (currentSpan.isRecording()) { + this.parentTraceContext = Context.current(); + } + } + } } - public ComponentInvocation getInvocation() { return invocation; @@ -101,6 +125,10 @@ public List getThreadContextRestorers() { return threadContextRestorers; } + public Context getParentTraceContext() { + return parentTraceContext; + } + /** * Used to make duplicate of the InvocationContext. */ diff --git a/appserver/ejb/ejb-container/pom.xml b/appserver/ejb/ejb-container/pom.xml index d9fb291869a..77d3236ecb9 100755 --- a/appserver/ejb/ejb-container/pom.xml +++ b/appserver/ejb/ejb-container/pom.xml @@ -257,5 +257,10 @@ org.mockito mockito-core + + io.opentelemetry + opentelemetry-api + provided + diff --git a/appserver/ejb/ejb-container/src/main/java/com/sun/ejb/containers/BaseContainer.java b/appserver/ejb/ejb-container/src/main/java/com/sun/ejb/containers/BaseContainer.java index 4de9a98f75d..668c76696e6 100644 --- a/appserver/ejb/ejb-container/src/main/java/com/sun/ejb/containers/BaseContainer.java +++ b/appserver/ejb/ejb-container/src/main/java/com/sun/ejb/containers/BaseContainer.java @@ -90,6 +90,10 @@ import fish.payara.notification.requesttracing.RequestTraceSpanLog; import fish.payara.nucleus.requesttracing.RequestTracingService; import fish.payara.opentracing.OpenTracingService; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.ejb.AccessLocalException; @@ -4094,6 +4098,29 @@ final void onEjbMethodEnd(String method_sig, Throwable th) { private void addEjbMethodTraceLog(CallFlowInfo info, boolean callEnter) { if (openTracingService.isEnabled()) { + Tracer tracer = openTracingService.getTracer(openTracingService.getApplicationName(invocationManager)); + if (tracer != null) { + String eventName = callEnter ? "enterEjbMethodEvent" : "exitEjbMethodEvent"; + Span span = tracer.spanBuilder(info.getMethod().getName()).startSpan(); + try(Scope scope = span.makeCurrent()) { + if (span.isRecording()) { + span.addEvent(eventName, Attributes.builder() + .put("ApplicationName", info.getApplicationName()) + .put("ComponentName", info.getComponentName()) + .put("ComponentType", info.getComponentType().toString()) + .put("ModuleName", info.getModuleName()) + .put("EJBClass", ejbClass.getCanonicalName()) + .put("EJBMethod", info.getMethod().getName()) + .put("CallerPrincipal", String.valueOf(info.getCallerPrincipal())) + .put("TX-ID", String.valueOf(info.getTransactionId())) + .build()); + } else { + _logger.log(Level.FINE, "Tracing is not enabled, skipping event {0}", eventName); + } + } finally { + span.end(); + } + } RequestTraceSpanLog spanLog = constructEjbMethodSpanLog(info, callEnter); requestTracingService.addSpanLog(spanLog); } diff --git a/appserver/payara-appserver-modules/microprofile/telemetry/pom.xml b/appserver/payara-appserver-modules/microprofile/telemetry/pom.xml index 2f4f2e7c45f..6872fd4d338 100644 --- a/appserver/payara-appserver-modules/microprofile/telemetry/pom.xml +++ b/appserver/payara-appserver-modules/microprofile/telemetry/pom.xml @@ -121,5 +121,10 @@ true provided + + fish.payara.server.internal.orb + orb-connector + ${project.version} + \ No newline at end of file diff --git a/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/TracedInterceptor.java b/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/TracedInterceptor.java index e4973672aa5..3be125c6a57 100644 --- a/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/TracedInterceptor.java +++ b/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/TracedInterceptor.java @@ -109,17 +109,36 @@ public Object traceCdiCall(final InvocationContext invocationContext) throws Exc final String applicationName = payaraTracingServices.getApplicationName(); final Tracer tracer = payaraTracingServices.getActiveTracer(); final String operationName = getOperationName(invocationContext, traced); - Span span = tracer.spanBuilder(operationName).setAttribute("otel.service.name", applicationName).startSpan(); - try (Scope scope = span.makeCurrent()) { - try { - return invocationContext.proceed(); - } catch (Exception e) { - LOG.log(Level.FINEST, "Setting the error to the active span ...", e); - span.recordException(e); - throw e; + + Span span = Span.current(); + if (span.isRecording()) { + span = span.updateName(operationName).setAttribute("otel.service.name", applicationName); + try (Scope scope = span.makeCurrent()) { + try { + return invocationContext.proceed(); + } catch (Exception e) { + LOG.log(Level.FINEST, "Setting the error to the active span ...", e); + span.recordException(e); + throw e; + } + } finally { + span.end(); + LOG.log(Level.FINEST, "Don't close the span for now"); + } + } else { + span = tracer.spanBuilder(operationName).setAttribute("otel.service.name", applicationName).startSpan(); + try (Scope scope = span.makeCurrent()) { + try { + return invocationContext.proceed(); + } catch (Exception e) { + LOG.log(Level.FINEST, "Setting the error to the active span ...", e); + span.recordException(e); + throw e; + } + } finally { + span.end(); + LOG.log(Level.FINEST, "Don't close the span for now"); } - } finally { - span.end(); } } diff --git a/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/ejb/iiop/OpenTelemetryIiopClientInterceptor.java b/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/ejb/iiop/OpenTelemetryIiopClientInterceptor.java new file mode 100644 index 00000000000..94c2d067aa5 --- /dev/null +++ b/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/ejb/iiop/OpenTelemetryIiopClientInterceptor.java @@ -0,0 +1,124 @@ +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright (c) 2020-2026 Payara Foundation and/or its affiliates. All rights reserved. + * + * The contents of this file are subject to the terms of either the GNU + * General Public License Version 2 only ("GPL") or the Common Development + * and Distribution License("CDDL") (collectively, the "License"). You + * may not use this file except in compliance with the License. You can + * obtain a copy of the License at + * https://github.com/payara/Payara/blob/main/LICENSE.txt + * See the License for the specific + * language governing permissions and limitations under the License. + * + * When distributing the software, include this License Header Notice in each + * file and include the License file at legal/OPEN-SOURCE-LICENSE.txt. + * + * GPL Classpath Exception: + * The Payara Foundation designates this particular file as subject to the "Classpath" + * exception as provided by the Payara Foundation in the GPL Version 2 section of the License + * file that accompanied this code. + * + * Modifications: + * If applicable, add the following below the License Header, with the fields + * enclosed by brackets [] replaced by your own identifying information: + * "Portions Copyright [year] [name of copyright owner]" + * + * Contributor(s): + * If you wish your version of this file to be governed by only the CDDL or + * only the GPL Version 2, indicate your decision by adding "[Contributor] + * elects to include this software in this distribution under the [CDDL or GPL + * Version 2] license." If you don't indicate a single choice of license, a + * recipient has the option to distribute your version of this file under + * either the CDDL, the GPL Version 2 or to extend the choice of license to + * its licensees as provided above. However, if you add GPL Version 2 code + * and therefore, elected the GPL Version 2 license, then the option applies + * only if the new code is made subject to such option by the copyright + * holder. + */ +package fish.payara.microprofile.telemetry.tracing.ejb.iiop; + +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.context.Context; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.ObjectOutputStream; +import java.util.HashMap; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.omg.CORBA.LocalObject; +import org.omg.IOP.ServiceContext; +import org.omg.PortableInterceptor.ClientRequestInfo; +import org.omg.PortableInterceptor.ClientRequestInterceptor; +import org.omg.PortableInterceptor.ForwardRequest; + +/** + * IIOP Client Interceptor for propagating OpenTracing SpanContext to Payara Server. + * + * @author Andrew Pielage + */ +public class OpenTelemetryIiopClientInterceptor extends LocalObject implements ClientRequestInterceptor { + + private static final Logger logger = Logger.getLogger(OpenTelemetryIiopClientInterceptor.class.getName()); + static final int OPENTELEMETRY_IIOP_ID = 3226428; + + @Override + public void send_request(ClientRequestInfo clientRequestInfo) throws ForwardRequest { + Span currentSpan = Span.current(); + if (!currentSpan.getSpanContext().isValid()) { + return; + } + + OpenTelemetry openTelemetry = GlobalOpenTelemetry.get(); + HashMap contextMap = new HashMap<>(); + openTelemetry.getPropagators().getTextMapPropagator() + .inject(Context.current(), contextMap, HashMap::put); + + if (contextMap.isEmpty()) { + return; + } + + try (ByteArrayOutputStream bos = new ByteArrayOutputStream(); + ObjectOutputStream out = new ObjectOutputStream(bos)) { + out.writeObject(contextMap); + out.flush(); + clientRequestInfo.add_request_service_context( + new ServiceContext(OPENTELEMETRY_IIOP_ID, bos.toByteArray()), true); + } catch (IOException ex) { + logger.log(Level.SEVERE, "Exception propagating OTel span context over IIOP"); + } + } + + @Override + public void send_poll(ClientRequestInfo clientRequestInfo) { + + } + + @Override + public void receive_reply(ClientRequestInfo clientRequestInfo) { + + } + + @Override + public void receive_exception(ClientRequestInfo clientRequestInfo) throws ForwardRequest { + + } + + @Override + public void receive_other(ClientRequestInfo clientRequestInfo) throws ForwardRequest { + + } + + @Override + public String name() { + return this.getClass().getSimpleName(); + } + + @Override + public void destroy() { + + } +} diff --git a/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/ejb/iiop/OpenTelemetryIiopInterceptorFactory.java b/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/ejb/iiop/OpenTelemetryIiopInterceptorFactory.java new file mode 100644 index 00000000000..59b8adad0fa --- /dev/null +++ b/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/ejb/iiop/OpenTelemetryIiopInterceptorFactory.java @@ -0,0 +1,119 @@ +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright (c) 2020-2026 Payara Foundation and/or its affiliates. All rights reserved. + * + * The contents of this file are subject to the terms of either the GNU + * General Public License Version 2 only ("GPL") or the Common Development + * and Distribution License("CDDL") (collectively, the "License"). You + * may not use this file except in compliance with the License. You can + * obtain a copy of the License at + * https://github.com/payara/Payara/blob/main/LICENSE.txt + * See the License for the specific + * language governing permissions and limitations under the License. + * + * When distributing the software, include this License Header Notice in each + * file and include the License file at legal/OPEN-SOURCE-LICENSE.txt. + * + * GPL Classpath Exception: + * The Payara Foundation designates this particular file as subject to the "Classpath" + * exception as provided by the Payara Foundation in the GPL Version 2 section of the License + * file that accompanied this code. + * + * Modifications: + * If applicable, add the following below the License Header, with the fields + * enclosed by brackets [] replaced by your own identifying information: + * "Portions Copyright [year] [name of copyright owner]" + * + * Contributor(s): + * If you wish your version of this file to be governed by only the CDDL or + * only the GPL Version 2, indicate your decision by adding "[Contributor] + * elects to include this software in this distribution under the [CDDL or GPL + * Version 2] license." If you don't indicate a single choice of license, a + * recipient has the option to distribute your version of this file under + * either the CDDL, the GPL Version 2 or to extend the choice of license to + * its licensees as provided above. However, if you add GPL Version 2 code + * and therefore, elected the GPL Version 2 license, then the option applies + * only if the new code is made subject to such option by the copyright + * holder. + */ +package fish.payara.microprofile.telemetry.tracing.ejb.iiop; + +import fish.payara.opentracing.OpenTracingService; +import jakarta.inject.Singleton; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.glassfish.enterprise.iiop.api.IIOPInterceptorFactory; +import org.glassfish.hk2.api.ServiceLocator; +import org.glassfish.internal.api.Globals; +import org.jvnet.hk2.annotations.Service; +import org.omg.IOP.Codec; +import org.omg.PortableInterceptor.ClientRequestInterceptor; +import org.omg.PortableInterceptor.ORBInitInfo; +import org.omg.PortableInterceptor.ServerRequestInterceptor; + +/** + * Factory for creating IIOP client and server interceptors that propagate OpenTracing SpanContext. + * + * @author Andrew Pielage + */ +@Service(name = "OpenTracingIiopInterceptorFactory") +@Singleton +public class OpenTelemetryIiopInterceptorFactory implements IIOPInterceptorFactory { + + private static final Logger logger = Logger.getLogger(OpenTelemetryIiopInterceptorFactory.class.getName()); + + private ClientRequestInterceptor clientRequestInterceptor; + private ServerRequestInterceptor serverRequestInterceptor; + + private ServiceLocator serviceLocator; + private OpenTracingService openTracingService; + + @Override + public ClientRequestInterceptor createClientRequestInterceptor(ORBInitInfo info, Codec codec) { + if (clientRequestInterceptor == null) { + if (attemptCreation()) { + try { + clientRequestInterceptor = new OpenTelemetryIiopClientInterceptor(); + } catch (NullPointerException nullPointerException) { + logger.log(Level.WARNING, "Could not create OpenTracing IIOP Client Interceptor - Remote EJBs will not be traced"); + return null; + } + } + } + + return clientRequestInterceptor; + } + + @Override + public ServerRequestInterceptor createServerRequestInterceptor(ORBInitInfo info, Codec codec) { + if (serverRequestInterceptor == null) { + if (attemptCreation()) { + try { + serverRequestInterceptor = new OpenTelemetryIiopServerInterceptor(openTracingService); + } catch (NullPointerException nullPointerException) { + logger.log(Level.WARNING, "Could not create OpenTracing IIOP Server Interceptor - Remote EJBs will not be traced"); + return null; + } + } + } + + return serverRequestInterceptor; + } + + private boolean attemptCreation() { + if (serviceLocator == null) { + serviceLocator = Globals.getStaticBaseServiceLocator(); + if (serviceLocator == null) { + return false; + } + } + + if (openTracingService == null) { + openTracingService = serviceLocator.getService(OpenTracingService.class); + return openTracingService != null; + } + + return true; + } +} diff --git a/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/ejb/iiop/OpenTelemetryIiopServerInterceptor.java b/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/ejb/iiop/OpenTelemetryIiopServerInterceptor.java new file mode 100644 index 00000000000..6d59fd20335 --- /dev/null +++ b/appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/ejb/iiop/OpenTelemetryIiopServerInterceptor.java @@ -0,0 +1,189 @@ +/* + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. + * + * Copyright (c) 2020-2026 Payara Foundation and/or its affiliates. All rights reserved. + * + * The contents of this file are subject to the terms of either the GNU + * General Public License Version 2 only ("GPL") or the Common Development + * and Distribution License("CDDL") (collectively, the "License"). You + * may not use this file except in compliance with the License. You can + * obtain a copy of the License at + * https://github.com/payara/Payara/blob/main/LICENSE.txt + * See the License for the specific + * language governing permissions and limitations under the License. + * + * When distributing the software, include this License Header Notice in each + * file and include the License file at legal/OPEN-SOURCE-LICENSE.txt. + * + * GPL Classpath Exception: + * The Payara Foundation designates this particular file as subject to the "Classpath" + * exception as provided by the Payara Foundation in the GPL Version 2 section of the License + * file that accompanied this code. + * + * Modifications: + * If applicable, add the following below the License Header, with the fields + * enclosed by brackets [] replaced by your own identifying information: + * "Portions Copyright [year] [name of copyright owner]" + * + * Contributor(s): + * If you wish your version of this file to be governed by only the CDDL or + * only the GPL Version 2, indicate your decision by adding "[Contributor] + * elects to include this software in this distribution under the [CDDL or GPL + * Version 2] license." If you don't indicate a single choice of license, a + * recipient has the option to distribute your version of this file under + * either the CDDL, the GPL Version 2 or to extend the choice of license to + * its licensees as provided above. However, if you add GPL Version 2 code + * and therefore, elected the GPL Version 2 license, then the option applies + * only if the new code is made subject to such option by the copyright + * holder. + */ +package fish.payara.microprofile.telemetry.tracing.ejb.iiop; + +import fish.payara.opentracing.OpenTelemetryService; +import fish.payara.opentracing.OpenTracingService; +import io.opentelemetry.api.GlobalOpenTelemetry; +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.context.propagation.TextMapGetter; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.util.HashMap; +import java.util.logging.Logger; +import org.omg.CORBA.BAD_PARAM; +import org.omg.CORBA.LocalObject; +import org.omg.IOP.ServiceContext; +import org.omg.PortableInterceptor.ForwardRequest; +import org.omg.PortableInterceptor.ServerRequestInfo; +import org.omg.PortableInterceptor.ServerRequestInterceptor; + +import static fish.payara.microprofile.telemetry.tracing.ejb.iiop.OpenTelemetryIiopClientInterceptor.OPENTELEMETRY_IIOP_ID; + + +/** + * IIOP Server Interceptor for propagating OpenTracing SpanContext to Payara Server. + * + * @author Andrew Pielage + */ +public class OpenTelemetryIiopServerInterceptor extends LocalObject implements ServerRequestInterceptor { + + private static final Logger LOGGER = Logger.getLogger(OpenTelemetryIiopServerInterceptor.class.getName()); + + private OpenTracingService openTracingService; + private final ThreadLocal currentScope = new ThreadLocal<>(); + private final ThreadLocal currentSpan = new ThreadLocal<>(); + + public OpenTelemetryIiopServerInterceptor(OpenTracingService openTracingService) { + this.openTracingService = openTracingService; + } + + @Override + public void receive_request_service_contexts(ServerRequestInfo serverRequestInfo) throws ForwardRequest { + // Noop + } + + @Override + public void receive_request(ServerRequestInfo serverRequestInfo) throws ForwardRequest { + ServiceContext serviceContext; + + if (!tracerAvailable()) { + return; + } + + try { + serviceContext = serverRequestInfo.get_request_service_context(OPENTELEMETRY_IIOP_ID); + if (serviceContext == null) { + return; + } + } catch (BAD_PARAM e) { + // No OTel context was propagated by the client + return; + } + + HashMap contextMap; + try (ByteArrayInputStream bis = new ByteArrayInputStream(serviceContext.context_data); + ObjectInputStream in = new ObjectInputStream(bis)) { + contextMap = (HashMap) in.readObject(); + } catch (IOException | ClassNotFoundException e) { + throw new ForwardRequest(e.getMessage(), serverRequestInfo); + } + + OpenTelemetry openTelemetry = GlobalOpenTelemetry.get(); + TextMapGetter> getter = new TextMapGetter>() { + @Override + public Iterable keys(HashMap carrier) { + return carrier.keySet(); + } + + @Override + public String get(HashMap carrier, String key) { + return carrier.get(key); + } + }; + + Context parentContext = openTelemetry.getPropagators().getTextMapPropagator() + .extract(Context.current(), contextMap, getter); + Tracer tracer = openTelemetry.getTracerProvider().get(OpenTelemetryService.INSTRUMENTATION_SCOPE_NAME); + Span span = tracer.spanBuilder("rmi") + .setParent(parentContext) + .setSpanKind(SpanKind.SERVER) + .setAttribute("component", "ejb") + .startSpan(); + + if (currentScope.get() != null) { + LOGGER.warning("Overlapping traced RMI operations identified, please report"); + } + + currentSpan.set(span); + currentScope.set(span.makeCurrent()); + } + + @Override + public void send_reply(ServerRequestInfo serverRequestInfo) { + closeScope(); + } + + @Override + public void send_exception(ServerRequestInfo serverRequestInfo) throws ForwardRequest { + closeScope(); + } + + @Override + public void send_other(ServerRequestInfo serverRequestInfo) throws ForwardRequest { + closeScope(); + } + + @Override + public String name() { + return this.getClass().getSimpleName(); + } + + @Override + public void destroy() { + + } + + private void closeScope() { + Scope scope = currentScope.get(); + if (scope != null) { + scope.close(); + currentScope.remove(); + } + Span span = currentSpan.get(); + if (span != null) { + span.end(); + currentSpan.remove(); + } + } + + private boolean tracerAvailable() { + if (openTracingService == null) { + return false; + } + return openTracingService.isEnabled(); + } +} diff --git a/appserver/tests/payara-samples/samples/pom.xml b/appserver/tests/payara-samples/samples/pom.xml index f45193ebc76..adec2ef5aee 100644 --- a/appserver/tests/payara-samples/samples/pom.xml +++ b/appserver/tests/payara-samples/samples/pom.xml @@ -78,7 +78,7 @@ http logging ejb-http-remoting - + remote-ejb-tracing rolesallowed-unprotected-methods multiple-keystores diff --git a/appserver/tests/payara-samples/samples/remote-ejb-tracing/pom.xml b/appserver/tests/payara-samples/samples/remote-ejb-tracing/pom.xml index 4b4a01ddf87..a613773737f 100644 --- a/appserver/tests/payara-samples/samples/remote-ejb-tracing/pom.xml +++ b/appserver/tests/payara-samples/samples/remote-ejb-tracing/pom.xml @@ -95,6 +95,23 @@ samples-test-utils test + + io.opentelemetry + opentelemetry-api + provided + + + fish.payara.server.internal.payara-appserver-modules + microprofile-telemetry + ${project.version} + provided + + + fish.payara.server.internal.payara-appserver-modules + jaxrs-client-tracing + ${project.version} + provided + diff --git a/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/main/java/fish/payara/samples/remote/ejb/tracing/EjbRemote.java b/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/main/java/fish/payara/samples/remote/ejb/tracing/EjbRemote.java index a74d62d464c..062e55d9632 100644 --- a/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/main/java/fish/payara/samples/remote/ejb/tracing/EjbRemote.java +++ b/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/main/java/fish/payara/samples/remote/ejb/tracing/EjbRemote.java @@ -3,7 +3,7 @@ /* * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. * - * Copyright (c) 2020-2021 Payara Foundation and/or its affiliates. All rights reserved. + * Copyright (c) 2020-2026 Payara Foundation and/or its affiliates. All rights reserved. * * The contents of this file are subject to the terms of either the GNU * General Public License Version 2 only ("GPL") or the Common Development @@ -53,4 +53,5 @@ public interface EjbRemote { String shouldNotBeTraced(); String editBaggageItems(); + } diff --git a/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/main/java/fish/payara/samples/remote/ejb/tracing/server/Ejb.java b/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/main/java/fish/payara/samples/remote/ejb/tracing/server/Ejb.java index 3584d17668c..ca675e242f8 100644 --- a/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/main/java/fish/payara/samples/remote/ejb/tracing/server/Ejb.java +++ b/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/main/java/fish/payara/samples/remote/ejb/tracing/server/Ejb.java @@ -40,19 +40,22 @@ package fish.payara.samples.remote.ejb.tracing.server; import fish.payara.samples.remote.ejb.tracing.EjbRemote; -import org.eclipse.microprofile.opentracing.Traced; - +import fish.payara.microprofile.telemetry.tracing.Traced; +import io.opentelemetry.api.baggage.Baggage; +import io.opentelemetry.context.Context; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.common.AttributeKey; import jakarta.ejb.Stateless; import jakarta.inject.Inject; import java.util.Map; import java.util.Random; + @Stateless public class Ejb implements EjbRemote { - @Inject - Tracer tracer; - /** * This method should not be traced, but the baggage items should still be available. * @@ -61,12 +64,14 @@ public class Ejb implements EjbRemote { @Override public String nonAnnotatedMethod() { randomSleep(); - Span activeSpan = tracer.activeSpan(); - if (activeSpan != null) { - return getBaggageItems(activeSpan); - } else { - return "Nothing found!"; + Baggage baggage = Baggage.builder() + .put("Wibbles", "Wobbles") + .put("Nibbles", "Nobbles") + .build(); + try (Scope scope = baggage.storeInContext(Context.current()).makeCurrent()) { + return getBaggageItems(); } + } /** @@ -78,11 +83,11 @@ public String nonAnnotatedMethod() { @Traced(operationName = "customName") public String annotatedMethod() { randomSleep(); - Span activeSpan = tracer.activeSpan(); - if (activeSpan != null) { - return getBaggageItems(activeSpan); - } else { - return "Nothing found!"; + Baggage baggage = Baggage.builder() + .put("Wibbles", "Wobbles") + .build(); + try (Scope scope = baggage.storeInContext(Context.current()).makeCurrent()) { + return getBaggageItems(); } } @@ -95,25 +100,33 @@ public String annotatedMethod() { @Traced(false) public String shouldNotBeTraced() { randomSleep(); - Span activeSpan = tracer.activeSpan(); - if (activeSpan != null) { - return getBaggageItems(activeSpan); - } else { - return "Nothing found!"; + Baggage baggage = Baggage.builder() + .put("Wibbles", "Wobbles") + .put("Nibbles", "Nobbles") + .put("Bibbles", "Bobbles") + .build(); + try (Scope scope = baggage.storeInContext(Context.current()).makeCurrent()) { + return getBaggageItems(); } } @Override public String editBaggageItems() { randomSleep(); - Span activeSpan = tracer.activeSpan(); - if (activeSpan != null) { - activeSpan.setBaggageItem("Wibbles", "Wabbles"); - activeSpan.setBaggageItem("Nibbles", "Nabbles"); - activeSpan.setBaggageItem("Bibbles", "Babbles"); - return getBaggageItems(activeSpan); - } else { - return "Nothing found!"; + Baggage initial = Baggage.builder() + .put("Wibbles", "Wobbles") + .put("Nibbles", "Nobbles") + .put("Bibbles", "Bobbles") + .build(); + try (Scope scope = initial.storeInContext(Context.current()).makeCurrent()) { + Baggage edited = Baggage.current().toBuilder() + .put("Wibbles", "Wabbles") + .put("Nibbles", "Nabbles") + .put("Bibbles", "Babbles") + .build(); + try (Scope scope2 = edited.storeInContext(Context.current()).makeCurrent()) { + return getBaggageItems(); + } } } @@ -125,11 +138,20 @@ private void randomSleep() { } } - private String getBaggageItems(Span activeSpan) { - String baggageItems = "\n"; - for (Map.Entry baggageItem : activeSpan.context().baggageItems()) { - baggageItems += baggageItem.getKey() + " : " + baggageItem.getValue() + "\n"; + private String getBaggageItems() { + StringBuilder sb = new StringBuilder("\n"); + Baggage.current().asMap().forEach((key, entry) -> + sb.append(key).append(" : ").append(entry.getValue()).append("\n")); + Span currentSpan = Span.current(); + if (currentSpan.isRecording()) { + if (currentSpan instanceof ReadableSpan) { + ReadableSpan readableSpan = (ReadableSpan) currentSpan; + String tcID = readableSpan.getAttribute(AttributeKey.stringKey("TX-ID")); + if (tcID != null) { + sb.append("TX-ID : ").append(tcID).append("\n"); + } + } } - return baggageItems; + return sb.toString(); } } diff --git a/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/test/java/fish/payara/samples/remote/ejb/tracing/RemoteEjbClientIT.java b/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/test/java/fish/payara/samples/remote/ejb/tracing/RemoteEjbClientIT.java index 3d227e0aa45..942896769d9 100644 --- a/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/test/java/fish/payara/samples/remote/ejb/tracing/RemoteEjbClientIT.java +++ b/appserver/tests/payara-samples/samples/remote-ejb-tracing/src/test/java/fish/payara/samples/remote/ejb/tracing/RemoteEjbClientIT.java @@ -77,43 +77,32 @@ public void executeRemoteEjbMethodIT() throws NamingException { contextProperties.setProperty("org.omg.CORBA.ORBInitialPort", "3700"); // enable OpenTelemetry tracing so we get our OpenTracing instance System.setProperty("otel.sdk.disabled", "false"); - - + Context context = new InitialContext(contextProperties); EjbRemote ejb = (EjbRemote) context.lookup(String.format("java:global%sEjb", uri.getPath())); + String baggageItems = ejb.annotatedMethod(); + Assert.assertTrue("Baggage items didn't match, received: " + baggageItems, + baggageItems.contains("Wibbles : Wobbles")); + + baggageItems = ejb.nonAnnotatedMethod(); + Assert.assertTrue("Baggage items didn't match, received: " + baggageItems, + baggageItems.contains("Wibbles : Wobbles") + && baggageItems.contains("Nibbles : Nobbles")); + + baggageItems = ejb.shouldNotBeTraced(); + Assert.assertTrue("Baggage items didn't match, received: " + baggageItems, + baggageItems.contains("Wibbles : Wobbles") + && baggageItems.contains("Nibbles : Nobbles") + && baggageItems.contains("Bibbles : Bobbles")); + + baggageItems = ejb.editBaggageItems(); + Assert.assertTrue("Baggage items didn't match, received: " + baggageItems, + baggageItems.contains("Wibbles : Wabbles") + && baggageItems.contains("Nibbles : Nabbles") + && baggageItems.contains("Bibbles : Babbles")); - - Tracer tracer = GlobalTracer.get(); - Span span = tracer.buildSpan("ExecuteEjb").start(); - try (Scope scope = tracer.activateSpan(span)) { - span.setBaggageItem("Wibbles", "Wobbles"); - String baggageItems = ejb.annotatedMethod(); - Assert.assertTrue("Baggage items didn't match, received: " + baggageItems, - baggageItems.contains("\nWibbles : Wobbles\n")); - - span.setBaggageItem("Nibbles", "Nobbles"); - baggageItems = ejb.nonAnnotatedMethod(); - Assert.assertTrue("Baggage items didn't match, received: " + baggageItems, - baggageItems.contains("Wibbles : Wobbles") - && baggageItems.contains("Nibbles : Nobbles")); - - span.setBaggageItem("Bibbles", "Bobbles"); - baggageItems = ejb.shouldNotBeTraced(); - Assert.assertTrue("Baggage items didn't match, received: " + baggageItems, - baggageItems.contains("Wibbles : Wobbles") - && baggageItems.contains("Nibbles : Nobbles") - && baggageItems.contains("Bibbles : Bobbles")); - - baggageItems = ejb.editBaggageItems(); - Assert.assertTrue("Baggage items didn't match, received: " + baggageItems, - baggageItems.contains("Wibbles : Wabbles") - && baggageItems.contains("Nibbles : Nabbles") - && baggageItems.contains("Bibbles : Babbles")); - } finally { - span.finish(); - } } - + @Test public void transactionIdAddedAsBaggageIT() throws NamingException { Properties contextProperties = new Properties(); @@ -125,19 +114,12 @@ public void transactionIdAddedAsBaggageIT() throws NamingException { Context context = new InitialContext(contextProperties); EjbRemote ejb = (EjbRemote) context.lookup(String.format("java:global%sEjb", uri.getPath())); - - Tracer tracer = GlobalTracer.get(); - - Span span = tracer.buildSpan("ExecuteEjb").start(); - try(Scope scope = tracer.activateSpan(span)) { - String baggageItems = ejb.annotatedMethod(); - Assert.assertTrue("Baggage items didn't contain transaction ID, received: " + baggageItems, + + String baggageItems = ejb.annotatedMethod(); + Assert.assertTrue("Baggage items didn't contain transaction ID, received: " + baggageItems, baggageItems.contains("TX-ID")); - } finally { - span.finish(); - } } - + @Deployment public static WebArchive deploy() { return ShrinkWrap.create(WebArchive.class).addClasses(EjbRemote.class, Ejb.class); diff --git a/appserver/tests/payara-samples/test-domain-setup/src/test/java/fish/payara/samples/setuptests/EjbRequestTracingTest.java b/appserver/tests/payara-samples/test-domain-setup/src/test/java/fish/payara/samples/setuptests/EjbRequestTracingTest.java index 26808f7d805..38e635a1b8d 100644 --- a/appserver/tests/payara-samples/test-domain-setup/src/test/java/fish/payara/samples/setuptests/EjbRequestTracingTest.java +++ b/appserver/tests/payara-samples/test-domain-setup/src/test/java/fish/payara/samples/setuptests/EjbRequestTracingTest.java @@ -1,7 +1,7 @@ /* * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. * - * Copyright (c) 2020 Payara Foundation and/or its affiliates. All rights reserved. + * Copyright (c) 2020-2026 Payara Foundation and/or its affiliates. All rights reserved. * * The contents of this file are subject to the terms of either the GNU * General Public License Version 2 only ("GPL") or the Common Development @@ -59,6 +59,10 @@ public void turnOnRequestTracing() { "--enabled", "true", "--dynamic", "true") != 0) { throw new IllegalStateException("Can't enable request tracing"); } + + if (CliCommands.payaraGlassFish("create-system-properties", "otel.sdk.disabled=false") != 0) { + throw new IllegalStateException("Can't set system property"); + } } } } diff --git a/appserver/transaction/jta/pom.xml b/appserver/transaction/jta/pom.xml index 5a82882f4e6..e39b17f1649 100644 --- a/appserver/transaction/jta/pom.xml +++ b/appserver/transaction/jta/pom.xml @@ -145,5 +145,16 @@ ${project.version} provided + + io.opentelemetry + opentelemetry-api + provided + + + fish.payara.server.internal.payara-appserver-modules + jaxrs-client-tracing + ${project.version} + provided + diff --git a/appserver/transaction/jta/src/main/java/com/sun/enterprise/transaction/JavaEETransactionManagerSimplified.java b/appserver/transaction/jta/src/main/java/com/sun/enterprise/transaction/JavaEETransactionManagerSimplified.java index 872db22883e..3db10f7005a 100644 --- a/appserver/transaction/jta/src/main/java/com/sun/enterprise/transaction/JavaEETransactionManagerSimplified.java +++ b/appserver/transaction/jta/src/main/java/com/sun/enterprise/transaction/JavaEETransactionManagerSimplified.java @@ -60,6 +60,11 @@ import fish.payara.notification.requesttracing.RequestTraceSpanLog; import fish.payara.nucleus.requesttracing.RequestTracingService; import fish.payara.opentracing.OpenTracingService; +import fish.payara.requesttracing.jaxrs.client.PayaraTracingServices; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; @@ -679,6 +684,14 @@ private JavaEETransactionImpl initJavaEETransaction(int timeout) { } setCurrentTransaction(tx); + + if (openTracingServiceProvider != null) { + OpenTracingService openTracingService = getOpenTracing(); + if (openTracingService != null && openTracingService.isEnabled()) { + addJtaEventTraceLog(constructJTABeginSpanLog(tx), tx); + } + } + return tx; } @@ -965,6 +978,13 @@ public void commit() throws RollbackException, } } } + + if (openTracingServiceProvider != null) { + OpenTracingService openTracingService = getOpenTracing(); + if (openTracingService != null && openTracingService.isEnabled()) { + addJtaEventTraceLog(constructJTAEndSpanLog(tx), tx); + } + } } finally { setCurrentTransaction(null); // clear current thread's tx @@ -999,6 +1019,13 @@ public void rollback() throws IllegalStateException, SecurityException, } } } + + if (openTracingServiceProvider != null) { + OpenTracingService openTracingService = getOpenTracing(); + if (openTracingService != null && openTracingService.isEnabled()) { + addJtaEventTraceLog(constructJTAEndSpanLog(tx), tx); + } + } } finally { setCurrentTransaction(null); // clear current thread's tx delegates.set(null); @@ -1719,6 +1746,44 @@ public JavaEETransaction createImportedTransaction(TransactionInternal jtsTx) return tx; } + private void addJtaEventTraceLog(RequestTraceSpanLog spanLog, JavaEETransaction tx) { + Span span = Span.current(); + if (span.isRecording()) { + AttributesBuilder attrsBuilder = Attributes.builder(); + spanLog.getLogEntries().forEach(attrsBuilder::put); + String eventName = spanLog.getLogEntries().getOrDefault("logEvent", "jtaTransactionEvent"); + span.addEvent(eventName, attrsBuilder.build(), spanLog.getTimeMillis(), TimeUnit.MILLISECONDS); + // Add transaction ID as baggage item + if (tx != null) { + if (tx.getClass().equals(JavaEETransactionImpl.class)) { + span.setAttribute("TX-ID", ((JavaEETransactionImpl) tx).getTransactionId()); + } else { + span.setAttribute("TransactionInfo", tx.toString()); + } + } + } else { + final PayaraTracingServices payaraTracingServices = new PayaraTracingServices(); + final Tracer tracer = payaraTracingServices.getActiveTracer(); + AttributesBuilder attrsBuilder = Attributes.builder(); + spanLog.getLogEntries().forEach(attrsBuilder::put); + if (tx != null && tracer != null) { + span = tracer.spanBuilder("addJtaEventTraceLog").startSpan(); + span.makeCurrent(); + String eventName = spanLog.getLogEntries().getOrDefault("logEvent", "jtaTransactionEvent"); + span.addEvent(eventName, attrsBuilder.build(), spanLog.getTimeMillis(), TimeUnit.MILLISECONDS); + // Add transaction ID as baggage item + if (tx != null) { + if (tx.getClass().equals(JavaEETransactionImpl.class)) { + span.setAttribute("TX-ID", ((JavaEETransactionImpl) tx).getTransactionId()); + } else { + span.setAttribute("TransactionInfo", tx.toString()); + } + } + } + getRequestTracing().addSpanLog(spanLog); + } + } + private RequestTraceSpanLog constructJTABeginSpanLog(JavaEETransactionImpl transaction) { RequestTraceSpanLog spanLog = new RequestTraceSpanLog("jtaContextBeginEvent"); diff --git a/appserver/web/web-core/src/main/java/org/apache/catalina/core/StandardWrapper.java b/appserver/web/web-core/src/main/java/org/apache/catalina/core/StandardWrapper.java index 06328e20ac4..1504ea8a2ee 100644 --- a/appserver/web/web-core/src/main/java/org/apache/catalina/core/StandardWrapper.java +++ b/appserver/web/web-core/src/main/java/org/apache/catalina/core/StandardWrapper.java @@ -1572,7 +1572,7 @@ void service(ServletRequest request, ServletResponse response, Servlet servlet) if (tracer != null && response.isCommitted()) { // If response is not committed, it is likely async SpanBuilder spanBuilder = tracer.spanBuilder(applicationName); - spanBuilder.setAttribute(HttpAttributes.HTTP_RESPONSE_STATUS_CODE, ((HttpServletResponse) response).getStatus()); + spanBuilder.setAttribute(HttpAttributes.HTTP_RESPONSE_STATUS_CODE, ((HttpServletResponse) response).getStatus()).startSpan(); } // TODO: clear OpenTelemetry context once we move to natively using it. if (requestTracing.isRequestTracingEnabled() && span != null) {