Fish 13751 reimplementing ejb tracing with opentelemetry

appserver/concurrent/concurrent-impl/pom.xml

@@ -150,6 +150,11 @@
             <artifactId>opentelemetry-tracing-starter</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-api</artifactId>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 
     <build>

appserver/concurrent/concurrent-impl/src/main/java/org/glassfish/concurrent/runtime/ContextSetupProviderImpl.java

@@ -49,6 +49,12 @@
 import com.sun.enterprise.security.SecurityContext;
 import com.sun.enterprise.transaction.api.JavaEETransactionManager;
 import com.sun.enterprise.util.Utility;
+import io.opentelemetry.api.baggage.Baggage;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanBuilder;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Context;
+import io.opentelemetry.context.Scope;
 import org.glassfish.api.invocation.ComponentInvocation;
 import org.glassfish.api.invocation.InvocationManager;
 import org.glassfish.concurrent.LogFacade;
@@ -107,6 +113,9 @@ public class ContextSetupProviderImpl implements ContextSetupProvider {
     public static final String CONTEXT_TYPE_NAMING = "NAMING"; // Concurrency 3.0: APPLICATION
     public static final String CONTEXT_TYPE_WORKAREA = "WORKAREA"; // Concurrency 3.0: TRANSACTION
 
+    private static final ThreadLocal<Span> currentConcurrentSpan = new ThreadLocal<>();
+    private static final ThreadLocal<Scope> currentConcurrentScope = new ThreadLocal<>();
+
     // TODO: do we need these booleans if we have sets?
     private boolean classloading, security, naming, workArea;
     private final Set<String> contextPropagate;
@@ -306,6 +315,10 @@ public ContextHandle setup(ContextHandle contextHandle) throws IllegalStateExcep
             transactionManager.clearThreadTx();
         }
 
+        if (requestTracing != null && requestTracing.isRequestTracingEnabled()) {
+            startConcurrentContextSpan(invocation, handle);
+        }
+
         if (stuckThreads != null) {
             stuckThreads.registerThread(Thread.currentThread().threadId());
         }
@@ -322,6 +335,50 @@ public ContextHandle setup(ContextHandle contextHandle) throws IllegalStateExcep
                 Collections.EMPTY_LIST, restorers);
     }
 
+    private void startConcurrentContextSpan(ComponentInvocation invocation, InvocationContext handle) {
+        Tracer tracer = openTracing.getTracer(openTracing.getApplicationName(
+                Globals.getDefaultBaseServiceLocator().getService(InvocationManager.class)));
+        SpanBuilder builder = tracer.spanBuilder("executeConcurrentContext");
+        Context parentContext = handle.getParentTraceContext();
+        if (parentContext != null) {
+            builder.setParent(parentContext);
+
+            String parentOperationName = Baggage.fromContext(parentContext).getEntryValue("operation.name");
+            if (parentOperationName != null) {
+                builder.setAttribute("Parent Operation Name", parentOperationName);
+            }
+        }
+
+        if (invocation != null) {
+            builder.setAttribute("App Name", invocation.getAppName())
+                    .setAttribute("Component ID", invocation.getComponentId())
+                    .setAttribute("Module Name", invocation.getModuleName());
+
+            Object instance = invocation.getInstance();
+            if (instance != null) {
+                builder.setAttribute("Class Name", instance.getClass().getName());
+            }
+        }
+
+        builder.setAttribute("Thread Name", Thread.currentThread().getName());
+        Span span = builder.startSpan();
+        currentConcurrentSpan.set(span);
+        currentConcurrentScope.set(span.makeCurrent());
+    }
+
+    private void stopConcurrentContextSpan() {
+        Scope scope = currentConcurrentScope.get();
+        Span span = currentConcurrentSpan.get();
+        if (scope != null) {
+            scope.close();
+            currentConcurrentScope.remove();
+        }
+        if (span != null) {
+            span.end();
+            currentConcurrentSpan.remove();
+        }
+    }
+
     @Override
     public void reset(ContextHandle contextHandle) {
         if (! (contextHandle instanceof InvocationContext)) {
@@ -362,6 +419,7 @@ public void reset(ContextHandle contextHandle) {
         }
 
         if (requestTracing != null && requestTracing.isRequestTracingEnabled()) {
+            stopConcurrentContextSpan();
             requestTracing.endTrace();
         }
         if (stuckThreads != null) {

appserver/concurrent/concurrent-impl/src/main/java/org/glassfish/concurrent/runtime/InvocationContext.java

@@ -42,10 +42,16 @@
 package org.glassfish.concurrent.runtime;
 
 import com.sun.enterprise.security.SecurityContext;
+import fish.payara.nucleus.requesttracing.RequestTracingService;
+import fish.payara.opentracing.OpenTracingService;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.context.Context;
 import jakarta.enterprise.concurrent.spi.ThreadContextRestorer;
 import jakarta.enterprise.concurrent.spi.ThreadContextSnapshot;
 import org.glassfish.api.invocation.ComponentInvocation;
 import org.glassfish.concurro.spi.ContextHandle;
+import org.glassfish.hk2.api.ServiceLocator;
+import org.glassfish.internal.api.Globals;
 import org.glassfish.internal.data.ApplicationInfo;
 import org.glassfish.internal.data.ApplicationRegistry;
 
@@ -63,7 +69,8 @@ public class InvocationContext implements ContextHandle {
 
     private List<ThreadContextSnapshot> threadContextSnapshots;
     private List<ThreadContextRestorer> threadContextRestorers;
-
+
+    private Context parentTraceContext;
 
     public InvocationContext(ComponentInvocation invocation, ClassLoader contextClassLoader, SecurityContext securityContext,
             boolean useTransactionOfExecutionThread, List<ThreadContextSnapshot> threadContextSnapshots,
@@ -74,8 +81,25 @@ public InvocationContext(ComponentInvocation invocation, ClassLoader contextClas
         this.useTransactionOfExecutionThread = useTransactionOfExecutionThread;
         this.threadContextSnapshots = threadContextSnapshots;
         this.threadContextRestorers = threadContextRestorers;
+        saveTracingContext();
+    }
+
+    private void saveTracingContext() {
+        ServiceLocator serviceLocator = Globals.getDefaultBaseServiceLocator();
+
+        if (serviceLocator != null) {
+            RequestTracingService requestTracing = serviceLocator.getService(RequestTracingService.class);
+            OpenTracingService openTracing = serviceLocator.getService(OpenTracingService.class);
+
+            if (requestTracing != null && requestTracing.isRequestTracingEnabled()
+                    && requestTracing.isTraceInProgress() && openTracing != null) {
+                Span currentSpan = Span.current();
+                if (currentSpan.isRecording()) {
+                    this.parentTraceContext = Context.current();
+                }
+            }
+        }
     }
-
 
     public ComponentInvocation getInvocation() {
         return invocation;
@@ -101,6 +125,10 @@ public List<ThreadContextRestorer> getThreadContextRestorers() {
         return threadContextRestorers;
     }
 
+    public Context getParentTraceContext() {
+        return parentTraceContext;
+    }
+
     /**
      * Used to make duplicate of the InvocationContext.
      */

appserver/ejb/ejb-container/pom.xml

@@ -257,5 +257,10 @@
             <groupId>org.mockito</groupId>
             <artifactId>mockito-core</artifactId>
         </dependency>
+        <dependency>
+            <groupId>io.opentelemetry</groupId>
+            <artifactId>opentelemetry-api</artifactId>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 </project>

appserver/ejb/ejb-container/src/main/java/com/sun/ejb/containers/BaseContainer.java

@@ -90,6 +90,10 @@
 import fish.payara.notification.requesttracing.RequestTraceSpanLog;
 import fish.payara.nucleus.requesttracing.RequestTracingService;
 import fish.payara.opentracing.OpenTracingService;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.context.Scope;
 import jakarta.annotation.PostConstruct;
 import jakarta.annotation.PreDestroy;
 import jakarta.ejb.AccessLocalException;
@@ -4094,6 +4098,29 @@ final void onEjbMethodEnd(String method_sig, Throwable th) {
 
     private void addEjbMethodTraceLog(CallFlowInfo info, boolean callEnter) {
         if (openTracingService.isEnabled()) {
+            Tracer tracer = openTracingService.getTracer(openTracingService.getApplicationName(invocationManager));
+            if (tracer != null) {
+                String eventName = callEnter ? "enterEjbMethodEvent" : "exitEjbMethodEvent";
+                Span span = tracer.spanBuilder(info.getMethod().getName()).startSpan();
+                try(Scope scope = span.makeCurrent()) {
+                    if (span.isRecording()) {
+                        span.addEvent(eventName, Attributes.builder()
+                                .put("ApplicationName", info.getApplicationName())
+                                .put("ComponentName", info.getComponentName())
+                                .put("ComponentType", info.getComponentType().toString())
+                                .put("ModuleName", info.getModuleName())
+                                .put("EJBClass", ejbClass.getCanonicalName())
+                                .put("EJBMethod", info.getMethod().getName())
+                                .put("CallerPrincipal", String.valueOf(info.getCallerPrincipal()))
+                                .put("TX-ID", String.valueOf(info.getTransactionId()))
+                                .build());
+                    } else {
+                        _logger.log(Level.FINE, "Tracing is not enabled, skipping event {0}", eventName);
+                    }
+                } finally {
+                   span.end();
+                }
+            }
             RequestTraceSpanLog spanLog = constructEjbMethodSpanLog(info, callEnter);
             requestTracingService.addSpanLog(spanLog);
         }

appserver/payara-appserver-modules/microprofile/telemetry/pom.xml

@@ -121,5 +121,10 @@
             <optional>true</optional>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>fish.payara.server.internal.orb</groupId>
+            <artifactId>orb-connector</artifactId>
+            <version>${project.version}</version>
+        </dependency>
     </dependencies>
 </project>

appserver/payara-appserver-modules/microprofile/telemetry/src/main/java/fish/payara/microprofile/telemetry/tracing/TracedInterceptor.java

@@ -109,17 +109,36 @@ public Object traceCdiCall(final InvocationContext invocationContext) throws Exc
         final String applicationName = payaraTracingServices.getApplicationName();
         final Tracer tracer = payaraTracingServices.getActiveTracer();
         final String operationName = getOperationName(invocationContext, traced);
-        Span span = tracer.spanBuilder(operationName).setAttribute("otel.service.name", applicationName).startSpan();
-        try (Scope scope = span.makeCurrent()) {
-            try {
-                return invocationContext.proceed();
-            } catch (Exception e) {
-                LOG.log(Level.FINEST, "Setting the error to the active span ...", e);
-                span.recordException(e);
-                throw e;
+
+        Span span = Span.current();
+        if (span.isRecording()) {
+            span = span.updateName(operationName).setAttribute("otel.service.name", applicationName);
+            try (Scope scope = span.makeCurrent()) {
+                try {
+                    return invocationContext.proceed();
+                } catch (Exception e) {
+                    LOG.log(Level.FINEST, "Setting the error to the active span ...", e);
+                    span.recordException(e);
+                    throw e;
+                }
+            } finally {
+                span.end();
+                LOG.log(Level.FINEST, "Don't close the span for now");
+            }
+        } else {
+            span = tracer.spanBuilder(operationName).setAttribute("otel.service.name", applicationName).startSpan();
+            try (Scope scope = span.makeCurrent()) {
+                try {
+                    return invocationContext.proceed();
+                } catch (Exception e) {
+                    LOG.log(Level.FINEST, "Setting the error to the active span ...", e);
+                    span.recordException(e);
+                    throw e;
+                }
+            } finally {
+                span.end();
+                LOG.log(Level.FINEST, "Don't close the span for now");
             }
-        } finally {
-            span.end();
         }
     }