diff --git a/service/history/interfaces/mutable_state.go b/service/history/interfaces/mutable_state.go index 40db09ed5f2..bae87c91f42 100644 --- a/service/history/interfaces/mutable_state.go +++ b/service/history/interfaces/mutable_state.go @@ -48,6 +48,7 @@ type ( AddActivityTaskCancelRequestedEvent(int64, int64, string) (*historypb.HistoryEvent, *persistencespb.ActivityInfo, error) AddActivityTaskCanceledEvent(int64, int64, int64, *commonpb.Payloads, string) (*historypb.HistoryEvent, error) AddWorkerCommandsTasks(commands []*workerpb.WorkerCommand, controlQueue string) error + GenerateActivityCancelCommandsForClose() error AddActivityTaskCompletedEvent(int64, int64, *workflowservice.RespondActivityTaskCompletedRequest) (*historypb.HistoryEvent, error) AddActivityTaskFailedEvent(int64, int64, *failurepb.Failure, enumspb.RetryState, string, *commonpb.WorkerVersionStamp) (*historypb.HistoryEvent, error) AddActivityTaskScheduledEvent(int64, *commandpb.ScheduleActivityTaskCommandAttributes, bool) (*historypb.HistoryEvent, *persistencespb.ActivityInfo, error) diff --git a/service/history/interfaces/mutable_state_mock.go b/service/history/interfaces/mutable_state_mock.go index 741f37e46e5..512e5ea002d 100644 --- a/service/history/interfaces/mutable_state_mock.go +++ b/service/history/interfaces/mutable_state_mock.go @@ -647,6 +647,20 @@ func (mr *MockMutableStateMockRecorder) AddWorkerCommandsTasks(commands, control return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddWorkerCommandsTasks", reflect.TypeOf((*MockMutableState)(nil).AddWorkerCommandsTasks), commands, controlQueue) } +// GenerateActivityCancelCommandsForClose mocks base method. +func (m *MockMutableState) GenerateActivityCancelCommandsForClose() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateActivityCancelCommandsForClose") + ret0, _ := ret[0].(error) + return ret0 +} + +// GenerateActivityCancelCommandsForClose indicates an expected call of GenerateActivityCancelCommandsForClose. +func (mr *MockMutableStateMockRecorder) GenerateActivityCancelCommandsForClose() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateActivityCancelCommandsForClose", reflect.TypeOf((*MockMutableState)(nil).GenerateActivityCancelCommandsForClose)) +} + // AddWorkflowExecutionCancelRequestedEvent mocks base method. func (m *MockMutableState) AddWorkflowExecutionCancelRequestedEvent(arg0 *historyservice.RequestCancelWorkflowExecutionRequest) (*history.HistoryEvent, error) { m.ctrl.T.Helper() diff --git a/service/history/ndc/events_reapplier_test.go b/service/history/ndc/events_reapplier_test.go index ce3c4b88a43..4ecfb3b9630 100644 --- a/service/history/ndc/events_reapplier_test.go +++ b/service/history/ndc/events_reapplier_test.go @@ -459,6 +459,7 @@ func (s *nDCEventReapplicationSuite) TestReapplyEvents_AppliedEvent_Termination( false, nil, ).Return(nil, nil) + msCurrent.EXPECT().GenerateActivityCancelCommandsForClose().Return(nil) events := []*historypb.HistoryEvent{ {EventType: enumspb.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED}, event, diff --git a/service/history/ndc/workflow_resetter_test.go b/service/history/ndc/workflow_resetter_test.go index 6a2b1ee1014..a0c1a639eaf 100644 --- a/service/history/ndc/workflow_resetter_test.go +++ b/service/history/ndc/workflow_resetter_test.go @@ -561,6 +561,7 @@ func (s *workflowResetterSuite) TestTerminateWorkflow() { false, nil, ).Return(&historypb.HistoryEvent{}, nil) + mutableState.EXPECT().GenerateActivityCancelCommandsForClose().Return(nil) err := s.workflowResetter.terminateWorkflow(mutableState, terminateReason) s.NoError(err) @@ -1255,6 +1256,7 @@ func (s *workflowResetterSuite) TestReapplyEvents() { false, event.Links, ).Return(&historypb.HistoryEvent{}, nil) + ms.EXPECT().GenerateActivityCancelCommandsForClose().Return(nil) } } } diff --git a/service/history/workflow/mutable_state_impl.go b/service/history/workflow/mutable_state_impl.go index 2513b30643c..52c3e0f1962 100644 --- a/service/history/workflow/mutable_state_impl.go +++ b/service/history/workflow/mutable_state_impl.go @@ -61,6 +61,7 @@ import ( "go.temporal.io/server/common/searchattribute/sadefs" serviceerrors "go.temporal.io/server/common/serviceerror" "go.temporal.io/server/common/softassert" + "go.temporal.io/server/common/tasktoken" "go.temporal.io/server/common/util" "go.temporal.io/server/common/worker_versioning" "go.temporal.io/server/components/callbacks" @@ -4556,6 +4557,68 @@ func (ms *MutableStateImpl) AddWorkerCommandsTasks(commands []*workerpb.WorkerCo return ms.taskGenerator.GenerateWorkerCommandsTasks(commands, controlQueue) } +// GenerateActivityCancelCommandsForClose generates WorkerCommandsTasks to cancel all +// in-flight activities that have a worker control queue. Called when the workflow is being +// terminated (or otherwise forcefully closed) to proactively notify workers. +func (ms *MutableStateImpl) GenerateActivityCancelCommandsForClose() error { + serializer := tasktoken.NewSerializer() + wfKey := ms.GetWorkflowKey() + nsID := ms.GetNamespaceEntry().ID().String() + + commandsByQueue := make(map[string][]*workerpb.WorkerCommand) + for _, ai := range ms.pendingActivityInfoIDs { + if ai.WorkerControlTaskQueue == "" { + continue + } + if ai.StartedClock == nil { + // StartedClock may be nil for activities started before this feature was deployed. + // Skip cancel command; the activity will time out normally. + ms.logger.Debug("Skipping worker cancel command: activity missing StartedClock (pre-deploy)", + tag.WorkflowNamespaceID(wfKey.NamespaceID), + tag.WorkflowID(wfKey.WorkflowID), + tag.WorkflowRunID(wfKey.RunID), + tag.WorkflowScheduledEventID(ai.ScheduledEventId), + ) + continue + } + + taskToken, err := serializer.Serialize(tasktoken.NewActivityTaskToken( + nsID, + wfKey.WorkflowID, + wfKey.RunID, + ai.ScheduledEventId, + ai.ActivityId, + ai.ActivityType.GetName(), + ai.Attempt, + ai.StartedClock, + ai.Version, + ai.StartVersion, + nil, + )) + if err != nil { + return err + } + + commandsByQueue[ai.WorkerControlTaskQueue] = append( + commandsByQueue[ai.WorkerControlTaskQueue], + &workerpb.WorkerCommand{ + Type: &workerpb.WorkerCommand_CancelActivity{ + CancelActivity: &workerpb.CancelActivityCommand{ + TaskToken: taskToken, + }, + }, + }, + ) + } + + for controlQueue, commands := range commandsByQueue { + if err := ms.AddWorkerCommandsTasks(commands, controlQueue); err != nil { + return err + } + } + return nil +} + func (ms *MutableStateImpl) ApplyActivityTaskCancelRequestedEvent( event *historypb.HistoryEvent, ) error { diff --git a/service/history/workflow/mutable_state_impl_test.go b/service/history/workflow/mutable_state_impl_test.go index 955cba32f2a..d23cebd30ae 100644 --- a/service/history/workflow/mutable_state_impl_test.go +++ b/service/history/workflow/mutable_state_impl_test.go @@ -7285,3 +7285,161 @@ func (s *mutableStateSuite) TestApplyWorkflowExecutionOptionsUpdatedEvent_TimeSk }) } } + +func TestGenerateActivityCancelCommandsForClose(t *testing.T) { + t.Parallel() + + startedClock := &clockspb.VectorClock{ShardId: 1, Clock: 100} + + testCases := []struct { + name string + featureEnabled bool + activities map[int64]*persistencespb.ActivityInfo + expectedQueues map[string]int // controlQueue -> expected command count + expectedNoTasks bool + }{ + { + name: "activities with control queue and started clock", + featureEnabled: true, + activities: map[int64]*persistencespb.ActivityInfo{ + 1: { + ScheduledEventId: 1, + ActivityId: "act-1", + ActivityType: &commonpb.ActivityType{Name: "type1"}, + WorkerControlTaskQueue: "control-queue-1", + StartedClock: startedClock, + Attempt: 1, + }, + }, + expectedQueues: map[string]int{"control-queue-1": 1}, + }, + { + name: "skips activities without control queue", + featureEnabled: true, + activities: map[int64]*persistencespb.ActivityInfo{ + 1: { + ScheduledEventId: 1, + ActivityId: "act-1", + ActivityType: &commonpb.ActivityType{Name: "type1"}, + StartedClock: startedClock, + Attempt: 1, + }, + }, + expectedNoTasks: true, + }, + { + name: "skips activities without started clock", + featureEnabled: true, + activities: map[int64]*persistencespb.ActivityInfo{ + 1: { + ScheduledEventId: 1, + ActivityId: "act-1", + ActivityType: &commonpb.ActivityType{Name: "type1"}, + WorkerControlTaskQueue: "control-queue-1", + Attempt: 1, + }, + }, + expectedNoTasks: true, + }, + { + name: "multiple activities batched by control queue", + featureEnabled: true, + activities: map[int64]*persistencespb.ActivityInfo{ + 1: { + ScheduledEventId: 1, + ActivityId: "act-1", + ActivityType: &commonpb.ActivityType{Name: "type1"}, + WorkerControlTaskQueue: "queue-A", + StartedClock: startedClock, + Attempt: 1, + }, + 2: { + ScheduledEventId: 2, + ActivityId: "act-2", + ActivityType: &commonpb.ActivityType{Name: "type2"}, + WorkerControlTaskQueue: "queue-A", + StartedClock: startedClock, + Attempt: 1, + }, + 3: { + ScheduledEventId: 3, + ActivityId: "act-3", + ActivityType: &commonpb.ActivityType{Name: "type3"}, + WorkerControlTaskQueue: "queue-B", + StartedClock: startedClock, + Attempt: 1, + }, + }, + expectedQueues: map[string]int{"queue-A": 2, "queue-B": 1}, + }, + { + name: "feature flag disabled - no tasks generated", + featureEnabled: false, + activities: map[int64]*persistencespb.ActivityInfo{ + 1: { + ScheduledEventId: 1, + ActivityId: "act-1", + ActivityType: &commonpb.ActivityType{Name: "type1"}, + WorkerControlTaskQueue: "control-queue-1", + StartedClock: startedClock, + Attempt: 1, + }, + }, + expectedNoTasks: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockEventsCache := events.NewMockCache(ctrl) + mockConfig := tests.NewDynamicConfig() + mockConfig.EnableCancelActivityWorkerCommand = dynamicconfig.GetBoolPropertyFn(tc.featureEnabled) + + mockShard := shard.NewTestContext( + ctrl, + &persistencespb.ShardInfo{ShardId: 0, RangeId: 1}, + mockConfig, + ) + defer mockShard.StopForTest() + reg := hsm.NewRegistry() + require.NoError(t, RegisterStateMachine(reg)) + require.NoError(t, callbacks.RegisterStateMachine(reg)) + require.NoError(t, nexusoperations.RegisterStateMachines(reg)) + mockShard.SetStateMachineRegistry(reg) + mockShard.SetEventsCacheForTesting(mockEventsCache) + + namespaceEntry := tests.GlobalNamespaceEntry + mockShard.Resource.NamespaceCache.EXPECT().GetNamespaceByID(tests.NamespaceID).Return(namespaceEntry, nil).AnyTimes() + mockShard.Resource.ClusterMetadata.EXPECT().ClusterNameForFailoverVersion(gomock.Any(), gomock.Any()).Return(cluster.TestCurrentClusterName).AnyTimes() + mockShard.Resource.ClusterMetadata.EXPECT().GetCurrentClusterName().Return(cluster.TestCurrentClusterName).AnyTimes() + mockShard.Resource.ClusterMetadata.EXPECT().GetClusterID().Return(int64(1)).AnyTimes() + + ms := NewMutableState(mockShard, mockEventsCache, log.NewTestLogger(), namespaceEntry, tests.WorkflowID, tests.RunID, time.Now().UTC()) + ms.pendingActivityInfoIDs = tc.activities + + err := ms.GenerateActivityCancelCommandsForClose() + require.NoError(t, err) + + if tc.expectedNoTasks { + require.Empty(t, ms.InsertTasks[tasks.CategoryOutbound]) + return + } + + // Verify tasks were generated by checking outbound task messages + var workerCommandTasks []*tasks.WorkerCommandsTask + for _, task := range ms.InsertTasks[tasks.CategoryOutbound] { + if wct, ok := task.(*tasks.WorkerCommandsTask); ok { + workerCommandTasks = append(workerCommandTasks, wct) + } + } + + // Verify each expected queue got the right number of commands + tasksByQueue := make(map[string]int) + for _, wct := range workerCommandTasks { + tasksByQueue[wct.Destination] = len(wct.Commands) + } + require.Equal(t, tc.expectedQueues, tasksByQueue) + }) + } +} diff --git a/service/history/workflow/util.go b/service/history/workflow/util.go index cb5091dfc44..d5fbe09dec2 100644 --- a/service/history/workflow/util.go +++ b/service/history/workflow/util.go @@ -95,7 +95,11 @@ func TimeoutWorkflow( retryState, continuedRunID, ) - return err + if err != nil { + return err + } + + return mutableState.GenerateActivityCancelCommandsForClose() } // TerminateWorkflow will write a WorkflowExecutionTerminated event with a fresh @@ -142,8 +146,13 @@ func TerminateWorkflow( deleteAfterTerminate, links, ) + if err != nil { + return err + } - return err + // Proactively cancel in-flight activities by dispatching worker commands to workers + // that support control queues, so they don't run uselessly after the workflow is closed. + return mutableState.GenerateActivityCancelCommandsForClose() } // FindAutoResetPoint returns the auto reset point diff --git a/tests/worker_commands_task_test.go b/tests/worker_commands_task_test.go index 187062797b5..a50e5b572c8 100644 --- a/tests/worker_commands_task_test.go +++ b/tests/worker_commands_task_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/google/uuid" commandpb "go.temporal.io/api/command/v1" commonpb "go.temporal.io/api/common/v1" enumspb "go.temporal.io/api/enums/v1" @@ -163,3 +164,232 @@ func TestDispatchCancelToWorker(t *testing.T) { "Cancel command task token must match the activity's original task token") t.Log("SUCCESS: Received ExecuteCommands Nexus request on control queue with matching CancelActivity task token") } + +// TestDispatchCancelOnWorkflowTermination tests that when a workflow is terminated, +// the server proactively dispatches cancel commands for in-flight activities +// that have a worker control queue. +func TestDispatchCancelOnWorkflowTermination(t *testing.T) { + env := testcore.NewEnv(t, testcore.WithDynamicConfig(dynamicconfig.EnableCancelActivityWorkerCommand, true)) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + tv := env.Tv() + poller := env.TaskPoller() + + controlQueueName := tv.ControlQueueName(env.Namespace().String()) + + // Start the workflow + startResp, err := env.FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ + RequestId: tv.Any().String(), + Namespace: env.Namespace().String(), + WorkflowId: tv.WorkflowID(), + WorkflowType: tv.WorkflowType(), + TaskQueue: tv.TaskQueue(), + WorkflowExecutionTimeout: durationpb.New(60 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + }) + env.NoError(err) + + // Schedule the activity + _, err = poller.PollAndHandleWorkflowTask(tv, + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ + ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: tv.ActivityID(), + ActivityType: tv.ActivityType(), + TaskQueue: tv.TaskQueue(), + ScheduleToCloseTimeout: durationpb.New(60 * time.Second), + StartToCloseTimeout: durationpb.New(60 * time.Second), + }, + }, + }, + }, + }, nil + }) + env.NoError(err) + + // Poll for activity task and start it with a worker control queue + activityPollResp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ + Namespace: env.Namespace().String(), + TaskQueue: tv.TaskQueue(), + Identity: tv.WorkerIdentity(), + WorkerInstanceKey: tv.WorkerInstanceKey(), + WorkerControlTaskQueue: controlQueueName, + }) + env.NoError(err) + env.NotNil(activityPollResp) + env.NotEmpty(activityPollResp.TaskToken) + + // Terminate the workflow directly (no cancellation request, no workflow task) + _, err = env.FrontendClient().TerminateWorkflowExecution(ctx, &workflowservice.TerminateWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tv.WorkflowID(), + RunId: startResp.RunId, + }, + Reason: "test termination", + }) + env.NoError(err) + + // Poll Nexus control queue - should receive cancel command for the in-flight activity + var nexusPollResp *workflowservice.PollNexusTaskQueueResponse + env.Eventually(func() bool { + pollCtx, pollCancel := context.WithTimeout(ctx, 5*time.Second) + defer pollCancel() + resp, err := env.FrontendClient().PollNexusTaskQueue(pollCtx, &workflowservice.PollNexusTaskQueueRequest{ + Namespace: env.Namespace().String(), + TaskQueue: &taskqueuepb.TaskQueue{Name: controlQueueName, Kind: enumspb.TASK_QUEUE_KIND_WORKER_COMMANDS}, + Identity: tv.WorkerIdentity(), + }) + if err == nil && resp != nil && resp.Request != nil { + nexusPollResp = resp + return true + } + return false + }, 120*time.Second, 100*time.Millisecond, "Timed out waiting for cancel command on control queue after termination") + + // Verify the cancel command + startOp := nexusPollResp.Request.GetStartOperation() + env.NotNil(startOp, "Expected StartOperation in Nexus request") + env.Equal("temporal.api.nexusservices.workerservice.v1.WorkerService", startOp.Service) + env.Equal("ExecuteCommands", startOp.Operation) + + var executeReq workerservicepb.ExecuteCommandsRequest + err = payload.Decode(startOp.Payload, &executeReq) + env.NoError(err) + env.Len(executeReq.Commands, 1, "Expected exactly 1 command") + cancelCmd := executeReq.Commands[0].GetCancelActivity() + env.NotNil(cancelCmd, "Expected CancelActivity command") + env.Equal(activityPollResp.TaskToken, cancelCmd.TaskToken, + "Cancel command task token must match the activity's original task token") + t.Log("SUCCESS: Received cancel command on control queue after workflow termination") +} + +// TestDispatchCancelOnWorkflowReset tests that when a workflow is reset, +// the old run is terminated and cancel commands are dispatched for its in-flight activities. +func TestDispatchCancelOnWorkflowReset(t *testing.T) { + env := testcore.NewEnv(t, testcore.WithDynamicConfig(dynamicconfig.EnableCancelActivityWorkerCommand, true)) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + tv := env.Tv() + poller := env.TaskPoller() + + controlQueueName := tv.ControlQueueName(env.Namespace().String()) + + // Start the workflow + startResp, err := env.FrontendClient().StartWorkflowExecution(ctx, &workflowservice.StartWorkflowExecutionRequest{ + RequestId: tv.Any().String(), + Namespace: env.Namespace().String(), + WorkflowId: tv.WorkflowID(), + WorkflowType: tv.WorkflowType(), + TaskQueue: tv.TaskQueue(), + WorkflowExecutionTimeout: durationpb.New(60 * time.Second), + WorkflowTaskTimeout: durationpb.New(10 * time.Second), + }) + env.NoError(err) + + // Schedule the activity + _, err = poller.PollAndHandleWorkflowTask(tv, + func(task *workflowservice.PollWorkflowTaskQueueResponse) (*workflowservice.RespondWorkflowTaskCompletedRequest, error) { + return &workflowservice.RespondWorkflowTaskCompletedRequest{ + Commands: []*commandpb.Command{ + { + CommandType: enumspb.COMMAND_TYPE_SCHEDULE_ACTIVITY_TASK, + Attributes: &commandpb.Command_ScheduleActivityTaskCommandAttributes{ + ScheduleActivityTaskCommandAttributes: &commandpb.ScheduleActivityTaskCommandAttributes{ + ActivityId: tv.ActivityID(), + ActivityType: tv.ActivityType(), + TaskQueue: tv.TaskQueue(), + ScheduleToCloseTimeout: durationpb.New(60 * time.Second), + StartToCloseTimeout: durationpb.New(60 * time.Second), + }, + }, + }, + }, + }, nil + }) + env.NoError(err) + + // Poll for activity task and start it with a worker control queue + activityPollResp, err := env.FrontendClient().PollActivityTaskQueue(ctx, &workflowservice.PollActivityTaskQueueRequest{ + Namespace: env.Namespace().String(), + TaskQueue: tv.TaskQueue(), + Identity: tv.WorkerIdentity(), + WorkerInstanceKey: tv.WorkerInstanceKey(), + WorkerControlTaskQueue: controlQueueName, + }) + env.NoError(err) + env.NotNil(activityPollResp) + env.NotEmpty(activityPollResp.TaskToken) + + // Find the WFT completed event ID for the reset point + histResp, err := env.FrontendClient().GetWorkflowExecutionHistory(ctx, &workflowservice.GetWorkflowExecutionHistoryRequest{ + Namespace: env.Namespace().String(), + Execution: &commonpb.WorkflowExecution{ + WorkflowId: tv.WorkflowID(), + RunId: startResp.RunId, + }, + }) + env.NoError(err) + var wftCompletedEventID int64 + for _, event := range histResp.History.Events { + if event.EventType == enumspb.EVENT_TYPE_WORKFLOW_TASK_COMPLETED { + wftCompletedEventID = event.EventId + break + } + } + env.NotZero(wftCompletedEventID) + + // Reset the workflow — this terminates the current run + _, err = env.FrontendClient().ResetWorkflowExecution(ctx, &workflowservice.ResetWorkflowExecutionRequest{ + Namespace: env.Namespace().String(), + WorkflowExecution: &commonpb.WorkflowExecution{ + WorkflowId: tv.WorkflowID(), + RunId: startResp.RunId, + }, + Reason: "test reset", + RequestId: uuid.NewString(), + WorkflowTaskFinishEventId: wftCompletedEventID, + }) + env.NoError(err) + + // Poll Nexus control queue - should receive cancel command for the in-flight activity + var nexusPollResp *workflowservice.PollNexusTaskQueueResponse + env.Eventually(func() bool { + pollCtx, pollCancel := context.WithTimeout(ctx, 5*time.Second) + defer pollCancel() + resp, err := env.FrontendClient().PollNexusTaskQueue(pollCtx, &workflowservice.PollNexusTaskQueueRequest{ + Namespace: env.Namespace().String(), + TaskQueue: &taskqueuepb.TaskQueue{Name: controlQueueName, Kind: enumspb.TASK_QUEUE_KIND_WORKER_COMMANDS}, + Identity: tv.WorkerIdentity(), + }) + if err == nil && resp != nil && resp.Request != nil { + nexusPollResp = resp + return true + } + return false + }, 120*time.Second, 100*time.Millisecond, "Timed out waiting for cancel command on control queue after reset") + + // Verify the cancel command + startOp := nexusPollResp.Request.GetStartOperation() + env.NotNil(startOp, "Expected StartOperation in Nexus request") + env.Equal("temporal.api.nexusservices.workerservice.v1.WorkerService", startOp.Service) + env.Equal("ExecuteCommands", startOp.Operation) + + var executeReq workerservicepb.ExecuteCommandsRequest + err = payload.Decode(startOp.Payload, &executeReq) + env.NoError(err) + env.Len(executeReq.Commands, 1, "Expected exactly 1 command") + cancelCmd := executeReq.Commands[0].GetCancelActivity() + env.NotNil(cancelCmd, "Expected CancelActivity command") + env.Equal(activityPollResp.TaskToken, cancelCmd.TaskToken, + "Cancel command task token must match the activity's original task token") + t.Log("SUCCESS: Received cancel command on control queue after workflow reset") +}