diff --git a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml index 22daf2988b0a..b2473a5341b5 100644 --- a/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml +++ b/.github/skills/azsdk-common-apiview-feedback-resolution/evals/eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-apiview-feedback-resolution type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: gpt-6.1-sol diff --git a/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml b/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml index 3ce10aed89b6..5856f2c07a64 100644 --- a/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml +++ b/.github/skills/azsdk-common-generate-sdk-locally/evals/eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-generate-sdk-locally type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: gpt-6.1-sol diff --git a/.github/skills/azsdk-common-generate-sdk-locally/evals/trigger.eval.yaml b/.github/skills/azsdk-common-generate-sdk-locally/evals/trigger.eval.yaml index 2ebf85d89760..b33524cda990 100644 --- a/.github/skills/azsdk-common-generate-sdk-locally/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-generate-sdk-locally/evals/trigger.eval.yaml @@ -20,6 +20,10 @@ scoring: stimuli: - name: trigger-generate-python-sdk-locally prompt: "Generate the Python SDK locally from my TypeSpec project at specification/compute/resource-manager/Microsoft.Compute/ComputeRP/tspconfig.yaml." + environment: + skills: + - .. + - ../../azsdk-common-sdk-breaking-change graders: - type: skill-invocation config: @@ -28,7 +32,12 @@ stimuli: # Should NOT trigger - name: anti-trigger-mitigate-breaking-change prompt: "Fix the SDK breaking changes" + environment: + skills: + - .. + - ../../azsdk-common-sdk-breaking-change graders: - type: skill-invocation config: + required: ["azsdk-common-sdk-breaking-change"] disallowed: ["azsdk-common-generate-sdk-locally"] diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/eval.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/eval.yaml index 548a435dd598..615883a4d2e1 100644 --- a/.github/skills/azsdk-common-prepare-release-plan/evals/eval.yaml +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-prepare-release-plan type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: gpt-6.1-sol diff --git a/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml b/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml index bc5e8a9bca30..a87fdcf1f099 100644 --- a/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml +++ b/.github/skills/azsdk-common-prepare-release-plan/evals/schedule-risk.eval.yaml @@ -58,6 +58,7 @@ stimuli: config: substring: "https://github.com/Azure/azure-rest-api-specs/pull/12345" + # The new target stays future-proof; the mock's older plan supplies the warning. - name: create-relays-past-due-guidance environment: skills: @@ -65,7 +66,8 @@ stimuli: prompt: | Create a public preview release plan for TypeSpec project specification/contosowidgetmanager/Contoso.WidgetManager with target - release month June 2026, spec PR + release month next calendar month (resolve to an explicit month and year + using the current date), spec PR https://github.com/Azure/azure-rest-api-specs/pull/38387, service tree ID a7f2b8e4-9c1d-4a3e-b6f9-2d8e5a7c3b1f, and product tree ID f1a8c5d2-6e4b-4f7a-9c2d-8b5e1f3a6c9e. Use the Azure SDK MCP tools. diff --git a/.github/skills/azsdk-common-sdk-breaking-change/evals/trigger.eval.yaml b/.github/skills/azsdk-common-sdk-breaking-change/evals/trigger.eval.yaml index 933bdcfb40f5..6a0ba6107815 100644 --- a/.github/skills/azsdk-common-sdk-breaking-change/evals/trigger.eval.yaml +++ b/.github/skills/azsdk-common-sdk-breaking-change/evals/trigger.eval.yaml @@ -20,6 +20,9 @@ scoring: stimuli: - name: trigger-detect-sdk-breaking-changes-for-service prompt: "Detect and mitigate SDK breaking changes for the service webpubsub" + environment: + skills: + - .. graders: - type: skill-invocation config: diff --git a/.github/skills/azsdk-common-sdk-release/evals/eval.yaml b/.github/skills/azsdk-common-sdk-release/evals/eval.yaml index e4604b9e2b05..b73345200b0d 100644 --- a/.github/skills/azsdk-common-sdk-release/evals/eval.yaml +++ b/.github/skills/azsdk-common-sdk-release/evals/eval.yaml @@ -8,7 +8,7 @@ tags: area: azsdk-common-sdk-release type: ci-gate -config: +defaults: runs: 1 timeout: "90s" model: gpt-6.1-sol @@ -60,12 +60,16 @@ stimuli: - name: basic-trigger prompt: "I need to release a new version of my Azure SDK package. Walk me through the process." + environment: + skills: + - .. graders: - type: output-contains config: substring: "release" - constraints: - expect_skills: ["azsdk-common-sdk-release"] + - type: skill-invocation + config: + required: ["azsdk-common-sdk-release"] - name: anti-trigger prompt: "How do I write unit tests for my Azure SDK client?"