Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ tags:
area: azsdk-common-apiview-feedback-resolution
type: ci-gate

config:
defaults:
runs: 1
timeout: "90s"
model: gpt-6.1-sol
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ tags:
area: azsdk-common-generate-sdk-locally
type: ci-gate

config:
defaults:
runs: 1
timeout: "90s"
model: gpt-6.1-sol
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ scoring:
stimuli:
- name: trigger-generate-python-sdk-locally
prompt: "Generate the Python SDK locally from my TypeSpec project at specification/compute/resource-manager/Microsoft.Compute/ComputeRP/tspconfig.yaml."
environment:
skills:
- ..
- ../../azsdk-common-sdk-breaking-change
graders:
- type: skill-invocation
config:
Expand All @@ -28,7 +32,12 @@ stimuli:
# Should NOT trigger
- name: anti-trigger-mitigate-breaking-change
prompt: "Fix the SDK breaking changes"
environment:
skills:
- ..
- ../../azsdk-common-sdk-breaking-change
graders:
- type: skill-invocation
config:
required: ["azsdk-common-sdk-breaking-change"]
disallowed: ["azsdk-common-generate-sdk-locally"]
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ tags:
area: azsdk-common-prepare-release-plan
type: ci-gate

config:
defaults:
runs: 1
timeout: "90s"
model: gpt-6.1-sol
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,16 @@ stimuli:
config:
substring: "https://github.com/Azure/azure-rest-api-specs/pull/12345"

# The new target stays future-proof; the mock's older plan supplies the warning.
- name: create-relays-past-due-guidance
environment:
skills:
- ..
prompt: |
Create a public preview release plan for TypeSpec project
specification/contosowidgetmanager/Contoso.WidgetManager with target
release month June 2026, spec PR
release month next calendar month (resolve to an explicit month and year
using the current date), spec PR
https://github.com/Azure/azure-rest-api-specs/pull/38387, service tree ID
a7f2b8e4-9c1d-4a3e-b6f9-2d8e5a7c3b1f, and product tree ID
f1a8c5d2-6e4b-4f7a-9c2d-8b5e1f3a6c9e. Use the Azure SDK MCP tools.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,9 @@ scoring:
stimuli:
- name: trigger-detect-sdk-breaking-changes-for-service
prompt: "Detect and mitigate SDK breaking changes for the service webpubsub"
environment:
skills:
- ..
graders:
- type: skill-invocation
config:
Expand Down
10 changes: 7 additions & 3 deletions .github/skills/azsdk-common-sdk-release/evals/eval.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ tags:
area: azsdk-common-sdk-release
type: ci-gate

config:
defaults:
runs: 1
timeout: "90s"
model: gpt-6.1-sol
Expand Down Expand Up @@ -60,12 +60,16 @@ stimuli:

- name: basic-trigger
prompt: "I need to release a new version of my Azure SDK package. Walk me through the process."
environment:
skills:
- ..
graders:
- type: output-contains
config:
substring: "release"
constraints:
expect_skills: ["azsdk-common-sdk-release"]
- type: skill-invocation
config:
required: ["azsdk-common-sdk-release"]

- name: anti-trigger
prompt: "How do I write unit tests for my Azure SDK client?"
Expand Down
Loading