diff --git a/cli/gh-student/accept.go b/cli/gh-student/accept.go index a3f443b1..b1425110 100644 --- a/cli/gh-student/accept.go +++ b/cli/gh-student/accept.go @@ -57,18 +57,57 @@ const ( defaultConfigRepoBranch = "main" ) +// shimBranchTriggerLine is the exact `on.push.branches` line of the embedded +// shim (before placeholder substitution). Tag submission mode removes it, so +// the shim triggers only on submit/* tag pushes. Pinned by the accept shim +// tests so an embed edit can't silently break the line surgery. +const shimBranchTriggerLine = " branches: [\"" + shimBranchPlaceholder + "\"]\n" + +// shimTagsTriggerLine is the exact `on.push.tags` line of the embedded shim. +// Assignments with submission_tags replace it with the union of the +// teacher's milestone patterns and submit/* (contract.ShimTagsList); the +// default keeps it verbatim. Pinned by the accept shim tests like +// shimBranchTriggerLine, and single-occurrence-guarded for the same reason. +const shimTagsTriggerLine = " tags: [\"submit/*\"]\n" + // renderEmbeddedShim returns the embedded shim with the org, submission-branch, // and config-branch placeholders substituted. The shim never changes after // accept — runtime customization, runner edits, and teacher overrides all flow // through the runner workflow + assignments.json on the teacher's side. -func renderEmbeddedShim(org, branch, configBranch string) string { +// +// submissionMode contract.SubmissionModeTag drops the branch-push trigger line +// so only submission-tag pushes grade (`gh student submit` creates the tag; a +// hand-pushed submit/* tag works too). Every other value — including "" and an +// explicit "every-push" — takes the identical code path as before the field +// existed, keeping the default shim byte-identical. +// +// submissionTags (teacher-named milestone patterns, e.g. phase1) widen the +// tags trigger to their union with the always-on submit/* namespace; empty +// keeps the tags line verbatim (again byte-identical). Orthogonal to +// submissionMode: an every-push assignment can also name milestone tags. +func renderEmbeddedShim(org, branch, configBranch, submissionMode string, submissionTags []string) string { if branch == "" { branch = defaultConfigRepoBranch } if configBranch == "" { configBranch = defaultConfigRepoBranch } - out := strings.ReplaceAll(embeddedShimContent, shimOrgPlaceholder, org) + shim := embeddedShimContent + if submissionMode == contract.SubmissionModeTag { + // Exact-line surgery, not a template branch: the embed stays one + // lintable file and every-push output can't drift. If the embed's + // trigger line ever changes shape, the tests pin this constant and + // the fallback below keeps accept emitting a valid (every-push) shim + // rather than garbage. + shim = strings.Replace(shim, shimBranchTriggerLine, "", 1) + } + if len(submissionTags) > 0 { + // Same exact-line surgery for the tags trigger: milestone patterns + // union submit/*, so the canonical namespace always fires. + shim = strings.Replace(shim, shimTagsTriggerLine, + " tags: ["+contract.ShimTagsList(submissionTags)+"]\n", 1) + } + out := strings.ReplaceAll(shim, shimOrgPlaceholder, org) out = strings.ReplaceAll(out, shimBranchPlaceholder, branch) out = strings.ReplaceAll(out, shimConfigBranchPlaceholder, configBranch) return out @@ -458,7 +497,7 @@ func acceptAssignment(cmd *cobra.Command, client githubapi.Client, u *ui.UI, out } configBranch = commitBranch } - shim = renderEmbeddedShim(org, commitBranch, configBranch) + shim = renderEmbeddedShim(org, commitBranch, configBranch, entry.SubmissionMode, entry.SubmissionTags) } repoName := reponame.Name(classroom, assignment, username) diff --git a/cli/gh-student/accept_shim_test.go b/cli/gh-student/accept_shim_test.go index 50a3bd6f..a39951a5 100644 --- a/cli/gh-student/accept_shim_test.go +++ b/cli/gh-student/accept_shim_test.go @@ -6,6 +6,8 @@ import ( "net/http/httptest" "strings" "testing" + + "github.com/foundation50/classroom50-cli-shared/contract" ) func TestRenderEmbeddedShim(t *testing.T) { @@ -13,7 +15,7 @@ func TestRenderEmbeddedShim(t *testing.T) { // accept drops into every student repo. {{ORG}}, the submission branch, and // the config-repo branch are the per-repo substitutions; everything else is // fixed. - got := renderEmbeddedShim("cs50-fall-2026", "main", "main") + got := renderEmbeddedShim("cs50-fall-2026", "main", "main", "", nil) // Trigger contract: branch pushes auto-grade; manual submit/* tag pushes // still work (the runner detects which fired and creates or reuses the tag). @@ -74,7 +76,7 @@ func TestRenderEmbeddedShim_OrgSubstitution(t *testing.T) { // matching anything else. for _, org := range []string{"cs50-fall-2026", "foundation50", "very-long-org-name-2026"} { t.Run(org, func(t *testing.T) { - got := renderEmbeddedShim(org, "main", "main") + got := renderEmbeddedShim(org, "main", "main", "", nil) wantUses := `uses: "` + org + `/classroom50/.github/workflows/autograde-runner.yaml@main"` if !strings.Contains(got, wantUses) { t.Errorf("expected %q in shim, got:\n%s", wantUses, got) @@ -90,7 +92,7 @@ func TestRenderEmbeddedShim_BranchSubstitution(t *testing.T) { // A master-default assignment repo must trigger on `master`; a config repo // that stayed on `master` (rename didn't land) must be referenced via // `@master` so the reusable-workflow ref resolves. - got := renderEmbeddedShim("cs50", "master", "master") + got := renderEmbeddedShim("cs50", "master", "master", "", nil) if !strings.Contains(got, `branches: ["master"]`) { t.Errorf("expected branches: [\"master\"], got:\n%s", got) } @@ -100,7 +102,7 @@ func TestRenderEmbeddedShim_BranchSubstitution(t *testing.T) { } // Empty branch/configBranch default to main. - def := renderEmbeddedShim("cs50", "", "") + def := renderEmbeddedShim("cs50", "", "", "", nil) if !strings.Contains(def, `branches: ["main"]`) { t.Errorf("empty branch should default to main, got:\n%s", def) } @@ -109,6 +111,69 @@ func TestRenderEmbeddedShim_BranchSubstitution(t *testing.T) { } } +func TestRenderEmbeddedShim_TagMode(t *testing.T) { + got := renderEmbeddedShim("cs50-fall-2026", "main", "main", contract.SubmissionModeTag, nil) + + // Tag mode keeps ONLY the submit/* tag trigger: a plain `git push` must + // not grade. The branch trigger line is removed whole — no leftover key. + if strings.Contains(got, "branches:") { + t.Errorf("tag-mode shim must not contain a branches: trigger:\n%s", got) + } + if !strings.Contains(got, `tags: ["submit/*"]`) { + t.Errorf("tag-mode shim missing the submit/* tag trigger:\n%s", got) + } + + // Everything else is unchanged: uses: line, permissions, no placeholders. + wantUses := `uses: "cs50-fall-2026/classroom50/.github/workflows/autograde-runner.yaml@main"` + if !strings.Contains(got, wantUses) { + t.Errorf("tag-mode shim missing %q\nfull:\n%s", wantUses, got) + } + for _, ph := range []string{"{{ORG}}", "{{BRANCH}}", "{{CONFIG_BRANCH}}"} { + if strings.Contains(got, ph) { + t.Errorf("tag-mode shim still contains unsubstituted %s:\n%s", ph, got) + } + } + for _, perm := range []string{"contents: write", "statuses: write"} { + if !strings.Contains(got, perm) { + t.Errorf("tag-mode shim missing required permission %q\nfull:\n%s", perm, got) + } + } + + // The removal is exactly one line: tag-mode output equals every-push + // output minus the substituted branch trigger line. + everyPush := renderEmbeddedShim("cs50-fall-2026", "main", "main", "", nil) + wantTag := strings.Replace(everyPush, " branches: [\"main\"]\n", "", 1) + if got != wantTag { + t.Errorf("tag-mode shim is not every-push minus the branch line:\ngot:\n%s\nwant:\n%s", got, wantTag) + } +} + +// TestRenderEmbeddedShim_EveryPushByteIdentical pins that every non-tag mode +// value — absent, the explicit wire default, and junk (validated upstream) — +// renders the identical bytes, so introducing submission_mode changed nothing +// for existing assignments. +func TestRenderEmbeddedShim_EveryPushByteIdentical(t *testing.T) { + base := renderEmbeddedShim("cs50", "main", "main", "", nil) + for _, mode := range []string{contract.SubmissionModeEveryPush, "unvalidated-junk"} { + if got := renderEmbeddedShim("cs50", "main", "main", mode, nil); got != base { + t.Errorf("mode %q rendered different bytes than the default", mode) + } + } +} + +// TestShimBranchTriggerLine_MatchesEmbed guards the line-surgery constant +// against embed drift: if autograde-shim.yaml's trigger line changes shape, +// tag mode would silently stop removing it (falling back to an every-push +// shim). Fail here instead. +func TestShimBranchTriggerLine_MatchesEmbed(t *testing.T) { + if !strings.Contains(embeddedShimContent, shimBranchTriggerLine) { + t.Fatalf("embed/autograde-shim.yaml no longer contains the exact branch trigger line %q — update shimBranchTriggerLine in lockstep", shimBranchTriggerLine) + } + if strings.Count(embeddedShimContent, shimBranchTriggerLine) != 1 { + t.Fatalf("branch trigger line appears more than once in the embed; single-occurrence surgery would remove the wrong one") + } +} + func TestResolveConfigRepoBranch(t *testing.T) { newServer := func(t *testing.T, handler http.HandlerFunc) *httptest.Server { server := httptest.NewServer(handler) @@ -152,3 +217,60 @@ func TestResolveConfigRepoBranch(t *testing.T) { } }) } + +func TestRenderEmbeddedShim_SubmissionTags(t *testing.T) { + // Milestone patterns widen the tags trigger to their union with the + // always-on submit/* namespace — plain `git tag phase1 && git push origin + // phase1` grades, and `gh student submit` keeps working unchanged. + got := renderEmbeddedShim("cs50", "main", "main", "", []string{"phase1", "v*"}) + wantTags := `tags: ["phase1", "v*", "submit/*"]` + if !strings.Contains(got, wantTags) { + t.Errorf("shim missing widened tags trigger %q\nfull:\n%s", wantTags, got) + } + // The branch trigger stays (every-push assignment with milestone tags). + if !strings.Contains(got, `branches: ["main"]`) { + t.Errorf("milestone tags must not drop the branch trigger:\n%s", got) + } + + // Orthogonal to tag mode: both together drop the branch line AND widen + // the tags line. + both := renderEmbeddedShim("cs50", "main", "main", contract.SubmissionModeTag, []string{"phase1"}) + if strings.Contains(both, "branches:") { + t.Errorf("tag mode + milestone tags must drop the branch trigger:\n%s", both) + } + if !strings.Contains(both, `tags: ["phase1", "submit/*"]`) { + t.Errorf("tag mode + milestone tags missing the widened trigger:\n%s", both) + } + + // The widening is exactly one line: tags-mode output equals the default + // output with only the tags line swapped. + base := renderEmbeddedShim("cs50", "main", "main", "", nil) + want := strings.Replace(base, " tags: [\"submit/*\"]\n", + " tags: [\"phase1\", \"v*\", \"submit/*\"]\n", 1) + if got != want { + t.Errorf("submission-tags shim is not the default minus one line swap:\ngot:\n%s\nwant:\n%s", got, want) + } +} + +// TestRenderEmbeddedShim_NoTagsByteIdentical pins that an empty/nil pattern +// list renders the identical bytes — introducing submission_tags changed +// nothing for existing assignments. +func TestRenderEmbeddedShim_NoTagsByteIdentical(t *testing.T) { + base := renderEmbeddedShim("cs50", "main", "main", "", nil) + if got := renderEmbeddedShim("cs50", "main", "main", "", []string{}); got != base { + t.Error("empty submission_tags rendered different bytes than nil") + } +} + +// TestShimTagsTriggerLine_MatchesEmbed guards the tags line-surgery constant +// against embed drift, exactly like TestShimBranchTriggerLine_MatchesEmbed: +// if the embed's tags line changes shape, milestone patterns would silently +// stop being rendered (falling back to submit/*-only). Fail here instead. +func TestShimTagsTriggerLine_MatchesEmbed(t *testing.T) { + if !strings.Contains(embeddedShimContent, shimTagsTriggerLine) { + t.Fatalf("embed/autograde-shim.yaml no longer contains the exact tags trigger line %q — update shimTagsTriggerLine in lockstep", shimTagsTriggerLine) + } + if strings.Count(embeddedShimContent, shimTagsTriggerLine) != 1 { + t.Fatalf("tags trigger line appears more than once in the embed; single-occurrence surgery would replace the wrong one") + } +} diff --git a/cli/gh-student/internal/assignments/assignments.go b/cli/gh-student/internal/assignments/assignments.go index c4d49cd1..aaee7d23 100644 --- a/cli/gh-student/internal/assignments/assignments.go +++ b/cli/gh-student/internal/assignments/assignments.go @@ -75,6 +75,28 @@ type Entry struct { // own create default template-less), an explicit true/false forces the // feature on/off. See accept.go's repo-feature PATCH. RepoFeatures *RepoFeatures `json:"repo_features,omitempty"` + + // SubmissionMode picks when the autograder fires: absent/"every-push" + // (the wire default) keeps the shim's every-push trigger; "tag" makes the + // shim trigger ONLY on submit/* tag pushes, which `gh student submit` + // creates after the branch push. Consumed at accept time (shim rendering) + // and at submit time (tag push). + SubmissionMode string `json:"submission_mode,omitempty"` + + // SubmissionTags is the teacher-named milestone tag patterns (e.g. + // phase1, v*) that ALSO trigger grading, alongside the always-on submit/* + // namespace. Consumed at accept time: the shim's tags trigger renders as + // the union of these patterns and submit/*. Empty/absent means no + // milestone tags (the default shim, byte-identical to before the field + // existed). + SubmissionTags []string `json:"submission_tags,omitempty"` +} + +// IsTagSubmissionMode reports whether the entry grades only on submit/* tag +// pushes. Centralized so accept (shim rendering) and submit (tag push) can't +// drift on the absent-means-every-push default. +func (e Entry) IsTagSubmissionMode() bool { + return e.SubmissionMode == contract.SubmissionModeTag } // RepoFeatures is the tri-state Issues/Wiki/Projects/Pull-requests override; a nil pointer diff --git a/cli/gh-student/internal/assignments/assignments_test.go b/cli/gh-student/internal/assignments/assignments_test.go index 9460f135..47f846c7 100644 --- a/cli/gh-student/internal/assignments/assignments_test.go +++ b/cli/gh-student/internal/assignments/assignments_test.go @@ -330,6 +330,55 @@ func TestEntryDecodesFeedbackPR(t *testing.T) { } } +func TestEntryDecodesSubmissionMode(t *testing.T) { + // submission_mode decodes when present and defaults to every-push + // semantics when absent — accept (shim rendering) and submit (tag push) + // both branch on IsTagSubmissionMode, so the wire contract matters. + var file assignmentsFile + if err := json.Unmarshal([]byte(`{ + "schema": "classroom50/assignments/v1", + "assignments": [ + {"slug": "tagged", "name": "Tagged", "mode": "individual", "autograder": "default", "submission_mode": "tag"}, + {"slug": "explicit", "name": "Explicit", "mode": "individual", "autograder": "default", "submission_mode": "every-push"}, + {"slug": "hello", "name": "Hello", "mode": "individual", "autograder": "default"} + ] +}`), &file); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if !file.Assignments[0].IsTagSubmissionMode() { + t.Error("tagged.IsTagSubmissionMode() = false, want true") + } + if file.Assignments[1].IsTagSubmissionMode() { + t.Error("explicit(every-push).IsTagSubmissionMode() = true, want false") + } + if file.Assignments[2].IsTagSubmissionMode() { + t.Error("hello(absent).IsTagSubmissionMode() = true, want false") + } +} + +func TestEntryDecodesSubmissionTags(t *testing.T) { + // submission_tags decodes when present and reads empty when absent — + // accept renders the shim's tags trigger from it, so the wire contract + // matters. + var file assignmentsFile + if err := json.Unmarshal([]byte(`{ + "schema": "classroom50/assignments/v1", + "assignments": [ + {"slug": "proj", "name": "Project", "mode": "individual", "autograder": "default", "submission_tags": ["phase1", "v*"]}, + {"slug": "hello", "name": "Hello", "mode": "individual", "autograder": "default"} + ] +}`), &file); err != nil { + t.Fatalf("unmarshal: %v", err) + } + got := file.Assignments[0].SubmissionTags + if len(got) != 2 || got[0] != "phase1" || got[1] != "v*" { + t.Errorf("proj.SubmissionTags = %v, want [phase1 v*]", got) + } + if len(file.Assignments[1].SubmissionTags) != 0 { + t.Errorf("hello(absent).SubmissionTags = %v, want empty", file.Assignments[1].SubmissionTags) + } +} + func TestEntryDecodesLocked(t *testing.T) { // locked decodes when present and defaults to false when absent — the // accept flow refuses a locked assignment, so the wire contract matters. diff --git a/cli/gh-student/internal/submitcmd/copysubmittable_test.go b/cli/gh-student/internal/submitcmd/copysubmittable_test.go index 371556db..64bbaad2 100644 --- a/cli/gh-student/internal/submitcmd/copysubmittable_test.go +++ b/cli/gh-student/internal/submitcmd/copysubmittable_test.go @@ -140,10 +140,11 @@ func TestCopySubmittableFiles_ControlFilesKeptEvenUnderStarIgnore(t *testing.T) } } -func TestFetchAllowedFiles_FetchFailureReturnsNil(t *testing.T) { +func TestFetchSubmitEntry_FetchFailureReturnsNil(t *testing.T) { // Best-effort guarantee: a manifest fetch failure must never block - // submission — fetchAllowedFiles returns nil (submit all) since the - // runner enforces the allowlist authoritatively. + // submission — fetchSubmitEntry returns (nil, err); the caller submits + // all files (the runner enforces allowed_files authoritatively) and + // warns about the unknown submission mode post-push. orig := fetchEntryFn t.Cleanup(func() { fetchEntryFn = orig }) fetchEntryFn = func(ctx context.Context, org, classroom, secret, assignment string) (assignments.Entry, error) { @@ -152,24 +153,37 @@ func TestFetchAllowedFiles_FetchFailureReturnsNil(t *testing.T) { u := ui.NewForced(os.Stderr, false) cfg := &classroomcfg.Config{Classroom: "cs-principles", Assignment: "hello"} - got := fetchAllowedFiles(context.Background(), "o", cfg, u, false) - if got != nil { - t.Errorf("fetchAllowedFiles on fetch failure = %#v, want nil (submit all)", got) + entry, err := fetchSubmitEntry(context.Background(), "o", cfg, u, false) + if entry != nil { + t.Errorf("fetchSubmitEntry on fetch failure = %#v, want nil (submit all)", entry) + } + if err == nil { + t.Error("fetchSubmitEntry on fetch failure must surface the error for the post-push warning") } } -func TestFetchAllowedFiles_SuccessReturnsPatterns(t *testing.T) { +func TestFetchSubmitEntry_SuccessReturnsEntry(t *testing.T) { orig := fetchEntryFn t.Cleanup(func() { fetchEntryFn = orig }) fetchEntryFn = func(ctx context.Context, org, classroom, secret, assignment string) (assignments.Entry, error) { - return assignments.Entry{AllowedFiles: []string{"*", "!hello.py"}}, nil + return assignments.Entry{ + AllowedFiles: []string{"*", "!hello.py"}, + SubmissionMode: "tag", + }, nil } u := ui.NewForced(os.Stderr, false) cfg := &classroomcfg.Config{Classroom: "cs-principles", Assignment: "hello"} - got := fetchAllowedFiles(context.Background(), "o", cfg, u, false) + entry, err := fetchSubmitEntry(context.Background(), "o", cfg, u, false) + if err != nil || entry == nil { + t.Fatalf("fetchSubmitEntry = (%#v, %v), want entry", entry, err) + } + got := entry.AllowedFiles if len(got) != 2 || got[0] != "*" || got[1] != "!hello.py" { - t.Errorf("fetchAllowedFiles = %#v, want [* !hello.py]", got) + t.Errorf("AllowedFiles = %#v, want [* !hello.py]", got) + } + if !entry.IsTagSubmissionMode() { + t.Error("IsTagSubmissionMode() = false, want true") } } diff --git a/cli/gh-student/internal/submitcmd/finish_submission_test.go b/cli/gh-student/internal/submitcmd/finish_submission_test.go new file mode 100644 index 00000000..2a5f1404 --- /dev/null +++ b/cli/gh-student/internal/submitcmd/finish_submission_test.go @@ -0,0 +1,247 @@ +package submitcmd + +import ( + "bytes" + "context" + "errors" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/foundation50/classroom50-cli-shared/contract" + "github.com/foundation50/gh-student/internal/assignments" + "github.com/foundation50/gh-student/internal/ui" +) + +// _finishHarness drives finishSubmission with a shared out/err buffer so the +// CONFIRM marker (announce) and any warning are ordering-assertable by index, +// plus a counting retry stub. +type _finishHarness struct { + buf bytes.Buffer + u *ui.UI + retryCalls int +} + +func _newFinishHarness() *_finishHarness { + h := &_finishHarness{} + h.u = ui.NewForced(&h.buf, false) + return h +} + +func (h *_finishHarness) announce() func() { + return func() { h.buf.WriteString("CONFIRM\n") } +} + +func (h *_finishHarness) retry(entry *assignments.Entry, err error) func(context.Context) (*assignments.Entry, error) { + return func(context.Context) (*assignments.Entry, error) { + h.retryCalls++ + return entry, err + } +} + +func _tagModeEntry() *assignments.Entry { + return &assignments.Entry{SubmissionMode: contract.SubmissionModeTag} +} + +// _deadRemote repoints the fixture's origin at a nonexistent path so any tag +// push fails (mirrors TestPushSubmitTag_PushFailureSurfacesError). +func _deadRemote(t *testing.T, local string) { + t.Helper() + if out, err := exec.Command( + "git", "--git-dir", local, "remote", "set-url", "origin", + filepath.Join(t.TempDir(), "gone.git"), + ).CombinedOutput(); err != nil { + t.Fatalf("set-url: %v\n%s", err, out) + } +} + +func TestFinishSubmission_TagModePushesTag(t *testing.T) { + local, remote, sha := _tagTestRepos(t) + h := _newFinishHarness() + + err := finishSubmission(context.Background(), _tagModeEntry(), nil, + h.retry(nil, errors.New("retry must not run")), + local, sha, "https://x", false, h.u, h.announce()) + if err != nil { + t.Fatalf("finishSubmission: %v", err) + } + if h.retryCalls != 0 { + t.Errorf("retryCalls = %d, want 0 (entry already resolved)", h.retryCalls) + } + if tags := _remoteTags(t, remote); len(tags) != 1 || !strings.HasPrefix(tags[0], "submit/") { + t.Errorf("remote tags = %v, want exactly one submit/* tag", tags) + } + out := h.buf.String() + if !strings.Contains(out, "CONFIRM") { + t.Errorf("confirmation not printed:\n%s", out) + } + if strings.Contains(out, "could not determine") { + t.Errorf("unexpected mode-unknown warning:\n%s", out) + } +} + +func TestFinishSubmission_EveryPushNeverTags(t *testing.T) { + // Both the absent wire default and an explicit every-push: no tag, no + // retry, no warning — the branch push already grades. + for _, mode := range []string{"", contract.SubmissionModeEveryPush} { + t.Run("mode="+mode, func(t *testing.T) { + local, remote, sha := _tagTestRepos(t) + h := _newFinishHarness() + + entry := &assignments.Entry{SubmissionMode: mode} + err := finishSubmission(context.Background(), entry, nil, + h.retry(nil, errors.New("retry must not run")), + local, sha, "https://x", false, h.u, h.announce()) + if err != nil { + t.Fatalf("finishSubmission: %v", err) + } + if h.retryCalls != 0 { + t.Errorf("retryCalls = %d, want 0", h.retryCalls) + } + if tags := _remoteTags(t, remote); len(tags) != 0 { + t.Errorf("remote tags = %v, want none", tags) + } + if out := h.buf.String(); !strings.Contains(out, "CONFIRM") || strings.Contains(out, "could not determine") { + t.Errorf("want confirmation and no warning, got:\n%s", out) + } + }) + } +} + +func TestFinishSubmission_NilEntryNilErrNoOps(t *testing.T) { + // Defensive shape (fetchSubmitEntry never returns it): no retry, no tag, + // no warning. + h := _newFinishHarness() + err := finishSubmission(context.Background(), nil, nil, + h.retry(nil, errors.New("retry must not run")), + "/nonexistent-git-dir", "deadbeef", "https://x", false, h.u, h.announce()) + if err != nil { + t.Fatalf("finishSubmission: %v", err) + } + if h.retryCalls != 0 { + t.Errorf("retryCalls = %d, want 0", h.retryCalls) + } + if out := h.buf.String(); !strings.Contains(out, "CONFIRM") || strings.Contains(out, "could not determine") { + t.Errorf("want confirmation only, got:\n%s", out) + } +} + +func TestFinishSubmission_RetryResolvesTagMode(t *testing.T) { + // The headline regression: pre-push fetch failed, the post-push retry + // resolves tag mode — the tag is pushed, grading proceeds, NO warning. + local, remote, sha := _tagTestRepos(t) + h := _newFinishHarness() + + err := finishSubmission(context.Background(), nil, errors.New("pages blip"), + h.retry(_tagModeEntry(), nil), + local, sha, "https://x", false, h.u, h.announce()) + if err != nil { + t.Fatalf("finishSubmission: %v", err) + } + if h.retryCalls != 1 { + t.Errorf("retryCalls = %d, want exactly 1", h.retryCalls) + } + if tags := _remoteTags(t, remote); len(tags) != 1 || !strings.HasPrefix(tags[0], "submit/") { + t.Errorf("remote tags = %v, want the submit/* tag from the retried entry", tags) + } + if out := h.buf.String(); strings.Contains(out, "could not determine") { + t.Errorf("retry succeeded — no warning expected:\n%s", out) + } +} + +func TestFinishSubmission_RetryResolvesEveryPush(t *testing.T) { + local, remote, sha := _tagTestRepos(t) + h := _newFinishHarness() + + err := finishSubmission(context.Background(), nil, errors.New("pages blip"), + h.retry(&assignments.Entry{}, nil), + local, sha, "https://x", false, h.u, h.announce()) + if err != nil { + t.Fatalf("finishSubmission: %v", err) + } + if h.retryCalls != 1 { + t.Errorf("retryCalls = %d, want 1", h.retryCalls) + } + if tags := _remoteTags(t, remote); len(tags) != 0 { + t.Errorf("remote tags = %v, want none (every-push)", tags) + } + if out := h.buf.String(); strings.Contains(out, "could not determine") { + t.Errorf("mode resolved — no warning expected:\n%s", out) + } +} + +func TestFinishSubmission_RetryFailsWarnsAfterConfirmation(t *testing.T) { + // Both fetches failed: no tag (never tag blind), and the warning prints + // strictly AFTER the confirmation — the deliberate ordering so it's the + // last thing a tag-mode student sees. + h := _newFinishHarness() + retryErr := errors.New("pages still down") + + err := finishSubmission(context.Background(), nil, errors.New("pages blip"), + h.retry(nil, retryErr), + "/nonexistent-git-dir", "deadbeef", "https://x", false, h.u, h.announce()) + if err != nil { + t.Fatalf("finishSubmission: %v", err) + } + if h.retryCalls != 1 { + t.Errorf("retryCalls = %d, want exactly 1 (no retry loop)", h.retryCalls) + } + out := h.buf.String() + ci := strings.Index(out, "CONFIRM") + wi := strings.Index(out, "could not determine") + if ci == -1 || wi == -1 { + t.Fatalf("want both confirmation and warning, got:\n%s", out) + } + if wi < ci { + t.Errorf("warning printed before the confirmation:\n%s", out) + } + if !strings.Contains(out, retryErr.Error()) { + t.Errorf("warning should carry the RETRY's error, got:\n%s", out) + } +} + +func TestFinishSubmission_TagPushFailureIsFatal(t *testing.T) { + // A tag-mode tag-push failure returns the wrapped re-run guidance and + // never prints the confirmation (the submission is not "done"). + local, _, sha := _tagTestRepos(t) + _deadRemote(t, local) + h := _newFinishHarness() + + err := finishSubmission(context.Background(), _tagModeEntry(), nil, + h.retry(nil, errors.New("retry must not run")), + local, sha, "https://x", false, h.u, h.announce()) + if err == nil { + t.Fatal("want the wrapped tag-push error") + } + if !strings.Contains(err.Error(), "re-run `gh student submit`") { + t.Errorf("error missing the re-run guidance: %v", err) + } + if out := h.buf.String(); strings.Contains(out, "CONFIRM") { + t.Errorf("confirmation must not print on a fatal tag-push failure:\n%s", out) + } +} + +func TestFinishSubmission_RetryThenTagPushFailure(t *testing.T) { + // The retry path feeds the same fatal contract: retry resolves tag mode, + // the push fails, the wrapped error surfaces, no confirmation. + local, _, sha := _tagTestRepos(t) + _deadRemote(t, local) + h := _newFinishHarness() + + err := finishSubmission(context.Background(), nil, errors.New("pages blip"), + h.retry(_tagModeEntry(), nil), + local, sha, "https://x", false, h.u, h.announce()) + if err == nil { + t.Fatal("want the wrapped tag-push error") + } + if h.retryCalls != 1 { + t.Errorf("retryCalls = %d, want 1", h.retryCalls) + } + if !strings.Contains(err.Error(), "re-run `gh student submit`") { + t.Errorf("error missing the re-run guidance: %v", err) + } + if out := h.buf.String(); strings.Contains(out, "CONFIRM") { + t.Errorf("confirmation must not print on a fatal tag-push failure:\n%s", out) + } +} diff --git a/cli/gh-student/internal/submitcmd/submit.go b/cli/gh-student/internal/submitcmd/submit.go index c2b3b402..e2cbdeae 100644 --- a/cli/gh-student/internal/submitcmd/submit.go +++ b/cli/gh-student/internal/submitcmd/submit.go @@ -39,11 +39,17 @@ func NewCmd() *cobra.Command { Use: "submit", Short: "Submit the current assignment to its remote", Long: "Snapshot the current branch and push it as a new commit on top\n" + - "of the assignment repo's default branch. The autograde workflow\n" + - "in the student repo listens for pushes to that branch and (a)\n" + - "creates its own `submit/-` tag at the\n" + - "pushed commit and (b) publishes a scored Release at that tag a\n" + - "minute or two later.\n\n" + + "of the assignment repo's default branch. For an every-push\n" + + "assignment (the default) the autograde workflow listens for\n" + + "pushes to that branch and (a) creates its own\n" + + "`submit/-` tag at the pushed commit\n" + + "and (b) publishes a scored Release at that tag a minute or two\n" + + "later.\n\n" + + "For a tag-mode assignment (submission_mode: tag) plain pushes\n" + + "are not graded; this command additionally pushes the\n" + + "`submit/-` tag itself, which is what\n" + + "triggers grading. (Pushing your own `submit/*` tag by hand\n" + + "works too.)\n\n" + "Before snapshotting, the latest teacher `.gitignore` and\n" + "`.github/` (both optional) are fetched from the template repo\n" + "recorded in `.classroom50.yaml` so any teacher-side updates\n" + @@ -155,9 +161,15 @@ func submitAssignment(ctx context.Context, client githubapi.Client, verbose bool u.Detail("Preparing submission snapshot from %s", root) } - // Resolve allowed_files (best-effort): a fetch failure never blocks - // submission — the runner enforces authoritatively at grade time. - allowedFiles := fetchAllowedFiles(ctx, repoOwner, config, u, verbose) + // Resolve the manifest entry once (best-effort): allowed_files fails open + // (the runner enforces authoritatively at grade time); submission_mode + // failing to resolve is warned about after the push (a tag-mode + // assignment would then need a re-run to grade). + entry, entryErr := fetchSubmitEntry(ctx, repoOwner, config, u, verbose) + var allowedFiles []string + if entry != nil { + allowedFiles = entry.AllowedFiles + } if err := copySubmittableFiles(root, workTree, allowedFiles, u, verbose); err != nil { return err @@ -229,13 +241,79 @@ func submitAssignment(ctx context.Context, client githubapi.Client, verbose bool sp.Stop("Submission pushed") } - // Confirmation on stdout: the assignment's full name (falls back to the - // slug — see resolveAssignmentName), the local submission time, then a - // link to the submitted commit. - displayName := resolveAssignmentName(ctx, repoOwner, config.Classroom, config.Secret, config.Assignment) - localTime := time.Now().Local().Format("2006-01-02 15:04:05 MST") - _, _ = fmt.Fprintf(out, "Submitted assignment %q at %s\n", displayName, localTime) - _, _ = fmt.Fprintf(out, "View your submission at: %s/commit/%s\n", repoHTMLURL, sha) + // Everything after the branch push — the retry of a failed manifest fetch, + // the tag-mode submit/* tag push, the stdout confirmation, and the + // mode-unknown warning — lives in finishSubmission so the ordering + // contract (warning strictly after the confirmation) is structural. + retryFetch := func(c context.Context) (*assignments.Entry, error) { + return fetchSubmitEntry(c, repoOwner, config, u, verbose) + } + announce := func() { + // Confirmation on stdout: the assignment's full name (falls back to + // the slug — see resolveAssignmentName), the local submission time, + // then a link to the submitted commit. + displayName := resolveAssignmentName(ctx, repoOwner, config.Classroom, config.Secret, config.Assignment) + localTime := time.Now().Local().Format("2006-01-02 15:04:05 MST") + _, _ = fmt.Fprintf(out, "Submitted assignment %q at %s\n", displayName, localTime) + _, _ = fmt.Fprintf(out, "View your submission at: %s/commit/%s\n", repoHTMLURL, sha) + } + return finishSubmission(ctx, entry, entryErr, retryFetch, gitDir, sha, repoHTMLURL, verbose, u, announce) +} + +// finishSubmission is the post-push tail of submit. Order matters and is +// enforced here, not at the call site: +// +// 1. If the pre-push manifest fetch failed, retry it ONCE — Pages blips and +// cold caches are common, and on a tag-mode assignment the entry decides +// whether the grading tag is pushed at all. The pre-push result is still +// fetched early (allowed_files filtering needs it before the snapshot). +// 2. Tag-mode assignments grade ONLY on submit/* tag pushes, so push the tag +// with the user's token (user pushes fire workflows; the runner's own +// github.token pushes deliberately don't). Every-push assignments must +// NOT get a tag here — the branch push already grades, and a second push +// event would double-grade the same commit. A tag-push failure is fatal +// (no confirmation): the work is safe on the branch, and the error says +// how to retry. +// 3. announce() prints the stdout success confirmation. +// 4. If the mode is STILL unknown (both fetches failed), warn — after the +// confirmation, so it's the last thing a tag-mode student sees: their +// push may not grade. Never push a tag blind (double-grades every-push). +func finishSubmission( + ctx context.Context, + entry *assignments.Entry, + entryErr error, + retryFetch func(context.Context) (*assignments.Entry, error), + gitDir, sha, repoHTMLURL string, + verbose bool, + u *ui.UI, + announce func(), +) error { + if entry == nil && entryErr != nil { + if verbose { + u.Detail("Retrying assignment manifest fetch (pre-push attempt failed: %v)", entryErr) + } + entry, entryErr = retryFetch(ctx) + } + + if entry != nil && entry.IsTagSubmissionMode() { + tag, err := pushSubmitTag(gitDir, sha) + if err != nil { + return fmt.Errorf( + "submission pushed (%s/commit/%s) but the submit tag failed: %w\n"+ + "this assignment grades only on submit/* tags — re-run `gh student submit` to retry (the pushed work is safe)", + repoHTMLURL, sha, err, + ) + } + if verbose { + u.Detail("Pushed submission tag %s", tag) + } + } + + announce() + + if entry == nil && entryErr != nil { + u.Warn("could not determine the assignment's submission mode (%v); if this assignment grades on submit tags, re-run `gh student submit`", entryErr) + } return nil } @@ -244,26 +322,76 @@ func submitAssignment(ctx context.Context, client githubapi.Client, verbose bool // exercise the success and failure paths without a live Pages fetch. var fetchEntryFn = assignments.FetchEntry -// fetchAllowedFiles resolves the assignment's allowed_files patterns -// from the manifest. Best-effort: any failure returns nil and warns, -// since the runner enforces the list authoritatively. Bounded by -// assignmentNameTimeout. -func fetchAllowedFiles(ctx context.Context, org string, config *classroomcfg.Config, u *ui.UI, verbose bool) []string { - ctx, cancel := context.WithTimeout(ctx, assignmentNameTimeout) +// fetchSubmitEntry resolves the assignment's manifest entry (one Pages fetch +// for allowed_files + submission_mode). Best-effort: any failure returns +// (nil, err) and the caller decides — allowed_files fails open (the runner +// enforces authoritatively), submission_mode failure warns post-push. +// Bounded by submitEntryTimeout: unlike the cosmetic name lookup, this fetch +// decides whether a tag-mode submission pushes the tag that grades it, so it +// gets a generous bound (it also runs BEFORE the slow clone/push, where a few +// extra seconds stall nothing). +func fetchSubmitEntry(ctx context.Context, org string, config *classroomcfg.Config, u *ui.UI, verbose bool) (*assignments.Entry, error) { + ctx, cancel := context.WithTimeout(ctx, submitEntryTimeout) defer cancel() entry, err := fetchEntryFn(ctx, org, config.Classroom, config.Secret, config.Assignment) if err != nil { if verbose { - u.Detail("Could not resolve allowed_files (%v); submitting all files — the autograder enforces the list", err) + u.Detail("Could not resolve the assignment entry (%v); submitting all files — the autograder enforces allowed_files", err) } - return nil + return nil, err } if len(entry.AllowedFiles) > 0 && verbose { u.Detail("Applying allowed_files filter (%d pattern(s))", len(entry.AllowedFiles)) } - return entry.AllowedFiles + return &entry, nil } +// pushSubmitTag creates submit/- at sha and pushes +// it from the temp bare clone left by commitWorkTreeOnRemoteBranch. If a +// submit/* tag already points at sha (a retry after a tag-push failure, or a +// hand-pushed tag), it is reused — mirroring the runner's ls-remote +// idempotency check — so the same commit never grades twice. +func pushSubmitTag(gitDir, sha string) (string, error) { + existing, err := existingSubmitTagAt(gitDir, sha) + if err == nil && existing != "" { + return existing, nil + } + // A failed reuse probe falls through to pushing a fresh tag. The fresh + // tag's timestamped name never collides with an existing one, so the + // worst case is a second submit/* tag at the same SHA (one extra graded + // run of identical work) — preferred over failing the submission when + // the probe hiccups but the push would succeed. + tag := contract.BuildSubmitTag(timeNow(), sha) + if _, err := gitOutputWithGitDir(gitDir, "push", "origin", sha+":refs/tags/"+tag); err != nil { + return "", err + } + return tag, nil +} + +// existingSubmitTagAt returns the first submit/* tag pointing at sha on +// origin, or "" when none. `--refs` filters the peeled-ref (^{}) rows +// annotated tags emit, mirroring the runner's awk pipeline. +func existingSubmitTagAt(gitDir, sha string) (string, error) { + out, err := gitOutputWithGitDir(gitDir, "ls-remote", "--refs", "--tags", "origin") + if err != nil { + return "", err + } + for _, line := range strings.Split(out, "\n") { + fields := strings.Fields(line) + if len(fields) != 2 { + continue + } + refSHA, ref := fields[0], fields[1] + if refSHA == sha && strings.HasPrefix(ref, "refs/tags/"+contract.SubmitTagPrefix) { + return strings.TrimPrefix(ref, "refs/tags/"), nil + } + } + return "", nil +} + +// timeNow is stubbed in tests to pin the generated tag name. +var timeNow = time.Now + // resolveAssignmentName returns the assignment's full name from the published // manifest, falling back to the slug on any error/timeout. The fetch is // bounded (assignmentNameTimeout) and runs after the push succeeded, so submit @@ -282,6 +410,13 @@ func resolveAssignmentName(ctx context.Context, org, classroom, secret, slug str // Pages CDN can't stall the terminal after the submission already landed. const assignmentNameTimeout = 3 * time.Second +// submitEntryTimeout bounds the pre-push manifest fetch (allowed_files + +// submission_mode). Deliberately larger than assignmentNameTimeout: on a +// tag-mode assignment this fetch gates the submit/* tag push — timing out +// means the submission silently doesn't grade — and it runs before the +// clone/push, so the extra allowance never stalls a completed submission. +const submitEntryTimeout = 15 * time.Second + // resolveRepoDefaultBranch reads the assignment repo's default branch. A GET // failure is returned as an error (submitting to the wrong branch would skip // grading); an empty value falls back to "main" (matches an auto_init repo). diff --git a/cli/gh-student/internal/submitcmd/submit_tag_test.go b/cli/gh-student/internal/submitcmd/submit_tag_test.go new file mode 100644 index 00000000..48df919a --- /dev/null +++ b/cli/gh-student/internal/submitcmd/submit_tag_test.go @@ -0,0 +1,143 @@ +package submitcmd + +import ( + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// _tagTestRepos builds the fixture pushSubmitTag runs against: a bare +// "remote" with one commit on main, and a local bare clone of it (the shape +// commitWorkTreeOnRemoteBranch leaves behind). Returns (localGitDir, +// remoteGitDir, commitSHA). +func _tagTestRepos(t *testing.T) (string, string, string) { + t.Helper() + tmp := t.TempDir() + + run := func(dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + + // Seed a work repo, then serve it as a bare remote. + seed := filepath.Join(tmp, "seed") + run(tmp, "init", "-q", "-b", "main", seed) + run(seed, "-c", "user.name=t", "-c", "user.email=t@example.com", + "commit", "-q", "--allow-empty", "-m", "Submit hello") + sha := run(seed, "rev-parse", "HEAD") + + remote := filepath.Join(tmp, "remote.git") + run(tmp, "clone", "-q", "--bare", seed, remote) + + local := filepath.Join(tmp, "local.git") + run(tmp, "clone", "-q", "--bare", remote, local) + + return local, remote, sha +} + +func _remoteTags(t *testing.T, remote string) []string { + t.Helper() + out, err := exec.Command("git", "--git-dir", remote, "tag", "--list").Output() + if err != nil { + t.Fatalf("list remote tags: %v", err) + } + var tags []string + for _, l := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if l != "" { + tags = append(tags, l) + } + } + return tags +} + +func TestPushSubmitTag_PushesCanonicalTag(t *testing.T) { + local, remote, sha := _tagTestRepos(t) + + origNow := timeNow + t.Cleanup(func() { timeNow = origNow }) + timeNow = func() time.Time { + return time.Date(2026, 8, 3, 14, 30, 5, 0, time.UTC) + } + + tag, err := pushSubmitTag(local, sha) + if err != nil { + t.Fatalf("pushSubmitTag: %v", err) + } + want := "submit/2026-08-03T14-30-05Z-" + sha[:7] + if tag != want { + t.Errorf("tag = %q, want %q", tag, want) + } + tags := _remoteTags(t, remote) + if len(tags) != 1 || tags[0] != want { + t.Errorf("remote tags = %v, want [%s]", tags, want) + } +} + +func TestPushSubmitTag_ReusesExistingTagAtSHA(t *testing.T) { + // A retry after a tag-push failure — or a hand-pushed submit/* tag — + // must be reused, never duplicated: one grading run per commit. + local, remote, sha := _tagTestRepos(t) + + pre := "submit/hand-pushed" + if out, err := exec.Command( + "git", "--git-dir", local, "push", "origin", sha+":refs/tags/"+pre, + ).CombinedOutput(); err != nil { + t.Fatalf("seed existing tag: %v\n%s", err, out) + } + + tag, err := pushSubmitTag(local, sha) + if err != nil { + t.Fatalf("pushSubmitTag: %v", err) + } + if tag != pre { + t.Errorf("tag = %q, want reused %q", tag, pre) + } + if tags := _remoteTags(t, remote); len(tags) != 1 { + t.Errorf("remote tags = %v, want exactly the pre-existing one", tags) + } +} + +func TestPushSubmitTag_NonSubmitTagAtSHAIsIgnored(t *testing.T) { + // Only submit/* tags count as submissions; an unrelated tag at the same + // SHA must not suppress the canonical submit tag. + local, remote, sha := _tagTestRepos(t) + + if out, err := exec.Command( + "git", "--git-dir", local, "push", "origin", sha+":refs/tags/v1.0", + ).CombinedOutput(); err != nil { + t.Fatalf("seed unrelated tag: %v\n%s", err, out) + } + + tag, err := pushSubmitTag(local, sha) + if err != nil { + t.Fatalf("pushSubmitTag: %v", err) + } + if !strings.HasPrefix(tag, "submit/") { + t.Errorf("tag = %q, want a fresh submit/* tag", tag) + } + if tags := _remoteTags(t, remote); len(tags) != 2 { + t.Errorf("remote tags = %v, want v1.0 plus the submit tag", tags) + } +} + +func TestPushSubmitTag_PushFailureSurfacesError(t *testing.T) { + local, _, sha := _tagTestRepos(t) + // Point origin at a nonexistent path so the push fails. + if out, err := exec.Command( + "git", "--git-dir", local, "remote", "set-url", "origin", + filepath.Join(t.TempDir(), "gone.git"), + ).CombinedOutput(); err != nil { + t.Fatalf("set-url: %v\n%s", err, out) + } + if _, err := pushSubmitTag(local, sha); err == nil { + t.Fatal("pushSubmitTag against a dead remote must error (submit surfaces the re-run guidance)") + } +} diff --git a/cli/gh-teacher/autograders_tests/test_inline_validator.py b/cli/gh-teacher/autograders_tests/test_inline_validator.py index df9a861d..68783e4f 100644 --- a/cli/gh-teacher/autograders_tests/test_inline_validator.py +++ b/cli/gh-teacher/autograders_tests/test_inline_validator.py @@ -180,7 +180,9 @@ def _classroom_yaml(classroom: str = "cs-test", assignment: str = "hello") -> st def _manifest(*, slug: str = "hello", runtime: dict | None = None, tests: list | None = None, - release_assets: object = _MISSING) -> dict: + release_assets: object = _MISSING, + submission_mode: object = _MISSING, + submission_tags: object = _MISSING) -> dict: """Minimum assignments.json with one entry, optional runtime/tests.""" entry = { "slug": slug, @@ -195,6 +197,10 @@ def _manifest(*, slug: str = "hello", runtime: dict | None = None, entry["tests"] = tests if release_assets is not _MISSING: entry["release_assets"] = release_assets + if submission_mode is not _MISSING: + entry["submission_mode"] = submission_mode + if submission_tags is not _MISSING: + entry["submission_tags"] = submission_tags return { "schema": "classroom50/assignments/v1", "assignments": [entry], @@ -852,3 +858,283 @@ def test_empty_repo_hard_stops_before_detection(self, inline_script, tmp_path): assert rc != 0 assert "empty-repository assignment" in stderr assert "no-autograder" not in outputs + + +# --------------------------------------------------------------------------- +# Submission mode — stale-shim defense +# --------------------------------------------------------------------------- + + +class TestSubmissionMode: + # The read step emits submission-mode (absent → every-push) and + # branch-push-suppressed (tag mode + branch-triggered run). Suppression + # only fires for a stale/hand-edited every-push shim on a tag-mode + # assignment — a correctly retrofitted shim never branch-triggers. + + _BRANCH_REF = {"REF": "refs/heads/main"} + _TAG_REF = {"REF": "refs/tags/submit/2026-06-01T14-32-05Z-a1b2c3d"} + + def test_absent_mode_defaults_to_every_push(self, inline_script, tmp_path): + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(), + extra_env=self._BRANCH_REF, + ) + assert rc == 0 + assert outputs["submission-mode"] == "every-push" + assert outputs["branch-push-suppressed"] == "false" + + def test_explicit_every_push_not_suppressed(self, inline_script, tmp_path): + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_mode="every-push"), + extra_env=self._BRANCH_REF, + ) + assert rc == 0 + assert outputs["submission-mode"] == "every-push" + assert outputs["branch-push-suppressed"] == "false" + + def test_tag_mode_branch_run_suppressed(self, inline_script, tmp_path): + # The stale-shim case: mode flipped to tag but this repo's shim + # still branch-triggers. Suppress so the push costs nothing. + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_mode="tag"), + extra_env=self._BRANCH_REF, + ) + assert rc == 0 + assert outputs["submission-mode"] == "tag" + assert outputs["branch-push-suppressed"] == "true" + + def test_tag_mode_tag_run_grades(self, inline_script, tmp_path): + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_mode="tag"), + extra_env=self._TAG_REF, + ) + assert rc == 0 + assert outputs["submission-mode"] == "tag" + assert outputs["branch-push-suppressed"] == "false" + + def test_invalid_mode_hard_fails(self, inline_script, tmp_path): + # Mirrors the other per-entry fields: junk in a hand-edited + # manifest is a setup-time error, not a silent default. + rc, _stdout, stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_mode="on-demand"), + extra_env=self._BRANCH_REF, + ) + assert rc != 0 + assert "submission_mode" in stderr + assert "branch-push-suppressed" not in outputs + + def test_every_push_tag_run_not_suppressed(self, inline_script, tmp_path): + # A manual submit/* tag on an every-push assignment always grades + # (today's behavior, unchanged). + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(), + extra_env=self._TAG_REF, + ) + assert rc == 0 + assert outputs["branch-push-suppressed"] == "false" + + +# --------------------------------------------------------------------------- +# Submission tags — milestone-tag classification +# --------------------------------------------------------------------------- + + +class TestSubmissionTags: + # The read step classifies TAG runs: canonical submit/* (grades, reuse), + # a configured milestone pattern (grades; trigger-tag emitted so the tag + # step mints the canonical record tag), or neither (foreign-tag + # suppression — stale/hand-edited shim; on.push.tags normally + # pre-filters). Branch runs are untouched by the patterns. + + _BRANCH_REF = {"REF": "refs/heads/main", "REF_NAME": "main"} + _SUBMIT_REF = { + "REF": "refs/tags/submit/2026-06-01T14-32-05Z-a1b2c3d", + "REF_NAME": "submit/2026-06-01T14-32-05Z-a1b2c3d", + } + _MILESTONE_REF = {"REF": "refs/tags/phase1", "REF_NAME": "phase1"} + _FOREIGN_REF = {"REF": "refs/tags/v9.9", "REF_NAME": "v9.9"} + + def test_milestone_tag_grades_with_trigger(self, inline_script, tmp_path): + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_tags=["phase1", "phase2"]), + extra_env=self._MILESTONE_REF, + ) + assert rc == 0 + assert outputs["trigger-tag"] == "phase1" + assert outputs["foreign-tag-suppressed"] == "false" + assert outputs["branch-push-suppressed"] == "false" + + def test_glob_pattern_matches(self, inline_script, tmp_path): + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_tags=["v*"]), + extra_env=self._FOREIGN_REF, + ) + assert rc == 0 + assert outputs["trigger-tag"] == "v9.9" + assert outputs["foreign-tag-suppressed"] == "false" + + def test_canonical_submit_tag_unaffected(self, inline_script, tmp_path): + # A submit/* push never reads as a milestone trigger. + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_tags=["phase1"]), + extra_env=self._SUBMIT_REF, + ) + assert rc == 0 + assert outputs["trigger-tag"] == "" + assert outputs["foreign-tag-suppressed"] == "false" + + def test_foreign_tag_suppressed(self, inline_script, tmp_path): + # A tag matching neither submit/* nor any pattern: suppressed + # gracefully (stale/hand-edited shim), never a hard error. + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_tags=["phase1"]), + extra_env=self._FOREIGN_REF, + ) + assert rc == 0 + assert outputs["foreign-tag-suppressed"] == "true" + assert outputs["trigger-tag"] == "" + + def test_foreign_tag_without_patterns_suppressed(self, inline_script, tmp_path): + # No patterns configured: only submit/* counts (replaces the old + # hard-error in the tag step with graceful suppression). + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(), + extra_env=self._FOREIGN_REF, + ) + assert rc == 0 + assert outputs["foreign-tag-suppressed"] == "true" + + def test_branch_run_ignores_patterns(self, inline_script, tmp_path): + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_tags=["phase1"]), + extra_env=self._BRANCH_REF, + ) + assert rc == 0 + assert outputs["trigger-tag"] == "" + assert outputs["foreign-tag-suppressed"] == "false" + assert outputs["branch-push-suppressed"] == "false" + + def test_invalid_pattern_hard_fails(self, inline_script, tmp_path): + # Junk in a hand-edited manifest is a setup-time error, mirroring + # submission_mode and the other per-entry fields. + rc, _stdout, stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_tags=['ta"g']), + extra_env=self._BRANCH_REF, + ) + assert rc != 0 + assert "submission_tags" in stderr + + def test_non_list_hard_fails(self, inline_script, tmp_path): + rc, _stdout, stderr, _outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_tags="phase1"), + extra_env=self._BRANCH_REF, + ) + assert rc != 0 + assert "submission_tags" in stderr + + @pytest.mark.parametrize("pattern", ["v*+", "a++", "x?+", "m**+", "+lead", "?lead"]) + def test_stacked_quantifier_hard_fails(self, inline_script, tmp_path, pattern): + # Stacked/leading quantifiers compile as POSSESSIVE quantifiers in + # Python (this very validator's dialect) but are compile errors in + # Go/JS — the one construct where the four matcher copies would + # diverge, so the read step rejects them like the write-side + # validators do (contract.stackedQuantifierRE). + rc, _stdout, stderr, _outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_tags=[pattern]), + extra_env=self._BRANCH_REF, + ) + assert rc != 0 + assert "submission_tags" in stderr + + def test_tag_mode_milestone_tag_grades(self, inline_script, tmp_path): + # Orthogonality: tag mode + milestone patterns — a milestone push + # grades (it IS a tag run), only branch pushes are suppressed. + rc, _stdout, _stderr, outputs = _run_validator( + inline_script, tmp_path, + classroom50_yaml=_classroom_yaml(), + manifest=_manifest(submission_mode="tag", submission_tags=["phase1"]), + extra_env=self._MILESTONE_REF, + ) + assert rc == 0 + assert outputs["trigger-tag"] == "phase1" + assert outputs["branch-push-suppressed"] == "false" + assert outputs["foreign-tag-suppressed"] == "false" + + +class TestInlineMatcherFixtureParity: + # The read step's matches_submission_tag is a by-value copy of the shared + # matcher (Go contract.MatchesSubmissionTag / web matchesSubmissionTag / + # regrade_repos.py) with NO import link. Run the whole golden fixture + # through the copy extracted from the live workflow YAML so drift in the + # inline implementation fails here, exactly like the other three sides. + + def test_inline_matcher_matches_golden_fixture(self, inline_script): + # Execute just the matcher's dependencies from the inline script in an + # isolated namespace: the two functions are self-contained (re only). + import re as _re + namespace: dict = {"re": _re} + src = inline_script + # The matcher depends on the _TAG_PATTERN charset gate and the + # _STACKED_QUANTIFIER guard defined earlier in the read step — + # extract those lines first so the sliced functions run against the + # live regexes. + charset_match = _re.search(r"_TAG_PATTERN = re\.compile\([^\n]+\)", src) + assert charset_match, "inline validator lost its _TAG_PATTERN charset gate" + exec(charset_match.group(0), namespace) # noqa: S102 — test-only, our own YAML + quant_match = _re.search(r"_STACKED_QUANTIFIER = re\.compile\([^\n]+\)", src) + assert quant_match, "inline validator lost its _STACKED_QUANTIFIER guard" + exec(quant_match.group(0), namespace) # noqa: S102 — test-only, our own YAML + # Slice from the compile helper through the end of the matcher. + start = src.index("def _compile_tag_pattern") + end = src.index("\n# ", start) if "\n# " in src[start:] else None + block_lines = [] + for line in src[start:].splitlines(): + if block_lines and line and not line.startswith((" ", "\t", "def ", "#")): + break + block_lines.append(line) + if line.strip() == "return False" and "def matches_submission_tag" in "\n".join(block_lines): + break + exec("\n".join(block_lines), namespace) # noqa: S102 — test-only, our own YAML + matches = namespace["matches_submission_tag"] + + fixture = json.loads( + (_REPO_ROOT / "cli" / "shared" / "testdata" / "submission_tag_match_cases.json") + .read_text() + ) + for case in fixture["cases"]: + got = matches(case["patterns"], case["tag"]) + assert got is case["matches"], ( + f"inline matcher drift: patterns={case['patterns']} tag={case['tag']!r} " + f"got {got}, fixture expects {case['matches']} — keep the workflow's " + f"by-value copy in lockstep with contract.MatchesSubmissionTag" + ) diff --git a/cli/gh-teacher/autograders_tests/test_runner.py b/cli/gh-teacher/autograders_tests/test_runner.py index e15b08bc..f86250c4 100644 --- a/cli/gh-teacher/autograders_tests/test_runner.py +++ b/cli/gh-teacher/autograders_tests/test_runner.py @@ -478,8 +478,84 @@ def fake_run(cmd, *args, **kwargs): monkeypatch.setattr(ag.subprocess, "run", fake_run) assert ag.is_acceptance_commit(repo, shas[1]) is False - # `runner.py --detect-acceptance`: writes is-acceptance to - # $GITHUB_OUTPUT from cwd, always exits 0. + + +class TestIsShimUpdateCommit: + # True only when the tip commit touches ONLY the shim path + # (SHIM_UPDATE_COMMIT_PATHS); everything uncertain fails open + # (False -> grade). The [skip ci] on retrofit commits is the primary + # suppression; this is the backstop. + + def _head(self, path): + return _git(path, "rev-parse", "HEAD").stdout.strip() + + def _commit_shim_only(self, repo, subject="[Classroom 50] Update autograder trigger to tag (submission-mode)"): + wf = repo / ".github" / "workflows" + wf.mkdir(parents=True, exist_ok=True) + (wf / "autograde.yaml").write_text(f"name: Autograde\n# {subject}\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", subject) + + def test_shim_only_commit_is_shim_update(self, tmp_path): + repo = tmp_path / "repo" + _make_repo(repo, ["Initial commit", ACCEPT, "Submit hello"]) + self._commit_shim_only(repo) + assert ag.is_shim_update_commit(repo, self._head(repo)) is True + + def test_subject_is_irrelevant(self, tmp_path): + # Path-based like acceptance detection: a student hand-editing + # their shim gets the skip too (the edit alone is never gradeable). + repo = tmp_path / "repo" + _make_repo(repo, ["Initial commit", ACCEPT]) + self._commit_shim_only(repo, subject="tweak my workflow") + assert ag.is_shim_update_commit(repo, self._head(repo)) is True + + def test_shim_plus_other_file_grades(self, tmp_path): + # A commit smuggling real work alongside the shim edit must grade. + repo = tmp_path / "repo" + _make_repo(repo, ["Initial commit", ACCEPT]) + wf = repo / ".github" / "workflows" + wf.mkdir(parents=True) + (wf / "autograde.yaml").write_text("name: Autograde\n") + (repo / "solution.py").write_text("print('graded work')\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "Update trigger (and sneak in work)") + assert ag.is_shim_update_commit(repo, self._head(repo)) is False + + def test_ordinary_submission_grades(self, tmp_path): + repo = tmp_path / "repo" + shas = _make_repo(repo, ["Initial commit", ACCEPT, "Submit hello"]) + assert ag.is_shim_update_commit(repo, shas[2]) is False + + def test_empty_head_sha_grades(self, tmp_path): + repo = tmp_path / "repo" + _make_repo(repo, ["Initial commit", ACCEPT]) + assert ag.is_shim_update_commit(repo, "") is False + + def test_non_repo_grades(self, tmp_path): + (tmp_path / "plain").mkdir() + assert ag.is_shim_update_commit(tmp_path / "plain", "deadbeef") is False + + def test_git_error_grades(self, tmp_path, monkeypatch): + repo = tmp_path / "repo" + _make_repo(repo, ["Initial commit", ACCEPT]) + self._commit_shim_only(repo) + head = self._head(repo) + real_run = subprocess.run + + def fake_run(cmd, *args, **kwargs): + if (isinstance(cmd, (list, tuple)) and cmd[0] == "git" + and "--name-only" in cmd): + return subprocess.CompletedProcess(cmd, 128, stdout="", stderr="boom") + return real_run(cmd, *args, **kwargs) + + monkeypatch.setattr(ag.subprocess, "run", fake_run) + assert ag.is_shim_update_commit(repo, head) is False + + +class TestDetectAcceptanceMode: + # `runner.py --detect-acceptance`: writes is-acceptance and + # is-shim-update to $GITHUB_OUTPUT from cwd, always exits 0. def _run(self, repo, head_sha, tmp_path, monkeypatch): out = tmp_path / "ghout" out.write_text("") @@ -495,6 +571,9 @@ def test_acceptance_commit_emits_true(self, tmp_path, monkeypatch): rc, text = self._run(repo, shas[1], tmp_path, monkeypatch) assert rc == 0 assert "is-acceptance=true\n" in text + # Acceptance takes precedence — the accept commit also touches the + # shim path, but must never double-report as a shim update. + assert "is-shim-update=false\n" in text def test_submission_emits_false(self, tmp_path, monkeypatch): repo = tmp_path / "repo" @@ -503,6 +582,25 @@ def test_submission_emits_false(self, tmp_path, monkeypatch): rc, text = self._run(repo, head, tmp_path, monkeypatch) assert rc == 0 assert "is-acceptance=false\n" in text + assert "is-shim-update=false\n" in text + + def test_shim_update_commit_emits_true(self, tmp_path, monkeypatch): + # A submission-mode retrofit: tip commit touches ONLY the shim. + # Normally [skip ci] keeps the workflow from running at all; this + # detection is the backstop. + repo = tmp_path / "repo" + _make_repo(repo, ["Initial commit", ACCEPT, "Submit hello"]) + wf = repo / ".github" / "workflows" + wf.mkdir(parents=True) + (wf / "autograde.yaml").write_text("name: Autograde\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", + "[Classroom 50] Update autograder trigger to tag (submission-mode)") + head = _git(repo, "rev-parse", "HEAD").stdout.strip() + rc, text = self._run(repo, head, tmp_path, monkeypatch) + assert rc == 0 + assert "is-acceptance=false\n" in text + assert "is-shim-update=true\n" in text def test_main_dispatches_on_flag(self, tmp_path, monkeypatch): # The flag short-circuits before main()'s env checks. diff --git a/cli/gh-teacher/init_skeleton_test.go b/cli/gh-teacher/init_skeleton_test.go index 1e61fca7..1a2d9805 100644 --- a/cli/gh-teacher/init_skeleton_test.go +++ b/cli/gh-teacher/init_skeleton_test.go @@ -217,6 +217,24 @@ func TestSkeletonFiles_AutogradeRunner(t *testing.T) { // a dropped output would make every gate below read empty and // re-grade the acceptance commit. "is-acceptance", + // is-shim-update gates the skip for a teacher-side submission-mode + // shim retrofit commit — the backstop behind the [skip ci] in the + // retrofit commit message. + "is-shim-update", + // branch-push-suppressed gates the tag-mode stale-shim defense: a + // branch-triggered run on a tag-mode assignment must not tag or + // grade. A dropped output would read empty and grade every push, + // defeating the cost lever the mode exists for. (The step-level + // submission-mode emit stays for run-log debugging, but it is NOT a + // job output — branch-push-suppressed is the only gate.) + "branch-push-suppressed", + // foreign-tag-suppressed gates the milestone-tags twin: a pushed tag + // matching neither submit/* nor any configured submission_tags + // pattern must not tag or grade. + "foreign-tag-suppressed", + // trigger-tag carries the milestone tag that triggered the run so + // the Release title can note it ("via phase1"). + "trigger-tag", // no-autograder no longer skips the grade job (the submission is // still recorded via runner.py's vacuous pass); the output is retained // so the toolchain-setup steps can skip provisioning a toolchain the @@ -238,15 +256,23 @@ func TestSkeletonFiles_AutogradeRunner(t *testing.T) { // every acceptance commit and republish the spurious 0/0 release — // exactly what this guard exists to prevent. // - // grade is gated at the job level ONLY on the acceptance commit. The - // no-autograder case still runs the grade job so the submission is - // recorded: runner.py synthesizes a vacuous-pass (0/0 success) result and - // the Release step publishes it, keeping the submission visible on the - // teacher dashboard (which reads submit/* releases). Toolchain setup is - // separately gated off no-autograder so the recording path spends no - // minutes provisioning unused toolchains. - if got, _ := nested(doc, "jobs", "grade", "if"); got != "needs.setup.outputs.is-acceptance != 'true'" { - t.Errorf("grade.if = %v, want the acceptance-commit skip gate", got) + // grade is gated at the job level (set-latest needs grade, so it skips + // transitively). The gate carries three skips: the acceptance commit, a + // teacher-side shim-retrofit commit (is-shim-update, the [skip ci] + // backstop), and a suppressed run (branch-push-suppressed / + // foreign-tag-suppressed, the stale-shim defenses). The no-autograder + // case still runs the grade job so the submission is recorded: runner.py + // synthesizes a vacuous-pass (0/0 success) result and the Release step + // publishes it, keeping the submission visible on the teacher dashboard + // (which reads submit/* releases). Toolchain setup is separately gated + // off no-autograder so the recording path spends no minutes provisioning + // unused toolchains. + wantGradeIf := "needs.setup.outputs.is-acceptance != 'true' && " + + "needs.setup.outputs.is-shim-update != 'true' && " + + "needs.setup.outputs.branch-push-suppressed != 'true' && " + + "needs.setup.outputs.foreign-tag-suppressed != 'true'" + if got, _ := nested(doc, "jobs", "grade", "if"); got != wantGradeIf { + t.Errorf("grade.if = %v, want the acceptance + shim-update + suppressed-push + foreign-tag skip gate", got) } // The setup checkout must use full history: _baseline_scan walks back // to the commit that added .classroom50.yaml, and a shallow clone @@ -256,9 +282,66 @@ func TestSkeletonFiles_AutogradeRunner(t *testing.T) { t.Errorf("autograde-runner.yaml setup checkout missing fetch-depth: 0 (acceptance scan needs full history)") } // The tag and read steps are step-gated off the same detection so the - // acceptance commit produces no submit/* tag and no metadata read. - if !strings.Contains(body, "if: steps.acceptance.outputs.is-acceptance != 'true'") { - t.Errorf("autograde-runner.yaml tag/read steps not gated on the acceptance detection") + // acceptance commit produces no submit/* tag and no metadata read. Both + // use folded (>-) multi-line ifs now, so assert on the parsed step + // conditions rather than a raw substring. The tag step additionally + // gates on branch-push-suppressed: it runs AFTER the read step so a + // tag-mode branch push (stale every-push shim) never mints a tag for a + // run that won't grade. + setupSteps, _ := nested(doc, "jobs", "setup", "steps") + stepIf := func(id string) string { + steps, _ := setupSteps.([]any) + for _, s := range steps { + m, _ := s.(map[string]any) + if m["id"] == id { + cond, _ := m["if"].(string) + return cond + } + } + return "" + } + for _, id := range []string{"read", "tag"} { + cond := stepIf(id) + for _, gate := range []string{ + "steps.acceptance.outputs.is-acceptance != 'true'", + "steps.acceptance.outputs.is-shim-update != 'true'", + } { + if !strings.Contains(cond, gate) { + t.Errorf("autograde-runner.yaml %s step if = %q, missing gate %q", id, cond, gate) + } + } + } + if cond := stepIf("tag"); !strings.Contains(cond, "steps.read.outputs.branch-push-suppressed != 'true'") { + t.Errorf("autograde-runner.yaml tag step if = %q, missing the branch-push-suppressed gate (a suppressed tag-mode push must not mint a tag)", cond) + } + if cond := stepIf("tag"); !strings.Contains(cond, "steps.read.outputs.foreign-tag-suppressed != 'true'") { + t.Errorf("autograde-runner.yaml tag step if = %q, missing the foreign-tag-suppressed gate (a non-submission tag must not mint a canonical tag)", cond) + } + // The read step's suppression decision classifies branch vs tag runs from + // $REF, so the env mapping is load-bearing: dropped, REF reads empty, + // every run classifies as branch-triggered, and tag-mode submit/* TAG + // pushes get suppressed — grading stops for tag mode entirely. The Python + // tests inject REF themselves, so only this pin catches the mapping. + stepEnv := func(id string) map[string]any { + steps, _ := setupSteps.([]any) + for _, s := range steps { + m, _ := s.(map[string]any) + if m["id"] == id { + env, _ := m["env"].(map[string]any) + return env + } + } + return nil + } + if got := stepEnv("read")["REF"]; got != "${{ github.ref }}" { + t.Errorf("read step env.REF = %v, want ${{ github.ref }} (suppression classifies branch vs tag runs from it)", got) + } + // submission-tag was rewired from the read step to the tag step (which + // now runs after read); pin the expression VALUE, not just key presence — + // a typo'd expression would silently feed an empty TAG to the release + // and set-latest jobs. + if got := outputsMap["submission-tag"]; got != "${{ steps.tag.outputs.tag }}" { + t.Errorf("setup.outputs.submission-tag = %v, want ${{ steps.tag.outputs.tag }}", got) } // Branch trigger only — a tag push is always a submission, so the // detection step must not run on tag pushes (its absence leaves @@ -276,7 +359,7 @@ func TestSkeletonFiles_AutogradeRunner(t *testing.T) { t.Errorf("autograde-runner.yaml acceptance step doesn't invoke runner.py --detect-acceptance") } - // === set-latest job: serialized + commit-time-based === + // === set-latest job: serialized + submission-event ordering === if got, _ := nested(doc, "jobs", "set-latest", "concurrency", "group"); got != "classroom50-set-latest-${{ github.repository }}" { t.Errorf("set-latest concurrency group = %v, want per-repo serialization", got) } @@ -383,6 +466,15 @@ func TestSkeletonFiles_AutogradeRunner(t *testing.T) { if !strings.Contains(body, `context="classroom50/autograde"`) { t.Errorf("autograde-runner.yaml doesn't post the classroom50/autograde commit status") } + // Suppressed runs (foreign tag, tag-mode branch push) report under a + // DISTINCT context: the student's work exists but was NOT graded, and a + // green classroom50/autograde would read as "graded successfully" to any + // human or client checking that context. Exactly the two suppression + // statuses use it; the nothing-to-grade skips (acceptance, shim-update, + // no-autograder) stay on the main context. + if got := strings.Count(body, `context="classroom50/autograde-skipped"`); got != 2 { + t.Errorf("classroom50/autograde-skipped context count = %d, want 2 (foreign-tag + suppressed-push statuses)", got) + } if !strings.Contains(body, "if: success() && steps.autograde.outputs.status != 'error'") { t.Errorf("release step not gated on success() && status != 'error'") } @@ -416,6 +508,12 @@ func TestSkeletonFiles_AutogradeRunner(t *testing.T) { `EXTRA_ASSETS+=("$ASSET")`, `gh release delete "$TAG" --repo "$GITHUB_REPOSITORY" --yes`, `gh release create "$TAG" result.json ${EXTRA_ASSETS[@]+"${EXTRA_ASSETS[@]}"}`, + // Immutable-release tolerance (regrade failed live 2026-08-07): a + // rejected delete re-checks existence; if the release survived, + // warn and keep it instead of failing the grade job. + `if ! DELETE_ERR="$(gh release delete "$TAG" --repo "$GITHUB_REPOSITORY" --yes 2>&1)"; then`, + `the release at $TAG is immutable (org ruleset) and cannot be refreshed`, + `gradebook collection keeps reading the OLD release's result.json`, } { if !strings.Contains(releaseRun, want) { t.Errorf("Release shell is missing %q", want) @@ -532,20 +630,144 @@ printf '%s\n' "$*" >> "$GH_LOG" t.Errorf("Release shell output missing invalid-basename warning:\n%s", output) } }) - } - // set-latest uses commit-time-based comparison (not lexical tag - // compare) so two pushes in the same UTC second still order - // correctly, and a non-submit/* "latest" can't permanently block - // future submissions from claiming latest. - if !strings.Contains(body, `if [[ -z "$CURRENT" || "$CURRENT" != submit/* ]]`) { - t.Errorf("set-latest job missing non-submit/* fallback (cascade-block protection)") - } - if !strings.Contains(body, `commit.committer.date`) { - t.Errorf("set-latest job not using commit-time-based comparison") + // Immutable-release tolerance (regrade failed live 2026-08-07): when the + // delete is rejected AND the release still exists (immutable ruleset), + // the step warns, skips the create, and exits 0 — grading already + // posted its commit status, so record-keeping GitHub forbids must not + // fail the job. + t.Run("ReleaseShellToleratesImmutableRuleset", func(t *testing.T) { + tmp := t.TempDir() + binDir := filepath.Join(tmp, "bin") + if err := os.MkdirAll(binDir, 0o700); err != nil { + t.Fatal(err) + } + ghLog := filepath.Join(tmp, "gh.log") + // view succeeds (release exists, before AND after), delete is + // rejected the way an immutable-releases ruleset rejects it. + fakeGH := []byte(`#!/bin/sh +printf '%s\n' "$*" >> "$GH_LOG" +case "$1 $2" in + "release delete") + echo "HTTP 422: Release is immutable (ruleset)" >&2 + exit 1 + ;; +esac +exit 0 +`) + if err := os.WriteFile(filepath.Join(binDir, "gh"), fakeGH, 0o700); err != nil { + t.Fatal(err) + } + for _, name := range []string{"result.json", "release-body.md"} { + if err := os.WriteFile(filepath.Join(tmp, name), []byte("fixture"), 0o600); err != nil { + t.Fatal(err) + } + } + + cmd := exec.Command("bash", "-c", releaseRun) + cmd.Dir = tmp + cmd.Env = append(os.Environ(), + "PATH="+binDir+string(os.PathListSeparator)+os.Getenv("PATH"), + "GH_LOG="+ghLog, + "GH_TOKEN=test-token", + "GITHUB_REPOSITORY=example/classroom-assignment-student", + "STAGED_RELEASE_BASENAMES=", + "TAG=submit/test", + ) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Release shell must exit 0 when the delete is immutable-blocked: %v\n%s", err, output) + } + if !strings.Contains(string(output), "::warning::the release at $TAG is immutable") && + !strings.Contains(string(output), "::warning::the release at submit/test is immutable") { + t.Errorf("Release shell output missing the immutable-release warning:\n%s", output) + } + log, err := os.ReadFile(ghLog) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(log), "release create") { + t.Errorf("immutable-blocked path must skip `gh release create` (the old release stays):\n%s", log) + } + }) + + // Race fallback: the delete fails but the release is GONE on re-check, + // so the step proceeds to create instead of warning-and-skipping. + t.Run("ReleaseShellCreatesWhenFailedDeleteLeftNoRelease", func(t *testing.T) { + tmp := t.TempDir() + binDir := filepath.Join(tmp, "bin") + if err := os.MkdirAll(binDir, 0o700); err != nil { + t.Fatal(err) + } + ghLog := filepath.Join(tmp, "gh.log") + viewCount := filepath.Join(tmp, "view.count") + // First view: exists. Delete: fails. Second view: gone (race). + fakeGH := []byte(`#!/bin/sh +printf '%s\n' "$*" >> "$GH_LOG" +case "$1 $2" in + "release view") + if [ -f "$VIEW_COUNT" ]; then + exit 1 + fi + : > "$VIEW_COUNT" + exit 0 + ;; + "release delete") + echo "HTTP 500: something transient" >&2 + exit 1 + ;; +esac +exit 0 +`) + if err := os.WriteFile(filepath.Join(binDir, "gh"), fakeGH, 0o700); err != nil { + t.Fatal(err) + } + for _, name := range []string{"result.json", "release-body.md"} { + if err := os.WriteFile(filepath.Join(tmp, name), []byte("fixture"), 0o600); err != nil { + t.Fatal(err) + } + } + + cmd := exec.Command("bash", "-c", releaseRun) + cmd.Dir = tmp + cmd.Env = append(os.Environ(), + "PATH="+binDir+string(os.PathListSeparator)+os.Getenv("PATH"), + "GH_LOG="+ghLog, + "VIEW_COUNT="+viewCount, + "GH_TOKEN=test-token", + "GITHUB_REPOSITORY=example/classroom-assignment-student", + "STAGED_RELEASE_BASENAMES=", + "TAG=submit/test", + ) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Release shell must create when the failed delete left no release: %v\n%s", err, output) + } + log, err := os.ReadFile(ghLog) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(log), "release create submit/test result.json") { + t.Errorf("race path must fall through to `gh release create`:\n%s", log) + } + }) } - if !strings.Contains(body, `gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --latest=true`) { - t.Errorf("set-latest job missing forward-only latest pointer flip") + + // set-latest: every published submission claims the badge (latest = + // most recent SUBMISSION EVENT, matching the collector and the web + // views), best-effort — a rejected edit (e.g. an org ruleset enforcing + // immutable releases 422s edits) warns and never fails the pipeline. + // The old commit-time comparator must stay gone: it read its own + // just-published release back as CURRENT, self-compared, and never + // acted (dead code confirmed live on 2026-08-05). + if !strings.Contains(body, `if ! gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --latest=true; then`) { + t.Errorf("set-latest job missing the best-effort latest claim") + } + if !strings.Contains(body, "could not mark $TAG as the latest release") { + t.Errorf("set-latest job missing the rejected-edit warning (immutable-release rulesets)") + } + if strings.Contains(body, `commit.committer.date`) { + t.Errorf("set-latest job still carries the dead commit-time comparator (self-compares its own release; see 2026-08-05 finding)") } } diff --git a/cli/gh-teacher/internal/assignment/assignments_json.go b/cli/gh-teacher/internal/assignment/assignments_json.go index d26e0e3a..87ee746d 100644 --- a/cli/gh-teacher/internal/assignment/assignments_json.go +++ b/cli/gh-teacher/internal/assignment/assignments_json.go @@ -59,6 +59,19 @@ func ValidateStudentPermission(p string) error { return nil } +// ValidateSubmissionMode accepts "" (the wire default, every-push) or one of +// contract.SubmissionModes. An explicit "every-push" is legal on read (other +// writers may emit it) even though this CLI normalizes it to absent on write. +func ValidateSubmissionMode(m string) error { + if m == "" { + return nil + } + if !contract.IsValidSubmissionMode(m) { + return fmt.Errorf("invalid submission_mode %q: must be one of %v", m, contract.SubmissionModes) + } + return nil +} + // LargeAssignmentsWarnBytes is the encoded-size threshold above which // `assignment add` warns on stderr. Set well below GitHub's ~1 MiB // contents-API limit (past which encoding flips to "none", wedging every @@ -119,6 +132,14 @@ type AssignmentsJSON struct { // explicit true/false = force on/off), so absent inherits the template's // feature on a templated assignment and leaves GitHub's own create default // template-less. See RepoFeatures. +// +// SubmissionMode picks when the autograder fires: "" or +// contract.SubmissionModeEveryPush (the wire default — writers omit it) keeps +// today's shim triggers (every default-branch push plus submit/* tags); +// contract.SubmissionModeTag makes the shim trigger ONLY on submit/* tag +// pushes, which the submit clients create. Baked into the shim at accept time; +// changing it later requires retrofitting existing repos' shims (`gh teacher +// assignment submission-mode`). Mutually exclusive with EmptyRepo (no shim). type AssignmentEntry struct { Slug string `json:"slug"` Name string `json:"name"` @@ -140,6 +161,8 @@ type AssignmentEntry struct { ReleaseAssets []string `json:"release_assets,omitempty"` PassThreshold *int `json:"pass_threshold,omitempty"` StudentPermission string `json:"student_permission,omitempty"` + SubmissionMode string `json:"submission_mode,omitempty"` + SubmissionTags []string `json:"submission_tags,omitempty"` RepoFeatures *RepoFeatures `json:"repo_features,omitempty"` MigratedFrom *MigratedFromRef `json:"migrated_from,omitempty"` @@ -157,7 +180,8 @@ var knownEntryKeys = map[string]struct{}{ "runtime": {}, "tests": {}, "feedback_pr": {}, "empty_repo": {}, "locked": {}, "allowed_files": {}, "release_assets": {}, "pass_threshold": {}, "migrated_from": {}, "available_from": {}, "available_from_meta": {}, - "student_permission": {}, "repo_features": {}, + "student_permission": {}, "submission_mode": {}, "submission_tags": {}, + "repo_features": {}, } // UnmarshalJSON captures unknown top-level keys into Extra, then strictly @@ -835,6 +859,12 @@ func ValidateAssignmentEntry(entry AssignmentEntry) error { if err := ValidateStudentPermission(entry.StudentPermission); err != nil { return err } + if err := ValidateSubmissionMode(entry.SubmissionMode); err != nil { + return err + } + if err := ValidateSubmissionTags(entry.SubmissionTags); err != nil { + return err + } if entry.EmptyRepo { if err := validateEmptyRepoExclusions(entry); err != nil { return err @@ -866,6 +896,12 @@ func validateEmptyRepoExclusions(entry AssignmentEntry) error { if entry.PassThreshold != nil { return errors.New("empty_repo is mutually exclusive with pass_threshold (--empty-repo vs --pass-threshold): a bare repo never autogrades") } + if entry.SubmissionMode != "" { + return errors.New("empty_repo is mutually exclusive with submission_mode (--empty-repo vs --submission-mode): a bare repo has no autograde shim to trigger") + } + if len(entry.SubmissionTags) > 0 { + return errors.New("empty_repo is mutually exclusive with submission_tags (--empty-repo vs --submission-tag): a bare repo has no autograde shim to trigger") + } return nil } @@ -965,6 +1001,12 @@ func ValidateExistingEntry(entry AssignmentEntry) error { if err := ValidateStudentPermission(entry.StudentPermission); err != nil { return fmt.Errorf("entry %q: %w", entry.Slug, err) } + if err := ValidateSubmissionMode(entry.SubmissionMode); err != nil { + return fmt.Errorf("entry %q: %w", entry.Slug, err) + } + if err := ValidateSubmissionTags(entry.SubmissionTags); err != nil { + return fmt.Errorf("entry %q: %w", entry.Slug, err) + } if entry.EmptyRepo { if err := validateEmptyRepoExclusions(entry); err != nil { return fmt.Errorf("entry %q: %w", entry.Slug, err) diff --git a/cli/gh-teacher/internal/assignment/assignments_json_test.go b/cli/gh-teacher/internal/assignment/assignments_json_test.go index f80c2972..18bf9789 100644 --- a/cli/gh-teacher/internal/assignment/assignments_json_test.go +++ b/cli/gh-teacher/internal/assignment/assignments_json_test.go @@ -51,6 +51,123 @@ func TestStudentPermissionEnumParity(t *testing.T) { } } +// TestSubmissionModeEnumParity pins the submission_mode allow-list across its +// hand-mirrored sources: the JSON schema enum (declared source of truth) and +// the Go contract.SubmissionModes (what ValidateSubmissionMode enforces). The +// web mirror (SUBMISSION_MODES) is pinned against the same schema enum by a +// vitest, and the runner's inline validator carries a by-value copy. +func TestSubmissionModeEnumParity(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..", "..", "..")) + if err != nil { + t.Fatalf("resolve repo root: %v", err) + } + raw, err := os.ReadFile(filepath.Join(root, "schemas", "assignments-v1.schema.json")) + if err != nil { + t.Fatalf("read schema: %v", err) + } + var schema struct { + Defs struct { + Assignment struct { + Properties struct { + SubmissionMode struct { + Enum []string `json:"enum"` + } `json:"submission_mode"` + } `json:"properties"` + } `json:"assignment"` + } `json:"$defs"` + } + if err := json.Unmarshal(raw, &schema); err != nil { + t.Fatalf("parse schema: %v", err) + } + schemaEnum := schema.Defs.Assignment.Properties.SubmissionMode.Enum + if len(schemaEnum) == 0 { + t.Fatalf("schema submission_mode.enum not found; did the $defs shape change?") + } + if !reflect.DeepEqual(schemaEnum, contract.SubmissionModes) { + t.Errorf("submission_mode drift: schema enum %v != contract.SubmissionModes %v — update every mirror in lockstep (schema, Go contract, web SUBMISSION_MODES, runner inline validator)", + schemaEnum, contract.SubmissionModes) + } +} + +// TestValidateSubmissionMode pins the read/write validator: absent (empty) and +// both enum values pass; anything else is a hard error. An explicit +// "every-push" is legal on read even though this CLI normalizes it to absent +// on write (other writers may emit it). +func TestValidateSubmissionMode(t *testing.T) { + for _, ok := range []string{"", "every-push", "tag"} { + if err := ValidateSubmissionMode(ok); err != nil { + t.Errorf("ValidateSubmissionMode(%q) = %v, want nil", ok, err) + } + } + for _, bad := range []string{"Tag", "every_push", "push", "submit"} { + if err := ValidateSubmissionMode(bad); err == nil { + t.Errorf("ValidateSubmissionMode(%q) = nil, want error", bad) + } + } +} + +// TestSubmissionMode_RoundTrip pins two wire behaviors: submission_mode is a +// KNOWN key (decodes onto the struct, not Extra), and an explicit +// "every-push" written by another client survives a read-modify-write +// verbatim (EncodeAssignments never normalizes it away — only this CLI's own +// write paths collapse their own input). +func TestSubmissionMode_RoundTrip(t *testing.T) { + in := []byte(`{ + "schema": "classroom50/assignments/v1", + "assignments": [ + { "slug": "tagged", "name": "Tagged", "mode": "individual", "autograder": "default", "submission_mode": "tag" }, + { "slug": "pushy", "name": "Pushy", "mode": "individual", "autograder": "default", "submission_mode": "every-push" } + ] +}`) + file, err := ParseAssignments(in) + if err != nil { + t.Fatalf("ParseAssignments: %v", err) + } + if got := file.Assignments[0].SubmissionMode; got != contract.SubmissionModeTag { + t.Errorf("SubmissionMode = %q, want %q", got, contract.SubmissionModeTag) + } + if len(file.Assignments[0].Extra) != 0 { + t.Errorf("submission_mode leaked into Extra: %v", file.Assignments[0].Extra) + } + out, err := EncodeAssignments(file) + if err != nil { + t.Fatalf("EncodeAssignments: %v", err) + } + if !strings.Contains(string(out), `"submission_mode": "tag"`) { + t.Errorf("encoded output lost submission_mode=tag:\n%s", out) + } + if !strings.Contains(string(out), `"submission_mode": "every-push"`) { + t.Errorf("encoded output normalized away explicit every-push (must round-trip verbatim):\n%s", out) + } +} + +// TestParseAssignments_InvalidSubmissionMode pins the read-side hard error. +func TestParseAssignments_InvalidSubmissionMode(t *testing.T) { + in := []byte(`{ + "schema": "classroom50/assignments/v1", + "assignments": [ + { "slug": "hello", "name": "Hello", "mode": "individual", "autograder": "default", "submission_mode": "on-demand" } + ] +}`) + if _, err := ParseAssignments(in); err == nil { + t.Fatal("ParseAssignments accepted invalid submission_mode") + } else if !strings.Contains(err.Error(), "submission_mode") { + t.Errorf("error %v does not mention submission_mode", err) + } +} + +// TestValidateAssignmentEntry_SubmissionModeEmptyRepo pins the write-side +// mutual exclusion: a bare repo has no shim to trigger. +func TestValidateAssignmentEntry_SubmissionModeEmptyRepo(t *testing.T) { + entry := AssignmentEntry{ + Slug: "bare", Name: "Bare", Mode: "individual", Autograder: "default", + EmptyRepo: true, SubmissionMode: contract.SubmissionModeTag, + } + if err := ValidateAssignmentEntry(entry); err == nil { + t.Fatal("ValidateAssignmentEntry accepted empty_repo + submission_mode") + } +} + func TestParseAssignments_Canonical(t *testing.T) { in := []byte(`{ "schema": "classroom50/assignments/v1", diff --git a/cli/gh-teacher/internal/assignment/submission_tags.go b/cli/gh-teacher/internal/assignment/submission_tags.go new file mode 100644 index 00000000..0e3aeb26 --- /dev/null +++ b/cli/gh-teacher/internal/assignment/submission_tags.go @@ -0,0 +1,56 @@ +package assignment + +import ( + "fmt" + "strings" + + "github.com/foundation50/classroom50-cli-shared/contract" +) + +// submission_tags validation: the writer-side gate for teacher-named +// milestone tag patterns (see the assignments-v1 schema description). The +// values are rendered verbatim into the shim's quoted-YAML `on.push.tags` +// line AND compiled by the shared matcher (contract.MatchesSubmissionTag and +// its web/Python mirrors), so both rules live in the shared contract package +// (charset + no stacked quantifiers — see contract.IsSafeSubmissionTagPattern +// for why stacked quantifiers are the one cross-language divergence risk). +// This wrapper only adds the human-facing error messages. Keep the constants +// in lockstep with the schema's submission_tags maxItems/items.pattern and +// the web SUBMISSION_TAGS_CAP / SUBMISSION_TAG_PATTERN_RE +// (web/src/util/submissionTags.ts) — parity-pinned by +// TestSubmissionTagsSchemaParity. + +// SubmissionTagsCap is the maximum number of milestone tag patterns. +// Single-sourced in the shared contract package. +const SubmissionTagsCap = contract.SubmissionTagsCap + +// ValidateSubmissionTags accepts an empty list (no milestone tags — the +// canonical submit/* namespace always triggers) or up to SubmissionTagsCap +// unique, charset-safe patterns. `!` excludes are rejected: tags-ignore is +// deferred, and a silently-dropped exclude would grade tags the teacher +// meant to exclude. A quantifier with nothing to repeat (leading `?`/`+`, or +// `+` stacked on another quantifier like `v*+`) is rejected because the four +// matcher implementations would disagree on it (possessive quantifier in +// Python, compile error in Go/JS). +func ValidateSubmissionTags(patterns []string) error { + if len(patterns) > SubmissionTagsCap { + return fmt.Errorf("too many submission_tags patterns (%d): %d max", len(patterns), SubmissionTagsCap) + } + seen := make(map[string]struct{}, len(patterns)) + for _, pattern := range patterns { + if strings.HasPrefix(pattern, "!") { + return fmt.Errorf("invalid submission_tags pattern %q: exclude patterns (\"!\") are not supported", pattern) + } + if !contract.SubmissionTagCharsetRE.MatchString(pattern) { + return fmt.Errorf("invalid submission_tags pattern %q: only letters, digits, . _ / - and the glob characters * ? + [ ] are allowed", pattern) + } + if !contract.IsSafeSubmissionTagPattern(pattern) { + return fmt.Errorf("invalid submission_tags pattern %q: `?` and `+` repeat the preceding character and cannot start a pattern or follow another glob quantifier", pattern) + } + if _, dup := seen[pattern]; dup { + return fmt.Errorf("submission_tags pattern %q is listed more than once", pattern) + } + seen[pattern] = struct{}{} + } + return nil +} diff --git a/cli/gh-teacher/internal/assignment/submission_tags_test.go b/cli/gh-teacher/internal/assignment/submission_tags_test.go new file mode 100644 index 00000000..59cdca7c --- /dev/null +++ b/cli/gh-teacher/internal/assignment/submission_tags_test.go @@ -0,0 +1,162 @@ +package assignment + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/foundation50/classroom50-cli-shared/contract" +) + +// TestValidateSubmissionTags pins the writer-side gate: empty and valid +// pattern lists pass; excludes, bad charset, duplicates, and over-cap fail. +func TestValidateSubmissionTags(t *testing.T) { + for _, ok := range [][]string{ + nil, + {}, + {"phase1", "phase2", "complete"}, + {"v*", "release-[0-9]", "a/b?", "vv+", "milestone.**"}, + } { + if err := ValidateSubmissionTags(ok); err != nil { + t.Errorf("ValidateSubmissionTags(%v) = %v, want nil", ok, err) + } + } + for _, tc := range []struct { + patterns []string + wantSub string + }{ + {[]string{"!v*"}, "exclude"}, + {[]string{`ta"g`}, "only letters"}, + {[]string{"has space"}, "only letters"}, + {[]string{"dup", "dup"}, "more than once"}, + {[]string{""}, "only letters"}, + // Stacked/leading quantifiers: possessive in Python, compile error + // in Go/JS — the one construct where the four matcher copies would + // diverge, so the writer refuses it (see contract.stackedQuantifierRE). + {[]string{"v*+"}, "follow another glob quantifier"}, + {[]string{"a++"}, "follow another glob quantifier"}, + {[]string{"x?+"}, "follow another glob quantifier"}, + {[]string{"m**+"}, "follow another glob quantifier"}, + {[]string{"+lead"}, "cannot start a pattern"}, + {[]string{"?lead"}, "cannot start a pattern"}, + } { + err := ValidateSubmissionTags(tc.patterns) + if err == nil || !strings.Contains(err.Error(), tc.wantSub) { + t.Errorf("ValidateSubmissionTags(%v) = %v, want error containing %q", tc.patterns, err, tc.wantSub) + } + } + over := make([]string, SubmissionTagsCap+1) + for i := range over { + over[i] = "t" + strings.Repeat("x", i+1) + } + if err := ValidateSubmissionTags(over); err == nil { + t.Error("ValidateSubmissionTags(over-cap) = nil, want error") + } +} + +// TestSubmissionTagsSchemaParity pins the hand-mirrored constants against the +// schema (declared source of truth): maxItems vs SubmissionTagsCap and +// items.pattern vs submissionTagPatternRE. The web mirror +// (SUBMISSION_TAGS_CAP / SUBMISSION_TAG_PATTERN_RE) is pinned by its own +// vitest against the same schema. +func TestSubmissionTagsSchemaParity(t *testing.T) { + root, err := filepath.Abs(filepath.Join("..", "..", "..", "..")) + if err != nil { + t.Fatalf("resolve repo root: %v", err) + } + raw, err := os.ReadFile(filepath.Join(root, "schemas", "assignments-v1.schema.json")) + if err != nil { + t.Fatalf("read schema: %v", err) + } + var schema struct { + Defs struct { + Assignment struct { + Properties struct { + SubmissionTags struct { + MaxItems int `json:"maxItems"` + Items struct { + Pattern string `json:"pattern"` + } `json:"items"` + } `json:"submission_tags"` + } `json:"properties"` + } `json:"assignment"` + } `json:"$defs"` + } + if err := json.Unmarshal(raw, &schema); err != nil { + t.Fatalf("parse schema: %v", err) + } + st := schema.Defs.Assignment.Properties.SubmissionTags + if st.MaxItems != SubmissionTagsCap { + t.Errorf("schema submission_tags.maxItems = %d, want SubmissionTagsCap %d — update every mirror in lockstep", st.MaxItems, SubmissionTagsCap) + } + // The schema pattern is the same charset class; compare modulo the JSON + // escaping of [ and ]. (The stacked-quantifier rule is validator-only — + // JSON Schema patterns must stay Go-RE2-compilable, which rules out the + // lookahead an ECMA encoding of that rule would need.) + wantPattern := `^[A-Za-z0-9._/*?+\[\]-]+$` + if st.Items.Pattern != wantPattern { + t.Errorf("schema submission_tags.items.pattern = %q, want %q (mirror of contract.SubmissionTagCharsetRE)", st.Items.Pattern, wantPattern) + } + if got := contract.SubmissionTagCharsetRE.String(); got != wantPattern { + t.Errorf("contract.SubmissionTagCharsetRE = %q, want %q", got, wantPattern) + } +} + +// TestSubmissionTags_RoundTrip pins the wire behavior: submission_tags is a +// KNOWN key (decodes onto the struct, not Extra) and survives a +// read-modify-write verbatim. +func TestSubmissionTags_RoundTrip(t *testing.T) { + in := []byte(`{ + "schema": "classroom50/assignments/v1", + "assignments": [ + { "slug": "proj", "name": "Project", "mode": "individual", "autograder": "default", "submission_tags": ["phase1", "phase2", "complete"] } + ] +}`) + file, err := ParseAssignments(in) + if err != nil { + t.Fatalf("ParseAssignments: %v", err) + } + got := file.Assignments[0].SubmissionTags + if len(got) != 3 || got[0] != "phase1" || got[2] != "complete" { + t.Errorf("SubmissionTags = %v, want [phase1 phase2 complete]", got) + } + if len(file.Assignments[0].Extra) != 0 { + t.Errorf("submission_tags leaked into Extra: %v", file.Assignments[0].Extra) + } + out, err := EncodeAssignments(file) + if err != nil { + t.Fatalf("EncodeAssignments: %v", err) + } + if !strings.Contains(string(out), `"submission_tags"`) { + t.Errorf("encoded output lost submission_tags:\n%s", out) + } +} + +// TestParseAssignments_InvalidSubmissionTags pins the read-side hard error. +func TestParseAssignments_InvalidSubmissionTags(t *testing.T) { + in := []byte(`{ + "schema": "classroom50/assignments/v1", + "assignments": [ + { "slug": "proj", "name": "Project", "mode": "individual", "autograder": "default", "submission_tags": ["!v*"] } + ] +}`) + if _, err := ParseAssignments(in); err == nil { + t.Fatal("ParseAssignments accepted an exclude submission_tags pattern") + } else if !strings.Contains(err.Error(), "submission_tags") { + t.Errorf("error %v does not mention submission_tags", err) + } +} + +// TestValidateAssignmentEntry_SubmissionTagsEmptyRepo pins the write-side +// mutual exclusion: a bare repo has no shim to trigger. +func TestValidateAssignmentEntry_SubmissionTagsEmptyRepo(t *testing.T) { + entry := AssignmentEntry{ + Slug: "bare", Name: "Bare", Mode: "individual", Autograder: "default", + EmptyRepo: true, SubmissionTags: []string{"phase1"}, + } + if err := ValidateAssignmentEntry(entry); err == nil { + t.Fatal("ValidateAssignmentEntry accepted empty_repo + submission_tags") + } +} diff --git a/cli/gh-teacher/internal/assignmentcmd/assignment.go b/cli/gh-teacher/internal/assignmentcmd/assignment.go index 5a3185d0..f223e176 100644 --- a/cli/gh-teacher/internal/assignmentcmd/assignment.go +++ b/cli/gh-teacher/internal/assignmentcmd/assignment.go @@ -55,6 +55,7 @@ func NewCmd() *cobra.Command { cmd.AddCommand(assignmentRemoveCmd()) cmd.AddCommand(assignmentListCmd()) cmd.AddCommand(assignmentLockCmd()) + cmd.AddCommand(assignmentSubmissionModeCmd()) cmd.AddCommand(assignmentTestCmd()) cmd.AddCommand(feedbackpr.NewCmd()) return cmd @@ -65,21 +66,23 @@ func NewCmd() *cobra.Command { // students join (direct GitHub-UI invites can bypass it — documented). func assignmentAddCmd() *cobra.Command { var ( - name string - template string - description string - due string - availableFrom string - mode string - maxGroupSize int - autograder string - runtimeFile string - testsFile string - feedbackPR bool - emptyRepo bool - allowedFiles []string - passThreshold int - studentPerm string + name string + template string + description string + due string + availableFrom string + mode string + maxGroupSize int + autograder string + runtimeFile string + testsFile string + feedbackPR bool + emptyRepo bool + allowedFiles []string + passThreshold int + studentPerm string + submissionMd string + submissionTags []string ) cmd := &cobra.Command{ @@ -102,7 +105,8 @@ func assignmentAddCmd() *cobra.Command { "autograding later would require retrofitting control files into\n" + "every already-accepted repo, which classroom50 does not do.\n" + "Mutually exclusive with --template, --tests, --feedback-pr,\n" + - "--allowed-files, and --pass-threshold.\n\n" + + "--allowed-files, --pass-threshold, --submission-mode, and\n" + + "--submission-tag.\n\n" + "--template parses `/` or `/@`.\n" + "When the branch is omitted, the template repo's default branch is\n" + "used. The template repo must be marked `is_template: true` (set\n" + @@ -200,6 +204,29 @@ func assignmentAddCmd() *cobra.Command { if err := assignment.ValidateStudentPermission(studentPermVal); err != nil { return err } + // Normalize the wire default away so an every-push assignment's + // entry stays byte-identical to one written before the field + // existed. --empty-repo excludes it (no shim to trigger). + submissionModeVal := strings.TrimSpace(submissionMd) + if submissionModeVal == contract.SubmissionModeEveryPush { + submissionModeVal = "" + } + if submissionModeVal != "" { + if err := assignment.ValidateSubmissionMode(submissionModeVal); err != nil { + return err + } + if emptyRepo { + return errors.New("--empty-repo is mutually exclusive with --submission-mode: a bare repo has no autograde shim to trigger") + } + } + if len(submissionTags) > 0 { + if err := assignment.ValidateSubmissionTags(submissionTags); err != nil { + return err + } + if emptyRepo { + return errors.New("--empty-repo is mutually exclusive with --submission-tag: a bare repo has no autograde shim to trigger") + } + } if err := autograderseam.ValidateName(autograderVal); err != nil { return err } @@ -237,26 +264,30 @@ func assignmentAddCmd() *cobra.Command { } return runAssignmentAdd(client, cmd.OutOrStdout(), cmd.ErrOrStderr(), addAssignmentParams{ - Org: org, - Classroom: classroom, - Slug: slug, - Name: nameVal, - Description: strings.TrimSpace(description), - Tmpl: tmplArg, - Due: dueVal, - DueMeta: dueMetaVal, - AvailableFrom: availableFromVal, - AvailableFromMeta: availableFromMetaVal, - Mode: modeVal, - MaxGroupSize: maxGroupSize, - Autograder: autograderVal, - Runtime: runtime, - Tests: tests, - FeedbackPR: feedbackPRVal, - EmptyRepo: emptyRepo, - AllowedFiles: allowedFiles, - PassThreshold: passThresholdPtr, - StudentPermission: studentPermVal, + Org: org, + Classroom: classroom, + Slug: slug, + Name: nameVal, + Description: strings.TrimSpace(description), + Tmpl: tmplArg, + Due: dueVal, + DueMeta: dueMetaVal, + AvailableFrom: availableFromVal, + AvailableFromMeta: availableFromMetaVal, + Mode: modeVal, + MaxGroupSize: maxGroupSize, + Autograder: autograderVal, + Runtime: runtime, + Tests: tests, + FeedbackPR: feedbackPRVal, + EmptyRepo: emptyRepo, + AllowedFiles: allowedFiles, + PassThreshold: passThresholdPtr, + StudentPermission: studentPermVal, + SubmissionMode: submissionModeVal, + SubmissionModeChanged: cmd.Flags().Changed("submission-mode"), + SubmissionTags: submissionTags, + SubmissionTagsChanged: cmd.Flags().Changed("submission-tag"), }) }, } @@ -272,10 +303,12 @@ func assignmentAddCmd() *cobra.Command { cmd.Flags().StringVar(&runtimeFile, "runtime", "", "Path to a JSON file describing the runtime environment (runs-on as a single label or an array of labels for self-hosted runners, python/node/java/go/rust versions, apt packages, or container image), or `-` to read from stdin. Omit for ubuntu-latest + Python 3.14.") cmd.Flags().StringVar(&testsFile, "tests", "", "Path to a JSON file with a bare array of declarative test specs (io/run/python), or `-` to read from stdin. Sets the assignment's `tests` block; mutually exclusive with a per-assignment autograder.py. See `gh teacher assignment test --help`.") cmd.Flags().BoolVar(&feedbackPR, "feedback-pr", true, "Open one long-lived Feedback pull request per student repo so you can leave inline review comments on the full starter→submission diff. Accept freezes a base branch at the baseline commit and opens the PR right away, so it exists even with GitHub Actions disabled; the autograde runner then adopts and maintains it (and opens it on the first submission if accept could not). Default on; pass --feedback-pr=false to disable. Requires `gh teacher init` to have set up the org prerequisites.") - cmd.Flags().BoolVar(&emptyRepo, "empty-repo", false, "Create truly bare student repos: no README/initial commit, no .classroom50.yaml marker, no autograde workflow — for assignments where students build the repo (including their own GitHub Actions) from scratch. Autograding and the Feedback PR are disabled and cannot be enabled later (the setting is immutable after creation). Mutually exclusive with --template, --tests, --feedback-pr, --allowed-files, and --pass-threshold.") + cmd.Flags().BoolVar(&emptyRepo, "empty-repo", false, "Create truly bare student repos: no README/initial commit, no .classroom50.yaml marker, no autograde workflow — for assignments where students build the repo (including their own GitHub Actions) from scratch. Autograding and the Feedback PR are disabled and cannot be enabled later (the setting is immutable after creation). Mutually exclusive with --template, --tests, --feedback-pr, --allowed-files, --pass-threshold, --submission-mode, and --submission-tag.") cmd.Flags().StringArrayVar(&allowedFiles, "allowed-files", nil, "Ordered .gitignore-style pattern (repeatable, order preserved) defining which files belong to the submission. Last match wins; `!` re-includes. Pass `--allowed-files '*' --allowed-files '!hello.py'` to allow only hello.py. The autograde runner removes disallowed files before grading (control files are always kept); `gh student submit` filters them too. Omit to allow every file.") cmd.Flags().IntVar(&passThreshold, "pass-threshold", 0, "Opt-in passing bar as a percentage of max score (0–100): at/above it a gradebook client shows a submission as passing. Advisory/display-only — it does not change a student's score. Omit to leave it off (no passing concept); pass --pass-threshold 0 for an explicit 0%.") cmd.Flags().StringVar(&studentPerm, "student-permission", "", "Optional collaborator role each student gets on their OWN assignment repo at accept time: one of pull, triage, push, maintain, admin. Omit for the default (push for individual, admin for group). Choose admin to let students manage repo settings and enable GitHub Pages. Applies to students who accept from now on; existing repos are unchanged. Caution: admin on a private repo also lets the student change its visibility.") + cmd.Flags().StringVar(&submissionMd, "submission-mode", contract.SubmissionModeEveryPush, "When the autograder fires: `every-push` (default; every push to the default branch grades) or `tag` (only submit/* tag pushes grade — `gh student submit` pushes the tag, or push any submit/* tag by hand; plain `git push` costs no Actions minutes). Baked into each student repo's shim at accept time; change it later with `gh teacher assignment submission-mode`, which also retrofits existing repos. Mutually exclusive with --empty-repo.") + cmd.Flags().StringArrayVar(&submissionTags, "submission-tag", nil, "Milestone tag pattern (repeatable) that ALSO triggers grading — e.g. --submission-tag phase1 --submission-tag phase2, or a glob like 'v*'. A student pushing a matching tag (`git tag phase1 && git push origin phase1`) gets that commit graded; the grading record still lives at the canonical submit/* tag the runner mints, so history and collection are unchanged. The canonical submit/* namespace always triggers too. Baked into the shim at accept time like --submission-mode (same retrofit to change later). Caution: a broad glob like 'v*' grades every matching tag a student pushes. Mutually exclusive with --empty-repo.") return cmd } @@ -549,6 +582,19 @@ type addAssignmentParams struct { AllowedFiles []string PassThreshold *int StudentPermission string + SubmissionMode string + // Whether --submission-mode was explicitly passed. Distinguishes "omitted" + // (carry a prior entry's mode forward, like Locked) from an explicit + // --submission-mode every-push (a deliberate reset). Without this a + // same-slug re-add would silently flip a tag-mode assignment back to + // every-push while its deployed shims still only fire on tags — submit + // would stop pushing tags and NOTHING would grade. + SubmissionModeChanged bool + SubmissionTags []string + // Same omitted-vs-explicit distinction for --submission-tag: an omitted + // flag carries a prior entry's patterns forward (deployed shims were + // rendered with them); passing the flag replaces the set. + SubmissionTagsChanged bool } // runAssignmentAdd validates template visibility and entry shape before the @@ -631,6 +677,8 @@ func runAssignmentAdd(client githubapi.Client, out, errOut io.Writer, p addAssig AllowedFiles: allowedFiles, PassThreshold: passThreshold, StudentPermission: p.StudentPermission, + SubmissionMode: p.SubmissionMode, + SubmissionTags: p.SubmissionTags, } if err := assignment.ValidateAssignmentEntry(entry); err != nil { return err @@ -750,6 +798,21 @@ func runAssignmentAdd(client githubapi.Client, out, errOut io.Writer, p addAssig attemptEntry.ReleaseAssets = append([]string(nil), previous.ReleaseAssets...) attemptEntry.Extra = previous.Extra attemptEntry.Locked = previous.Locked + // submission_mode is carried forward when --submission-mode was + // omitted: deployed shims were rendered under the prior mode, so a + // silent reset to every-push would strand a tag-mode assignment + // (tag-only shims + a submit client that stops pushing tags = + // nothing grades). An explicit flag is a deliberate change — the + // teacher owns retrofitting via `assignment submission-mode`. + if !p.SubmissionModeChanged { + attemptEntry.SubmissionMode = previous.SubmissionMode + } + // submission_tags gets the same treatment: deployed shims were + // rendered with the prior patterns, so an omitted flag must not + // silently drop them (milestone tags would stop grading). + if !p.SubmissionTagsChanged { + attemptEntry.SubmissionTags = append([]string(nil), previous.SubmissionTags...) + } } committedLocked = attemptEntry.Locked updated, replaced := assignment.UpsertAssignment(file.Assignments, attemptEntry) diff --git a/cli/gh-teacher/internal/assignmentcmd/lock_test.go b/cli/gh-teacher/internal/assignmentcmd/lock_test.go index 03061cae..869a7ddd 100644 --- a/cli/gh-teacher/internal/assignmentcmd/lock_test.go +++ b/cli/gh-teacher/internal/assignmentcmd/lock_test.go @@ -325,3 +325,157 @@ func TestRunAssignmentAdd_PreservesLockAndSkipsGrant(t *testing.T) { t.Errorf("expected a note that the assignment stayed locked, got %q", errOut.String()) } } + +// submissionModeAssignmentsBody is lockAssignmentsBody's tag-mode twin: one +// existing entry whose submission_mode is "tag", for re-add carry-forward tests. +func submissionModeAssignmentsBody() string { + return `{ + "schema": "classroom50/assignments/v1", + "assignments": [ + { + "slug": "hello", + "name": "Hello", + "template": { "owner": "o", "repo": "hello-template", "branch": "main" }, + "mode": "individual", + "autograder": "default", + "submission_mode": "tag", + "feedback_pr": true + } + ] +}` +} + +// TestRunAssignmentAdd_PreservesSubmissionMode is the regression guard for the +// carry-forward: a same-slug re-add WITHOUT --submission-mode must keep a prior +// "tag" mode. A silent reset to every-push would strand the assignment — +// deployed tag-only shims fire on nothing while the submit clients stop pushing +// tags, so NOTHING grades. Mirrors TestRunAssignmentAdd_PreservesLockAndSkipsGrant. +func TestRunAssignmentAdd_PreservesSubmissionMode(t *testing.T) { + server, fix := newLockServer(t, lockServerConfig{ + assignments: submissionModeAssignmentsBody(), + classroom: lockClassroomBody(), + }) + client := githubtest.NewTestClient(t, server) + + var out, errOut bytes.Buffer + err := runAssignmentAdd(client, &out, &errOut, addAssignmentParams{ + Org: "o", + Classroom: "dst", + Slug: "hello", + Name: "Hello", + Tmpl: &templateArg{Owner: "o", Repo: "hello-template", Branch: "main"}, + Mode: assignment.ModeIndividual, + Autograder: "default", + // --submission-mode omitted: SubmissionMode is the normalized zero value + // and SubmissionModeChanged is false — the carry-forward must fire. + }) + if err != nil { + t.Fatalf("runAssignmentAdd(re-add tag-mode): %v", err) + } + if got := decodeLock(t, fix).Assignments[0].SubmissionMode; got != "tag" { + t.Errorf("re-adding without --submission-mode must keep submission_mode=tag, got %q", got) + } +} + +// TestRunAssignmentAdd_ExplicitEveryPushResetsSubmissionMode pins the deliberate +// half of the contract: --submission-mode every-push on a re-add is an explicit +// reset (SubmissionModeChanged=true with the normalized empty value), so the +// prior "tag" must NOT be carried forward. +func TestRunAssignmentAdd_ExplicitEveryPushResetsSubmissionMode(t *testing.T) { + server, fix := newLockServer(t, lockServerConfig{ + assignments: submissionModeAssignmentsBody(), + classroom: lockClassroomBody(), + }) + client := githubtest.NewTestClient(t, server) + + var out, errOut bytes.Buffer + err := runAssignmentAdd(client, &out, &errOut, addAssignmentParams{ + Org: "o", + Classroom: "dst", + Slug: "hello", + Name: "Hello", + Tmpl: &templateArg{Owner: "o", Repo: "hello-template", Branch: "main"}, + Mode: assignment.ModeIndividual, + Autograder: "default", + // An explicit --submission-mode every-push normalizes to "" with + // Changed=true — a deliberate reset the carry-forward must honor. + SubmissionMode: "", + SubmissionModeChanged: true, + }) + if err != nil { + t.Fatalf("runAssignmentAdd(explicit every-push reset): %v", err) + } + if got := decodeLock(t, fix).Assignments[0].SubmissionMode; got != "" { + t.Errorf("explicit --submission-mode every-push must reset the field (absent on the wire), got %q", got) + } +} + +// TestRunAssignmentAdd_PreservesSubmissionTags is the submission_tags twin of +// the mode carry-forward guard: a same-slug re-add WITHOUT --submission-tag +// must keep prior milestone patterns (deployed shims were rendered with +// them); passing the flag replaces the set. +func TestRunAssignmentAdd_PreservesSubmissionTags(t *testing.T) { + assignments := `{ + "schema": "classroom50/assignments/v1", + "assignments": [ + { + "slug": "hello", + "name": "Hello", + "template": { "owner": "o", "repo": "hello-template", "branch": "main" }, + "mode": "individual", + "autograder": "default", + "submission_tags": ["phase1", "phase2"], + "feedback_pr": true + } + ] +}` + server, fix := newLockServer(t, lockServerConfig{ + assignments: assignments, + classroom: lockClassroomBody(), + }) + client := githubtest.NewTestClient(t, server) + + var out, errOut bytes.Buffer + err := runAssignmentAdd(client, &out, &errOut, addAssignmentParams{ + Org: "o", + Classroom: "dst", + Slug: "hello", + Name: "Hello", + Tmpl: &templateArg{Owner: "o", Repo: "hello-template", Branch: "main"}, + Mode: assignment.ModeIndividual, + Autograder: "default", + // --submission-tag omitted: the prior patterns must carry forward. + }) + if err != nil { + t.Fatalf("runAssignmentAdd(re-add with milestone tags): %v", err) + } + got := decodeLock(t, fix).Assignments[0].SubmissionTags + if len(got) != 2 || got[0] != "phase1" || got[1] != "phase2" { + t.Errorf("re-adding without --submission-tag must keep the patterns, got %v", got) + } + + // Passing the flag replaces the set. + server2, fix2 := newLockServer(t, lockServerConfig{ + assignments: assignments, + classroom: lockClassroomBody(), + }) + client2 := githubtest.NewTestClient(t, server2) + err = runAssignmentAdd(client2, &out, &errOut, addAssignmentParams{ + Org: "o", + Classroom: "dst", + Slug: "hello", + Name: "Hello", + Tmpl: &templateArg{Owner: "o", Repo: "hello-template", Branch: "main"}, + Mode: assignment.ModeIndividual, + Autograder: "default", + SubmissionTags: []string{"final"}, + SubmissionTagsChanged: true, + }) + if err != nil { + t.Fatalf("runAssignmentAdd(explicit tags replace): %v", err) + } + got = decodeLock(t, fix2).Assignments[0].SubmissionTags + if len(got) != 1 || got[0] != "final" { + t.Errorf("explicit --submission-tag must replace the set, got %v", got) + } +} diff --git a/cli/gh-teacher/internal/assignmentcmd/reuse.go b/cli/gh-teacher/internal/assignmentcmd/reuse.go index e48ce2fc..30e47eb4 100644 --- a/cli/gh-teacher/internal/assignmentcmd/reuse.go +++ b/cli/gh-teacher/internal/assignmentcmd/reuse.go @@ -46,9 +46,10 @@ func assignmentReuseCmd() *cobra.Command { "for rebuilding last term's assignments in a new classroom.\n\n" + "The source record is copied verbatim (template, due/due_meta,\n" + "mode, autograder, max_group_size, feedback_pr, runtime,\n" + - "allowed_files, release_assets, pass_threshold, tests, description);\n" + - "only the slug and name change. By default the source slug/name are\n" + - "reused; pass --slug and/or --name to override.\n\n" + + "allowed_files, release_assets, pass_threshold, student_permission,\n" + + "submission_mode, submission_tags, tests, description); only the\n" + + "slug and name change. By default the source slug/name are reused;\n" + + "pass --slug and/or --name to override.\n\n" + "In-org only (v1): a private template can only be shared with the\n" + "target classroom's team inside its own org, so cross-org reuse of\n" + "a private template is out of scope. When the copied assignment\n" + diff --git a/cli/gh-teacher/internal/assignmentcmd/submissionmode.go b/cli/gh-teacher/internal/assignmentcmd/submissionmode.go new file mode 100644 index 00000000..51d87834 --- /dev/null +++ b/cli/gh-teacher/internal/assignmentcmd/submissionmode.go @@ -0,0 +1,526 @@ +package assignmentcmd + +import ( + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + + "github.com/spf13/cobra" + + "github.com/foundation50/classroom50-cli-shared/contract" + "github.com/foundation50/gh-teacher/internal/assignment" + "github.com/foundation50/gh-teacher/internal/cliutil" + "github.com/foundation50/gh-teacher/internal/configrepo" + "github.com/foundation50/gh-teacher/internal/configwrite" + "github.com/foundation50/gh-teacher/internal/githubapi" + "github.com/foundation50/gh-teacher/internal/validate" +) + +// autogradeShimPath is the shim's path inside every student repo. Hand- +// mirrored with NO compile-time link from gh-student's +// classroomcfg.AutogradeWorkflowPath and runner.py's SHIM_UPDATE_COMMIT_PATHS +// — keep byte-identical. +const autogradeShimPath = ".github/workflows/autograde.yaml" + +// shimTriggerBlock matches the default shim's `on:` block in any mode/tags +// combination: the optional `branches:` line (group 1) followed by the tags +// line (group 2 — `["submit/*"]` on a default shim, a milestone-pattern union +// on a submission_tags one). Both accept clients emit exactly this shape +// (their comment headers differ, which is why the retrofit is line surgery on +// the trigger block and never a full re-render); anything else is teacher-/ +// student-authored and is never touched. Hand-mirrored with NO compile-time +// link in the web SHIM_TRIGGER_BLOCK +// (web/src/domain/assignments/submissionTrigger.ts) — keep in lockstep. +var shimTriggerBlock = regexp.MustCompile( + `(?m)^on:\n push:\n( branches: \[[^\n]*\]\n)?( tags: \[[^\n]*\]\n)`, +) + +// assignmentSubmissionModeCmd flips an assignment's `submission_mode` and, by +// default, retrofits the autograde shim in every existing student repo to +// match (the trigger lives in each repo's workflow file, which is otherwise +// frozen at accept time). +func assignmentSubmissionModeCmd() *cobra.Command { + var ( + everyPush bool + tagMode bool + updateShims bool + user string + dryRun bool + quiet bool + ) + cmd := &cobra.Command{ + Use: "submission-mode (--every-push | --tag)", + Short: "Set when the autograder fires (every push vs. submit tags) and retrofit existing repos", + Long: "Set the assignment's submission mode and update the autograde shim in\n" + + "every existing student repo to match.\n\n" + + "Modes:\n" + + " --every-push every push to the default branch grades (the default\n" + + " behavior); submit/* tag pushes grade too\n" + + " --tag ONLY submit/* tag pushes grade. `gh student submit`\n" + + " pushes the tag; a hand-pushed submit/* tag works too.\n" + + " Plain `git push` costs no Actions minutes — the cost\n" + + " lever for large cohorts.\n\n" + + "The trigger lives in each student repo's shim (GitHub evaluates a\n" + + "workflow's `on:` block before any job runs), so changing the mode must\n" + + "rewrite `.github/workflows/autograde.yaml` across existing repos. That\n" + + "retrofit runs by default: enrollment comes from the classroom team, each\n" + + "member's -- repo is updated idempotently, and the\n" + + "commit carries `[skip ci]` so it never triggers grading. Repos whose shim\n" + + "doesn't match a known default-shim trigger shape (e.g., student-edited)\n" + + "are reported and left untouched. Students must `git pull` afterward —\n" + + "stale clones will conflict on their next push.\n\n" + + "Committing workflow files needs the `workflow` OAuth scope\n" + + "(`gh auth refresh -s workflow` if missing).\n\n" + + "Custom-autograder assignments: the shim is teacher-authored, so this\n" + + "command refuses to rewrite it. Edit your autograder's `on:` block\n" + + "yourself, then re-run with --update-shims=false to flip only the field\n" + + "(which still controls whether submit clients push the tag).\n\n" + + "Pass --user to retrofit a single student's repo (e.g., one that was\n" + + "skipped or failed on a previous run); the field flip is idempotent.", + Example: " gh teacher assignment submission-mode cs50-fall-2026 cs-principles hello --tag\n" + + " gh teacher assignment submission-mode cs50-fall-2026 cs-principles hello --every-push\n" + + " gh teacher assignment submission-mode cs50-fall-2026 cs-principles hello --tag --user alice\n" + + " gh teacher assignment submission-mode cs50-fall-2026 cs-principles hello --tag --dry-run", + Args: cobra.ExactArgs(3), + RunE: func(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true + org := strings.TrimSpace(args[0]) + classroom := strings.TrimSpace(args[1]) + slug := strings.TrimSpace(args[2]) + if org == "" || classroom == "" || slug == "" { + return errors.New("org, classroom, and slug must all be non-empty") + } + if err := validate.ShortName(classroom, "classroom"); err != nil { + return err + } + if err := validate.ShortName(slug, "slug"); err != nil { + return err + } + if everyPush == tagMode { + return errors.New("pass exactly one of --every-push or --tag") + } + mode := contract.SubmissionModeEveryPush + if tagMode { + mode = contract.SubmissionModeTag + } + client, err := githubapi.RequireAuthClient(cmd) + if err != nil { + return err + } + verbose, _ := cmd.Flags().GetBool("verbose") + return runSubmissionMode(client, cmd.OutOrStdout(), cmd.ErrOrStderr(), submissionModeParams{ + org: org, classroom: classroom, slug: slug, + mode: mode, + updateShims: updateShims, + user: strings.TrimSpace(user), + dryRun: dryRun, + quiet: quiet, + verbose: verbose, + }) + }, + } + cmd.Flags().BoolVar(&everyPush, "every-push", false, "Grade every push to the default branch (the default behavior)") + cmd.Flags().BoolVar(&tagMode, "tag", false, "Grade only submit/* tag pushes (submit clients push the tag; plain `git push` does not grade)") + cmd.Flags().BoolVar(&updateShims, "update-shims", true, "Retrofit each existing student repo's autograde shim to the new trigger; pass --update-shims=false to flip only the assignments.json field") + cmd.Flags().StringVar(&user, "user", "", "Retrofit a single student's repo (their -- repo) instead of every team member's") + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "Report the field flip and per-repo shim changes without writing anything") + cmd.Flags().BoolVarP(&quiet, "quiet", "q", false, "Suppress informational output (per-repo and summary lines); errors still go to stderr") + return cmd +} + +type submissionModeParams struct { + org, classroom, slug string + mode string // contract.SubmissionModeEveryPush | contract.SubmissionModeTag + updateShims bool + user string + dryRun bool + quiet, verbose bool +} + +// shimOutcome is one repo's classified retrofit result. updated/current are +// the happy paths; unrecognized needs the teacher's judgment (never +// overwritten); notAccepted is a skip; failed is transient (re-run retries). +type shimOutcome int + +const ( + shimUpdated shimOutcome = iota // trigger block rewritten + shimCurrent // already on the target trigger — no commit + shimUnrecognized // content doesn't match a known default-shim shape — left untouched + shimNotAccepted // repo (or shim file) doesn't exist yet + shimFailed // transient error — re-running retries +) + +type shimResult struct { + repo string + outcome shimOutcome + reason string // set for unrecognized/failed +} + +// runSubmissionMode flips the field (lock.go's CommitTree pattern, idempotent) +// and then, unless disabled, retrofits each student repo's shim serially +// (feedbackpr.go's enumeration pattern; GitHub's secondary-rate-limit budget +// makes a concurrent fan-out a liability). +func runSubmissionMode(client githubapi.Client, out, errOut io.Writer, p submissionModeParams) error { + branch, err := configrepo.ResolveConfigRepoBranch(client, p.org) + if err != nil { + return err + } + + // Wire form: every-push collapses to absent (writers omit the default). + wireMode := p.mode + if wireMode == contract.SubmissionModeEveryPush { + wireMode = "" + } + + // Pre-read the entry for the gating checks (empty_repo, custom + // autograder) so a refused command writes nothing. The flip itself + // re-reads inside the commit loop for rebase safety. + preFile, err := loadAssignments(client, p.org, p.classroom, branch) + if err != nil { + return err + } + preIdx, ok := assignment.FindAssignment(preFile.Assignments, p.slug) + if !ok { + return fmt.Errorf("assignment %q not found in %s/%s/%s", + p.slug, p.org, configrepo.ConfigRepoName, assignmentsFilePath(p.classroom)) + } + preEntry := preFile.Assignments[preIdx] + if preEntry.EmptyRepo { + return fmt.Errorf("assignment %q is an empty_repo assignment: its repos carry no autograde shim, so submission_mode does not apply", p.slug) + } + customAutograder := preEntry.Autograder != "" && preEntry.Autograder != contract.DefaultAutograderName + if customAutograder && p.updateShims { + return fmt.Errorf("assignment %q uses the custom autograder %q — its shim is teacher-authored and this command never rewrites it. Edit that autograder's `on:` trigger yourself, then re-run with --update-shims=false to flip only the field (which controls whether submit clients push the tag)", + p.slug, preEntry.Autograder) + } + + if p.dryRun { + if preEntry.SubmissionMode == wireMode { + _, _ = fmt.Fprintf(out, "dry run: %s already has submission_mode %s — no field change\n", p.slug, p.mode) + } else { + _, _ = fmt.Fprintf(out, "dry run: would set submission_mode of %s to %s\n", p.slug, p.mode) + } + } else { + build := func(parentSHA string) (map[string]string, error) { + file, err := loadAssignments(client, p.org, p.classroom, parentSHA) + if err != nil { + return nil, err + } + idx, ok := assignment.FindAssignment(file.Assignments, p.slug) + if !ok { + return nil, fmt.Errorf("assignment %q disappeared from %s during the update — retry", + p.slug, assignmentsFilePath(p.classroom)) + } + entry := file.Assignments[idx] + if entry.SubmissionMode == wireMode { + return nil, nil // already in the desired state — no commit + } + entry.SubmissionMode = wireMode + file.Assignments[idx] = entry + data, err := assignment.EncodeAssignments(file) + if err != nil { + return nil, err + } + return map[string]string{assignmentsFilePath(p.classroom): string(data)}, nil + } + message := contract.PrefixCommit(fmt.Sprintf("assignment: set submission_mode of %s to %s in %s (gh teacher assignment submission-mode)", p.slug, p.mode, p.classroom)) + // Report from the commit OUTCOME (did a commit land?), never from + // build-attempt state: a rebase retry can no-op after a stale first + // attempt, and the pre-write read can lag a just-landed write + // (run-2 misreport, observed live 2026-08-05). + commitSHA, err := configwrite.CommitTree(client, p.org, configrepo.ConfigRepoName, branch, message, build) + if err != nil { + return err + } + if !p.quiet { + if commitSHA != "" { + _, _ = fmt.Fprintf(out, "%s/%s/%s: set submission_mode of %s to %s\n", + p.org, configrepo.ConfigRepoName, assignmentsFilePath(p.classroom), p.slug, p.mode) + } else { + _, _ = fmt.Fprintf(out, "%s/%s/%s: %s already has submission_mode %s\n", + p.org, configrepo.ConfigRepoName, assignmentsFilePath(p.classroom), p.slug, p.mode) + } + } + } + + if !p.updateShims { + if !p.quiet { + _, _ = fmt.Fprintln(out, "Shim retrofit skipped (--update-shims=false); existing repos keep their current trigger until updated") + } + return nil + } + + repos, err := submissionModeTargetRepos(client, p, branch) + if err != nil { + return err + } + if len(repos) == 0 { + if !p.quiet { + _, _ = fmt.Fprintf(out, "%s: no repos to process — the classroom's student team has no members (sync the roster, or target one repo with --user )\n", p.org) + } + return nil + } + + var results []shimResult + notAccepted := 0 + for _, repo := range repos { + res := retrofitShim(client, p.org, repo, p.mode, preEntry.SubmissionTags, p.dryRun) + if res.outcome == shimNotAccepted { + // Enrolled but not accepted yet. On the explicit --user path the + // teacher named this repo, so report it unconditionally; the bulk + // summary still counts it so "of N repo(s)" reflects the roster + // actually probed (a team full of non-accepters must not read as + // "0 repo(s)" — that looks like an enumeration failure). + notAccepted++ + if p.user != "" { + _, _ = fmt.Fprintf(out, "%s does not exist — %s has not accepted %s yet\n", repo, p.user, p.slug) + } else if p.verbose && !p.quiet { + _, _ = fmt.Fprintf(out, "Skipped %s (no repo — not accepted yet?)\n", repo) + } + continue + } + results = append(results, res) + reportShimResult(out, res, p) + } + + return summarizeShimResults(out, errOut, p, results, notAccepted) +} + +// submissionModeTargetRepos resolves the repo names to process: a single +// --user repo, or the derived repo for every classroom-team member. +func submissionModeTargetRepos(client githubapi.Client, p submissionModeParams, branch string) ([]string, error) { + if p.user != "" { + return []string{contract.AssignmentRepoName(p.classroom, p.slug, p.user)}, nil + } + teamSlug, err := configrepo.ResolveClassroomTeamSlug(client, p.org, p.classroom, branch) + if err != nil { + return nil, err + } + logins, err := configrepo.ListTeamMembers(client, p.org, teamSlug) + if err != nil { + return nil, fmt.Errorf("list team %q members: %w", teamSlug, err) + } + repos := make([]string, 0, len(logins)) + for _, login := range logins { + repos = append(repos, contract.AssignmentRepoName(p.classroom, p.slug, login)) + } + return repos, nil +} + +// retrofitShim rewrites one repo's shim trigger block to `mode` + `tags`. +// Line surgery only: the two accept clients' shim comment headers differ, so +// a full re-render would churn repos accepted by the other client — instead +// the known trigger block is swapped in place and everything else is +// preserved byte-for-byte. Unrecognized content is never overwritten. +func retrofitShim(client githubapi.Client, org, repo, mode string, tags []string, dryRun bool) shimResult { + branch, notFound, err := studentRepoDefaultBranch(client, org, repo) + if err != nil { + return shimResult{repo: repo, outcome: shimFailed, reason: err.Error()} + } + if notFound { + return shimResult{repo: repo, outcome: shimNotAccepted} + } + + // The read+rewrite runs INSIDE the commit build, against the exact parent + // SHA of each attempt — never the branch name. A branch-name contents read + // seconds after a prior write can serve stale (pre-write) content + // (GitHub read-after-write lag, observed live 2026-08-05: an immediate + // re-run re-reported "updated" off the stale read while correctly + // committing nothing). Reading at the parent SHA makes the no-op check + // authoritative and the rewrite rebase-safe in one move. + var unrecognized error + build := func(parentSHA string) (map[string]string, error) { + unrecognized = nil + current, exists, err := configrepo.ReadFileContents(client, org, repo, autogradeShimPath, parentSHA) + if err != nil { + return nil, err + } + if !exists { + // Repo exists but the shim never landed (a mid-flow accept + // failure). Accept's self-heal owns that case; nothing safe to + // rewrite here. + unrecognized = errors.New("no " + autogradeShimPath + " — accept may not have completed; re-accept heals it") + return nil, nil + } + updated, changed, err := rewriteShimTrigger(string(current), mode, branch, tags) + if err != nil { + unrecognized = err + return nil, nil + } + if !changed { + return nil, nil // already on the target trigger — no commit + } + return map[string]string{autogradeShimPath: updated}, nil + } + + if dryRun { + // One build pass against the current tip classifies without writing. + files, err := build(branch) + switch { + case err != nil: + return shimResult{repo: repo, outcome: shimFailed, reason: err.Error()} + case unrecognized != nil: + return shimResult{repo: repo, outcome: shimUnrecognized, reason: unrecognized.Error()} + case files == nil: + return shimResult{repo: repo, outcome: shimCurrent} + default: + return shimResult{repo: repo, outcome: shimUpdated} + } + } + + commitSHA, err := configwrite.CommitTree(client, org, repo, branch, contract.ShimUpdateCommitMessage(mode), build) + if err != nil { + if errors.Is(err, configwrite.ErrMissingWorkflowScope) { + return shimResult{repo: repo, outcome: shimFailed, reason: "token lacks the `workflow` OAuth scope — run `gh auth refresh -s workflow` and re-run"} + } + return shimResult{repo: repo, outcome: shimFailed, reason: err.Error()} + } + if unrecognized != nil { + return shimResult{repo: repo, outcome: shimUnrecognized, reason: unrecognized.Error()} + } + if commitSHA == "" { + // The build no-opped: classify by what actually landed, not by the + // pre-write read (the run-2 misreport this rewrite exists to prevent). + return shimResult{repo: repo, outcome: shimCurrent} + } + return shimResult{repo: repo, outcome: shimUpdated} +} + +// rewriteShimTrigger swaps the shim's trigger block to `mode` + `tags`, +// returning the rewritten content and whether anything changed. An error +// means the content doesn't carry a recognizable default-shim trigger block. +// +// every-push → tag removes the `branches:` line; tag → every-push inserts it +// with the repo's CURRENT default branch (better than any stale frozen name — +// the shim must fire on the branch pushes actually land on; an existing +// branches line is kept verbatim for the same reason). The tags line is +// reconciled to the union of the assignment's milestone patterns and submit/* +// (contract.ShimTagsList) — so the same retrofit that flips the mode also +// repairs a stale pattern set. +func rewriteShimTrigger(content, mode string, branch string, tags []string) (string, bool, error) { + loc := shimTriggerBlock.FindStringSubmatchIndex(content) + if loc == nil { + return "", false, errors.New("shim does not carry a recognizable default trigger block — left untouched (student-edited?)") + } + hasBranches := loc[2] != -1 + + var branchesLine string + switch mode { + case contract.SubmissionModeTag: + branchesLine = "" + case contract.SubmissionModeEveryPush: + if hasBranches { + // Keep the existing line verbatim: its (possibly stale) branch + // name is accept-time behavior, not this command's to correct. + branchesLine = content[loc[2]:loc[3]] + } else { + branchesLine = ` branches: ["` + branch + `"]` + "\n" + } + default: + return "", false, fmt.Errorf("unknown submission mode %q", mode) + } + tagsLine := " tags: [" + contract.ShimTagsList(tags) + "]\n" + + rebuilt := content[:loc[0]] + "on:\n push:\n" + branchesLine + tagsLine + content[loc[1]:] + return rebuilt, rebuilt != content, nil +} + +// studentRepoDefaultBranch reads the repo's settled default branch and +// reports whether the repo is missing (enrolled but not accepted). Mirrors +// feedbackpr.go's defaultBranch. +func studentRepoDefaultBranch(client githubapi.Client, org, repoName string) (branch string, notFound bool, err error) { + path := fmt.Sprintf("repos/%s/%s", url.PathEscape(org), url.PathEscape(repoName)) + var repo struct { + DefaultBranch string `json:"default_branch"` + } + if err := client.Get(path, &repo); err != nil { + if cliutil.IsHTTPStatus(err, http.StatusNotFound) { + return "", true, nil + } + return "", false, fmt.Errorf("GET %s: %w", path, err) + } + if repo.DefaultBranch == "" { + return "main", false, nil + } + return repo.DefaultBranch, false, nil +} + +// reportShimResult prints one repo's per-line outcome on the human channel. +func reportShimResult(out io.Writer, res shimResult, p submissionModeParams) { + if p.quiet { + return + } + prefix := "" + if p.dryRun { + prefix = "dry run: would have " + } + switch res.outcome { + case shimUpdated: + if p.dryRun { + _, _ = fmt.Fprintf(out, "%supdated the autograde trigger on %s\n", prefix, res.repo) + } else { + _, _ = fmt.Fprintf(out, "Updated autograde trigger on %s\n", res.repo) + } + case shimCurrent: + if p.verbose { + _, _ = fmt.Fprintf(out, "%s already has the target trigger\n", res.repo) + } + case shimUnrecognized: + _, _ = fmt.Fprintf(out, "Skipped %s: %s\n", res.repo, res.reason) + case shimFailed: + _, _ = fmt.Fprintf(out, "Failed: %s (%s)\n", res.repo, res.reason) + } +} + +// summarizeShimResults prints the aggregate counts plus the skipped/failed +// detail lists, and returns a non-nil error when any repo failed. +func summarizeShimResults(out, errOut io.Writer, p submissionModeParams, results []shimResult, notAccepted int) error { + var updated, current int + var unrecognized, failed []shimResult + for _, r := range results { + switch r.outcome { + case shimUpdated: + updated++ + case shimCurrent: + current++ + case shimUnrecognized: + unrecognized = append(unrecognized, r) + case shimFailed: + failed = append(failed, r) + } + } + + if !p.quiet { + verb := "updated" + if p.dryRun { + verb = "would update" + } + _, _ = fmt.Fprintf(out, "%s: %d %s, %d already current, %d skipped, %d failed (of %d repo(s))\n", + p.org, updated, verb, current, len(unrecognized), len(failed), len(results)+notAccepted) + if notAccepted > 0 { + _, _ = fmt.Fprintf(out, "%d enrolled student(s) have not accepted %s yet — their repos will get the new trigger at accept time\n", notAccepted, p.slug) + } + if updated > 0 && !p.dryRun { + _, _ = fmt.Fprintln(out, "Tell students to `git pull` — clones made before this change will conflict on their next push.") + } + } + + if len(unrecognized) > 0 { + _, _ = fmt.Fprintln(errOut, "Skipped (shim content not recognized — review and update by hand if intended):") + for _, r := range unrecognized { + _, _ = fmt.Fprintf(errOut, " %s: %s\n", r.repo, r.reason) + } + } + if len(failed) > 0 { + _, _ = fmt.Fprintln(errOut, "Failed (re-run to retry just these, or pass --user for one repo):") + for _, r := range failed { + _, _ = fmt.Fprintf(errOut, " %s: %s\n", r.repo, r.reason) + } + return fmt.Errorf("%d of %d repo(s) failed", len(failed), len(results)) + } + return nil +} diff --git a/cli/gh-teacher/internal/assignmentcmd/submissionmode_test.go b/cli/gh-teacher/internal/assignmentcmd/submissionmode_test.go new file mode 100644 index 00000000..1379633b --- /dev/null +++ b/cli/gh-teacher/internal/assignmentcmd/submissionmode_test.go @@ -0,0 +1,596 @@ +package assignmentcmd + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/foundation50/classroom50-cli-shared/contract" + "github.com/foundation50/gh-teacher/internal/assignment" + "github.com/foundation50/gh-teacher/internal/githubtest" +) + +// The two accept clients' shims share the trigger block but differ in their +// comment headers — the retrofit must survive both (and preserve them). +const cliShimEveryPush = `# Classroom50 autograder shim. +# +# This file should not be edited. + +name: Autograde + +on: + push: + branches: ["main"] + tags: ["submit/*"] + +jobs: + grade: + uses: "o/classroom50/.github/workflows/autograde-runner.yaml@main" + permissions: + contents: write + statuses: write + pull-requests: write +` + +const webShimTagMode = `name: Autograde + +on: + push: + tags: ["submit/*"] + +jobs: + grade: + uses: "o/classroom50/.github/workflows/autograde-runner.yaml@main" + permissions: + contents: write + statuses: write + # Lets the runner open the opt-in Feedback PR. + pull-requests: write +` + +// --------------------------------------------------------------------------- +// rewriteShimTrigger — pure line surgery +// --------------------------------------------------------------------------- + +func TestRewriteShimTrigger_EveryPushToTag(t *testing.T) { + got, changed, err := rewriteShimTrigger(cliShimEveryPush, contract.SubmissionModeTag, "main", nil) + if err != nil || !changed { + t.Fatalf("rewrite = (changed=%v, err=%v), want changed", changed, err) + } + if strings.Contains(got, "branches:") { + t.Errorf("tag-mode shim still has a branches: line:\n%s", got) + } + // Exactly the one line removed; comments and everything else preserved. + want := strings.Replace(cliShimEveryPush, " branches: [\"main\"]\n", "", 1) + if got != want { + t.Errorf("rewrite is not surgical:\ngot:\n%s\nwant:\n%s", got, want) + } +} + +func TestRewriteShimTrigger_TagToEveryPush(t *testing.T) { + // The branches line is inserted with the repo's CURRENT default branch + // (master here), not a hardcoded main. + got, changed, err := rewriteShimTrigger(webShimTagMode, contract.SubmissionModeEveryPush, "master", nil) + if err != nil || !changed { + t.Fatalf("rewrite = (changed=%v, err=%v), want changed", changed, err) + } + if !strings.Contains(got, " branches: [\"master\"]\n tags: [\"submit/*\"]\n") { + t.Errorf("every-push shim missing the inserted branches line:\n%s", got) + } + // Round trip: removing it again restores the original. + back, changed, err := rewriteShimTrigger(got, contract.SubmissionModeTag, "master", nil) + if err != nil || !changed || back != webShimTagMode { + t.Errorf("round trip failed:\n%s", back) + } +} + +func TestRewriteShimTrigger_Idempotent(t *testing.T) { + // Already on target -> no change (no commit at the call site). + if _, changed, err := rewriteShimTrigger(webShimTagMode, contract.SubmissionModeTag, "main", nil); err != nil || changed { + t.Errorf("tag->tag = (changed=%v, err=%v), want no change", changed, err) + } + if _, changed, err := rewriteShimTrigger(cliShimEveryPush, contract.SubmissionModeEveryPush, "main", nil); err != nil || changed { + t.Errorf("every-push->every-push = (changed=%v, err=%v), want no change", changed, err) + } +} + +func TestRewriteShimTrigger_UnrecognizedContent(t *testing.T) { + // Teacher-/student-authored triggers must never be rewritten. + for _, content := range []string{ + "name: Custom\non:\n workflow_dispatch: {}\njobs: {}\n", + "on:\n push:\n branches: [\"main\"]\n", // no submit/* tags line + "", + } { + if _, _, err := rewriteShimTrigger(content, contract.SubmissionModeTag, "main", nil); err == nil { + t.Errorf("rewriteShimTrigger accepted unrecognized content:\n%s", content) + } + } +} + +func TestRewriteShimTrigger_QuotedYamlHostileBranch(t *testing.T) { + // A branch named `off` stays quoted (matching the accept clients' quoting). + got, changed, err := rewriteShimTrigger(webShimTagMode, contract.SubmissionModeEveryPush, "off", nil) + if err != nil || !changed { + t.Fatalf("rewrite = (changed=%v, err=%v)", changed, err) + } + if !strings.Contains(got, `branches: ["off"]`) { + t.Errorf("branch not quoted:\n%s", got) + } +} + +// --------------------------------------------------------------------------- +// runSubmissionMode — end to end against a fake GitHub +// --------------------------------------------------------------------------- + +type smFixture struct { + mu sync.Mutex + // committedAssignments is the last assignments.json blob written to the + // config repo; committedShims maps student repo -> last shim blob. + committedAssignments []byte + committedShims map[string][]byte + commitMessages map[string]string // repo -> last commit message +} + +type smServerConfig struct { + assignments string + // student repos: name -> shim content ("" => repo exists, shim missing; + // absent from map => repo 404s) + repos map[string]string + // tree-write 404 with a scopes header, simulating a missing workflow scope + workflowScope404 bool +} + +func newSMServer(t *testing.T, cfg smServerConfig) (*httptest.Server, *smFixture) { + t.Helper() + fix := &smFixture{ + committedShims: map[string][]byte{}, + commitMessages: map[string]string{}, + } + mux := http.NewServeMux() + + serveJSON := func(w http.ResponseWriter, v any) { _ = json.NewEncoder(w).Encode(v) } + serve404 := func(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, `{"message":"Not Found"}`) + } + serveContents := func(w http.ResponseWriter, body string) { + serveJSON(w, map[string]any{ + "type": "file", "encoding": "base64", + "content": base64.StdEncoding.EncodeToString([]byte(body)), + }) + } + + // Config repo: branch, classroom.json (team), assignments.json, git-data. + mux.HandleFunc("/repos/o/classroom50", func(w http.ResponseWriter, _ *http.Request) { + serveJSON(w, map[string]string{"default_branch": "main"}) + }) + mux.HandleFunc("/repos/o/classroom50/contents/dst/assignments.json", func(w http.ResponseWriter, _ *http.Request) { + serveContents(w, cfg.assignments) + }) + mux.HandleFunc("/repos/o/classroom50/contents/dst/classroom.json", func(w http.ResponseWriter, _ *http.Request) { + serveContents(w, lockClassroomBody()) + }) + mux.HandleFunc("/orgs/o/teams/classroom50-dst/members", func(w http.ResponseWriter, _ *http.Request) { + var members []map[string]string + for repo := range cfg.repos { + login := strings.TrimPrefix(repo, "dst-hello-") + members = append(members, map[string]string{"login": login}) + } + // A member who never accepted (repo absent from cfg.repos). + members = append(members, map[string]string{"login": "ghost"}) + serveJSON(w, members) + }) + + // Per-repo git-data write endpoints (config repo + every student repo). + gitData := func(repoName string) { + base := "/repos/o/" + repoName + mux.HandleFunc(base+"/git/refs/heads/main", func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodPatch { + w.WriteHeader(http.StatusOK) + return + } + serveJSON(w, map[string]any{"object": map[string]string{"sha": "parent-sha"}}) + }) + mux.HandleFunc(base+"/git/commits/parent-sha", func(w http.ResponseWriter, _ *http.Request) { + serveJSON(w, map[string]any{"tree": map[string]string{"sha": "parent-tree"}}) + }) + mux.HandleFunc(base+"/git/blobs", func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var payload struct{ Content, Encoding string } + _ = json.Unmarshal(body, &payload) + decoded, _ := base64.StdEncoding.DecodeString(payload.Content) + fix.mu.Lock() + if repoName == "classroom50" { + fix.committedAssignments = decoded + } else { + fix.committedShims[repoName] = decoded + } + fix.mu.Unlock() + serveJSON(w, map[string]string{"sha": "blob-sha"}) + }) + mux.HandleFunc(base+"/git/trees", func(w http.ResponseWriter, _ *http.Request) { + if cfg.workflowScope404 && repoName != "classroom50" { + w.Header().Set("X-OAuth-Scopes", "repo, read:org") + serve404(w) + return + } + serveJSON(w, map[string]string{"sha": "new-tree-sha"}) + }) + mux.HandleFunc(base+"/git/commits", func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + var payload struct{ Message string } + _ = json.Unmarshal(body, &payload) + fix.mu.Lock() + fix.commitMessages[repoName] = payload.Message + fix.mu.Unlock() + serveJSON(w, map[string]string{"sha": "new-commit-sha"}) + }) + } + gitData("classroom50") + + for repoName, shim := range cfg.repos { + repoName, shim := repoName, shim + mux.HandleFunc("/repos/o/"+repoName, func(w http.ResponseWriter, _ *http.Request) { + serveJSON(w, map[string]string{"default_branch": "main"}) + }) + mux.HandleFunc("/repos/o/"+repoName+"/contents/.github/workflows/autograde.yaml", func(w http.ResponseWriter, _ *http.Request) { + if shim == "" { + serve404(w) + return + } + serveContents(w, shim) + }) + gitData(repoName) + } + // Unregistered /repos/o/ paths 404 via the default mux handler? No — + // net/http's mux would match the longest pattern; add an explicit 404 for + // the ghost student's repo. + mux.HandleFunc("/repos/o/dst-hello-ghost", func(w http.ResponseWriter, _ *http.Request) { + serve404(w) + }) + + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + return server, fix +} + +func smAssignmentsBody(mode, autograder string, emptyRepo bool) string { + entry := map[string]any{ + "slug": "hello", "name": "Hello", + "mode": "individual", "autograder": autograder, + } + if mode != "" { + entry["submission_mode"] = mode + } + if emptyRepo { + entry["empty_repo"] = true + } + doc := map[string]any{ + "schema": "classroom50/assignments/v1", + "assignments": []any{entry}, + } + b, _ := json.Marshal(doc) + return string(b) +} + +func smParams(mode string) submissionModeParams { + return submissionModeParams{ + org: "o", classroom: "dst", slug: "hello", + mode: mode, + updateShims: true, + } +} + +func TestRunSubmissionMode_FlipsFieldAndRetrofitsShims(t *testing.T) { + server, fix := newSMServer(t, smServerConfig{ + assignments: smAssignmentsBody("", "default", false), + repos: map[string]string{ + "dst-hello-alice": cliShimEveryPush, + "dst-hello-bob": webShimTagMode, // already tag-shaped -> current + }, + }) + client := githubtest.NewTestClient(t, server) + + var out, errOut bytes.Buffer + if err := runSubmissionMode(client, &out, &errOut, smParams(contract.SubmissionModeTag)); err != nil { + t.Fatalf("runSubmissionMode: %v\nstderr: %s", err, errOut.String()) + } + + fix.mu.Lock() + defer fix.mu.Unlock() + // Field flip landed with the wire collapse (tag written explicitly). + file, err := assignment.ParseAssignments(fix.committedAssignments) + if err != nil { + t.Fatalf("committed assignments.json does not parse: %v", err) + } + if got := file.Assignments[0].SubmissionMode; got != contract.SubmissionModeTag { + t.Errorf("committed submission_mode = %q, want tag", got) + } + // alice's shim rewritten (branches line removed); bob untouched. + aliceShim := string(fix.committedShims["dst-hello-alice"]) + if strings.Contains(aliceShim, "branches:") || !strings.Contains(aliceShim, `tags: ["submit/*"]`) { + t.Errorf("alice's retrofitted shim wrong:\n%s", aliceShim) + } + if _, wrote := fix.committedShims["dst-hello-bob"]; wrote { + t.Error("bob's already-current shim must not be rewritten") + } + // The retrofit commit carries [skip ci] (load-bearing for tag->every-push). + if msg := fix.commitMessages["dst-hello-alice"]; !strings.Contains(msg, "[skip ci]") { + t.Errorf("retrofit commit message missing [skip ci]: %q", msg) + } + if !strings.Contains(out.String(), "git pull") { + t.Errorf("summary should tell students to re-pull:\n%s", out.String()) + } +} + +func TestRunSubmissionMode_TagToEveryPushInsertsBranchLine(t *testing.T) { + server, fix := newSMServer(t, smServerConfig{ + assignments: smAssignmentsBody("tag", "default", false), + repos: map[string]string{"dst-hello-alice": webShimTagMode}, + }) + client := githubtest.NewTestClient(t, server) + + var out, errOut bytes.Buffer + if err := runSubmissionMode(client, &out, &errOut, smParams(contract.SubmissionModeEveryPush)); err != nil { + t.Fatalf("runSubmissionMode: %v", err) + } + + fix.mu.Lock() + defer fix.mu.Unlock() + // every-push collapses to absent on the wire. + file, err := assignment.ParseAssignments(fix.committedAssignments) + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := file.Assignments[0].SubmissionMode; got != "" { + t.Errorf("committed submission_mode = %q, want absent (collapsed)", got) + } + shim := string(fix.committedShims["dst-hello-alice"]) + if !strings.Contains(shim, `branches: ["main"]`) { + t.Errorf("branches line not restored:\n%s", shim) + } +} + +func TestRunSubmissionMode_IdempotentFieldNoCommit(t *testing.T) { + // Already tag mode + already tag-shaped shim: no commits anywhere. + server, fix := newSMServer(t, smServerConfig{ + assignments: smAssignmentsBody("tag", "default", false), + repos: map[string]string{"dst-hello-alice": webShimTagMode}, + }) + client := githubtest.NewTestClient(t, server) + + var out, errOut bytes.Buffer + if err := runSubmissionMode(client, &out, &errOut, smParams(contract.SubmissionModeTag)); err != nil { + t.Fatalf("runSubmissionMode: %v", err) + } + fix.mu.Lock() + defer fix.mu.Unlock() + if fix.committedAssignments != nil { + t.Error("idempotent flip must not commit assignments.json") + } + if len(fix.committedShims) != 0 { + t.Errorf("idempotent retrofit must not commit shims, got %v", fix.committedShims) + } +} + +func TestRunSubmissionMode_EmptyRepoRefused(t *testing.T) { + server, _ := newSMServer(t, smServerConfig{ + assignments: smAssignmentsBody("", "default", true), + }) + client := githubtest.NewTestClient(t, server) + var out, errOut bytes.Buffer + err := runSubmissionMode(client, &out, &errOut, smParams(contract.SubmissionModeTag)) + if err == nil || !strings.Contains(err.Error(), "empty_repo") { + t.Fatalf("expected empty_repo refusal, got %v", err) + } +} + +func TestRunSubmissionMode_CustomAutograderRefusesShimUpdate(t *testing.T) { + server, fix := newSMServer(t, smServerConfig{ + assignments: smAssignmentsBody("", "grader50", false), + }) + client := githubtest.NewTestClient(t, server) + var out, errOut bytes.Buffer + + // Default (update shims) -> refused outright, nothing written. + err := runSubmissionMode(client, &out, &errOut, smParams(contract.SubmissionModeTag)) + if err == nil || !strings.Contains(err.Error(), "grader50") { + t.Fatalf("expected custom-autograder refusal, got %v", err) + } + fix.mu.Lock() + if fix.committedAssignments != nil { + t.Error("refused command must not write the field") + } + fix.mu.Unlock() + + // --update-shims=false -> field flip allowed (mode still drives client + // tag pushing, which is autograder-independent). + p := smParams(contract.SubmissionModeTag) + p.updateShims = false + if err := runSubmissionMode(client, &out, &errOut, p); err != nil { + t.Fatalf("field-only flip should succeed for a custom autograder: %v", err) + } + fix.mu.Lock() + defer fix.mu.Unlock() + file, err := assignment.ParseAssignments(fix.committedAssignments) + if err != nil { + t.Fatalf("parse: %v", err) + } + if got := file.Assignments[0].SubmissionMode; got != contract.SubmissionModeTag { + t.Errorf("committed submission_mode = %q, want tag", got) + } +} + +func TestRunSubmissionMode_UnrecognizedShimSkipped(t *testing.T) { + custom := "name: Autograde\non:\n workflow_dispatch: {}\njobs: {}\n" + server, fix := newSMServer(t, smServerConfig{ + assignments: smAssignmentsBody("", "default", false), + repos: map[string]string{"dst-hello-alice": custom}, + }) + client := githubtest.NewTestClient(t, server) + var out, errOut bytes.Buffer + // Skips are not failures: exit 0, reported on stderr. + if err := runSubmissionMode(client, &out, &errOut, smParams(contract.SubmissionModeTag)); err != nil { + t.Fatalf("unrecognized shim must not fail the command: %v", err) + } + fix.mu.Lock() + defer fix.mu.Unlock() + if _, wrote := fix.committedShims["dst-hello-alice"]; wrote { + t.Error("unrecognized shim must never be rewritten") + } + if !strings.Contains(errOut.String(), "not recognized") { + t.Errorf("skip must be reported on stderr:\n%s", errOut.String()) + } +} + +func TestRunSubmissionMode_WorkflowScope404Classified(t *testing.T) { + server, _ := newSMServer(t, smServerConfig{ + assignments: smAssignmentsBody("", "default", false), + repos: map[string]string{"dst-hello-alice": cliShimEveryPush}, + workflowScope404: true, + }) + client := githubtest.NewTestClient(t, server) + var out, errOut bytes.Buffer + err := runSubmissionMode(client, &out, &errOut, smParams(contract.SubmissionModeTag)) + if err == nil { + t.Fatal("scope-404 must fail the command") + } + if !strings.Contains(errOut.String(), "workflow") { + t.Errorf("failure should carry the workflow-scope remediation:\n%s", errOut.String()) + } +} + +func TestRunSubmissionMode_UserTargetsSingleRepo(t *testing.T) { + server, fix := newSMServer(t, smServerConfig{ + assignments: smAssignmentsBody("", "default", false), + repos: map[string]string{ + "dst-hello-alice": cliShimEveryPush, + "dst-hello-bob": cliShimEveryPush, + }, + }) + client := githubtest.NewTestClient(t, server) + var out, errOut bytes.Buffer + p := smParams(contract.SubmissionModeTag) + p.user = "alice" + if err := runSubmissionMode(client, &out, &errOut, p); err != nil { + t.Fatalf("runSubmissionMode --user: %v", err) + } + fix.mu.Lock() + defer fix.mu.Unlock() + if _, wrote := fix.committedShims["dst-hello-alice"]; !wrote { + t.Error("alice's shim should be updated") + } + if _, wrote := fix.committedShims["dst-hello-bob"]; wrote { + t.Error("--user alice must not touch bob's repo") + } +} + +func TestRunSubmissionMode_DryRunWritesNothing(t *testing.T) { + server, fix := newSMServer(t, smServerConfig{ + assignments: smAssignmentsBody("", "default", false), + repos: map[string]string{"dst-hello-alice": cliShimEveryPush}, + }) + client := githubtest.NewTestClient(t, server) + var out, errOut bytes.Buffer + p := smParams(contract.SubmissionModeTag) + p.dryRun = true + if err := runSubmissionMode(client, &out, &errOut, p); err != nil { + t.Fatalf("dry run: %v", err) + } + fix.mu.Lock() + defer fix.mu.Unlock() + if fix.committedAssignments != nil || len(fix.committedShims) != 0 { + t.Error("dry run must write nothing") + } + if !strings.Contains(out.String(), "dry run") { + t.Errorf("dry run output missing marker:\n%s", out.String()) + } +} + +func TestRunSubmissionMode_MissingSlugErrors(t *testing.T) { + server, _ := newSMServer(t, smServerConfig{ + assignments: `{"schema":"classroom50/assignments/v1","assignments":[]}`, + }) + client := githubtest.NewTestClient(t, server) + var out, errOut bytes.Buffer + if err := runSubmissionMode(client, &out, &errOut, smParams(contract.SubmissionModeTag)); err == nil { + t.Fatal("missing slug must error") + } +} + +// --------------------------------------------------------------------------- +// rewriteShimTrigger — milestone submission_tags +// --------------------------------------------------------------------------- + +// A shim rendered WITH milestone patterns must still be recognized (never +// shimUnrecognized) and rewritable in both directions. +const cliShimWithTags = `# Classroom50 autograder shim. +# +# This file should not be edited. + +name: Autograde + +on: + push: + branches: ["main"] + tags: ["phase1", "v*", "submit/*"] + +jobs: + grade: + uses: "o/classroom50/.github/workflows/autograde-runner.yaml@main" + permissions: + contents: write + statuses: write + pull-requests: write +` + +func TestRewriteShimTrigger_TagsReconciled(t *testing.T) { + // Retrofitting with a pattern set widens the tags line (union with + // submit/*) while the mode surgery behaves as before. + got, changed, err := rewriteShimTrigger(cliShimEveryPush, contract.SubmissionModeEveryPush, "main", []string{"phase1", "v*"}) + if err != nil || !changed { + t.Fatalf("rewrite = (changed=%v, %v), want a change", changed, err) + } + if !strings.Contains(got, ` tags: ["phase1", "v*", "submit/*"]`) { + t.Errorf("tags line not widened:\n%s", got) + } + if !strings.Contains(got, ` branches: ["main"]`) { + t.Errorf("every-push branches line must be preserved:\n%s", got) + } + + // A pattern-bearing shim is recognized and can be narrowed back to the + // default set (patterns removed from the assignment). + back, changed, err := rewriteShimTrigger(cliShimWithTags, contract.SubmissionModeEveryPush, "main", nil) + if err != nil || !changed { + t.Fatalf("narrow = (changed=%v, %v), want a change", changed, err) + } + if !strings.Contains(back, ` tags: ["submit/*"]`) { + t.Errorf("tags line not narrowed to the default:\n%s", back) + } + + // Idempotent: rewriting a shim already in the target state is a no-op. + if _, changed, err := rewriteShimTrigger(cliShimWithTags, contract.SubmissionModeEveryPush, "main", []string{"phase1", "v*"}); err != nil || changed { + t.Errorf("already-current pattern shim must be a no-op (changed=%v, err=%v)", changed, err) + } +} + +func TestRewriteShimTrigger_TagModeWithTags(t *testing.T) { + // Tag mode + milestone patterns: branch line dropped, tags widened. + got, changed, err := rewriteShimTrigger(cliShimEveryPush, contract.SubmissionModeTag, "main", []string{"phase1"}) + if err != nil || !changed { + t.Fatalf("rewrite = (changed=%v, %v), want a change", changed, err) + } + if strings.Contains(got, "branches:") { + t.Errorf("tag mode must drop the branches line:\n%s", got) + } + if !strings.Contains(got, ` tags: ["phase1", "submit/*"]`) { + t.Errorf("tags line not widened:\n%s", got) + } +} diff --git a/cli/gh-teacher/internal/download/download.go b/cli/gh-teacher/internal/download/download.go index 0daece66..07cb1e0c 100644 --- a/cli/gh-teacher/internal/download/download.go +++ b/cli/gh-teacher/internal/download/download.go @@ -40,7 +40,10 @@ const dirTimestampFormat = "2006_01_02_T_15_04_05" // Cross-binary contract with collect_scores.py and autograde-runner.yaml // (which creates the submit-tag releases): asset name, submit-tag prefix, // per-asset size cap. Keep aligned with RESULT_ASSET_NAME, SUBMIT_TAG_PREFIX, -// and MAX_RESULT_BYTES in collect_scores.py. +// and MAX_RESULT_BYTES in collect_scores.py. submit/ is the RECORD namespace +// and stays fixed even with configurable submission_tags — milestone tags +// only trigger runs; the runner mints the canonical submit/* tag it releases +// at, so release discovery here never needs the pattern matcher. const ( resultAssetName = "result.json" submitTagPrefix = "submit/" diff --git a/cli/gh-teacher/skeleton/dotgithub/scripts/regrade_repos.py b/cli/gh-teacher/skeleton/dotgithub/scripts/regrade_repos.py index 43e42daf..fb09e925 100644 --- a/cli/gh-teacher/skeleton/dotgithub/scripts/regrade_repos.py +++ b/cli/gh-teacher/skeleton/dotgithub/scripts/regrade_repos.py @@ -88,6 +88,73 @@ _USERNAME_BAD_CHARS = re.compile(r"[^A-Za-z0-9-]") +def _compile_tag_pattern(pattern: str) -> re.Pattern[str] | None: + """One Actions tag-filter pattern -> an anchored regex, or None when it + can't compile (fail closed: matches nothing). Character by character so + `.` and other regex metacharacters in the pattern stay literal. Supported + subset: literal names, `*` (not crossing `/`), `**` (crossing), `?`/`+` + (zero-or-one / one-or-more of the preceding element), `[abc]` classes. + """ + out = ["^"] + i = 0 + while i < len(pattern): + ch = pattern[i] + if ch == "*": + if i + 1 < len(pattern) and pattern[i + 1] == "*": + out.append(".*") # ** crosses / + i += 1 + else: + out.append("[^/]*") # * stops at / + elif ch in ("?", "+"): + out.append(ch) + elif ch == "[": + close = pattern.find("]", i + 1) + if close != -1: + out.append(pattern[i : close + 1]) # class verbatim + i = close + else: + out.append(re.escape(ch)) # unclosed [ is literal + else: + out.append(re.escape(ch)) + i += 1 + out.append("$") + try: + return re.compile("".join(out)) + except re.error: + return None + + +# The safe-pattern charset — literal-name characters plus the glob +# metacharacters GitHub Actions tag filters support. Keep in lockstep with Go +# contract.SubmissionTagCharsetRE and the web SUBMISSION_TAG_PATTERN_RE. +_TAG_PATTERN = re.compile(r"^[A-Za-z0-9._/*?+\[\]-]+$") + +# A leading `?`/`+` (nothing to repeat) or a `+` stacked on another +# quantifier (`v*+`, `a++`). LOAD-BEARING here in the Python mirror: those +# translate to POSSESSIVE quantifiers, which Python 3.11+ compiles (and +# matches!) while Go RE2 and JS reject — without this guard the four matcher +# copies would diverge on exactly these patterns. Keep in lockstep with Go +# contract.stackedQuantifierRE and the web copies. +_STACKED_QUANTIFIER = re.compile(r"^[?+]|[*?+]\+") + + +def matches_submission_tag(patterns: list[str], tag: str) -> bool: + """Whether `tag` matches ANY of the Actions tag-filter `patterns`; an + empty list matches nothing. By-value copy of Go's + contract.MatchesSubmissionTag and the web matchesSubmissionTag — all + pinned to identical output by the shared golden fixture + cli/shared/testdata/submission_tag_match_cases.json. The same strings are + rendered into the shim's on.push.tags, so this matcher and GitHub's own + filter evaluation must agree on what fires. Keep in lockstep.""" + for pattern in patterns: + if not _TAG_PATTERN.fullmatch(pattern) or _STACKED_QUANTIFIER.search(pattern): + continue # fail closed, matching the Go/JS charset+compile guards + compiled = _compile_tag_pattern(pattern) + if compiled is not None and compiled.fullmatch(tag) is not None: + return True + return False + + # Top-level dispatch ---------------------------------------------------------- @@ -130,7 +197,7 @@ def main() -> int: classroom_dir = base_dir / classroom_filter try: - roster = load_roster(classroom_dir, assignment_filter, api_url, org, service_token) + roster, entry = load_roster(classroom_dir, assignment_filter, api_url, org, service_token) except EmptyRepoAssignment: # Successful no-op, not a failure: the teacher (or a stale button) # targeted an assignment whose repos are deliberately bare. @@ -189,6 +256,15 @@ def main() -> int: ) return 1 + # Tag-mode assignments introduce runs that complete green but grade + # nothing (a suppressed stale-shim branch push); regrade_repo must skip + # those when picking the run to replay. Milestone submission_tags runs + # are real graded runs, so the patterns ride along for the run filter. + tag_mode = is_tag_submission_mode(entry) + submission_tags = entry.get("submission_tags") or [] + if not isinstance(submission_tags, list): + submission_tags = [] + regraded = 0 # rerun an existing run (the true regrade) tagged = 0 # first-grade fallback (no prior run, tagged main HEAD) skipped = 0 # nothing to do (not accepted) or benign skip @@ -197,7 +273,9 @@ def main() -> int: for index, username in enumerate(targets, start=1): repo_name = assignment_repo_name(classroom_filter, assignment_filter, username) try: - outcome = regrade_repo(api_url, org, repo_name, service_token) + outcome = regrade_repo( + api_url, org, repo_name, service_token, tag_mode, submission_tags + ) except _SkipRepo: # Benign per-repo skip (e.g., the latest run can't be re-run right # now); already warned at the source. @@ -269,7 +347,14 @@ def main() -> int: AUTOGRADE_WORKFLOW = "autograde.yaml" -def regrade_repo(api_url: str, org: str, repo: str, token: str) -> str: +def regrade_repo( + api_url: str, + org: str, + repo: str, + token: str, + tag_mode: bool, + submission_tags: list[str] | None = None, +) -> str: """Re-run grading for `repo` on its existing latest submission, without creating a new one. Returns one of: @@ -277,18 +362,31 @@ def regrade_repo(api_url: str, org: str, repo: str, token: str) -> str: (re-fetching the current autograder), and because the runner stamps `datetime` from the commit's committer date, the submission time / late flag DON'T change — only the score. - "tagged" — no prior run, so a fresh submit/- tag was pushed to - first-grade the main HEAD. (Submission time is still the - commit's committer date; `graded_at` records the new run.) + "tagged" — no (usable) prior run, so a fresh submit/- tag was + pushed to first-grade the main HEAD. (Submission time is + still the commit's committer date; `graded_at` records the + new run.) "missing" — no prior run and no main HEAD (student hasn't accepted/pushed); nothing to do. + tag_mode narrows which run counts as "the latest submission": on a + tag-mode assignment a branch-triggered run is a SUPPRESSED run (a stale + every-push shim fired; the runner tagged and graded nothing), and + replaying it would re-suppress — regrade would report success while + grading nothing. So in tag mode only submit/* tag runs are candidates; + a repo with none (only suppressed pushes, or no runs at all) falls + through to the tag-at-HEAD path, which fires a REAL tag run (the + service token's tag push fires workflows). Every-push keeps today's + behavior exactly — its branch runs are real graded runs. + Raises urllib.error.HTTPError / ValueError on a hard failure the caller classifies (auth/network abort; other per-repo errors warn-and-skip). """ # Prefer re-running the existing run: a true "regrade the same commit" with # no new tag and no new submission event. - run_id = latest_autograde_run_id(api_url, org, repo, token) + run_id = latest_autograde_run_id( + api_url, org, repo, token, tag_only=tag_mode, submission_tags=submission_tags + ) if run_id is not None: rerun_workflow_run(api_url, org, repo, token, run_id) return "rerun" @@ -310,14 +408,33 @@ def regrade_repo(api_url: str, org: str, repo: str, token: str) -> str: def latest_autograde_run_id( - api_url: str, org: str, repo: str, token: str + api_url: str, + org: str, + repo: str, + token: str, + *, + tag_only: bool = False, + submission_tags: list[str] | None = None, ) -> int | None: """The id of the most recent autograde run on `repo`, or None when it has never run (or doesn't exist yet). Run ids are newest-first from the API, so - the first entry is the latest run — the one a regrade re-runs.""" + the first entry is the latest run — the one a regrade re-runs. + + tag_only=True (tag-mode assignments) considers only runs whose head_branch + names a real submission tag (GitHub sets head_branch to the tag on + tag-push runs): the canonical submit/* namespace, or a teacher-named + milestone pattern from `submission_tags` (a milestone run grades for real + — its record lives at the canonical tag the runner mints). Branch- + triggered runs on a tag-mode assignment are suppressed no-ops that must + never be replayed. One 100-run page is scanned, no pagination: if the + newest submission run has scrolled past 100 suppressed pushes, we return + None and the caller's tag-at-HEAD fallback freshly grades HEAD instead — + acceptable for that degenerate case. + """ + per_page = 100 if tag_only else 1 url = ( f"{_repo_url(api_url, org, repo)}/actions/workflows/" - f"{urllib.parse.quote(AUTOGRADE_WORKFLOW)}/runs?per_page=1" + f"{urllib.parse.quote(AUTOGRADE_WORKFLOW)}/runs?per_page={per_page}" ) try: body = _http_get(url, token, accept="application/vnd.github+json") @@ -330,7 +447,24 @@ def latest_autograde_run_id( runs = data.get("workflow_runs") if isinstance(data, dict) else None if not isinstance(runs, list) or not runs: return None - run = runs[0] + run: Any = None + if tag_only: + patterns = submission_tags or [] + for candidate in runs: + if not isinstance(candidate, dict): + continue + head_branch = candidate.get("head_branch") + if not isinstance(head_branch, str): + continue + if head_branch.startswith(SUBMIT_TAG_PREFIX) or matches_submission_tag( + patterns, head_branch + ): + run = candidate + break + if run is None: + return None + else: + run = runs[0] run_id = run.get("id") if isinstance(run, dict) else None if not isinstance(run_id, int): raise ValueError("workflow run object missing an integer id") @@ -563,20 +697,31 @@ def is_empty_repo(entry: dict[str, Any]) -> bool: return entry.get("empty_repo") is True +def is_tag_submission_mode(entry: dict[str, Any]) -> bool: + """Whether the assignment grades ONLY on submit/* tag pushes. Strict + equality mirroring the Go Entry.IsTagSubmissionMode: absent, "every-push", + and any junk value all read as every-push (fail open to today's regrade + behavior; the runner polices invalid modes at grade time).""" + return entry.get("submission_mode") == "tag" + + def load_roster( classroom_dir: pathlib.Path, assignment_slug: str, api_url: str, org: str, token: str, -) -> list[str]: - """Team members to regrade for an assignment registered in this classroom. +) -> tuple[list[str], dict[str, Any]]: + """(team members to regrade, the assignment's manifest entry) for an + assignment registered in this classroom. Validates the assignments.json schema and that the target slug is registered (so a typo'd slug fails loudly rather than tagging nothing), then enumerates the classroom GitHub team — the source of truth for enrollment. - Config problems raise RegradeInputError; a team-listing HTTP error - propagates so main() can classify it (hard auth/network vs. transient). + The entry rides along so main() can read submission_mode (regrade must not + replay a suppressed tag-mode branch run — see regrade_repo). Config + problems raise RegradeInputError; a team-listing HTTP error propagates so + main() can classify it (hard auth/network vs. transient). """ if not classroom_dir.is_dir(): raise RegradeInputError( @@ -645,7 +790,7 @@ def load_roster( continue seen.add(key) usernames.append(username) - return usernames + return usernames, entries[assignment_slug] def resolve_team_slug(classroom_meta: dict[str, Any], classroom_short: str) -> str: diff --git a/cli/gh-teacher/skeleton/dotgithub/scripts/runner.py b/cli/gh-teacher/skeleton/dotgithub/scripts/runner.py index 7232c48a..ad200b1a 100644 --- a/cli/gh-teacher/skeleton/dotgithub/scripts/runner.py +++ b/cli/gh-teacher/skeleton/dotgithub/scripts/runner.py @@ -147,6 +147,13 @@ } ) +# Paths a teacher-side submission-mode shim retrofit touches: exactly the shim, +# nothing else. Such a commit carries `[skip ci]` so the workflow normally +# never fires; is_shim_update_commit is the backstop for environments that +# strip it. Mirrors contract.ShimUpdateCommitMessage's write path — keep in +# lockstep with classroomcfg.AutogradeWorkflowPath. +SHIM_UPDATE_COMMIT_PATHS = frozenset({".github/workflows/autograde.yaml"}) + # `_baseline_scan` source discriminator. SOURCE_OPENABLE yields a usable # Feedback PR base (accept commit or root fallback); the others skip. SOURCE_ACCEPT = "accept" @@ -461,6 +468,11 @@ def validate_result( f"want {expected_type!r}" ) + # submit/* here is the RECORD namespace, deliberately not the configurable + # submission_tags patterns: a milestone-tag run (e.g. phase1) mints/reuses + # the canonical submit/- tag in the workflow's tag step BEFORE + # grading, so SUBMISSION_TAG — and thus result.json's `submission` — is + # always canonical. Custom tags trigger; submit/* records. submission = data.get("submission") if not isinstance(submission, str) or not submission.startswith("submit/"): return f"{RESULT_FILENAME} 'submission' must be a 'submit/*' string" @@ -648,6 +660,36 @@ def _accept_commit_is_setup_only(workspace: pathlib.Path, head_sha: str) -> bool empty path list, so a commit we can't fully inspect is treated as a submission rather than silently skipped. """ + return _commit_touches_only(workspace, head_sha, ACCEPT_COMMIT_PATHS) + + +def is_shim_update_commit(workspace: pathlib.Path, head_sha: str) -> bool: + """Whether head_sha is a teacher-side submission-mode shim retrofit: a tip + commit that touches ONLY .github/workflows/autograde.yaml. + + Such commits carry `[skip ci]` and normally never fire the workflow; this + is the defense-in-depth backstop for a client that forgot the marker or an + environment that strips it. Deliberately NOT gated on the accept scan: a + shim-only commit has nothing to grade regardless of who authored it, and a + student hand-editing their shim gets a skip either way (the edit alone is + never gradeable work). Fails open (False -> grade) on any uncertainty. + The acceptance check takes precedence at the call site — the accept commit + also touches the shim but additionally lands the marker, so the path sets + never overlap in practice. + """ + if not head_sha: + return False + return _commit_touches_only(workspace, head_sha, SHIM_UPDATE_COMMIT_PATHS) + + +def _commit_touches_only( + workspace: pathlib.Path, head_sha: str, allowed: frozenset[str] +) -> bool: + """True only when every path the commit touches is in `allowed`. Fails + open (False -> grade) on any git error or an empty path list, so a commit + we can't fully inspect is treated as a submission rather than silently + skipped. + """ def git(*args: str) -> subprocess.CompletedProcess[str]: return subprocess.run( @@ -668,7 +710,7 @@ def git(*args: str) -> subprocess.CompletedProcess[str]: paths = [p for p in changed.stdout.split("\0") if p] if not paths: return False - return all(p in ACCEPT_COMMIT_PATHS for p in paths) + return all(p in allowed for p in paths) except (OSError, subprocess.SubprocessError): return False @@ -2198,22 +2240,30 @@ def finalize_result(finalize: Finalizer, *, is_group: bool) -> int: def detect_acceptance_mode() -> int: - """`runner.py --detect-acceptance`: write is-acceptance=true|false to - $GITHUB_OUTPUT for the setup job's skip gate. Always exits 0; fails open - (False) on any uncertainty. + """`runner.py --detect-acceptance`: write is-acceptance=true|false and + is-shim-update=true|false to $GITHUB_OUTPUT for the setup job's skip + gates. Always exits 0; fails open (both False) on any uncertainty. + Acceptance takes precedence: the accept commit also touches the shim, so + is-shim-update is only computed for a non-acceptance tip. """ workspace = pathlib.Path.cwd() head_sha = os.environ.get("GITHUB_SHA", "").strip() is_acceptance = is_acceptance_commit(workspace, head_sha) + is_shim_update = not is_acceptance and is_shim_update_commit(workspace, head_sha) github_output = os.environ.get("GITHUB_OUTPUT") if github_output: with open(github_output, "a") as fh: fh.write(f"is-acceptance={'true' if is_acceptance else 'false'}\n") + fh.write(f"is-shim-update={'true' if is_shim_update else 'false'}\n") if is_acceptance: print( "::notice::acceptance commit detected — nothing to grade yet; " "submit work (gh student submit) to be graded" ) + elif is_shim_update: + print( + "::notice::autograder-trigger update detected — nothing to grade" + ) else: print("runner: not an acceptance commit; grading proceeds") return 0 diff --git a/cli/gh-teacher/skeleton/dotgithub/workflows/autograde-runner.yaml b/cli/gh-teacher/skeleton/dotgithub/workflows/autograde-runner.yaml index 21831249..2603871e 100644 --- a/cli/gh-teacher/skeleton/dotgithub/workflows/autograde-runner.yaml +++ b/cli/gh-teacher/skeleton/dotgithub/workflows/autograde-runner.yaml @@ -40,7 +40,22 @@ jobs: # 'true' -> pushed commit is the acceptance commit; skip the tag/read # steps and the grade job (no submission tag, no release). is-acceptance: ${{ steps.acceptance.outputs.is-acceptance }} - submission-tag: ${{ steps.read.outputs.submission-tag }} + # 'true' -> pushed commit is a teacher-side shim retrofit (touches only + # .github/workflows/autograde.yaml); nothing to grade. Defense-in-depth + # behind the [skip ci] in the retrofit commit message. + is-shim-update: ${{ steps.acceptance.outputs.is-shim-update }} + # 'true' -> the assignment is in tag submission mode but this run was + # branch-triggered (a stale every-push shim, or a hand-edited one); + # skip tagging + grading so a plain `git push` costs nothing. + branch-push-suppressed: ${{ steps.read.outputs.branch-push-suppressed }} + # 'true' -> a pushed tag matched neither submit/* nor any configured + # submission_tags pattern (stale/hand-edited shim); skip tagging + + # grading — the tag is not a submission trigger. + foreign-tag-suppressed: ${{ steps.read.outputs.foreign-tag-suppressed }} + # The teacher-named milestone tag that triggered this run ('' for a + # branch or canonical submit/* trigger); surfaces in the Release title. + trigger-tag: ${{ steps.read.outputs.trigger-tag }} + submission-tag: ${{ steps.tag.outputs.tag }} runs-on: ${{ steps.read.outputs.runs-on }} container: ${{ steps.read.outputs.container }} python: ${{ steps.read.outputs.python }} @@ -76,12 +91,15 @@ jobs: run: python3 -m pip install --quiet --user pyyaml working-directory: ${{ runner.temp }} - # === Skip the acceptance commit === + # === Skip the acceptance commit / a shim-retrofit commit === # The accept commit (adds .classroom50.yaml + the shim, student- - # authored, so it fires this workflow) has nothing to grade. Detect - # it via the Pages-fetched runner.py and skip the tag/read steps and - # grade job. Branch trigger only (a tag push is always a submission). - # Fails open (grades) on a fetch failure. + # authored, so it fires this workflow) has nothing to grade — and + # neither does a teacher-side submission-mode shim retrofit (touches + # only .github/workflows/autograde.yaml; its [skip ci] normally keeps + # this workflow from running at all, this is the backstop). Detect + # both via the Pages-fetched runner.py and skip the tag/read steps + # and grade job. Branch trigger only (a tag push is always a + # submission). Fails open (grades) on a fetch failure. - name: Detect acceptance commit id: acceptance if: github.ref_type == 'branch' @@ -102,10 +120,12 @@ jobs: python3 "$RUNNER_SCRIPT" --detect-acceptance || { echo "::warning::acceptance detection errored; grading proceeds" echo "is-acceptance=false" >> "$GITHUB_OUTPUT" + echo "is-shim-update=false" >> "$GITHUB_OUTPUT" } else echo "::warning::could not fetch runner.py for acceptance check; grading proceeds" echo "is-acceptance=false" >> "$GITHUB_OUTPUT" + echo "is-shim-update=false" >> "$GITHUB_OUTPUT" fi # === Acceptance-skip commit status === @@ -129,49 +149,6 @@ jobs: -f description="acceptance commit — nothing to grade yet; run gh student submit" \ -f target_url="$RUN_URL" >/dev/null 2>&1 || true - # === Tag the submission === - # Branch trigger: create submit/- at - # the pushed SHA. Tag trigger: reuse the pushed tag. Tags pushed - # with github.token don't fire workflows (anti-recursion). - - name: Tag submission - id: tag - if: steps.acceptance.outputs.is-acceptance != 'true' - env: - GH_TOKEN: ${{ github.token }} - REF: ${{ github.ref }} - REF_NAME: ${{ github.ref_name }} - SHA: ${{ github.sha }} - run: | - set -euo pipefail - if [[ "$REF" == refs/tags/* ]]; then - if [[ "$REF_NAME" != submit/* ]]; then - echo "::error::tag $REF_NAME does not match submit/* prefix" >&2 - exit 1 - fi - echo "tag=$REF_NAME" >> "$GITHUB_OUTPUT" - exit 0 - fi - # Idempotency: if a submit/* tag already points at this SHA - # (workflow re-run of the same commit), reuse it instead of - # racing on a fresh timestamp. `--refs` filters out the - # `refs/tags/X^{}` peeled-ref rows that annotated tags emit - # — without it, awk could capture e.g., `submit/foo^{}` and - # produce an invalid pseudo-tag. - EXISTING=$(git ls-remote --refs --tags origin \ - | awk -v sha="$SHA" '$1 == sha && $2 ~ /^refs\/tags\/submit\// { sub("refs/tags/", "", $2); print $2; exit }') - if [[ -n "$EXISTING" ]]; then - echo "tag=$EXISTING" >> "$GITHUB_OUTPUT" - exit 0 - fi - # Short-SHA suffix prevents collisions when two pushes land - # in the same UTC second. - TAG="submit/$(date -u +%Y-%m-%dT%H-%M-%SZ)-$(echo "$SHA" | cut -c1-7)" - git config user.name "classroom50-bot" - git config user.email "bot@classroom50" - git tag "$TAG" "$SHA" - git push origin "refs/tags/$TAG" - echo "tag=$TAG" >> "$GITHUB_OUTPUT" - # === Read metadata + validate runtime === # `.classroom50.yaml` is in the student repo (untrusted) — every # field validated against a strict regex; org pinned to the @@ -181,9 +158,12 @@ jobs: # cli/gh-teacher/tests.go (declarative tests block). - name: Read assignment config id: read - if: steps.acceptance.outputs.is-acceptance != 'true' + if: >- + steps.acceptance.outputs.is-acceptance != 'true' && + steps.acceptance.outputs.is-shim-update != 'true' env: - SUBMISSION_TAG: ${{ steps.tag.outputs.tag }} + REF: ${{ github.ref }} + REF_NAME: ${{ github.ref_name }} REPO_OWNER: ${{ github.repository_owner }} shell: python3 {0} run: | @@ -562,11 +542,105 @@ jobs: release_assets, separators=(",", ":"), ensure_ascii=False ) - submission_tag = os.environ.get("SUBMISSION_TAG", "").strip() - if not submission_tag: - submission_tag = os.environ.get("GITHUB_REF_NAME", "") - if not submission_tag.startswith("submit/"): - fail(f"submission tag {submission_tag!r} does not match submit/*") + # === Submission mode === + # Mirror Go's ValidateSubmissionMode: absent/empty means every-push; + # a non-empty value must be one of the enum (contract.SubmissionModes, + # by-value copy — keep in lockstep). In tag mode a BRANCH-triggered + # run is suppressed (no tag, no grade): the shim shouldn't have fired + # at all, so this only happens with a stale every-push shim (mode + # changed, retrofit pending/failed) or a hand-edited one. Suppressing + # here keeps a plain `git push` from grading — the cost lever the + # mode exists for — while a submission-tag push always grades. + submission_mode = entry.get("submission_mode") or "every-push" + if submission_mode not in ("every-push", "tag"): + fail(f"entry {assignment!r}: submission_mode {submission_mode!r} must be 'every-push' or 'tag'") + is_branch_run = not os.environ.get("REF", "").startswith("refs/tags/") + branch_push_suppressed = submission_mode == "tag" and is_branch_run + + # === Submission tags (teacher-named milestone patterns) === + # Validate like allowed_files (opaque pattern strings, charset + # mirrors the schema/Go/web validators), then classify a TAG run: + # - submit/* tag -> the canonical namespace, grades (reuse) + # - matches a pattern -> milestone submission; grades, and the + # tag step mints/reuses the canonical + # submit/* tag at this commit (the RECORD + # stays immutable one-release-per- + # submission; the trigger tag is only the + # trigger) + # - matches nothing -> not a submission trigger; suppress + # gracefully (stale/hand-edited shim — + # same defense-in-depth rationale as the + # stale-shim branch suppression; the shim + # normally pre-filters via on.push.tags) + # matches_submission_tag is a by-value copy of Go's + # contract.MatchesSubmissionTag / the web matchesSubmissionTag, + # pinned to identical output by the shared golden fixture + # cli/shared/testdata/submission_tag_match_cases.json — keep in + # lockstep. + submission_tags = entry.get("submission_tags") or [] + if not isinstance(submission_tags, list): + fail(f"entry {assignment!r}: submission_tags must be an array") + _TAG_PATTERN = re.compile(r"^[A-Za-z0-9._/*?+\[\]-]+$") + # Stacked/leading quantifiers (`v*+`, `+lead`) compile as POSSESSIVE + # quantifiers in Python but are compile errors in Go/JS — the one + # construct where the four matcher copies would diverge, so reject + # like the write-side validators (contract.stackedQuantifierRE). + _STACKED_QUANTIFIER = re.compile(r"^[?+]|[*?+]\+") + for pattern in submission_tags: + if not isinstance(pattern, str) or not _TAG_PATTERN.fullmatch(pattern) or _STACKED_QUANTIFIER.search(pattern): + fail(f"entry {assignment!r}: invalid submission_tags pattern {pattern!r}") + + def _compile_tag_pattern(pattern): + out = ["^"] + i = 0 + while i < len(pattern): + ch = pattern[i] + if ch == "*": + if i + 1 < len(pattern) and pattern[i + 1] == "*": + out.append(".*") + i += 1 + else: + out.append("[^/]*") + elif ch in ("?", "+"): + out.append(ch) + elif ch == "[": + close = pattern.find("]", i + 1) + if close != -1: + out.append(pattern[i : close + 1]) + i = close + else: + out.append(re.escape(ch)) + else: + out.append(re.escape(ch)) + i += 1 + out.append("$") + try: + return re.compile("".join(out)) + except re.error: + return None + + def matches_submission_tag(patterns, tag): + for pattern in patterns: + if not _TAG_PATTERN.fullmatch(pattern) or _STACKED_QUANTIFIER.search(pattern): + continue # fail closed like the Go/JS charset+compile guards + compiled = _compile_tag_pattern(pattern) + if compiled is not None and compiled.fullmatch(tag) is not None: + return True + return False + + # REF_NAME with a defensive fallback derived from REF (both are + # set in a real Actions run; the fallback keeps the classification + # sound if the env plumbing ever drops one). + ref_name = os.environ.get("REF_NAME") or os.environ.get("REF", "").removeprefix("refs/tags/") + trigger_tag = "" + foreign_tag_suppressed = False + if not is_branch_run: + if ref_name.startswith("submit/"): + pass # canonical namespace; the tag step reuses it + elif matches_submission_tag(submission_tags, ref_name): + trigger_tag = ref_name + else: + foreign_tag_suppressed = True # === No-autograder detection === # Skip grade + set-latest when nothing is configured to grade, @@ -605,7 +679,10 @@ jobs: default_py = f"{base_url}/{seg}/autograder.py" no_autograder = probe_absent(bundle) and probe_absent(default_py) - emit("submission-tag", submission_tag) + emit("submission-mode", submission_mode) + emit("branch-push-suppressed", "true" if branch_push_suppressed else "false") + emit("foreign-tag-suppressed", "true" if foreign_tag_suppressed else "false") + emit("trigger-tag", trigger_tag) emit("runs-on", runs_on_json) emit("container", container_json) emit("python", python) @@ -624,15 +701,146 @@ jobs: emit("release-assets", release_assets_json) emit("no-autograder", "true" if no_autograder else "false") + # === Tag the submission === + # Branch trigger: create submit/- at + # the pushed SHA. Canonical submit/* tag trigger: reuse the pushed + # tag. Milestone-tag trigger (a teacher-named submission_tags + # pattern): mint/reuse the canonical tag exactly like a branch run — + # the milestone tag TRIGGERS grading but the immutable submit/* tag + # IS the record (a reusable tag like `phase1` moves between commits; + # parking the release on it would overwrite prior submissions). + # Tags pushed with github.token don't fire workflows + # (anti-recursion). Runs AFTER the read step so a suppressed run + # (tag-mode branch push, or a tag matching no submission pattern) + # never mints a tag for a run that won't grade. + - name: Tag submission + id: tag + if: >- + steps.acceptance.outputs.is-acceptance != 'true' && + steps.acceptance.outputs.is-shim-update != 'true' && + steps.read.outputs.branch-push-suppressed != 'true' && + steps.read.outputs.foreign-tag-suppressed != 'true' + env: + GH_TOKEN: ${{ github.token }} + REF: ${{ github.ref }} + REF_NAME: ${{ github.ref_name }} + SHA: ${{ github.sha }} + TRIGGER_TAG: ${{ steps.read.outputs.trigger-tag }} + run: | + set -euo pipefail + # A canonical submit/* tag push IS the record — reuse it verbatim. + # A milestone-tag push (TRIGGER_TAG set) falls through to the + # mint/reuse path below, same as a branch push. Any other tag + # was suppressed by the read step (foreign-tag-suppressed), so + # reaching here with one is a wiring bug — fail loudly. + if [[ "$REF" == refs/tags/* && -z "$TRIGGER_TAG" ]]; then + if [[ "$REF_NAME" != submit/* ]]; then + echo "::error::tag $REF_NAME reached the tag step without matching submit/* or a submission pattern" >&2 + exit 1 + fi + echo "tag=$REF_NAME" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Idempotency: if a submit/* tag already points at this SHA + # (workflow re-run of the same commit, a submit client that + # already tagged it, or a re-pushed milestone tag), reuse it + # instead of racing on a fresh timestamp. `--refs` filters out + # the `refs/tags/X^{}` peeled-ref rows that annotated tags emit + # — without it, awk could capture e.g., `submit/foo^{}` and + # produce an invalid pseudo-tag. + EXISTING=$(git ls-remote --refs --tags origin \ + | awk -v sha="$SHA" '$1 == sha && $2 ~ /^refs\/tags\/submit\// { sub("refs/tags/", "", $2); print $2; exit }') + if [[ -n "$EXISTING" ]]; then + echo "tag=$EXISTING" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Short-SHA suffix prevents collisions when two pushes land + # in the same UTC second. + TAG="submit/$(date -u +%Y-%m-%dT%H-%M-%SZ)-$(echo "$SHA" | cut -c1-7)" + git config user.name "classroom50-bot" + git config user.email "bot@classroom50" + git tag "$TAG" "$SHA" + git push origin "refs/tags/$TAG" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + + # === Foreign-tag commit status === + # A pushed tag matched neither submit/* nor any configured + # submission_tags pattern (stale/hand-edited shim — the on.push.tags + # filter normally pre-empts this run entirely). Nothing tags or + # grades; say why. DISTINCT CONTEXT (classroom50/autograde-skipped), + # unlike the nothing-to-grade skips above: here the student's real + # work exists but was NOT graded, and a green classroom50/autograde + # would read as "graded successfully" to both humans and any client + # polling that context. The main context stays absent — accurate, + # since no grade will ever appear for this commit. Best-effort; + # never fails the job. + - name: Foreign-tag commit status + if: steps.read.outputs.foreign-tag-suppressed == 'true' + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + gh api "repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" \ + -f state="success" \ + -f context="classroom50/autograde-skipped" \ + -f description="tag is not a submission trigger — not graded" \ + -f target_url="$RUN_URL" >/dev/null 2>&1 || true + + # === Suppressed-push commit status === + # A tag-mode assignment got a branch push (stale every-push shim or a + # hand-edited trigger): nothing tags or grades. Same DISTINCT CONTEXT + # rationale as the foreign-tag status: ungraded work must never show + # green under classroom50/autograde. state stays success (a red X on + # every work-in-progress push would teach students to ignore + # failures); the separate context is what keeps graded and not-graded + # machine-distinguishable. Best-effort; never fails the job. + - name: Suppressed-push commit status + if: steps.read.outputs.branch-push-suppressed == 'true' + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + gh api "repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" \ + -f state="success" \ + -f context="classroom50/autograde-skipped" \ + -f description="tag-mode assignment — push not graded; run gh student submit" \ + -f target_url="$RUN_URL" >/dev/null 2>&1 || true + + # === Shim-update commit status === + # Parallel to the acceptance-skip status, for the [skip ci] backstop + # path: a shim-retrofit commit that still fired the workflow skips + # everything; say so explicitly. Best-effort; never fails the job. + - name: Shim-update commit status + if: steps.acceptance.outputs.is-shim-update == 'true' + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + gh api "repos/${GITHUB_REPOSITORY}/statuses/${GITHUB_SHA}" \ + -f state="success" \ + -f context="classroom50/autograde" \ + -f description="autograder trigger updated — nothing to grade" \ + -f target_url="$RUN_URL" >/dev/null 2>&1 || true + grade: needs: setup - # Skip only the acceptance commit. The no-autograder case still runs so the - # submission is RECORDED: runner.py synthesizes a vacuous-pass (0/0 success) - # result and the Release step publishes it, keeping the submission visible on - # the teacher dashboard (which reads submit/* releases). Language-toolchain - # setup is separately gated off no-autograder below, so this cheap path spends - # no minutes installing toolchains it won't use. - if: needs.setup.outputs.is-acceptance != 'true' + # Skip the acceptance commit, a shim-retrofit commit, a suppressed + # tag-mode branch push, and a suppressed foreign tag (set-latest needs + # grade, so it skips too). The no-autograder case still runs so the + # submission is RECORDED: runner.py synthesizes a vacuous-pass (0/0 + # success) result and the Release step publishes it, keeping the + # submission visible on the teacher dashboard (which reads submit/* + # releases). Language-toolchain setup is separately gated off + # no-autograder below, so this cheap path spends no minutes installing + # toolchains it won't use. + if: >- + needs.setup.outputs.is-acceptance != 'true' && + needs.setup.outputs.is-shim-update != 'true' && + needs.setup.outputs.branch-push-suppressed != 'true' && + needs.setup.outputs.foreign-tag-suppressed != 'true' runs-on: ${{ fromJSON(needs.setup.outputs.runs-on) }} container: ${{ fromJSON(needs.setup.outputs.container) }} timeout-minutes: 15 @@ -783,6 +991,9 @@ jobs: env: GH_TOKEN: ${{ github.token }} TAG: ${{ env.SUBMISSION_TAG }} + # The milestone tag that triggered this run ('' otherwise); noted in + # the Release title so the teacher sees WHICH milestone was graded. + TRIGGER_TAG: ${{ needs.setup.outputs.trigger-tag }} # Distinct name from the job-level RELEASE_ASSETS (a JSON array fed to # runner.py): this is the comma-joined list of accepted basenames the # runner emits post-staging. Different wire shape, different name, so a @@ -831,12 +1042,38 @@ jobs: # clobbered or new assets uploaded, so a re-run must delete and # recreate to attach the current result + extras atomically. Keep the # git tag (--cleanup-tag omitted) so the submission ref is unchanged. + # + # But an org ruleset enforcing immutable releases (same rule noted + # above for post-create uploads) ALSO rejects the delete itself with + # HTTP 422/403 — seen live on a teacher regrade 2026-08-07: the + # delete failed, this step went red, and set-latest was skipped even + # though grading PASSED and the commit status posted success. gh's + # exit codes can't reliably distinguish that ruleset rejection from + # a transient error, so re-check existence instead: if the delete + # failed and the release is STILL there, treat it as + # immutable-blocked — warn and keep the old release rather than + # failing the job over record-keeping GitHub forbids. If the delete + # failed but the release is GONE (a race removed it), proceed to + # create as usual. if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then - gh release delete "$TAG" --repo "$GITHUB_REPOSITORY" --yes + if ! DELETE_ERR="$(gh release delete "$TAG" --repo "$GITHUB_REPOSITORY" --yes 2>&1)"; then + if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo "$DELETE_ERR" + echo "::warning::the release at $TAG is immutable (org ruleset) and cannot be refreshed; the regraded score is in the commit status and the job summary, but gradebook collection keeps reading the OLD release's result.json for this submission (GitHub's rule, not ours)" + exit 0 + fi + # Release vanished between delete and re-check: surface why the + # delete complained, then create as if it had never existed. + echo "$DELETE_ERR" + fi + fi + TITLE="Submission $TAG" + if [[ -n "${TRIGGER_TAG:-}" ]]; then + TITLE="Submission $TAG (via $TRIGGER_TAG)" fi gh release create "$TAG" result.json ${EXTRA_ASSETS[@]+"${EXTRA_ASSETS[@]}"} \ --repo "$GITHUB_REPOSITORY" \ - --title "Submission $TAG" \ + --title "$TITLE" \ --notes-file release-body.md \ --latest=false @@ -903,19 +1140,29 @@ jobs: -f target_url="$RUN_URL" >/dev/null 2>&1 || true # === Move the "latest" pointer forward === - # Serialized across the whole repo so concurrent submissions can't - # race on the read-modify-write and leave an older release marked - # latest. The comparison is commit-time-based (not tag-name-based) - # so two pushes within the same UTC second still order correctly, - # and a non-submit/* release accidentally marked latest (e.g., a - # teacher upload) doesn't permanently block submissions from - # reclaiming latest. + # Latest = the most recent SUBMISSION EVENT, not the newest commit: + # every published submission release claims the badge, so grading an + # older commit on purpose (a milestone tag like phase1 re-pointed at + # earlier work, or a teacher regrade) makes THAT release latest. This + # matches the gradebook collector and the web submission views, which + # both order by release publish time — one definition everywhere. + # (An earlier commit-time comparator tried "newest code wins", but it + # read the just-published release back as the current latest, compared + # it against itself, and never acted — GitHub's default publish-order + # badge is what actually shipped, and it is the semantics we want.) + # Serialized per-repo so concurrent submissions can't interleave edits. set-latest: needs: [setup, grade] # Advance latest for any recorded submission — including the no-autograder # vacuous pass (status 'success'), so its release becomes the latest the - # dashboard surfaces. Gated only on a clean grade result with a real status. - if: needs.grade.result == 'success' && (needs.grade.outputs.status == 'success' || needs.grade.outputs.status == 'failure') + # dashboard surfaces. Explicit suppressed-push / foreign-tag / shim-update + # guards, defensive against a future change to grade's result semantics. + if: >- + needs.setup.outputs.branch-push-suppressed != 'true' && + needs.setup.outputs.foreign-tag-suppressed != 'true' && + needs.setup.outputs.is-shim-update != 'true' && + needs.grade.result == 'success' && + (needs.grade.outputs.status == 'success' || needs.grade.outputs.status == 'failure') runs-on: ubuntu-latest concurrency: group: classroom50-set-latest-${{ github.repository }} @@ -925,61 +1172,12 @@ jobs: env: GH_TOKEN: ${{ github.token }} TAG: ${{ needs.setup.outputs.submission-tag }} - SHA: ${{ github.sha }} run: | set -euo pipefail - # Locale-stable string compares (LC_COLLATE varies otherwise). - export LC_ALL=C - - CURRENT=$(gh release view --repo "$GITHUB_REPOSITORY" \ - --json tagName -q .tagName 2>/dev/null || echo "") - - # Empty CURRENT (no latest yet) or non-submit/* CURRENT (e.g., - # a teacher hand-uploaded a release marked latest) — claim - # latest unconditionally so a sticky non-submit release can't - # block future grading from being visible. - if [[ -z "$CURRENT" || "$CURRENT" != submit/* ]]; then - gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --latest=true - exit 0 - fi - - # Both are submit/* tags. Compare by commit time (the SHA - # the tag points at) — tag names tie within a UTC second - # and the short-SHA suffix isn't chronologically meaningful. - OBJ_TYPE=$(gh api "repos/$GITHUB_REPOSITORY/git/refs/tags/$CURRENT" \ - --jq .object.type 2>/dev/null || echo "") - CURRENT_REF=$(gh api "repos/$GITHUB_REPOSITORY/git/refs/tags/$CURRENT" \ - --jq .object.sha 2>/dev/null || echo "") - # Annotated tags resolve to a tag-object SHA — dereference - # it to the actual commit. - if [[ "$OBJ_TYPE" == "tag" && -n "$CURRENT_REF" ]]; then - CURRENT_REF=$(gh api "repos/$GITHUB_REPOSITORY/git/tags/$CURRENT_REF" \ - --jq .object.sha 2>/dev/null || echo "$CURRENT_REF") - fi - - if [[ -z "$CURRENT_REF" ]]; then - # Tag ref doesn't resolve (deleted, race, transient API). - # Fall back to claiming latest — same behavior as if the - # tag didn't exist. - gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --latest=true - exit 0 - fi - - CURRENT_TIME=$(gh api "repos/$GITHUB_REPOSITORY/commits/$CURRENT_REF" \ - --jq .commit.committer.date 2>/dev/null || echo "") - OUR_TIME=$(gh api "repos/$GITHUB_REPOSITORY/commits/$SHA" \ - --jq .commit.committer.date 2>/dev/null || echo "") - - if [[ -z "$CURRENT_TIME" || -z "$OUR_TIME" ]]; then - # Couldn't read both commit times. Conservative: leave the - # current latest alone rather than risk a wrong flip. - echo "::warning::could not resolve commit times (CURRENT='$CURRENT_TIME' OUR='$OUR_TIME'); leaving latest pointer unchanged" - exit 0 - fi - - # ISO 8601 UTC timestamps from GitHub are 'YYYY-MM-DDTHH:MM:SSZ' - # — pure ASCII digits + a couple of fixed separators, so - # lexical compare under LC_ALL=C is chronologically correct. - if [[ "$OUR_TIME" > "$CURRENT_TIME" ]]; then - gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --latest=true + # Best-effort: the badge is cosmetic and GitHub's own default + # (most recently published) already matches this rule, so a + # rejected edit must never fail the pipeline. Known rejection: + # an org ruleset enforcing immutable releases 422s any edit. + if ! gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --latest=true; then + echo "::warning::could not mark $TAG as the latest release (immutable-release ruleset?); GitHub's default latest (most recently published) matches anyway" fi diff --git a/cli/gh-teacher/skeleton_tests/test_assignments_schema.py b/cli/gh-teacher/skeleton_tests/test_assignments_schema.py index 2f639e0c..63dfa54d 100644 --- a/cli/gh-teacher/skeleton_tests/test_assignments_schema.py +++ b/cli/gh-teacher/skeleton_tests/test_assignments_schema.py @@ -100,6 +100,24 @@ def test_allowed_files_accepted(self): assert _errors(_manifest(_entry(allowed_files=["*", "!hello.py"]))) == [] assert _errors(_manifest(_entry(allowed_files=[]))) == [] + def test_submission_mode_accepted(self): + # Both enum values are legal: writers omit every-push (the wire + # default) but other clients may write it explicitly, and readers + # must accept it. Absent is covered by test_minimal_manifest. + assert _errors(_manifest(_entry(submission_mode="tag"))) == [] + assert _errors(_manifest(_entry(submission_mode="every-push"))) == [] + + def test_submission_tags_accepted(self): + # Milestone tag patterns: literal names and the supported glob + # characters. Absent is covered by test_minimal_manifest. + assert _errors(_manifest(_entry(submission_tags=["phase1", "phase2"]))) == [] + assert ( + _errors( + _manifest(_entry(submission_tags=["v*", "release-[0-9]", "a/b?", "m.**"])) + ) + == [] + ) + def test_container_with_ubuntu_runs_on(self): entry = _entry(runtime={"container": {"image": "x"}, "runs-on": "ubuntu-22.04"}) assert _errors(_manifest(entry)) == [] @@ -348,6 +366,33 @@ def test_wrong_schema_sentinel(self): def test_locked_must_be_boolean(self): assert _errors(_manifest(_entry(locked="yes"))) != [] + @pytest.mark.parametrize( + "submission_mode", ["Tag", "every_push", "push", "", None, True] + ) + def test_bad_submission_mode(self, submission_mode): + # Only the two enum values are legal; the Go parser normalizes + # nothing here (unlike autograder), so clients must write exact + # values. Mirrors contract.SubmissionModes. + assert _errors(_manifest(_entry(submission_mode=submission_mode))) != [] + + @pytest.mark.parametrize( + "submission_tags", + [ + ["!v*"], # excludes are deferred/rejected + ['ta"g'], # quote breaks the YAML tags line + ["has space"], # whitespace forbidden + [""], # empty pattern + ["a", "a"], # uniqueItems + "phase1", # must be an array, not a bare string + [f"t{i}" for i in range(21)], # over maxItems (20) + ], + ) + def test_bad_submission_tags(self, submission_tags): + # Mirrors gh-teacher's ValidateSubmissionTags and the web + # validateSubmissionTags — the charset is restricted because the + # patterns are spliced into the shim's quoted-YAML tags line. + assert _errors(_manifest(_entry(submission_tags=submission_tags))) != [] + class TestEmptyRepo: def _bare_entry(self, **overrides): @@ -392,6 +437,21 @@ def test_empty_repo_rejects_allowed_files(self): def test_empty_repo_rejects_pass_threshold(self): assert _errors(_manifest(self._bare_entry(pass_threshold=70))) != [] + def test_empty_repo_rejects_submission_mode(self): + # A bare repo carries no autograde shim, so there is no trigger for + # submission_mode to configure. Mirrors Go's + # validateEmptyRepoExclusions. + assert _errors(_manifest(self._bare_entry(submission_mode="tag"))) != [] + assert ( + _errors(_manifest(self._bare_entry(submission_mode="every-push"))) != [] + ) + + def test_empty_repo_rejects_submission_tags(self): + # Same shim-less reasoning as submission_mode. + assert ( + _errors(_manifest(self._bare_entry(submission_tags=["phase1"]))) != [] + ) + def _release_assets_errors(value): return _errors(_manifest(_entry(release_assets=value))) diff --git a/cli/gh-teacher/skeleton_tests/test_regrade_repos.py b/cli/gh-teacher/skeleton_tests/test_regrade_repos.py index b59b17df..a41136fb 100644 --- a/cli/gh-teacher/skeleton_tests/test_regrade_repos.py +++ b/cli/gh-teacher/skeleton_tests/test_regrade_repos.py @@ -74,7 +74,7 @@ def fake_rerun(api_url, org, repo, token, run_id): rr, "main_head_sha", lambda *a, **k: (_ for _ in ()).throw(AssertionError()) ) - assert rr.regrade_repo("https://api", "cs50", "cs50-hello-alice", "tok") == "rerun" + assert rr.regrade_repo("https://api", "cs50", "cs50-hello-alice", "tok", False) == "rerun" assert calls["run_id"] == 4242 @@ -90,7 +90,7 @@ def fake_create(api_url, org, repo, token, tag, sha): monkeypatch.setattr(rr, "create_tag_ref", fake_create) - assert rr.regrade_repo("https://api", "cs50", "cs50-hello-alice", "tok") == "tagged" + assert rr.regrade_repo("https://api", "cs50", "cs50-hello-alice", "tok", False) == "tagged" assert calls["sha"] == "deadbeefcafe" assert calls["tag"].startswith("submit/") @@ -106,7 +106,7 @@ def boom(*a, **k): raise AssertionError("create_tag_ref called despite an existing tag") monkeypatch.setattr(rr, "create_tag_ref", boom) - assert rr.regrade_repo("https://api", "cs50", "cs50-hello-alice", "tok") == "tagged" + assert rr.regrade_repo("https://api", "cs50", "cs50-hello-alice", "tok", False) == "tagged" def test_regrade_repo_missing_repo(monkeypatch): @@ -115,7 +115,7 @@ def test_regrade_repo_missing_repo(monkeypatch): monkeypatch.setattr( rr, "existing_submit_tag_at", lambda *a, **k: (_ for _ in ()).throw(AssertionError()) ) - assert rr.regrade_repo("https://api", "cs50", "cs50-hello-alice", "tok") == "missing" + assert rr.regrade_repo("https://api", "cs50", "cs50-hello-alice", "tok", False) == "missing" def test_latest_autograde_run_id_parses_newest(monkeypatch): @@ -362,10 +362,10 @@ def test_main_returns_1_on_missing_required_input(monkeypatch, missing): def test_main_all_success_returns_0(monkeypatch): _set_main_env(monkeypatch) - monkeypatch.setattr(rr, "load_roster", lambda *a, **k: ["alice", "bob"]) + monkeypatch.setattr(rr, "load_roster", lambda *a, **k: (["alice", "bob"], {"slug": "hello"})) outcomes = {"cs50-hello-alice": "rerun", "cs50-hello-bob": "tagged"} - def fake_regrade(api_url, org, repo, token): + def fake_regrade(api_url, org, repo, token, tag_mode, submission_tags=None): return outcomes[repo] monkeypatch.setattr(rr, "regrade_repo", fake_regrade) @@ -374,10 +374,10 @@ def fake_regrade(api_url, org, repo, token): def test_main_hard_http_error_aborts_immediately(monkeypatch): _set_main_env(monkeypatch) - monkeypatch.setattr(rr, "load_roster", lambda *a, **k: ["alice", "bob"]) + monkeypatch.setattr(rr, "load_roster", lambda *a, **k: (["alice", "bob"], {"slug": "hello"})) seen = [] - def fake_regrade(api_url, org, repo, token): + def fake_regrade(api_url, org, repo, token, tag_mode, submission_tags=None): seen.append(repo) raise _http_error(403) # hard error -> abort the whole run @@ -389,9 +389,9 @@ def fake_regrade(api_url, org, repo, token): def test_main_soft_http_error_skips_and_exits_1(monkeypatch): _set_main_env(monkeypatch) - monkeypatch.setattr(rr, "load_roster", lambda *a, **k: ["alice", "bob"]) + monkeypatch.setattr(rr, "load_roster", lambda *a, **k: (["alice", "bob"], {"slug": "hello"})) - def fake_regrade(api_url, org, repo, token): + def fake_regrade(api_url, org, repo, token, tag_mode, submission_tags=None): if repo.endswith("alice"): raise _http_error(500) # non-hard -> warn-and-skip, continue return "rerun" @@ -462,7 +462,7 @@ def test_main_empty_team_warns_and_exits_0(monkeypatch, capsys): # but emit an empty-team warning so a green 0-repo run isn't mistaken for a # real regrade. _set_main_env(monkeypatch) - monkeypatch.setattr(rr, "load_roster", lambda *a, **k: []) + monkeypatch.setattr(rr, "load_roster", lambda *a, **k: ([], {"slug": "hello"})) monkeypatch.setattr( rr, "regrade_repo", lambda *a, **k: (_ for _ in ()).throw(AssertionError()) ) @@ -473,9 +473,9 @@ def test_main_empty_team_warns_and_exits_0(monkeypatch, capsys): def test_main_skiprepo_counts_as_skipped_not_failed(monkeypatch): _set_main_env(monkeypatch) - monkeypatch.setattr(rr, "load_roster", lambda *a, **k: ["alice", "bob"]) + monkeypatch.setattr(rr, "load_roster", lambda *a, **k: (["alice", "bob"], {"slug": "hello"})) - def fake_regrade(api_url, org, repo, token): + def fake_regrade(api_url, org, repo, token, tag_mode, submission_tags=None): if repo.endswith("alice"): raise rr._SkipRepo() # benign per-repo skip return "rerun" @@ -487,10 +487,10 @@ def fake_regrade(api_url, org, repo, token): def test_main_owner_filter_narrows_to_one(monkeypatch): _set_main_env(monkeypatch, OWNER_FILTER="Bob") # case-insensitive match - monkeypatch.setattr(rr, "load_roster", lambda *a, **k: ["alice", "bob"]) + monkeypatch.setattr(rr, "load_roster", lambda *a, **k: (["alice", "bob"], {"slug": "hello"})) seen = [] - def fake_regrade(api_url, org, repo, token): + def fake_regrade(api_url, org, repo, token, tag_mode, submission_tags=None): seen.append(repo) return "rerun" @@ -501,7 +501,7 @@ def fake_regrade(api_url, org, repo, token): def test_main_owner_filter_no_match_returns_1(monkeypatch): _set_main_env(monkeypatch, OWNER_FILTER="carol") - monkeypatch.setattr(rr, "load_roster", lambda *a, **k: ["alice", "bob"]) + monkeypatch.setattr(rr, "load_roster", lambda *a, **k: (["alice", "bob"], {"slug": "hello"})) monkeypatch.setattr( rr, "regrade_repo", lambda *a, **k: (_ for _ in ()).throw(AssertionError()) ) @@ -513,7 +513,7 @@ def test_main_logs_incremental_progress(monkeypatch, capsys): # fan-out emits a progress line every PROGRESS_EVERY repos (and on the last). monkeypatch.setattr(rr, "PROGRESS_EVERY", 2) _set_main_env(monkeypatch) - monkeypatch.setattr(rr, "load_roster", lambda *a, **k: ["a", "b", "c"]) + monkeypatch.setattr(rr, "load_roster", lambda *a, **k: (["a", "b", "c"], {"slug": "hello"})) monkeypatch.setattr(rr, "regrade_repo", lambda *a, **k: "rerun") assert rr.main() == 0 @@ -547,7 +547,7 @@ def _write_classroom(tmp_path: pathlib.Path, *, slug="hello", team=None): def test_load_roster_returns_team_members(monkeypatch, tmp_path): cdir = _write_classroom(tmp_path) monkeypatch.setattr(rr, "list_team_member_logins", lambda *a, **k: ["alice", "bob"]) - assert rr.load_roster(cdir, "hello", "https://api", "cs50org", "tok") == ["alice", "bob"] + assert rr.load_roster(cdir, "hello", "https://api", "cs50org", "tok")[0] == ["alice", "bob"] def test_load_roster_uses_persisted_team_slug(monkeypatch, tmp_path): @@ -561,7 +561,7 @@ def fake_members(api_url, org, team_slug, token): return ["alice"] monkeypatch.setattr(rr, "list_team_member_logins", fake_members) - assert rr.load_roster(cdir, "hello", "https://api", "cs50org", "tok") == ["alice"] + assert rr.load_roster(cdir, "hello", "https://api", "cs50org", "tok")[0] == ["alice"] assert seen["team_slug"] == "classroom50-cs-1" @@ -584,7 +584,7 @@ def test_load_roster_dedupes_case_insensitively(monkeypatch, tmp_path): rr, "list_team_member_logins", lambda *a, **k: ["Alice", "alice", "BOB"] ) # First-seen casing wins; the case-insensitive duplicate is dropped. - assert rr.load_roster(cdir, "hello", "https://api", "cs50org", "tok") == ["Alice", "BOB"] + assert rr.load_roster(cdir, "hello", "https://api", "cs50org", "tok")[0] == ["Alice", "BOB"] def test_load_roster_skips_malformed_login(monkeypatch, tmp_path): @@ -593,7 +593,7 @@ def test_load_roster_skips_malformed_login(monkeypatch, tmp_path): rr, "list_team_member_logins", lambda *a, **k: ["alice", "bad/name", "bob"] ) # A malformed login is skipped with a warning; valid members survive. - assert rr.load_roster(cdir, "hello", "https://api", "cs50org", "tok") == ["alice", "bob"] + assert rr.load_roster(cdir, "hello", "https://api", "cs50org", "tok")[0] == ["alice", "bob"] def test_load_roster_propagates_team_http_error(monkeypatch, tmp_path): @@ -840,7 +840,7 @@ def test_load_roster_empty_repo_false_proceeds(monkeypatch, tmp_path): ) ) monkeypatch.setattr(rr, "list_team_member_logins", lambda *a, **k: ["alice"]) - assert rr.load_roster(cdir, "hello", "https://api", "cs50org", "tok") == ["alice"] + assert rr.load_roster(cdir, "hello", "https://api", "cs50org", "tok")[0] == ["alice"] def test_main_empty_repo_assignment_is_successful_noop(monkeypatch, capsys): @@ -860,3 +860,197 @@ def raise_empty(*a, **k): out = capsys.readouterr().out assert "empty_repo" in out assert "nothing to regrade" in out + + +# Tag-mode regrade: suppressed branch runs are never replayed ------------------ + + +def test_is_tag_submission_mode_strict(): + # Strict equality mirroring Go's Entry.IsTagSubmissionMode: only the exact + # string "tag" counts; absent / every-push / junk fail open to every-push. + assert rr.is_tag_submission_mode({"submission_mode": "tag"}) is True + for entry in ({}, {"submission_mode": "every-push"}, {"submission_mode": "Tag"}, + {"submission_mode": True}, {"submission_mode": 1}): + assert rr.is_tag_submission_mode(entry) is False + + +def test_latest_autograde_run_id_tag_only_skips_branch_runs(monkeypatch): + # Newest-first: the newest run is a suppressed branch push; the first + # submit/* run behind it is the real latest submission. + body = json.dumps({"workflow_runs": [ + {"id": 1, "head_branch": "main"}, + {"id": 2, "head_branch": "submit/2026-01-01T00-00-00Z-abcdefg"}, + {"id": 3, "head_branch": "main"}, + {"id": 4, "head_branch": "submit/2025-12-31T00-00-00Z-1234567"}, + ]}).encode("utf-8") + monkeypatch.setattr(rr, "_http_get", lambda *a, **k: body) + assert rr.latest_autograde_run_id("https://api", "cs50", "repo", "tok", tag_only=True) == 2 + + +def test_latest_autograde_run_id_tag_only_none_when_only_branch_runs(monkeypatch): + body = json.dumps({"workflow_runs": [ + {"id": 1, "head_branch": "main"}, + {"id": 2, "head_branch": "main"}, + ]}).encode("utf-8") + monkeypatch.setattr(rr, "_http_get", lambda *a, **k: body) + assert rr.latest_autograde_run_id("https://api", "cs50", "repo", "tok", tag_only=True) is None + + +def test_latest_autograde_run_id_tag_only_tolerates_missing_head_branch(monkeypatch): + # Malformed rows (absent / None / non-str head_branch) are skipped, not + # crashed on; a later well-formed submit run still matches. + body = json.dumps({"workflow_runs": [ + {"id": 1}, + {"id": 2, "head_branch": None}, + {"id": 3, "head_branch": 7}, + {"id": 4, "head_branch": "submit/2026-01-01T00-00-00Z-abcdefg"}, + ]}).encode("utf-8") + monkeypatch.setattr(rr, "_http_get", lambda *a, **k: body) + assert rr.latest_autograde_run_id("https://api", "cs50", "repo", "tok", tag_only=True) == 4 + # All-malformed: no crash, no match. + body = json.dumps({"workflow_runs": [{"id": 1}, {"id": 2, "head_branch": None}]}).encode("utf-8") + monkeypatch.setattr(rr, "_http_get", lambda *a, **k: body) + assert rr.latest_autograde_run_id("https://api", "cs50", "repo", "tok", tag_only=True) is None + + +def test_latest_autograde_run_id_request_shape_by_mode(monkeypatch): + # tag_only scans one 100-run page; the default keeps today's per_page=1 + # request byte-identical (pins that every-push behavior didn't change). + seen = {} + + def fake_get(url, token, *, accept, _retries=3): + seen["url"] = url + return json.dumps({"workflow_runs": []}).encode("utf-8") + + monkeypatch.setattr(rr, "_http_get", fake_get) + rr.latest_autograde_run_id("https://api", "cs50", "repo", "tok", tag_only=True) + assert "per_page=100" in seen["url"] + rr.latest_autograde_run_id("https://api", "cs50", "repo", "tok") + assert "per_page=1" in seen["url"] and "per_page=100" not in seen["url"] + + +def test_regrade_repo_tag_mode_reruns_latest_tag_run(monkeypatch): + calls = {} + + def fake_latest(api_url, org, repo, token, *, tag_only=False, submission_tags=None): + assert tag_only is True + return 4242 + + def fake_rerun(api_url, org, repo, token, run_id): + calls["run_id"] = run_id + + monkeypatch.setattr(rr, "latest_autograde_run_id", fake_latest) + monkeypatch.setattr(rr, "rerun_workflow_run", fake_rerun) + monkeypatch.setattr( + rr, "main_head_sha", lambda *a, **k: (_ for _ in ()).throw(AssertionError()) + ) + assert rr.regrade_repo("https://api", "cs50", "cs50-hello-alice", "tok", True) == "rerun" + assert calls["run_id"] == 4242 + + +def test_regrade_repo_tag_mode_suppressed_only_falls_to_tag_fallback(monkeypatch): + # A repo whose runs are ALL suppressed branch pushes must not replay one + # (it would re-suppress and grade nothing) — the tag-at-HEAD fallback + # fires a REAL tag run instead. + calls = {} + + monkeypatch.setattr(rr, "latest_autograde_run_id", lambda *a, **k: None) + monkeypatch.setattr( + rr, "rerun_workflow_run", lambda *a, **k: (_ for _ in ()).throw(AssertionError()) + ) + monkeypatch.setattr(rr, "main_head_sha", lambda *a, **k: "deadbeefcafe") + monkeypatch.setattr(rr, "existing_submit_tag_at", lambda *a, **k: None) + + def fake_create(api_url, org, repo, token, tag, sha): + calls["tag"] = tag + + monkeypatch.setattr(rr, "create_tag_ref", fake_create) + assert rr.regrade_repo("https://api", "cs50", "cs50-hello-alice", "tok", True) == "tagged" + assert calls["tag"].startswith("submit/") + + +def test_regrade_repo_every_push_uses_default_lookup(monkeypatch): + # Every-push must keep today's selection exactly: tag_only stays False. + def fake_latest(api_url, org, repo, token, *, tag_only=False, submission_tags=None): + assert tag_only is False + return 7 + + monkeypatch.setattr(rr, "latest_autograde_run_id", fake_latest) + monkeypatch.setattr(rr, "rerun_workflow_run", lambda *a, **k: None) + assert rr.regrade_repo("https://api", "cs50", "cs50-hello-alice", "tok", False) == "rerun" + + +def test_main_threads_tag_mode_from_manifest(monkeypatch, capsys): + # main() derives tag_mode from the loaded entry and passes it to every + # regrade_repo call. + for entry, want in ( + ({"slug": "hello", "submission_mode": "tag"}, True), + ({"slug": "hello"}, False), + ): + seen = {} + _set_main_env(monkeypatch) + monkeypatch.setattr(rr, "load_roster", lambda *a, **k: (["alice"], entry)) + + def fake_regrade(api_url, org, repo, token, tag_mode, submission_tags=None): + seen["tag_mode"] = tag_mode + return "rerun" + + monkeypatch.setattr(rr, "regrade_repo", fake_regrade) + assert rr.main() == 0 + capsys.readouterr() + assert seen["tag_mode"] is want + + +# Submission-tag matcher: shared-fixture parity -------------------------------- + + +def test_matches_submission_tag_shared_fixture_parity(): + # The Python half of the matcher lockstep: the same golden cases Go's + # contract.MatchesSubmissionTag and the web matchesSubmissionTag assert. + # The pattern strings go verbatim into the shim's on.push.tags, so every + # evaluator must agree on what fires. + fixture = ( + pathlib.Path(__file__).resolve().parents[2] + / "shared" + / "testdata" + / "submission_tag_match_cases.json" + ) + doc = json.loads(fixture.read_text()) + cases = doc["cases"] + assert cases, "shared fixture has no cases; did the file move?" + for case in cases: + got = rr.matches_submission_tag(case["patterns"], case["tag"]) + assert got is case["matches"], ( + f"matches_submission_tag({case['patterns']}, {case['tag']!r}) = {got}, " + f"want {case['matches']}" + ) + + +def test_matches_submission_tag_fails_closed_on_bad_pattern(): + # A reversed character-class range fails re.compile: the pattern matches + # nothing (never everything), and a later valid pattern still works. + assert rr.matches_submission_tag(["[z-a]"], "m") is False + assert rr.matches_submission_tag(["[z-a]", "good"], "good") is True + + +def test_latest_autograde_run_id_tag_only_accepts_milestone_runs(monkeypatch): + # A milestone-tag run (head_branch = the teacher-named tag) is a REAL + # graded run — its record lives at the canonical submit/* tag the runner + # minted — so tag-mode regrade may replay it. Foreign tags still skip. + body = json.dumps({"workflow_runs": [ + {"id": 1, "head_branch": "main"}, + {"id": 2, "head_branch": "v1.0"}, + {"id": 3, "head_branch": "phase1"}, + {"id": 4, "head_branch": "submit/2025-12-31T00-00-00Z-1234567"}, + ]}).encode("utf-8") + monkeypatch.setattr(rr, "_http_get", lambda *a, **k: body) + got = rr.latest_autograde_run_id( + "https://api", "cs50", "repo", "tok", + tag_only=True, submission_tags=["phase1", "phase2"], + ) + assert got == 3 + # Without patterns, only the canonical namespace counts. + got = rr.latest_autograde_run_id( + "https://api", "cs50", "repo", "tok", tag_only=True, + ) + assert got == 4 diff --git a/cli/shared/contract/contract.go b/cli/shared/contract/contract.go index 1cc0553b..cab78993 100644 --- a/cli/shared/contract/contract.go +++ b/cli/shared/contract/contract.go @@ -17,6 +17,7 @@ package contract import ( "fmt" "strings" + "time" ) const ( @@ -44,6 +45,22 @@ const ( ModeIndividual = "individual" ModeGroup = "group" + // SubmissionModeEveryPush and SubmissionModeTag are the assignment + // submission_mode values: every-push = the shim grades every push to the + // default branch plus submit/* tags (the wire default — writers omit it); + // tag = the shim grades ONLY submit/* tag pushes, which the submit clients + // create. Mirrored in the assignments-v1 schema enum and the web + // SUBMISSION_MODES; pinned by contract_test.go and the schema-parity tests. + SubmissionModeEveryPush = "every-push" + SubmissionModeTag = "tag" + + // SubmitTagPrefix is the tag namespace that marks a grading submission: + // only submit/* tag Releases count as submissions everywhere (runner, + // collect_scores.py SUBMIT_TAG_PREFIX, regrade_repos.py, the web + // SUBMISSION_TAG_PREFIX). Hand-mirrored with NO compile-time link — keep + // byte-identical; contract_test.go pins the Go half. + SubmitTagPrefix = "submit/" + // Repo collaborator permission levels, GitHub's low-to-high ladder. Used // for an assignment's optional student_permission (the access the enrolled // student gets on their own repo at accept time) and mirrored in the web @@ -281,6 +298,45 @@ func FeedbackLabelForMode(mode string) (name, color string) { return "Individual Assignment", "0E8A16" } +// SubmissionModes is every valid assignments.json submission_mode value. +// Single-sources the allow-list; the schema enum in assignments-v1.schema.json +// and the web SUBMISSION_MODES mirror it (parity-tested on both sides). +var SubmissionModes = []string{SubmissionModeEveryPush, SubmissionModeTag} + +// IsValidSubmissionMode reports whether m is one of the SubmissionModes. +func IsValidSubmissionMode(m string) bool { + for _, allowed := range SubmissionModes { + if m == allowed { + return true + } + } + return false +} + +// BuildSubmitTag is the canonical submission tag for a commit: +// submit/-. Byte-format-identical with the runner's +// tag-minting step in autograde-runner.yaml and regrade_repos.py's +// build_submit_tag — the short-SHA suffix prevents collisions when two +// submissions land in the same UTC second. +func BuildSubmitTag(now time.Time, sha string) string { + short := sha + if len(short) > 7 { + short = short[:7] + } + return SubmitTagPrefix + now.UTC().Format("2006-01-02T15-04-05Z") + "-" + short +} + +// ShimUpdateCommitMessage is the commit message for a submission-mode shim +// retrofit in a student repo. The `[skip ci]` body line is load-bearing: a +// tag→every-push retrofit commit carries the restored push trigger, and +// without it the shim would grade the retrofit commit itself (pushes with a +// user OAuth token DO fire workflows). The runner's shim-update detection is +// the backstop. Hand-mirrored with NO compile-time link in the web GUI +// (web/src/domain/assignments/submissionTrigger.ts) — keep byte-identical. +func ShimUpdateCommitMessage(mode string) string { + return PrefixCommit("Update autograder trigger to "+mode+" (submission-mode)") + "\n\n[skip ci]" +} + // RepoPermissions is GitHub's collaborator permission ladder, low to high. // Single-sources the assignment student_permission allow-list; the web mirror // is RepoAccessPermission and the schema enum in assignments-v1.schema.json. diff --git a/cli/shared/contract/contract_test.go b/cli/shared/contract/contract_test.go index 3a5503cc..7069743d 100644 --- a/cli/shared/contract/contract_test.go +++ b/cli/shared/contract/contract_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "strings" "testing" + "time" ) // TestContractLiterals is a change-detector pinning each cross-binary constant @@ -29,6 +30,17 @@ func TestContractLiterals(t *testing.T) { {"DefaultAutograderName", DefaultAutograderName, "default"}, {"ModeIndividual", ModeIndividual, "individual"}, {"ModeGroup", ModeGroup, "group"}, + // SubmissionMode values are mirrored, with NO compile-time link, in the + // assignments-v1 schema enum (parity-pinned by + // TestSubmissionModeEnumParity in gh-teacher), the web SUBMISSION_MODES, + // and the runner's inline validator. Update every copy in lockstep. + {"SubmissionModeEveryPush", SubmissionModeEveryPush, "every-push"}, + {"SubmissionModeTag", SubmissionModeTag, "tag"}, + // SubmitTagPrefix is mirrored, with NO compile-time link, in runner.py / + // collect_scores.py / regrade_repos.py (SUBMIT_TAG_PREFIX), the + // autograde-runner.yaml tag step, and the web SUBMISSION_TAG_PREFIX + // (releaseRunReads.ts). Update every copy in lockstep on change. + {"SubmitTagPrefix", SubmitTagPrefix, "submit/"}, {"ResultFilename", ResultFilename, "result.json"}, {"ReleaseBodyFilename", ReleaseBodyFilename, "release-body.md"}, // RosterFilename is mirrored, with NO compile-time link, in the web GUI @@ -210,6 +222,39 @@ func TestRequiredOAuthScopes(t *testing.T) { } } +// TestBuildSubmitTag pins the submit/- format +// byte-identical with the runner's tag-minting step (autograde-runner.yaml) +// and regrade_repos.py's build_submit_tag — no compile-time link. +func TestBuildSubmitTag(t *testing.T) { + at := time.Date(2026, 8, 3, 14, 30, 5, 0, time.UTC) + got := BuildSubmitTag(at, "abcdef0123456789") + want := "submit/2026-08-03T14-30-05Z-abcdef0" + if got != want { + t.Errorf("BuildSubmitTag = %q, want %q", got, want) + } + // Non-UTC input must normalize to UTC (the runner uses `date -u`). + est := time.FixedZone("EST", -5*60*60) + if got := BuildSubmitTag(at.In(est), "abcdef0123456789"); got != want { + t.Errorf("BuildSubmitTag(non-UTC) = %q, want %q", got, want) + } + // A short SHA is used as-is rather than sliced out of range. + if got := BuildSubmitTag(at, "abc"); got != "submit/2026-08-03T14-30-05Z-abc" { + t.Errorf("BuildSubmitTag(short sha) = %q", got) + } +} + +// TestShimUpdateCommitMessage pins the retrofit commit message: the [skip ci] +// body line is load-bearing (a tag→every-push retrofit commit carries the +// restored push trigger and must not grade itself), and the web GUI mirrors +// the whole string with NO compile-time link. +func TestShimUpdateCommitMessage(t *testing.T) { + got := ShimUpdateCommitMessage(SubmissionModeTag) + want := "[Classroom 50] Update autograder trigger to tag (submission-mode)\n\n[skip ci]" + if got != want { + t.Errorf("ShimUpdateCommitMessage = %q, want %q", got, want) + } +} + // TestPrefixCommit pins the canonical "[Classroom 50] " shape so the // separator (a single space) can't drift from the web GUI's prefixCommit. func TestPrefixCommit(t *testing.T) { diff --git a/cli/shared/contract/submissiontags.go b/cli/shared/contract/submissiontags.go new file mode 100644 index 00000000..d519b2b5 --- /dev/null +++ b/cli/shared/contract/submissiontags.go @@ -0,0 +1,160 @@ +package contract + +import ( + "regexp" + "strings" +) + +// Submission-tag pattern matching: the supported subset of GitHub Actions +// tag-filter patterns (the strings assignments.json's submission_tags carries +// are rendered verbatim into the shim's `on.push.tags`, so this matcher and +// GitHub's own filter evaluation MUST agree on what fires): +// +// - a literal name matches exactly (case-sensitive) +// - `*` matches zero or more characters, NOT crossing `/` +// - `**` matches zero or more characters, crossing `/` +// - `?` matches zero or one of the PRECEDING character +// - `+` matches one or more of the preceding character +// - `[abc]` / `[a-z]` character classes +// +// `!` negation and tags-ignore are deferred (writers reject them; see +// ValidateSubmissionTags in gh-teacher's assignment package). +// +// Hand-mirrored with NO compile-time link in the web +// (web/src/util/submissionTags.ts) and Python (autograde-runner.yaml read +// step, regrade_repos.py) — all four pinned to identical output by the shared +// golden fixture cli/shared/testdata/submission_tag_match_cases.json. + +// SubmissionTagsCap is the maximum number of milestone tag patterns. +const SubmissionTagsCap = 20 + +// SubmissionTagCharsetRE is the per-pattern charset: literal-name characters +// plus the glob metacharacters GitHub Actions tag filters support +// (* ? + [ ]). Deliberately excludes quotes, backslashes, whitespace, and +// control characters — patterns are spliced into the shim's quoted-YAML +// `tags:` line, so anything that could break out of that string context is +// rejected. Mirrors the schema items.pattern byte-for-byte (modulo JSON +// escaping) and the web SUBMISSION_TAG_PATTERN_RE — parity-pinned by +// TestSubmissionTagsSchemaParity and the web vitest. +var SubmissionTagCharsetRE = regexp.MustCompile(`^[A-Za-z0-9._/*?+\[\]-]+$`) + +// stackedQuantifierRE rejects a leading `?`/`+` (no literal to repeat) and — +// the load-bearing case — a `+` immediately following another quantifier +// (`v*+`, `a++`, `x?+`). Those translate to POSSESSIVE quantifiers, which +// Python 3.11+ compiles (and matches!) while Go RE2 and JS reject — the one +// construct where the four matcher copies would otherwise diverge. JSON +// Schema keeps the charset-only items.pattern (Go RE2 can't compile the +// lookahead an ECMA equivalent would need), so this rule is validator- and +// matcher-enforced. Keep the literal in lockstep with the web and Python +// copies (util/submissionTags.ts, autograde-runner.yaml read step, +// regrade_repos.py). +var stackedQuantifierRE = regexp.MustCompile(`^[?+]|[*?+]\+`) + +// IsSafeSubmissionTagPattern reports whether one milestone tag pattern is +// safe to render into the shim's quoted-YAML tags line AND compiles +// identically across the four matcher implementations. The write-side +// validators reject unsafe patterns with a friendly message; the render and +// match paths fail closed on them (defense-in-depth against a hand-edited +// published manifest that bypassed write validation). +func IsSafeSubmissionTagPattern(pattern string) bool { + return SubmissionTagCharsetRE.MatchString(pattern) && !stackedQuantifierRE.MatchString(pattern) +} + +// ShimTagsList renders the YAML flow sequence for the shim's +// `on.push.tags:` line: the configured milestone patterns (if any) UNION the +// always-on canonical submit/* namespace. No patterns -> `"submit/*"` alone, +// byte-identical to the pre-submission_tags shim. +// +// FAIL CLOSED: patterns are charset-validated at write time, but this is the +// render chokepoint for workflow files committed into student repos, so the +// manifest is not trusted here — any unsafe pattern (or an over-cap list) +// drops the ENTIRE milestone set and renders the canonical `"submit/*"` +// alone. A partially-filtered list would silently grade a different tag set +// than the teacher configured; all-or-nothing keeps the failure visible. +// Byte-format mirrored in the web's safeShimTagPatterns/shimTagsList +// (web/src/util/submissionTags.ts and its two call sites) — keep identical. +func ShimTagsList(patterns []string) string { + safe := len(patterns) <= SubmissionTagsCap + if safe { + for _, p := range patterns { + if !IsSafeSubmissionTagPattern(p) { + safe = false + break + } + } + } + if !safe { + patterns = nil + } + parts := make([]string, 0, len(patterns)+1) + for _, p := range patterns { + parts = append(parts, `"`+p+`"`) + } + parts = append(parts, `"`+SubmitTagPrefix+`*"`) + return strings.Join(parts, ", ") +} + +// MatchesSubmissionTag reports whether tag matches ANY of the patterns. An +// empty pattern list matches nothing. +func MatchesSubmissionTag(patterns []string, tag string) bool { + for _, pattern := range patterns { + // Unsafe patterns match nothing — fail closed. The explicit + // IsSafeSubmissionTagPattern gate (not just the compile error) is + // load-bearing in the PYTHON mirror, where a stacked quantifier like + // `v*+` compiles as a possessive quantifier and would MATCH; all + // four copies carry the same guard so they cannot diverge. + if !IsSafeSubmissionTagPattern(pattern) { + continue + } + re, err := compileTagPattern(pattern) + if err != nil { + continue + } + if re.MatchString(tag) { + return true + } + } + return false +} + +// compileTagPattern translates one Actions tag-filter pattern into an +// anchored regexp. Character-by-character so `.` and other regex +// metacharacters in the pattern stay literal. +func compileTagPattern(pattern string) (*regexp.Regexp, error) { + var b strings.Builder + b.WriteString("^") + runes := []rune(pattern) + for i := 0; i < len(runes); i++ { + switch r := runes[i]; r { + case '*': + if i+1 < len(runes) && runes[i+1] == '*' { + b.WriteString(".*") // ** crosses / + i++ + } else { + b.WriteString("[^/]*") // * stops at / + } + case '?': + // Zero-or-one of the preceding element: regexp `?` after the + // previous literal/class already emitted. + b.WriteString("?") + case '+': + b.WriteString("+") + case '[': + // Pass a character class through verbatim up to the closing ]. + j := i + 1 + for j < len(runes) && runes[j] != ']' { + j++ + } + if j < len(runes) { + b.WriteString(string(runes[i : j+1])) + i = j + } else { + b.WriteString(regexp.QuoteMeta(string(r))) // unclosed [ is literal + } + default: + b.WriteString(regexp.QuoteMeta(string(r))) + } + } + b.WriteString("$") + return regexp.Compile(b.String()) +} diff --git a/cli/shared/contract/submissiontags_test.go b/cli/shared/contract/submissiontags_test.go new file mode 100644 index 00000000..c1c8b7df --- /dev/null +++ b/cli/shared/contract/submissiontags_test.go @@ -0,0 +1,100 @@ +package contract + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" +) + +// TestMatchesSubmissionTag_SharedFixtureParity runs the shared golden cases so +// the Go matcher and its web/Python mirrors stay in lockstep — same role the +// control_path_cases.json fixture plays for the allowed_files keep-set. +func TestMatchesSubmissionTag_SharedFixtureParity(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "testdata", "submission_tag_match_cases.json")) + if err != nil { + t.Fatalf("read shared fixture: %v", err) + } + var fixture struct { + Cases []struct { + Patterns []string `json:"patterns"` + Tag string `json:"tag"` + Matches bool `json:"matches"` + } `json:"cases"` + } + if err := json.Unmarshal(raw, &fixture); err != nil { + t.Fatalf("parse shared fixture: %v", err) + } + if len(fixture.Cases) == 0 { + t.Fatal("shared fixture has no cases; did the file move?") + } + for _, c := range fixture.Cases { + if got := MatchesSubmissionTag(c.Patterns, c.Tag); got != c.Matches { + t.Errorf("MatchesSubmissionTag(%v, %q) = %v, want %v", c.Patterns, c.Tag, got, c.Matches) + } + } +} + +// TestMatchesSubmissionTag_UncompilablePatternFailsClosed pins the fail-closed +// contract: a pattern the translator can't compile matches nothing (never +// everything) — writer validation should prevent these, but the runner +// re-checks hand-edited manifests. +func TestMatchesSubmissionTag_UncompilablePatternFailsClosed(t *testing.T) { + // A reversed character-class range ([z-a]) fails regexp compilation. + if MatchesSubmissionTag([]string{"[z-a]"}, "m") { + t.Error("uncompilable pattern must match nothing") + } + // A bad pattern must not poison a later good one. + if !MatchesSubmissionTag([]string{"[z-a]", "good"}, "good") { + t.Error("a later valid pattern must still match") + } +} + +// TestShimTagsList_FailsClosedOnUnsafePatterns pins the render chokepoint: +// the shim writers (gh-student accept, gh-teacher retrofit, and the web +// mirrors) consume the PUBLISHED manifest, so an unsafe pattern that +// bypassed write-time validation must drop the ENTIRE milestone set — the +// rendered tags line falls back to the canonical `"submit/*"` alone rather +// than splicing hostile or divergent content into a student repo's workflow. +func TestShimTagsList_FailsClosedOnUnsafePatterns(t *testing.T) { + if got := ShimTagsList(nil); got != `"submit/*"` { + t.Errorf("ShimTagsList(nil) = %q, want the bare canonical entry", got) + } + if got := ShimTagsList([]string{"phase1", "v*"}); got != `"phase1", "v*", "submit/*"` { + t.Errorf("ShimTagsList(safe) = %q", got) + } + for _, patterns := range [][]string{ + {"v*+"}, // stacked quantifier (Python-divergent) + {"phase1", "a++"}, // one bad drops ALL (all-or-nothing) + {`ta"g`}, // quote would break the YAML string + {"has space"}, // charset + {"+lead"}, // leading quantifier + } { + if got := ShimTagsList(patterns); got != `"submit/*"` { + t.Errorf("ShimTagsList(%v) = %q, want fail-closed %q", patterns, got, `"submit/*"`) + } + } + over := make([]string, SubmissionTagsCap+1) + for i := range over { + over[i] = "t" + string(rune('a'+i%26)) + } + // Over-cap also fails closed (dups in `over` are irrelevant here). + if got := ShimTagsList(over); got != `"submit/*"` { + t.Errorf("ShimTagsList(over-cap) = %q, want fail-closed", got) + } +} + +// TestMatchesSubmissionTag_StackedQuantifiersFailClosed pins the guard the +// fixture also covers, plus the property that motivates it: these patterns +// must match NOTHING here (and in the web/Python mirrors) even though +// Python's regex dialect would happily compile them possessively. +func TestMatchesSubmissionTag_StackedQuantifiersFailClosed(t *testing.T) { + for _, pattern := range []string{"v*+", "a++", "x?+", "m**+", "+lead", "?lead"} { + if MatchesSubmissionTag([]string{pattern}, "v1") { + t.Errorf("MatchesSubmissionTag([%q], \"v1\") = true, want fail-closed false", pattern) + } + if !IsSafeSubmissionTagPattern("phase1") || IsSafeSubmissionTagPattern(pattern) { + t.Errorf("IsSafeSubmissionTagPattern(%q) should be false (and phase1 true)", pattern) + } + } +} diff --git a/cli/shared/testdata/submission_tag_match_cases.json b/cli/shared/testdata/submission_tag_match_cases.json new file mode 100644 index 00000000..750c906b --- /dev/null +++ b/cli/shared/testdata/submission_tag_match_cases.json @@ -0,0 +1,63 @@ +{ + "_comment": "Shared golden fixture pinning the submission-tag matcher to identical output across the four parallel implementations: Go contract.MatchesSubmissionTag (cli/shared/contract/submissiontags.go), TS matchesSubmissionTag (web/src/util/submissionTags.ts), and the Python matches_submission_tag copies (autograde-runner.yaml read step + regrade_repos.py). Semantics are the supported subset of GitHub Actions tag-filter patterns, because the same strings are rendered into the shim's on.push.tags — the matcher and GitHub must agree on what fires: literal names match exactly; '*' matches any run (including empty) NOT crossing '/'; '**' matches across '/'; '?' matches zero or one OF THE PRECEDING character (regex-like, NOT glob single-char: 'ab?' matches 'a' and 'ab', never 'ax'); '+' matches one or more of the preceding character; '[abc]'/'[a-z]' character classes. Matching is case-sensitive. An empty pattern list matches nothing ('!' negation and tags-ignore are deferred). Each case is (patterns, tag -> matches), asserted by all four sides so the copies can't drift. Charset-unsafe patterns (anything outside the schema charset [A-Za-z0-9._/*?+\\[\\]-], e.g. quotes or spaces — reachable only via a hand-edited manifest) fail closed and match nothing, mirroring the Go/TS IsSafeSubmissionTagPattern guard.", + "_stacked_quantifiers": "The v*+/a++/x?+/m**+/+lead/?lead cases pin the fail-closed guard for stacked or leading quantifiers: without it Python 3.11+ compiles them as POSSESSIVE quantifiers (v*+ would match v1) while Go RE2 and JS reject them — the one construct where the four copies diverge. All four implementations short-circuit these patterns to no-match before compiling.", + "cases": [ + { "patterns": ["submission"], "tag": "submission", "matches": true }, + { "patterns": ["submission"], "tag": "submissions", "matches": false }, + { "patterns": ["submission"], "tag": "Submission", "matches": false }, + { "patterns": ["submission"], "tag": "resubmission", "matches": false }, + { "patterns": ["phase1", "phase2", "complete"], "tag": "phase2", "matches": true }, + { "patterns": ["phase1", "phase2", "complete"], "tag": "phase3", "matches": false }, + { "patterns": ["phase1", "phase2", "complete"], "tag": "complete", "matches": true }, + { "patterns": [], "tag": "submission", "matches": false }, + { "patterns": ["submit/*"], "tag": "submit/2026-08-03T14-30-05Z-abcdef0", "matches": true }, + { "patterns": ["submit/*"], "tag": "submit/", "matches": true }, + { "patterns": ["submit/*"], "tag": "submit", "matches": false }, + { "patterns": ["submit/*"], "tag": "submit/a/b", "matches": false }, + { "patterns": ["v*"], "tag": "v1", "matches": true }, + { "patterns": ["v*"], "tag": "v1.2.3", "matches": true }, + { "patterns": ["v*"], "tag": "v", "matches": true }, + { "patterns": ["v*"], "tag": "version/1", "matches": false }, + { "patterns": ["v**"], "tag": "version/1", "matches": true }, + { "patterns": ["**"], "tag": "anything/at/all", "matches": true }, + { "patterns": ["*"], "tag": "no-slash", "matches": true }, + { "patterns": ["*"], "tag": "has/slash", "matches": false }, + { "patterns": ["v?"], "tag": "v", "matches": true }, + { "patterns": ["v?"], "tag": "v1", "matches": false }, + { "patterns": ["ab?"], "tag": "a", "matches": true }, + { "patterns": ["ab?"], "tag": "ab", "matches": true }, + { "patterns": ["ab?"], "tag": "ax", "matches": false }, + { "patterns": ["v+"], "tag": "v", "matches": true }, + { "patterns": ["vv+"], "tag": "v", "matches": false }, + { "patterns": ["vv+"], "tag": "vv", "matches": true }, + { "patterns": ["vv+"], "tag": "vvvv", "matches": true }, + { "patterns": ["release-[0-9]"], "tag": "release-3", "matches": true }, + { "patterns": ["release-[0-9]"], "tag": "release-x", "matches": false }, + { "patterns": ["phase[12]"], "tag": "phase1", "matches": true }, + { "patterns": ["phase[12]"], "tag": "phase3", "matches": false }, + { "patterns": ["milestone.*"], "tag": "milestone.alpha", "matches": true }, + { "patterns": ["milestone.*"], "tag": "milestoneXalpha", "matches": false }, + { "patterns": ["a*c"], "tag": "abc", "matches": true }, + { "patterns": ["a*c"], "tag": "ac", "matches": true }, + { "patterns": ["a*c"], "tag": "a/c", "matches": false }, + { "patterns": ["phase1", "submit/*"], "tag": "submit/2026-01-01T00-00-00Z-1234567", "matches": true }, + { "patterns": ["phase1", "submit/*"], "tag": "phase1", "matches": true }, + { "patterns": ["phase1", "submit/*"], "tag": "final", "matches": false }, + { "patterns": ["v*+"], "tag": "v1", "matches": false }, + { "patterns": ["v*+"], "tag": "v", "matches": false }, + { "patterns": ["a++"], "tag": "aa", "matches": false }, + { "patterns": ["x?+"], "tag": "x", "matches": false }, + { "patterns": ["m**+"], "tag": "m/anything", "matches": false }, + { "patterns": ["+lead"], "tag": "lead", "matches": false }, + { "patterns": ["?lead"], "tag": "lead", "matches": false }, + { "patterns": ["v*+", "phase1"], "tag": "phase1", "matches": true }, + { "patterns": ["ta\"g"], "tag": "ta\"g", "matches": false }, + { "patterns": ["pha se"], "tag": "pha se", "matches": false }, + { "patterns": ["v\"*"], "tag": "v1", "matches": false }, + { "patterns": ["ta\"g", "good"], "tag": "good", "matches": true }, + { "patterns": ["pha[se"], "tag": "pha[se", "matches": true }, + { "patterns": ["pha[se"], "tag": "phase", "matches": false }, + { "patterns": ["[z-a]"], "tag": "m", "matches": false }, + { "patterns": ["[z-a]", "good"], "tag": "good", "matches": true } + ] +} diff --git a/schemas/assignments-v1.schema.json b/schemas/assignments-v1.schema.json index 0a129121..9acf1191 100644 --- a/schemas/assignments-v1.schema.json +++ b/schemas/assignments-v1.schema.json @@ -175,6 +175,21 @@ "enum": ["pull", "triage", "push", "maintain", "admin"], "description": "The collaborator role the enrolled student is granted on their OWN assignment repo at accept time (the old GitHub Classroom \"grant students admin access\" checkbox, generalized to GitHub's full ladder). ABSENT means the built-in default: `push` for `individual`, `admin` for `group`. This is accept-time provisioning only — it governs what NEW accepters get; it does not retroactively change repos already accepted (teachers adjust those with the gradebook's per-repo/bulk access controls). Group coherence: a group founder must hold at least `admin` to add teammates via `gh student invite`, so for `mode: group` any configured value below `admin` is clamped up to `admin` by every accept client — prefer omitting it (or writing `admin`) for group assignments. Caution: on a PRIVATE in-org repo, `admin` also lets the student change repo visibility (e.g. make it public) and settings." }, + "submission_mode": { + "enum": ["every-push", "tag"], + "description": "When the autograder fires. ABSENT or \"every-push\" (the default): the student-repo shim triggers on every push to the default branch AND on submit/* tag pushes (the runner mints the submit/- tag itself on branch pushes). \"tag\": the shim triggers ONLY on submit/* tag pushes — `gh student submit` pushes the tag after the branch commit (a hand-pushed submit/* tag works too), so a plain `git push` does not grade (the primary Actions-cost lever for large cohorts). Applied to the shim at ACCEPT TIME; changing it later requires retrofitting each existing student repo's shim (`gh teacher assignment submission-mode` or the gradebook bulk action) and students must re-pull afterward. Default-autograder assignments only for the retrofit — teacher-authored (non-default) shims are never rewritten, though the field still governs client tag-pushing. Meaningless for empty_repo (no shim exists) — mutually exclusive (see the conditional at the assignment level). On the wire it is collapsed like feedback_pr: writers omit \"every-push\" (the CLI normalizes it away), so readers must treat an absent field as \"every-push\"; accept an explicit \"every-push\" too. See submission_tags for teacher-named milestone tags; additive room remains for future tags-ignore/branch-pattern fields." + }, + "submission_tags": { + "type": "array", + "maxItems": 20, + "uniqueItems": true, + "items": { + "type": "string", + "minLength": 1, + "pattern": "^[A-Za-z0-9._/*?+\\[\\]-]+$" + }, + "description": "Teacher-named milestone tag patterns that ALSO trigger grading — e.g. [\"phase1\", \"phase2\", \"complete\"], or globs like \"v*\". A student pushing a matching tag (`git tag phase1 && git push origin phase1`) gets that commit graded. TRIGGERS, not records: the runner still mints/reuses the canonical submit/- tag at the triggering commit and publishes the Release there (immutable one-release-per-submission history; the Release title notes the triggering tag), so collection, regrade, download, and web reads are unaffected. The canonical submit/* namespace ALWAYS triggers too — the generated shim trigger is the union of these patterns and submit/*, and the field is orthogonal to submission_mode (an every-push assignment can also name milestone tags; tag mode drops only the branch trigger). Patterns are the supported subset of GitHub Actions tag filters (literal, `*` not crossing `/`, `**` crossing, `?`, `+`, `[...]`; NO `!` excludes — rejected by writers) rendered verbatim into the shim's on.push.tags, hence the restricted charset (no quotes/whitespace — the values are spliced into a quoted-YAML line). One rule this schema's items.pattern cannot express (it must stay Go-RE2-compilable): validators also reject a LEADING `?`/`+` and a `+` stacked on another quantifier (e.g. `v*+`) — those compile as possessive quantifiers in Python but are errors in Go/JS, so every matcher fails closed on them; clients must apply the same rule (Go ValidateSubmissionTags / web validateSubmissionTags). Like submission_mode, the trigger is baked into each repo's shim at ACCEPT TIME; changing patterns later requires the same shim retrofit, and students must re-pull. Empty/absent means no milestone tags (exactly today's behavior). Caution: a broad glob like \"v*\" grades every matching release tag a student pushes. Mutually exclusive with empty_repo (no shim exists)." + }, "repo_features": { "type": "object", "additionalProperties": false, @@ -202,6 +217,8 @@ { "not": { "required": ["allowed_files"] } }, { "not": { "required": ["release_assets"] } }, { "not": { "required": ["pass_threshold"] } }, + { "not": { "required": ["submission_mode"] } }, + { "not": { "required": ["submission_tags"] } }, { "properties": { "feedback_pr": { "const": false } } } ] } diff --git a/web/src/components/SubmitUpload.tsx b/web/src/components/SubmitUpload.tsx index 9e38140a..d9a800ed 100644 --- a/web/src/components/SubmitUpload.tsx +++ b/web/src/components/SubmitUpload.tsx @@ -12,6 +12,7 @@ import { import { useSafeSubmit } from "@/hooks/useSafeSubmit" import { useToast } from "@/context/notifications/NotificationProvider" import { useSubmitAssignment } from "@/hooks/mutations/useSubmitAssignment" +import type { SubmissionMode } from "@/types/classroom" import { normalizeRepoPath, isReservedUploadPath, @@ -35,11 +36,15 @@ export function SubmitUpload({ org, repo, assignment, + submissionMode, onSubmitted, }: { org: string repo: string assignment: string + // The assignment's submission_mode from assignments.json; "tag" makes the + // upload also push the submit/* tag that triggers grading. + submissionMode?: SubmissionMode // Fired after a successful submit so the page can nudge the "grading runs in // the background" affordance. onSubmitted?: () => void @@ -47,7 +52,12 @@ export function SubmitUpload({ const { t } = useTranslation() const { notify } = useToast() const run = useSafeSubmit() - const mutation = useSubmitAssignment({ org, repo, assignment }) + const mutation = useSubmitAssignment({ + org, + repo, + assignment, + submissionMode, + }) const [open, setOpen] = useState(false) const [picked, setPicked] = useState([]) diff --git a/web/src/components/modals/BulkSubmissionTriggerModal.tsx b/web/src/components/modals/BulkSubmissionTriggerModal.tsx new file mode 100644 index 00000000..cab6b61d --- /dev/null +++ b/web/src/components/modals/BulkSubmissionTriggerModal.tsx @@ -0,0 +1,341 @@ +import { useEffect, useId, useMemo, useRef, useState } from "react" +import { useTranslation } from "react-i18next" +import type { TFunction } from "i18next" +import { GitBranch } from "lucide-react" + +import { Alert, Button, Modal } from "@/components/ui" +import { Spinner } from "@/components/Spinner" +import { + BulkResultSection, + type BulkPhase, + type BulkProgress, + type BulkResultView, +} from "@/components/bulk/resultView" +import { + updateShimSubmissionMode, + type ShimUpdateOutcome, +} from "@/domain/assignments/submissionTrigger" +import { useGitHubClient } from "@/context/github/GitHubProvider" +import { REPO_WRITE_CONCURRENCY } from "@/github-core/queries" +import { mapWithConcurrency } from "@/util/concurrency" +import { studentRepoName } from "@/util/studentRepo" +import { getName } from "@/util/students" +import { describeGitHubApiFailure } from "@/components/modals/collaboratorHelpers" +import { GitHubAPIError } from "@/github-core/errors" +import type { Student, SubmissionMode } from "@/types/classroom" + +type BulkSubmissionTriggerModalProps = { + open: boolean + onClose: () => void + org: string + classroom: string + assignment: string + // The assignment's STORED submission_mode — the source of truth the retrofit + // reconciles repos toward. Mode-setting itself lives on the settings form. + submissionMode: SubmissionMode + // The assignment's STORED milestone submission_tags (if any); the rewrite + // reconciles each shim's tags line to their union with submit/*. + submissionTags?: string[] + // Accepted students; each login is the owner segment of their own repo. + owners: string[] + students?: Student[] +} + +const describeFailure = (reason: unknown, t: TFunction): string | undefined => { + const shared = describeGitHubApiFailure(reason, t) + if (shared) return shared + if (reason instanceof GitHubAPIError) { + return t("components.modals.groupCollaborators.failure.httpStatus", { + status: reason.status, + }) + } + return reason instanceof Error ? reason.message : undefined +} + +// Whole-assignment autograding-trigger retrofit: rewrite each accepted +// student repo's shim to match the assignment's stored submission_mode, in +// one bounded fan-out. The way to reconcile existing repos after the mode is +// changed on the settings page, since the shim is baked at accept time. +// Sibling of BulkRepoFeaturesModal. +export function BulkSubmissionTriggerModal({ + open, + onClose, + org, + classroom, + assignment, + submissionMode, + submissionTags, + owners, + students = [], +}: BulkSubmissionTriggerModalProps) { + const titleId = useId() + const { t } = useTranslation() + const client = useGitHubClient() + const runningRef = useRef(false) + const mountedRef = useRef(true) + useEffect(() => { + mountedRef.current = true + return () => { + mountedRef.current = false + runningRef.current = false + } + }, []) + + const [phase, setPhase] = useState("idle") + const [progress, setProgress] = useState({ + processed: 0, + total: 0, + message: "", + }) + const [result, setResult] = useState(null) + + useEffect(() => { + if (!open) { + runningRef.current = false + setPhase("idle") + setResult(null) + setProgress({ processed: 0, total: 0, message: "" }) + } + }, [open]) + + const total = owners.length + const displayFor = (login: string) => getName(login, students) || login + const modeLabel = t( + submissionMode === "tag" + ? "assignments.form.submissionMode.choices.tag" + : "assignments.form.submissionMode.choices.everyPush", + ) + + type Outcome = { owner: string } & ( + | { status: ShimUpdateOutcome["status"]; detail?: string } + | { status: "deferred" } + | { status: "failed"; detail?: string } + ) + + const run = async () => { + if (runningRef.current || total === 0) return + runningRef.current = true + setPhase("working") + setResult(null) + let processed = 0 + setProgress({ processed: 0, total, message: "" }) + // Stop launching NEW writes on a secondary-rate-limit or a confirmed + // missing workflow scope (every remaining repo would fail identically). + let rateLimited = false + let missingScope = false + + const outcomes = await mapWithConcurrency( + owners, + // Each iteration is a 3-step git-data WRITE (tree + commit + ref) into + // a different repo — GitHub's secondary-rate-limit guidance is to avoid + // concurrent content writes (the CLI retrofit loop is serial for the + // same reason), unlike the sibling bulk modals' single PATCH/PUT calls, + // which safely share REPO_READ_CONCURRENCY. + REPO_WRITE_CONCURRENCY, + async (owner): Promise => { + if (rateLimited || missingScope || !mountedRef.current) { + processed += 1 + if (mountedRef.current) { + setProgress({ processed, total, message: displayFor(owner) }) + } + return { owner, status: "deferred" } + } + const repo = studentRepoName(classroom, assignment, owner) + try { + const outcome = await updateShimSubmissionMode({ + client, + org, + repo, + mode: submissionMode, + tags: submissionTags, + }) + if (outcome.status === "missingWorkflowScope") { + missingScope = true + return { owner, status: "missingWorkflowScope" } + } + if (outcome.status === "unrecognized") { + return { owner, status: "unrecognized", detail: outcome.reason } + } + return { owner, status: outcome.status } + } catch (err) { + if (err instanceof GitHubAPIError && err.isRateLimited) { + rateLimited = true + return { owner, status: "deferred" } + } + return { owner, status: "failed", detail: describeFailure(err, t) } + } finally { + processed += 1 + if (mountedRef.current) { + setProgress({ processed, total, message: displayFor(owner) }) + } + } + }, + ) + + if (!mountedRef.current) { + runningRef.current = false + return + } + + const updated = outcomes.filter((o) => o.status === "updated") + const current = outcomes.filter((o) => o.status === "current") + const notAccepted = outcomes.filter((o) => o.status === "notAccepted") + const unrecognized = outcomes.filter((o) => o.status === "unrecognized") + const deferred = outcomes.filter((o) => o.status === "deferred") + const scope = outcomes.filter((o) => o.status === "missingWorkflowScope") + const failed = outcomes.filter((o) => o.status === "failed") + + const section = ( + titleKey: string, + rows: Outcome[], + detailFallback?: string, + ) => + rows.length + ? [ + { + title: t(titleKey, { count: rows.length }), + rows: rows.map((o) => ({ + key: o.owner, + label: displayFor(o.owner), + detail: + ("detail" in o ? o.detail : undefined) ?? detailFallback, + })), + }, + ] + : [] + + setResult({ + headline: t("submissions.bulkTrigger.resultHeadline", { + updated: updated.length, + current: current.length, + total, + }), + sections: [ + ...section( + "submissions.bulkTrigger.scopeSection", + scope, + t("submissions.bulkTrigger.scopeDetail"), + ), + ...section("submissions.bulkTrigger.failedSection", failed), + ...section("submissions.bulkTrigger.unrecognizedSection", unrecognized), + ...section( + "submissions.bulkTrigger.deferredSection", + deferred, + t("submissions.bulkTrigger.deferredDetail"), + ), + ...section( + "submissions.bulkTrigger.notAcceptedSection", + notAccepted, + t("submissions.bulkTrigger.notAcceptedDetail"), + ), + ], + }) + setPhase( + failed.length || scope.length || deferred.length ? "error" : "complete", + ) + runningRef.current = false + } + + const busy = phase === "working" + const pct = useMemo( + () => + progress.total > 0 + ? Math.round((progress.processed / progress.total) * 100) + : 0, + [progress], + ) + + return ( + +
+
+
+
+

+ {t("submissions.bulkTrigger.title")} +

+

+ {t("submissions.bulkTrigger.subtitle", { + count: total, + mode: modeLabel, + })} +

+
+
+ + {phase === "idle" && ( +
+ {total === 0 ? ( + + {t("submissions.bulkTrigger.noRepos")} + + ) : ( + + {t("submissions.bulkTrigger.warning", { count: total })} + + )} +
+ )} + + {busy && ( +
+ + +

+ {t("submissions.bulkTrigger.progress", { + processed: progress.processed, + total: progress.total, + })} +

+
+ )} + + {(phase === "complete" || phase === "error") && result && ( +
+ + {result.headline} + + + {t("submissions.bulkTrigger.repullReminder")} + + {result.sections.map((section) => ( + + ))} +
+ )} + +
+ + {phase === "idle" && total > 0 && ( + + )} +
+ + ) +} + +export default BulkSubmissionTriggerModal diff --git a/web/src/domain/assignments.test.ts b/web/src/domain/assignments.test.ts index e90dd011..bcb57ce0 100644 --- a/web/src/domain/assignments.test.ts +++ b/web/src/domain/assignments.test.ts @@ -24,12 +24,14 @@ import { // Not on the @/domain/assignments barrel (the wrapper is internal scaffolding), // so reach the module directly rather than widening the public surface. import { withAcceptStep } from "./assignments/accessPrimitives" +import { defaultAutograderWorkflow } from "./assignments/autograderYaml" import { extractAssignments } from "@/github-core/queries" import { localizedError, localizedMessageOf } from "@/types/localizedMessage" import type { GitHubClient } from "@/github-core/client" import { GitHubAPIError } from "@/github-core/errors" import type { Assignment } from "@/types/classroom" -import { REPO_PERMISSIONS } from "@/types/classroom" +import { REPO_PERMISSIONS, SUBMISSION_MODES } from "@/types/classroom" +import type { SubmissionMode } from "@/types/classroom" const fullSource: Assignment = { slug: "hw1", @@ -915,6 +917,7 @@ describe("editAssignment (preserved-entry integration)", () => { [{ allowed_files: "*.py" }, /restrict allowed files/], [{ release_assets: "report.pdf" }, /release/], [{ pass_threshold: 70 }, /passing threshold/], + [{ submission_mode: "tag" }, /no autograde shim/], ] for (const [overrides, want] of cases) { const { client } = makeBareClient(bareEntry) @@ -924,6 +927,34 @@ describe("editAssignment (preserved-entry integration)", () => { } }) + // The write path's submission_mode branches (buildAssignmentEntry is not + // exported — assert through editAssignment, like the sibling tests above): + // "tag" lands in the entry; the wire default (explicit or absent) is + // omitted, mirroring the CLI's omitempty collapse; junk is rejected before + // a file the CLI would refuse to parse can be written. + it("writes submission_mode tag and omits the every-push wire default", async () => { + for (const [input, want] of [ + ["tag", "tag"], + ["every-push", undefined], + [undefined, undefined], + ] as const) { + const { client, committedContent } = makeClient() + await editAssignment(client, editInput({ submission_mode: input })) + const written = JSON.parse(committedContent()) as { + assignments: Assignment[] + } + const edited = written.assignments.find((a) => a.slug === SLUG)! + expect(edited.submission_mode).toBe(want) + } + }) + + it("rejects an out-of-enum submission_mode before writing", async () => { + const { client } = makeClient() + await expect( + editAssignment(client, editInput({ submission_mode: "on-demand" })), + ).rejects.toThrow(/submission_mode: must be one of every-push, tag/) + }) + // Route-table client like makeClient(), but seeded with a caller-supplied // existing entry (the empty_repo tests need a bare one). function makeBareClient(entry: Assignment): { @@ -2531,6 +2562,30 @@ describe("REPO_PERMISSIONS parity with assignments-v1 schema", () => { }) }) +// The web half of the submission_mode enum lockstep guard: SUBMISSION_MODES +// must equal the schema's submission_mode enum (the declared source of truth). +// The Go half (contract.SubmissionModes vs the same enum) is pinned by +// TestSubmissionModeEnumParity; the runner's inline validator carries a +// by-value copy. +describe("SUBMISSION_MODES parity with assignments-v1 schema", () => { + const schemaPath = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + "../../../schemas/assignments-v1.schema.json", + ) + const schema = JSON.parse(readFileSync(schemaPath, "utf-8")) as { + $defs: { + assignment: { + properties: { submission_mode: { enum: string[] } } + } + } + } + + it("matches the schema submission_mode enum exactly and in order", () => { + const schemaEnum = schema.$defs.assignment.properties.submission_mode.enum + expect(schemaEnum).toEqual([...SUBMISSION_MODES]) + }) +}) + describe("addFounderCollaborator — self grant (PUT only, no read-back)", () => { const owner = "cs50" const repo = "cs50-fall-2026-hello-alice" @@ -2744,6 +2799,48 @@ describe("resolveAutograderWorkflow default shim branch templating", () => { }), ).resolves.toContain('branches: ["main"]') }) + + it("tag mode drops the branch trigger and keeps only submit/* tags", async () => { + const yaml = await resolveAutograderWorkflow({ + org: "cs50", + classroom: "cs101", + autograder: "default", + branch: "main", + configBranch: "main", + submissionMode: "tag", + }) + expect(yaml).not.toContain("branches:") + expect(yaml).toContain('tags: ["submit/*"]') + expect(yaml).toContain( + 'uses: "cs50/classroom50/.github/workflows/autograde-runner.yaml@main"', + ) + // Exactly one line removed: tag mode equals every-push minus the branch + // trigger line. Mirrors the CLI's TestRenderEmbeddedShim_TagMode. + const everyPush = defaultAutograderWorkflow("cs50", "main", "main") + expect(yaml).toBe(everyPush.replace(' branches: ["main"]\n', "")) + }) + + it("every-push output is byte-identical for absent/explicit/junk modes", () => { + // Introducing submission_mode must change nothing for existing + // assignments: only the exact value "tag" alters the render. + const base = defaultAutograderWorkflow("cs50", "main", "main") + expect(defaultAutograderWorkflow("cs50", "main", "main", undefined)).toBe( + base, + ) + expect( + defaultAutograderWorkflow("cs50", "main", "main", "every-push"), + ).toBe(base) + // Junk is unrepresentable in the SubmissionMode union, so cast to pin the + // runtime contract: anything that isn't exactly "tag" renders every-push. + expect( + defaultAutograderWorkflow( + "cs50", + "main", + "main", + "junk" as SubmissionMode, + ), + ).toBe(base) + }) }) describe("createAssignmentRepo (bare / empty_repo)", () => { @@ -3502,3 +3599,74 @@ describe("setAssignmentLock", () => { expect(result.templateAccessWarning).toContain("tmpl") }) }) + +describe("defaultAutograderWorkflow — milestone submission_tags", () => { + it("widens the tags trigger to the union with submit/*", () => { + const yaml = defaultAutograderWorkflow("cs50", "main", "main", undefined, [ + "phase1", + "v*", + ]) + expect(yaml).toContain('tags: ["phase1", "v*", "submit/*"]') + expect(yaml).toContain('branches: ["main"]') + }) + + it("tag mode + patterns drops branches and widens tags", () => { + const yaml = defaultAutograderWorkflow("cs50", "main", "main", "tag", [ + "phase1", + ]) + expect(yaml).not.toContain("branches:") + expect(yaml).toContain('tags: ["phase1", "submit/*"]') + }) + + it("no patterns renders byte-identical output (empty and undefined)", () => { + const base = defaultAutograderWorkflow("cs50", "main", "main") + expect( + defaultAutograderWorkflow("cs50", "main", "main", undefined, []), + ).toBe(base) + expect( + defaultAutograderWorkflow("cs50", "main", "main", undefined, undefined), + ).toBe(base) + }) +}) + +// CLI-vs-web shim trigger parity: both accept clients must render the SAME +// on: block for the same inputs — the retrofit rewriters (Go shimTriggerBlock +// and the TS SHIM_TRIGGER_BLOCK) do line surgery on this exact shape, so a +// drift on either side would make one client's shims "unrecognized" to the +// retrofit. The CLI side is pinned against the embed by +// TestShimTagsTriggerLine_MatchesEmbed / TestShimBranchTriggerLine_MatchesEmbed; +// this pins the web render against the embed file itself. +describe("web shim trigger block parity with the CLI embed", () => { + const embedUrl = new URL( + "../../../cli/gh-student/embed/autograde-shim.yaml", + import.meta.url, + ) + const embed = readFileSync(fileURLToPath(embedUrl), "utf-8") + + function triggerBlock(yaml: string): string { + const match = + /^on:\n {2}push:\n(?: {4}branches: \[[^\n]*\]\n)?(?: {4}tags: \[[^\n]*\]\n)/m.exec( + yaml, + ) + if (!match) throw new Error(`no trigger block in:\n${yaml}`) + return match[0] + } + + it("default render matches the embed's trigger block (branch substituted)", () => { + const embedBlock = triggerBlock(embed).replace("{{BRANCH}}", "main") + expect(triggerBlock(defaultAutograderWorkflow("o", "main", "main"))).toBe( + embedBlock, + ) + }) + + it("tags render matches the embed's block with only the tags line widened", () => { + const embedBlock = triggerBlock(embed) + .replace("{{BRANCH}}", "main") + .replace('tags: ["submit/*"]', 'tags: ["phase1", "submit/*"]') + expect( + triggerBlock( + defaultAutograderWorkflow("o", "main", "main", undefined, ["phase1"]), + ), + ).toBe(embedBlock) + }) +}) diff --git a/web/src/domain/assignments/accept.ts b/web/src/domain/assignments/accept.ts index 48aa196d..502d1f64 100644 --- a/web/src/domain/assignments/accept.ts +++ b/web/src/domain/assignments/accept.ts @@ -651,6 +651,8 @@ export async function acceptAssignment(params: { // Preliminary branch; the default shim is re-rendered post-create // with the assignment repo's actual default branch (below). branch: sourceBranch || "main", + submissionMode: assignment.submission_mode, + submissionTags: assignment.submission_tags, }), ) if (isEmptyRepo) { @@ -812,9 +814,21 @@ export async function acceptAssignment(params: { org, resolvedBranch, ) - autogradeYaml = defaultAutograderWorkflow(org, resolvedBranch, configBranch) + autogradeYaml = defaultAutograderWorkflow( + org, + resolvedBranch, + configBranch, + assignment.submission_mode, + assignment.submission_tags, + ) rerenderShim = (branch: string) => - defaultAutograderWorkflow(org, branch, configBranch) + defaultAutograderWorkflow( + org, + branch, + configBranch, + assignment.submission_mode, + assignment.submission_tags, + ) } if (created.kind === "already-accepted") { diff --git a/web/src/domain/assignments/autograderYaml.ts b/web/src/domain/assignments/autograderYaml.ts index 407e91b1..e2eebddf 100644 --- a/web/src/domain/assignments/autograderYaml.ts +++ b/web/src/domain/assignments/autograderYaml.ts @@ -1,7 +1,9 @@ import { CONFIG_REPO, DEFAULT_BRANCH } from "@/util/configRepo" import { classroomPagesSegment } from "@/util/secret" +import { safeShimTagPatterns } from "@/util/submissionTags" import { fetchTextWithFriendlyErrors } from "../queries/assignments" import { localizedError } from "@/types/localizedMessage" +import type { SubmissionMode } from "@/types/classroom" export function createClassroom50Yaml(params: { classroom: string @@ -84,17 +86,45 @@ function pagesAutograderUrl(params: { return `https://${org}.github.io/${CONFIG_REPO}/${segment}/autograders/${name}.yaml` } +// The shim's on.push.tags flow sequence: the teacher's milestone patterns (if +// any) UNION the always-on canonical submit/* namespace. No patterns -> +// `"submit/*"` alone, byte-identical to the pre-submission_tags shim. +// Byte-format mirror of Go contract.ShimTagsList — keep identical. FAIL +// CLOSED: this renders a workflow file into a student repo from the +// PUBLISHED (hand-editable) manifest, so unsafe patterns drop the whole +// milestone set rather than trusting write-time validation +// (safeShimTagPatterns has the full rationale). +function shimTagsList(submissionTags?: string[]): string { + return [...safeShimTagPatterns(submissionTags), "submit/*"] + .map((p) => `"${p}"`) + .join(", ") +} + export function defaultAutograderWorkflow( org: string, branch: string, configBranch: string, + submissionMode?: SubmissionMode, + submissionTags?: string[], ) { + // Tag mode drops ONLY the branches: line, so the shim fires exclusively on + // submission-tag pushes (the submit flows create the tag; a hand-pushed + // submit/* tag works too). Every other value — undefined, an explicit + // "every-push", anything unvalidated — renders the identical bytes as + // before submission_mode existed. Milestone submission_tags widen the tags + // line to their union with submit/* and are orthogonal to the mode. + // Mirrors the CLI's renderEmbeddedShim. + const tagsLine = ` tags: [${shimTagsList(submissionTags)}]` + const pushTriggers = + submissionMode === "tag" + ? tagsLine + : ` branches: ["${branch}"] +${tagsLine}` return `name: Autograde on: push: - branches: ["${branch}"] - tags: ["submit/*"] +${pushTriggers} jobs: grade: @@ -125,13 +155,31 @@ export async function resolveAutograderWorkflow(params: { // built-in default shim; teacher-authored autograders are branch-agnostic. branch?: string configBranch?: string + // The assignment's submission_mode; "tag" drops the branch-push trigger. + // Only applies to the default shim — teacher-authored autograders own their + // triggers and are never rewritten. + submissionMode?: SubmissionMode + // The assignment's milestone submission_tags; rendered into the tags + // trigger as their union with submit/*. Default-shim only, like the mode. + submissionTags?: string[] }): Promise { - const { org, classroom, autograder, secret, branch, configBranch } = params + const { + org, + classroom, + autograder, + secret, + branch, + configBranch, + submissionMode, + submissionTags, + } = params if (isDefaultAutograder(autograder)) { return defaultAutograderWorkflow( org, branch || DEFAULT_BRANCH, configBranch || DEFAULT_BRANCH, + submissionMode, + submissionTags, ) } // Narrowed: isDefaultAutograder returns true for undefined/"default", so a diff --git a/web/src/domain/assignments/createEdit.ts b/web/src/domain/assignments/createEdit.ts index d8dae297..0a47036b 100644 --- a/web/src/domain/assignments/createEdit.ts +++ b/web/src/domain/assignments/createEdit.ts @@ -6,6 +6,7 @@ import { PASS_THRESHOLD_MAX, PASS_THRESHOLD_MIN, REPO_PERMISSIONS, + SUBMISSION_MODES, assertAssignmentMode, defaultStudentPermission, } from "@/types/classroom" @@ -42,6 +43,7 @@ import { } from "@/util/runtime" import { parseAllowedFiles, validateAllowedFiles } from "@/util/allowedFiles" import { parseReleaseAssets, validateReleaseAssets } from "@/util/releaseAssets" +import { validateSubmissionTags } from "@/util/submissionTags" import { addRepositoryToTeam, removeRepositoryFromTeam, @@ -115,6 +117,8 @@ const ASSIGNMENT_KEY_OWNERSHIP: Record< release_assets: "managed", pass_threshold: "managed", student_permission: "managed", + submission_mode: "managed", + submission_tags: "managed", repo_features: "managed", tests: "managed", // Written only by the CLI's `migrate`; the form never manages it, so it must @@ -628,6 +632,40 @@ async function buildAssignmentEntry( } } + // submission_mode: omit the wire default (every-push) so an untouched + // assignment stays byte-identical, mirroring the CLI's omitempty collapse. + // Validate against the enum so a bad value can't produce a file the CLI + // refuses to parse; reject alongside empty_repo (no shim exists to trigger). + if (input.submission_mode && input.submission_mode !== "every-push") { + if (!SUBMISSION_MODES.includes(input.submission_mode)) { + throw new Error( + `submission_mode: must be one of ${SUBMISSION_MODES.join(", ")} (got "${input.submission_mode}").`, + ) + } + if (input.empty_repo) { + throw new Error( + "submission_mode: mutually exclusive with empty_repo — a bare repo has no autograde shim to trigger.", + ) + } + entry.submission_mode = input.submission_mode + } + + // submission_tags: omit when empty (no milestone tags — today's behavior), + // mirroring the CLI's omitempty. Validate so a bad pattern can't produce a + // file the CLI refuses to parse; reject alongside empty_repo (no shim). + if (input.submission_tags && input.submission_tags.length > 0) { + const tagsError = validateSubmissionTags(input.submission_tags) + if (tagsError) { + throw new Error(`submission_tags: ${tagsError}`) + } + if (input.empty_repo) { + throw new Error( + "submission_tags: mutually exclusive with empty_repo — a bare repo has no autograde shim to trigger.", + ) + } + entry.submission_tags = [...input.submission_tags] + } + // repo_features: write only the keys the teacher set (undefined = inherit), // and omit the block entirely when no key is set — mirroring runtime's // omit-when-empty rule so an all-inherit assignment carries no repo_features. diff --git a/web/src/domain/assignments/repoCreation.ts b/web/src/domain/assignments/repoCreation.ts index ddca257f..886ffa22 100644 --- a/web/src/domain/assignments/repoCreation.ts +++ b/web/src/domain/assignments/repoCreation.ts @@ -1,6 +1,10 @@ import type { GitHubClient } from "@/github-core/client" import type { GitHubRepo } from "@/github-core/types" -import type { RepoPermission, RepoFeatures } from "@/types/classroom" +import type { + RepoPermission, + RepoFeatures, + SubmissionMode, +} from "@/types/classroom" import { GitHubAPIError } from "@/github-core/errors" import { getRepo } from "@/github-core/repoReads" import { DEFAULT_BRANCH } from "@/util/configRepo" @@ -355,6 +359,16 @@ export type CreateAssignmentInput = { // the mode default (push individual / admin group). buildAssignmentEntry // omits it when it equals the default and clamps group up to admin. student_permission?: RepoPermission + // When the autograder fires. Undefined or "every-push" = the wire default + // (buildAssignmentEntry omits it); "tag" = the shim grades only submit/* tag + // pushes. Mutually exclusive with empty_repo. Mirrors the CLI's + // --submission-mode. + submission_mode?: SubmissionMode + // Teacher-named milestone tag patterns that also trigger grading (union + // with the always-on submit/* namespace in the shim). Empty/undefined = + // none (buildAssignmentEntry omits the key). Mutually exclusive with + // empty_repo. Mirrors the CLI's --submission-tag. + submission_tags?: string[] // Per-assignment repo feature overrides (tri-state per key: undefined = // inherit, true = force on, false = force off). buildAssignmentEntry omits // the block when no key is set; accept resolves + applies it at fresh create. diff --git a/web/src/domain/assignments/submissionTrigger.test.ts b/web/src/domain/assignments/submissionTrigger.test.ts new file mode 100644 index 00000000..46eb2efb --- /dev/null +++ b/web/src/domain/assignments/submissionTrigger.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vitest" + +import { + rewriteShimTrigger, + shimUpdateCommitMessage, +} from "./submissionTrigger" +import { defaultAutograderWorkflow } from "./autograderYaml" + +// The CLI-accept shim differs from the web template only in its comment +// header — the retrofit must survive both and preserve everything around the +// trigger block. (The Go twin pins the same cases in submissionmode_test.go.) +const cliShimEveryPush = `# Classroom50 autograder shim. +# +# This file should not be edited. + +name: Autograde + +on: + push: + branches: ["main"] + tags: ["submit/*"] + +jobs: + grade: + uses: "o/classroom50/.github/workflows/autograde-runner.yaml@main" + permissions: + contents: write + statuses: write + pull-requests: write +` + +describe("rewriteShimTrigger", () => { + it("every-push → tag removes exactly the branches line (CLI-accepted shim)", () => { + const result = rewriteShimTrigger(cliShimEveryPush, "tag", "main") + if (result.kind !== "changed") throw new Error(`kind = ${result.kind}`) + expect(result.content).not.toContain("branches:") + expect(result.content).toBe( + cliShimEveryPush.replace(' branches: ["main"]\n', ""), + ) + }) + + it("every-push → tag works on the web-rendered shim too", () => { + const webShim = defaultAutograderWorkflow("o", "main", "main") + const result = rewriteShimTrigger(webShim, "tag", "main") + if (result.kind !== "changed") throw new Error(`kind = ${result.kind}`) + // Equals what the web renders natively in tag mode. + expect(result.content).toBe( + defaultAutograderWorkflow("o", "main", "main", "tag"), + ) + }) + + it("tag → every-push inserts the branches line with the repo's CURRENT branch", () => { + const tagShim = defaultAutograderWorkflow("o", "main", "main", "tag") + const result = rewriteShimTrigger(tagShim, "every-push", "master") + if (result.kind !== "changed") throw new Error(`kind = ${result.kind}`) + expect(result.content).toContain( + ' branches: ["master"]\n tags: ["submit/*"]', + ) + }) + + it("is idempotent: already on target → current, no content", () => { + const tagShim = defaultAutograderWorkflow("o", "main", "main", "tag") + expect(rewriteShimTrigger(tagShim, "tag", "main").kind).toBe("current") + expect( + rewriteShimTrigger(cliShimEveryPush, "every-push", "main").kind, + ).toBe("current") + }) + + it("refuses unrecognized content (never rewrites custom shims)", () => { + for (const content of [ + "name: Custom\non:\n workflow_dispatch: {}\njobs: {}\n", + 'on:\n push:\n branches: ["main"]\n', // no submit/* tags line + "", + ]) { + expect(rewriteShimTrigger(content, "tag", "main").kind).toBe( + "unrecognized", + ) + } + }) + + it("round-trips: tag → every-push → tag restores the original", () => { + const tagShim = defaultAutograderWorkflow("o", "main", "main", "tag") + const toPush = rewriteShimTrigger(tagShim, "every-push", "main") + if (toPush.kind !== "changed") throw new Error("expected change") + const back = rewriteShimTrigger(toPush.content, "tag", "main") + if (back.kind !== "changed") throw new Error("expected change") + expect(back.content).toBe(tagShim) + }) +}) + +describe("shimUpdateCommitMessage", () => { + it("is byte-identical to the Go contract.ShimUpdateCommitMessage", () => { + // Pinned on the Go side by TestShimUpdateCommitMessage; the [skip ci] is + // load-bearing (a tag→every-push retrofit commit must not grade itself). + expect(shimUpdateCommitMessage("tag")).toBe( + "[Classroom 50] Update autograder trigger to tag (submission-mode)\n\n[skip ci]", + ) + }) +}) + +describe("rewriteShimTrigger — milestone submission_tags", () => { + it("widens the tags line to the union with submit/* (every-push kept)", () => { + const result = rewriteShimTrigger(cliShimEveryPush, "every-push", "main", [ + "phase1", + "v*", + ]) + if (result.kind !== "changed") throw new Error(`kind = ${result.kind}`) + expect(result.content).toContain(' tags: ["phase1", "v*", "submit/*"]') + expect(result.content).toContain(' branches: ["main"]') + // Only the tags line changed — everything around the trigger block is + // preserved byte-for-byte (the CLI shim's comment header included). + expect(result.content).toBe( + cliShimEveryPush.replace( + ' tags: ["submit/*"]\n', + ' tags: ["phase1", "v*", "submit/*"]\n', + ), + ) + }) + + it("recognizes and narrows a pattern-bearing shim back to the default", () => { + const withTags = defaultAutograderWorkflow("o", "main", "main", undefined, [ + "phase1", + ]) + const result = rewriteShimTrigger(withTags, "every-push", "main", []) + if (result.kind !== "changed") throw new Error(`kind = ${result.kind}`) + expect(result.content).toBe(defaultAutograderWorkflow("o", "main", "main")) + }) + + it("tag mode + patterns drops branches and widens tags together", () => { + const result = rewriteShimTrigger(cliShimEveryPush, "tag", "main", [ + "phase1", + ]) + if (result.kind !== "changed") throw new Error(`kind = ${result.kind}`) + expect(result.content).not.toContain("branches:") + expect(result.content).toContain(' tags: ["phase1", "submit/*"]') + }) + + it("is idempotent with patterns: already-current is a no-op", () => { + const withTags = defaultAutograderWorkflow("o", "main", "main", undefined, [ + "phase1", + ]) + expect( + rewriteShimTrigger(withTags, "every-push", "main", ["phase1"]).kind, + ).toBe("current") + }) +}) diff --git a/web/src/domain/assignments/submissionTrigger.ts b/web/src/domain/assignments/submissionTrigger.ts new file mode 100644 index 00000000..a17d6358 --- /dev/null +++ b/web/src/domain/assignments/submissionTrigger.ts @@ -0,0 +1,240 @@ +// Retrofit a student repo's autograde shim to the assignment's +// submission_mode — the web twin of `gh teacher assignment submission-mode`'s +// per-repo loop. The trigger lives in each repo's workflow file (GitHub +// evaluates `on:` before any job runs), and the shim is otherwise frozen at +// accept time, so a mode change must rewrite the file in every existing repo. +// +// The rewrite is LINE SURGERY on the known trigger block, never a full +// re-render: the CLI-accept and web-accept shims share the trigger shape but +// differ in their comment headers, so re-rendering would churn repos accepted +// by the other client. Content that doesn't match a known default-shim +// trigger shape (student-edited, teacher-authored) is reported and never +// overwritten. Keep the block regex in lockstep with the Go twin +// (cli/gh-teacher/internal/assignmentcmd/submissionmode.go shimTriggerBlock). +import type { GitHubClient } from "@/github-core/client" +import { + createCommitForAssignment, + createTreeRepo, + updateRefForRepo, +} from "@/github-core/mutations" +import { getRepo } from "@/github-core/repoReads" +import { getBranchRefRepo, getCommitByRepo } from "@/github-core/queries" +import { GitHubAPIError } from "@/github-core/errors" +import { decodeBase64Utf8 } from "@/util/github" +import { prefixCommit } from "@/util/commit" +import { safeShimTagPatterns } from "@/util/submissionTags" + +// The shim's path in every student repo. Byte-mirror of the CLI's +// classroomcfg.AutogradeWorkflowPath and runner.py's SHIM_UPDATE_COMMIT_PATHS. +export const AUTOGRADE_SHIM_PATH = ".github/workflows/autograde.yaml" + +// The retrofit commit message. The `[skip ci]` body line is load-bearing: a +// tag→every-push retrofit commit carries the restored push trigger, and this +// commit is authored with the teacher's OAuth token (user pushes DO fire +// workflows). Byte-mirror of contract.ShimUpdateCommitMessage (Go), pinned by +// TestShimUpdateCommitMessage. +export function shimUpdateCommitMessage(mode: "every-push" | "tag"): string { + return ( + prefixCommit(`Update autograder trigger to ${mode} (submission-mode)`) + + "\n\n[skip ci]" + ) +} + +// The default shim's `on:` block in any mode/tags combination: an optional +// branches: line (group 1) followed by the tags line (group 2 — the default +// `["submit/*"]` or a milestone-pattern union). Mirror of the Go +// shimTriggerBlock regex. +const SHIM_TRIGGER_BLOCK = + /^on:\n {2}push:\n( {4}branches: \[[^\n]*\]\n)?( {4}tags: \[[^\n]*\]\n)/m + +export type ShimRewrite = + | { kind: "changed"; content: string } + | { kind: "current" } + | { kind: "unrecognized"; reason: string } + +// The shim's tags flow sequence: the milestone patterns (if any) union the +// always-on submit/* namespace. Byte-format mirror of Go +// contract.ShimTagsList and autograderYaml.ts's shimTagsList. FAIL CLOSED: +// the retrofit writes workflow files into student repos with the teacher's +// OAuth token from the (hand-editable) manifest, so unsafe patterns drop the +// whole milestone set (see safeShimTagPatterns). +function shimTagsList(tags: string[]): string { + return [...safeShimTagPatterns(tags), "submit/*"] + .map((p) => `"${p}"`) + .join(", ") +} + +// Swap the shim's trigger block to `mode` + `tags`. every-push → tag removes +// the branches: line; tag → every-push inserts it with the repo's CURRENT +// default branch (the branch pushes actually land on; an existing line is +// kept verbatim for the same reason). The tags line is reconciled to the +// union of the assignment's milestone patterns and submit/*, so the same +// retrofit that flips the mode also repairs a stale pattern set. +export function rewriteShimTrigger( + content: string, + mode: "every-push" | "tag", + branch: string, + tags: string[] = [], +): ShimRewrite { + const match = SHIM_TRIGGER_BLOCK.exec(content) + if (!match) { + return { + kind: "unrecognized", + reason: "shim does not carry a recognizable default trigger block", + } + } + const existingBranches = match[1] + + let branchesLine = "" + if (mode === "every-push") { + // Keep an existing line verbatim: its (possibly stale) branch name is + // accept-time behavior, not this action's to correct. + branchesLine = existingBranches ?? ` branches: ["${branch}"]\n` + } + const tagsLine = ` tags: [${shimTagsList(tags)}]\n` + + const rebuilt = + content.slice(0, match.index) + + "on:\n push:\n" + + branchesLine + + tagsLine + + content.slice(match.index + match[0].length) + return rebuilt === content + ? { kind: "current" } + : { kind: "changed", content: rebuilt } +} + +export type ShimUpdateOutcome = + | { status: "updated" } + | { status: "current" } + | { status: "unrecognized"; reason: string } + | { status: "notAccepted" } + | { status: "missingWorkflowScope" } + +// Update one repo's shim to `mode`, idempotently. Reads the live default +// branch (the shim is branch-specific), fetches the current shim, rewrites the +// trigger block, and commits with [skip ci]. A 404 tree write with the +// workflow scope absent from X-OAuth-Scopes is GitHub's signature for a token +// that can't touch .github/workflows/* — surfaced as its own outcome so the +// UI can show the re-auth remediation once, not per repo. +export async function updateShimSubmissionMode(params: { + client: GitHubClient + org: string + repo: string + mode: "every-push" | "tag" + // The assignment's milestone submission_tags; the rewrite reconciles the + // shim's tags line to their union with submit/*. Omit for none. + tags?: string[] +}): Promise { + const { client, org, repo, mode, tags } = params + + let branch: string + try { + const live = await getRepo(client, org, repo) + if (!live?.default_branch) return { status: "notAccepted" } + branch = live.default_branch + } catch (err) { + if (err instanceof GitHubAPIError && err.status === 404) { + return { status: "notAccepted" } + } + throw err + } + + // Pin the whole read-rewrite-commit cycle to one resolved tip SHA. Reading + // the shim at the branch NAME can lag a just-landed write (GitHub + // read-after-write lag — the misreport the Go retrofitShim fixed, observed + // live 2026-08-05), which would rewrite from stale content or misreport + // "current"; reading at the same SHA the commit builds on keeps the no-op + // check and the write consistent. + const ref = await getBranchRefRepo(client, org, repo, branch) + const parentSha = ref.object.sha + const parentCommit = await getCommitByRepo(client, org, repo, parentSha) + const baseTreeSha = parentCommit.tree?.sha + if (!parentSha || !baseTreeSha) { + throw new Error(`${org}/${repo}: could not resolve the branch tip`) + } + + let current: string + try { + const resp = await client.request<{ content?: string; encoding?: string }>( + `/repos/${org}/${repo}/contents/${AUTOGRADE_SHIM_PATH}?ref=${encodeURIComponent(parentSha)}`, + ) + if (!resp?.content || resp.encoding !== "base64") { + return { + status: "unrecognized", + reason: "unexpected contents response for the shim", + } + } + current = decodeBase64Utf8(resp.content) + } catch (err) { + if (err instanceof GitHubAPIError && err.status === 404) { + // Repo exists but the shim never landed (mid-flow accept failure); + // accept's self-heal owns that case. + return { + status: "unrecognized", + reason: "no autograde workflow — accept may not have completed", + } + } + throw err + } + + const rewrite = rewriteShimTrigger(current, mode, branch, tags ?? []) + if (rewrite.kind === "current") return { status: "current" } + if (rewrite.kind === "unrecognized") { + return { status: "unrecognized", reason: rewrite.reason } + } + + let tree: { sha: string } + try { + tree = await createTreeRepo(client, { + base_tree: baseTreeSha, + org, + repo, + tree: [ + { + path: AUTOGRADE_SHIM_PATH, + mode: "100644", + type: "blob", + content: rewrite.content, + }, + ], + }) + } catch (err) { + if ( + err instanceof GitHubAPIError && + err.status === 404 && + tokenLacksWorkflowScope(err) + ) { + return { status: "missingWorkflowScope" } + } + throw err + } + const commit = await createCommitForAssignment({ + client, + owner: org, + repo, + message: shimUpdateCommitMessage(mode), + treeSha: tree.sha, + parentSha, + }) + await updateRefForRepo({ + client, + owner: org, + repo, + branch, + commitSha: commit.sha, + }) + return { status: "updated" } +} + +// Whether the error's X-OAuth-Scopes header is present but missing +// `workflow`. Mirrors the Go tokenLacksWorkflowScope (configwrite): an absent +// header (fine-grained PAT) returns false so we don't guess. +function tokenLacksWorkflowScope(err: GitHubAPIError): boolean { + const scopes = err.oauthScopes + if (!scopes) return false + return !scopes + .split(",") + .map((s) => s.trim()) + .includes("workflow") +} diff --git a/web/src/domain/assignments/submit.test.ts b/web/src/domain/assignments/submit.test.ts index 4e71a658..770e621a 100644 --- a/web/src/domain/assignments/submit.test.ts +++ b/web/src/domain/assignments/submit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it, vi } from "vitest" -import { submitAssignment, normalizeRepoPath } from "./submit" +import { submitAssignment, normalizeRepoPath, buildSubmitTag } from "./submit" import type { GitHubClient } from "@/github-core/client" describe("normalizeRepoPath", () => { @@ -22,12 +22,16 @@ function makeClient(opts: { defaultBranch: string existingTree: { path: string; type: string; sha: string; mode: string }[] truncated?: boolean + // Pre-existing submit/* tags on the repo: tag name -> target sha, served + // from the matching-refs probe the tag-mode path runs before tagging. + existingSubmitTags?: Record }) { const created = { blobs: [] as { content: string; encoding: string }[], tree: null as null | { hasBaseTree: boolean; paths: string[] }, commit: null as null | { message: string; parents: string[] }, updatedRef: null as null | { branch: string; sha: string }, + tagRefs: [] as { ref: string; sha: string }[], } let blobN = 0 @@ -81,6 +85,27 @@ function makeClient(opts: { created.updatedRef = { branch, sha: body.sha } return { ref: "", object: { sha: body.sha, type: "commit", url: "" } } } + // findSubmitTagAtSha + if ( + path.includes("/git/matching-refs/tags/submit%2F") && + method === "GET" + ) { + return Object.entries(opts.existingSubmitTags ?? {}).map( + ([tag, sha]) => ({ + ref: `refs/tags/${tag}`, + object: { sha, type: "commit", url: "" }, + }), + ) + } + // createTagRefForRepo + if (path.endsWith("/git/refs") && method === "POST") { + const body = init?.body as { ref: string; sha: string } + created.tagRefs.push({ ref: body.ref, sha: body.sha }) + return { + ref: body.ref, + object: { sha: body.sha, type: "commit", url: "" }, + } + } throw new Error(`unexpected request: ${method} ${path}`) }, ) @@ -238,4 +263,73 @@ describe("submitAssignment", () => { }), ).rejects.toThrow(/No files/i) }) + + it("tag mode pushes a submit/* tag at the new commit", async () => { + const { client, created } = makeClient({ + defaultBranch: "main", + existingTree: [blob(".github/workflows/autograde.yaml", "wf-sha")], + }) + const result = await submitAssignment({ + client, + org: "acme", + repo: "r", + assignment: "hw", + files: [upload("main.py")], + submissionMode: "tag", + }) + expect(created.tagRefs).toHaveLength(1) + expect(created.tagRefs[0].sha).toBe("new-commit-sha") + expect(created.tagRefs[0].ref).toMatch(/^refs\/tags\/submit\//) + expect(result.tag).toBe(created.tagRefs[0].ref.replace("refs/tags/", "")) + }) + + it("tag mode reuses an existing submit/* tag at the same SHA", async () => { + // A retry after a failed tag push (or a hand-pushed tag) must not mint a + // second tag — one grading run per commit. + const { client, created } = makeClient({ + defaultBranch: "main", + existingTree: [], + existingSubmitTags: { "submit/hand-pushed": "new-commit-sha" }, + }) + const result = await submitAssignment({ + client, + org: "acme", + repo: "r", + assignment: "hw", + files: [upload("main.py")], + submissionMode: "tag", + }) + expect(created.tagRefs).toHaveLength(0) + expect(result.tag).toBe("submit/hand-pushed") + }) + + it("every-push mode (absent or explicit) never touches tag refs", async () => { + for (const submissionMode of [undefined, "every-push"] as const) { + const { client, created } = makeClient({ + defaultBranch: "main", + existingTree: [], + }) + const result = await submitAssignment({ + client, + org: "acme", + repo: "r", + assignment: "hw", + files: [upload("main.py")], + submissionMode, + }) + expect(created.tagRefs).toHaveLength(0) + expect(result.tag).toBeUndefined() + } + }) +}) + +describe("buildSubmitTag", () => { + it("formats submit/- (runner/CLI parity)", () => { + // Byte-format parity with the runner's `date -u +%Y-%m-%dT%H-%M-%SZ` and + // Go's contract.BuildSubmitTag (pinned by TestBuildSubmitTag). + const at = new Date(Date.UTC(2026, 7, 3, 14, 30, 5)) + expect(buildSubmitTag("abcdef0123456789", at)).toBe( + "submit/2026-08-03T14-30-05Z-abcdef0", + ) + }) }) diff --git a/web/src/domain/assignments/submit.ts b/web/src/domain/assignments/submit.ts index 7fbc546c..883a9800 100644 --- a/web/src/domain/assignments/submit.ts +++ b/web/src/domain/assignments/submit.ts @@ -2,11 +2,14 @@ import type { GitHubClient } from "@/github-core/client" import { createBlobForRepo, createCommitForAssignment, + createTagRefForRepo, createTreeFromFullEntries, + findSubmitTagAtSha, getRepoTreeRecursive, updateRefForRepo, type GitHubTreeEntryFull, } from "@/github-core/mutations" +import { SUBMISSION_TAG_PREFIX } from "@/github-core/queries/releaseRunReads" import { getRepo } from "@/github-core/repoReads" import { getBranchRefRepo, @@ -17,6 +20,7 @@ import { import { prefixCommit } from "@/util/commit" import { fileToBase64 } from "@/util/fileBytes" import { mapWithConcurrency } from "@/util/concurrency" +import type { SubmissionMode } from "@/types/classroom" // A file the student picked, with its repo-relative path. `path` is the drop's // relative path (or the bare name) — POSIX-normalized by the caller. @@ -38,11 +42,33 @@ export type SubmitAssignmentResult = { commitSha: string branch: string fileCount: number + // The submit/* tag pushed (tag mode only; created fresh or reused when the + // commit already carries one). + tag?: string +} + +// Tag-mode partial failure: the snapshot commit LANDED on the default branch +// but the submit/* tag push (the write that triggers grading) failed. Callers +// must not report this as an upload failure — the files are safe; only grading +// wasn't triggered, and a retry reuses the landed commit via the tag probe. +export class SubmitTagPushError extends Error { + readonly commitSha: string + constructor(commitSha: string, cause: unknown) { + super( + `Files were uploaded (commit ${commitSha}), but pushing the submit tag that triggers grading failed. Submit again to retry — the uploaded work is safe.`, + ) + this.name = "SubmitTagPushError" + this.commitSha = commitSha + this.cause = cause + } } // Commit the uploaded files as a replace-all snapshot on the student repo's // default branch — the browser equivalent of `gh student submit`. The push // (authored with the user's OAuth token) fires on:push and triggers autograding. +// For a tag-mode assignment (submission_mode: "tag") branch pushes are not +// graded, so a submit/- tag is additionally pushed — +// that tag push is what triggers grading. // // The new tree is AUTHORITATIVE (no base_tree), so prior files not re-uploaded // are dropped; the runner's control paths (.github/**, .classroom50.yaml) are @@ -54,8 +80,9 @@ export async function submitAssignment(params: { repo: string assignment: string files: UploadFile[] + submissionMode?: SubmissionMode }): Promise { - const { client, org, repo, assignment, files } = params + const { client, org, repo, assignment, files, submissionMode } = params if (files.length === 0) { throw new Error("No files selected to submit.") @@ -173,9 +200,57 @@ export async function submitAssignment(params: { } }) + // Tag-mode assignments grade ONLY on submit/* tag pushes, so push the tag + // with the user's token after the branch update (user pushes fire + // workflows). Outside the fresh-repo retry: the commit exists by now, and a + // tag-push failure must surface as its own actionable error, not re-run the + // whole snapshot. Reuse an existing submit/* tag at the same SHA (a retry, + // or a hand-pushed tag) so one commit never grades twice. Every-push + // assignments must NOT get a tag here — the branch push already grades. + if (submissionMode === "tag") { + const existing = await findSubmitTagAtSha({ + client, + owner: org, + repo, + sha: result.commitSha, + }).catch(() => null) // probe failure → fall through to a fresh tag + if (existing) { + result.tag = existing + } else { + const tag = buildSubmitTag(result.commitSha) + try { + await createTagRefForRepo({ + client, + owner: org, + repo, + tag, + commitSha: result.commitSha, + }) + } catch (err) { + // The commit landed; only the grading trigger failed. Surface a typed + // error so the UI can say "uploaded but not graded — retry" instead of + // the false "couldn't upload your files". + throw new SubmitTagPushError(result.commitSha, err) + } + result.tag = tag + } + } + return result } +// The canonical submission tag for a commit: submit/-. +// Byte-format-identical with the runner's tag-minting step and the CLI's +// contract.BuildSubmitTag — the short-SHA suffix prevents collisions when two +// submissions land in the same UTC second. +export function buildSubmitTag(sha: string, now: Date = new Date()): string { + const ts = now + .toISOString() + .replace(/\.\d{3}Z$/, "Z") + .replaceAll(":", "-") + return `${SUBMISSION_TAG_PREFIX}${ts}-${sha.slice(0, 7)}` +} + // Normalize a drop-relative path to a POSIX repo path: forward slashes, no // leading `./` or `/`, and reject `..` traversal (a path escaping the repo root). export function normalizeRepoPath(raw: string): string { diff --git a/web/src/github-core/mutations.ts b/web/src/github-core/mutations.ts index 65cb456d..d355868c 100644 --- a/web/src/github-core/mutations.ts +++ b/web/src/github-core/mutations.ts @@ -13,6 +13,8 @@ export { createCommitForAssignment, updateRef, updateRefForRepo, + createTagRefForRepo, + findSubmitTagAtSha, createGitTree, createGitCommit, createBlob, diff --git a/web/src/github-core/mutations/gitObjects.ts b/web/src/github-core/mutations/gitObjects.ts index cc2fd312..0ff2c52e 100644 --- a/web/src/github-core/mutations/gitObjects.ts +++ b/web/src/github-core/mutations/gitObjects.ts @@ -9,6 +9,8 @@ import type { CreateClassroomInput } from "@/domain/classrooms" import { STUDENT_CSV_FIELDS } from "@/util/rosterCsv" import { CONFIG_REPO, DEFAULT_BRANCH } from "@/util/configRepo" import { prefixCommit } from "@/util/commit" +import { paginateAll } from "../paginate" +import { SUBMISSION_TAG_PREFIX } from "../queries/releaseRunReads" import type { ClassroomTeamRef, StaffTeamRefs } from "./teams" // The branch a config repo's default is renamed TO when normalizing it. @@ -319,6 +321,57 @@ export function updateRefForRepo(params: { ) } +// Create a lightweight tag ref at a commit. Used by the tag-mode submit flow +// to push the submit/- tag with the user's token — +// user pushes fire workflows, which is exactly the point (the runner's own +// github.token tag pushes deliberately don't). +export function createTagRefForRepo(params: { + client: GitHubClient + owner: string + repo: string + tag: string + commitSha: string +}) { + const { client, owner, repo, tag, commitSha } = params + + return client.request(`/repos/${owner}/${repo}/git/refs`, { + method: "POST", + body: { + ref: `refs/tags/${tag}`, + sha: commitSha, + }, + }) +} + +// First submit/* tag pointing at `sha`, or null. The tag-mode submit flow +// checks this before creating a fresh tag — mirroring the runner's ls-remote +// idempotency check — so a retry after a tag-push failure reuses the existing +// tag and the same commit never grades twice. matching-refs is a prefix match +// (the prefix's slash needs encoding); the response carries each ref's target. +// Paginated: a tag-mode repo accrues one submit/* tag per submission, so a +// semester's worth easily exceeds one page — an unpaginated read would miss +// the existing tag and mint a duplicate (one redundant graded run). +export async function findSubmitTagAtSha(params: { + client: GitHubClient + owner: string + repo: string + sha: string +}): Promise { + const { client, owner, repo, sha } = params + const prefix = encodeURIComponent(SUBMISSION_TAG_PREFIX) + const refs = await paginateAll( + client, + (page) => + `/repos/${owner}/${repo}/git/matching-refs/tags/${prefix}?per_page=100&page=${page}`, + ) + for (const ref of refs) { + if (ref.object?.sha === sha) { + return ref.ref.replace(/^refs\/tags\//, "") + } + } + return null +} + // One entry in a git tree write. GitHub accepts either inline `content` or a // `sha` (existing blob, or `null` to delete the path). export type GitTreeFileMode = "100644" | "100755" | "120000" diff --git a/web/src/github-core/queries.ts b/web/src/github-core/queries.ts index 93c701f1..62595f63 100644 --- a/web/src/github-core/queries.ts +++ b/web/src/github-core/queries.ts @@ -15,6 +15,7 @@ export { isFreshRepoLagError, withFreshRepoRetry, REPO_READ_CONCURRENCY, + REPO_WRITE_CONCURRENCY, withGithubReadSlot, retryOnRateLimit, type FreshRepoRetryOptions, diff --git a/web/src/github-core/queries/shared.ts b/web/src/github-core/queries/shared.ts index 7c08d86b..f3f1b70e 100644 --- a/web/src/github-core/queries/shared.ts +++ b/web/src/github-core/queries/shared.ts @@ -13,6 +13,13 @@ export const log = logger.scope(LOG_SCOPE_QUERIES) // while still beating a strictly-sequential loop. export const REPO_READ_CONCURRENCY = 8 +// Max simultaneous per-repo CONTENT WRITES (tree/commit/ref chains). GitHub's +// secondary-rate-limit guidance is to avoid concurrent content writes — the +// CLI's retrofit loop is serial for the same reason — so bulk write fan-outs +// (e.g. the submission-trigger retrofit) stay effectively sequential while +// still reusing the mapWithConcurrency progress plumbing. +export const REPO_WRITE_CONCURRENCY = 1 + // A small FIFO counting semaphore. Independent per-repo fan-outs (the live // submissions hook and the group-member hook) can run on the same page load; // each capping *itself* at REPO_READ_CONCURRENCY still lets their union burst to diff --git a/web/src/hooks/mutations/useSubmitAssignment.ts b/web/src/hooks/mutations/useSubmitAssignment.ts index 6f408daf..55ba5fc6 100644 --- a/web/src/hooks/mutations/useSubmitAssignment.ts +++ b/web/src/hooks/mutations/useSubmitAssignment.ts @@ -2,27 +2,40 @@ import { useMutation, useQueryClient } from "@tanstack/react-query" import { submitAssignment, type UploadFile } from "@/domain/assignments" import { githubKeys } from "@/github-core/queries" import { useGitHubClient } from "@/context/github/GitHubProvider" +import type { SubmissionMode } from "@/types/classroom" // Submit uploaded files to the student's assignment repo (the browser // equivalent of `gh student submit`): commits a snapshot on the default branch, -// which triggers autograding. On success we invalidate the repo + releases -// queries so the submission page reflects the new HEAD and picks up the graded -// release once the background autograde run publishes it (grading is async, so -// the release won't appear on this tick — the invalidation just re-arms the -// list for the next refetch). +// which triggers autograding. We invalidate the repo + releases queries so the +// submission page reflects the new HEAD and picks up the graded release once +// the background autograde run publishes it (grading is async, so the release +// won't appear on this tick — the invalidation just re-arms the list for the +// next refetch). Invalidation runs on settled, not success: a tag-mode submit +// can fail AFTER the branch commit landed (the tag push is a separate write), +// and the page must still reflect the new HEAD in that case. export function useSubmitAssignment(params: { org: string repo: string assignment: string + // The assignment's submission_mode; "tag" makes submit also push the + // submit/* tag that triggers grading (branch pushes alone don't grade). + submissionMode?: SubmissionMode }) { const client = useGitHubClient() const queryClient = useQueryClient() - const { org, repo, assignment } = params + const { org, repo, assignment, submissionMode } = params return useMutation({ mutationFn: (files: UploadFile[]) => - submitAssignment({ client, org, repo, assignment, files }), - onSuccess: () => { + submitAssignment({ + client, + org, + repo, + assignment, + files, + submissionMode, + }), + onSettled: () => { void queryClient.invalidateQueries({ queryKey: githubKeys.repo(org, repo), }) diff --git a/web/src/locales/en.json b/web/src/locales/en.json index b494a64c..f0a9b2bf 100644 --- a/web/src/locales/en.json +++ b/web/src/locales/en.json @@ -1116,7 +1116,7 @@ "dueDateTz": "Times are in your local timezone ({{tz}}).", "availableFrom": "Release date ({{tz}})", "setAvailableFrom": "Set a release date", - "setAvailableFromTip": "Assignments are hidden from the student assignments page by default \u2014 students can only accept via the invite link. Set a release date to list it for everyone once the date passes. Students who already accepted always see it. This controls listing only, not access.", + "setAvailableFromTip": "Assignments are hidden from the student assignments page by default — students can only accept via the invite link. Set a release date to list it for everyone once the date passes. Students who already accepted always see it. This controls listing only, not access.", "availableFromTz": "Times are in your local timezone ({{tz}}).", "type": "Assignment type", "typeIndividual": "Individual", @@ -1143,6 +1143,22 @@ "admin": "Admin" } }, + "submissionMode": { + "label": "Autograding trigger", + "help": "When the autograder runs. \"Every push\" grades each push to the default branch. \"On submit only\" grades only when a student submits (with gh student submit) or pushes a submit/* tag — regular pushes cost no Actions minutes, which matters for large classes.", + "emptyRepoHelp": "Unavailable for empty repositories — they carry no autograding workflow.", + "editWarning": "Changing the trigger only affects repositories created from now on. Repositories students already accepted keep the old trigger until you update them — use \"Update autograding triggers\" on the submissions page or gh teacher assignment submission-mode. Students must pull after the update.", + "choices": { + "everyPush": "Every push (default)", + "tag": "On submit only" + } + }, + "submissionTags": { + "label": "Milestone submission tags", + "help": "Optional tag names (one per line) that also trigger grading — e.g. phase1, phase2, complete. A student pushing a matching tag (git tag phase1 && git push origin phase1) gets that commit graded; the graded record still appears as a submit/* release. Simple globs like v* work too. Leave empty for none.", + "wildcardCaution": "A broad pattern like v* grades every matching tag a student pushes, including unrelated release tags. Prefer exact milestone names like phase1.", + "commaHint": "Put one tag per line, not a comma-separated list — a comma becomes part of the tag name and will be rejected." + }, "repoFeatures": { "heading": "Repository features", "refresh": "Refresh template settings", @@ -1157,10 +1173,18 @@ "on": "On", "off": "Off" }, - "issues": { "label": "Issues" }, - "wiki": { "label": "Wiki" }, - "projects": { "label": "Projects" }, - "pull_requests": { "label": "Pull requests" } + "issues": { + "label": "Issues" + }, + "wiki": { + "label": "Wiki" + }, + "projects": { + "label": "Projects" + }, + "pull_requests": { + "label": "Pull requests" + } }, "advanced": "Advanced settings", "advancedHelp": "Optional grading and runtime settings. Blank fields use their documented defaults.", @@ -1231,6 +1255,7 @@ "setupTimeoutRange": "Setup timeout must be 0 or a whole number of seconds from 1 through {{max}}.", "passThresholdRange": "Pass threshold must be a whole number between {{min}} and {{max}}.", "studentPermissionInvalid": "Student repo access must be one of pull, triage, push, maintain, or admin.", + "submissionModeInvalid": "Autograding trigger must be \"every-push\" or \"tag\".", "releaseAssetsTooMany": "Choose at most {{max}} submission release files (currently {{count}}).", "releaseAssetsTooLarge": "Submission release file paths may total at most {{max}} UTF-8 bytes (8 KiB); currently {{bytes}}.", "releaseAssetsInvalidPath": "{{path}} is not an exact workspace-relative file path.", @@ -1604,7 +1629,7 @@ "step2Label": "GitHub Access Token", "expiryLabel": "Token expiry", "days": "days", - "expiryRange": "Enter {{min}}\u2013{{max}} days", + "expiryRange": "Enter {{min}}–{{max}} days", "generateOnGitHub": "Generate new access token", "generateHelp": "Opens GitHub's token form with the name, expiry, and required scopes pre-filled. Set Repository access to All repositories.", "learnMore": "Learn about service tokens", @@ -1616,7 +1641,7 @@ "saveButton": "Save token", "saveError": "Could not validate or save the token.", "saved": "Service token saved.", - "savedNoMetadata": "Service token saved, but its expiry and name couldn't be recorded — the health chip may show \u201Cexpiry not tracked\u201D until you save again." + "savedNoMetadata": "Service token saved, but its expiry and name couldn't be recorded — the health chip may show “expiry not tracked” until you save again." }, "audit": { "detail": { @@ -2494,6 +2519,48 @@ "deferredSection_other": "{{count}} repositories were not updated", "deferredDetail": "Skipped — re-run to update." }, + "bulkTrigger": { + "menuLabel": "Update autograding triggers", + "menuTitle": "Rewrite each student repository's autograding workflow to match the assignment's trigger setting", + "titleEmptyRoster": "No students have accepted yet", + "title": "Update autograding triggers", + "subtitle_one": "Sets the autograding trigger to \"{{mode}}\" on {{count}} student's repository.", + "subtitle_other": "Sets the autograding trigger to \"{{mode}}\" on {{count}} students' repositories.", + "noRepos": "No students have accepted this assignment yet, so there are no repositories to update.", + "warning_one": "This commits a change to the autograding workflow in {{count}} student's repository. The commit is marked so it doesn't trigger grading. Repositories whose workflow was hand-edited are reported and left untouched. Students must pull afterward — clones made before this change will conflict on their next push.", + "warning_other": "This commits a change to the autograding workflow in {{count}} students' repositories. The commit is marked so it doesn't trigger grading. Repositories whose workflow was hand-edited are reported and left untouched. Students must pull afterward — clones made before this change will conflict on their next push.", + "apply": "Update all", + "working": "Updating repositories", + "progress": "Processed {{processed}} of {{total}}", + "resultHeadline": "Updated {{updated}} of {{total}} repositories ({{current}} already current).", + "repullReminder": "Tell students to run git pull — clones made before this change will conflict on their next push.", + "scopeSection_one": "{{count}} repository needs a token with the workflow scope", + "scopeSection_other": "{{count}} repositories need a token with the workflow scope", + "scopeDetail": "Your GitHub authorization can't modify workflow files. Sign out and back in, then re-run.", + "failedSection_one": "{{count}} repository could not be updated", + "failedSection_other": "{{count}} repositories could not be updated", + "unrecognizedSection_one": "{{count}} repository was skipped (workflow not recognized)", + "unrecognizedSection_other": "{{count}} repositories were skipped (workflow not recognized)", + "deferredSection_one": "{{count}} repository was not processed", + "deferredSection_other": "{{count}} repositories were not processed", + "deferredDetail": "Skipped — re-run to update.", + "notAcceptedSection_one": "{{count}} student has not accepted yet", + "notAcceptedSection_other": "{{count}} students have not accepted yet", + "notAcceptedDetail": "No repository exists — nothing to update." + }, + "rowTrigger": { + "title": "Update autograding trigger", + "description": "Rewrite this repository's autograding workflow to match the assignment's trigger setting.", + "aria": "Update the autograding trigger on {{repo}}", + "outcome": { + "updated": "Autograding trigger updated. The student must pull before their next push.", + "current": "Already up to date — no change needed.", + "unrecognized": "Skipped: this repository's workflow was modified and can't be updated automatically.", + "notAccepted": "No repository exists yet — nothing to update.", + "missingWorkflowScope": "Your GitHub authorization can't modify workflow files. Sign out and back in, then retry.", + "failed": "Could not update the workflow — try again." + } + }, "rowRegrade": { "title": "Regrade this submission", "titleInFlight": "Regrade in progress…", diff --git a/web/src/pages/CreateAssignmentPage.tsx b/web/src/pages/CreateAssignmentPage.tsx index 8e173f48..6fe99fce 100644 --- a/web/src/pages/CreateAssignmentPage.tsx +++ b/web/src/pages/CreateAssignmentPage.tsx @@ -23,6 +23,7 @@ import { useOutageHint } from "@/lib/githubHealth" import { GitHubStatusNote } from "@/components/GitHubStatusNote" import { useState } from "react" import { useTranslation } from "react-i18next" +import { parseSubmissionTags } from "@/util/submissionTags" const log = logger.scope("CreateAssignmentPage") @@ -145,6 +146,8 @@ const CreateAssignmentPage = () => { ? values.pass_threshold : undefined, student_permission: values.student_permission || undefined, + submission_mode: values.submission_mode, + submission_tags: parseSubmissionTags(values.submission_tags), repo_features: formValuesToRepoFeatures(values), classroom, tests: values.tests, diff --git a/web/src/pages/SubmissionsPage.tsx b/web/src/pages/SubmissionsPage.tsx index 7a9ea2d4..9f56fd63 100644 --- a/web/src/pages/SubmissionsPage.tsx +++ b/web/src/pages/SubmissionsPage.tsx @@ -20,6 +20,8 @@ import { OpenAllFeedbackPrsModal } from "@/pages/submissions/OpenAllFeedbackPrsM import { DownloadAllSubmissionsModal } from "@/pages/submissions/DownloadAllSubmissionsModal" import { BulkRepoAccessModal } from "@/components/modals/BulkRepoAccessModal" import { BulkRepoFeaturesModal } from "@/components/modals/BulkRepoFeaturesModal" +import { BulkSubmissionTriggerModal } from "@/components/modals/BulkSubmissionTriggerModal" +import { isDefaultAutograder } from "@/domain/assignments/autograderYaml" import { DataFreshness } from "@/pages/submissions/DataFreshness" import { ConfirmModal } from "@/components/modals" import { @@ -260,6 +262,7 @@ const SubmissionsPageContent = () => { const [downloadAllOpen, setDownloadAllOpen] = useState(false) const [bulkAccessOpen, setBulkAccessOpen] = useState(false) const [bulkFeaturesOpen, setBulkFeaturesOpen] = useState(false) + const [bulkTriggerOpen, setBulkTriggerOpen] = useState(false) // Scope the collector's scores to the CURRENT roster (see rosterScopedRows). // Gate on a resolved roster so a transient load/permission failure falls back @@ -1051,6 +1054,24 @@ const SubmissionsPageContent = () => { ? () => setBulkFeaturesOpen(true) : undefined } + // Bulk retrofit autograding triggers: same gate as bulk features + // plus default-autograder only — teacher-authored (custom) shims + // are never rewritten. Reconciles existing repos with the + // assignment's submission_mode (baked into shims at accept time). + // Requires a RESOLVED assignmentInfo: isDefaultAutograder(undefined) + // is true, so gating on the optional chain alone would enable the + // action while the assignments query loads (or after it fails) and + // retrofit shims to the fallback every-push mode. + onBulkTrigger={ + isOwner && + !isGroupAssignment && + !isEmptyRepoAssignment && + assignmentInfo != null && + isDefaultAutograder(assignmentInfo.autograder) && + acceptedSet.size > 0 + ? () => setBulkTriggerOpen(true) + : undefined + } locked={isLockedAssignment} lockPending={setLock.isPending} // Lock/unlock is an authoring-tier action (teacher|hta), same gate @@ -1077,6 +1098,20 @@ const SubmissionsPageContent = () => { filtered={hasActiveFilter} onClearFilters={clearFilters} emptyRepo={isEmptyRepoAssignment} + // Per-row trigger retrofit: owner + default-autograder only (teacher- + // authored shims are never rewritten). Mirrors the bulk-action gate, + // including the resolved-assignmentInfo requirement: while the + // assignments query loads, isDefaultAutograder(undefined) is true and + // the ?? fallback would arm the action with "every-push" — clicking it + // would rewrite a tag-mode repo's shim to the wrong trigger. + submissionMode={ + isOwner && + assignmentInfo != null && + isDefaultAutograder(assignmentInfo.autograder) + ? (assignmentInfo.submission_mode ?? "every-push") + : undefined + } + submissionTags={assignmentInfo?.submission_tags} initialLoading={initialLoading} nonSubmittersLoading={ !nonSubmittersReady && @@ -1220,6 +1255,17 @@ const SubmissionsPageContent = () => { owners={acceptedOwners} students={students} /> + setBulkTriggerOpen(false)} + org={org} + classroom={classroom} + assignment={assignment} + submissionMode={assignmentInfo?.submission_mode ?? "every-push"} + submissionTags={assignmentInfo?.submission_tags} + owners={acceptedOwners} + students={students} + /> ) } diff --git a/web/src/pages/assignments/DetailsSection.tsx b/web/src/pages/assignments/DetailsSection.tsx index 801e1481..a420fcb4 100644 --- a/web/src/pages/assignments/DetailsSection.tsx +++ b/web/src/pages/assignments/DetailsSection.tsx @@ -23,6 +23,10 @@ import { REPO_PERMISSIONS, defaultStudentPermission, } from "@/types/classroom" +import { + parseSubmissionTags, + validateSubmissionTags, +} from "@/util/submissionTags" import type { AssignmentForm } from "./assignmentFormModel" // GitHub's own reference for the repo role ladder (read/triage/write/maintain/ @@ -385,6 +389,204 @@ export const DetailsSection = ({ )} + + {/* Submission trigger: every-push (the default) or tag mode + (only submit/* tags grade — the Actions-cost lever). A bare + repo has no shim, so the picker locks to the default. On + EDIT, a change only affects new accepts: warn that existing + repos need the retrofit action. */} + state.values.empty_repo}> + {(emptyRepo) => ( + + {(field) => ( +
+ + {({ id, describedById }) => ( + + )} + + {edit ? ( + state.values.submission_mode} + > + {(mode) => + mode !== + (form.options.defaultValues?.submission_mode ?? + "every-push") ? ( + + + {t( + "assignments.form.submissionMode.editWarning", + )} + + + ) : null + } + + ) : null} +
+ )} + + )} + + + {/* Milestone submission tags: teacher-named tag patterns (e.g. + phase1, phase2, complete) that ALSO trigger grading — a + student pushing a matching tag gets that commit graded, with + the record still living at the canonical submit/* tag the + runner mints. Union with submit/* in the shim, orthogonal to + the mode picker above; the same shim-retrofit warning applies + on edit. Locked like the mode picker for a bare repo. */} + state.values.empty_repo}> + {(emptyRepo) => ( + + {(field) => { + const error = field.state.meta.errors[0] as + string | undefined + return ( +
+ + {({ id, describedById }) => ( +