diff --git a/backend/make/apps/owasp.mk b/backend/make/apps/owasp.mk index 22e15ca8a0..fec8f33f70 100644 --- a/backend/make/apps/owasp.mk +++ b/backend/make/apps/owasp.mk @@ -2,9 +2,9 @@ owasp-aggregate-member-contributions owasp-aggregate-projects owasp-create-project-metadata-file \ owasp-enrich-chapters owasp-enrich-committees owasp-enrich-events owasp-enrich-projects \ owasp-generate-community-snapshot-video owasp-process-snapshots owasp-scrape-chapters \ - owasp-scrape-committees owasp-scrape-projects owasp-sync-posts owasp-update-events \ - owasp-update-leaders owasp-update-project-health-metrics owasp-update-project-health-requirements \ - owasp-update-project-health-scores owasp-update-sponsors + owasp-scrape-committees owasp-scrape-projects owasp-sync-board-activity owasp-sync-posts \ + owasp-update-events owasp-update-leaders owasp-update-project-health-metrics \ + owasp-update-project-health-requirements owasp-update-project-health-scores owasp-update-sponsors owasp-add-project-custom-tags: @echo "Adding project custom tags from $(FILE)" @@ -74,6 +74,10 @@ owasp-scrape-projects: @echo "Scraping OWASP site projects data" @CMD="python manage.py owasp_scrape_projects" $(MAKE) backend-exec-command +owasp-sync-board-activity: + @echo "Syncing OWASP board activity from www-board" + @CMD="python manage.py owasp_sync_board_activity $(ARGS)" $(MAKE) backend-exec-command + owasp-sync-posts: @CMD="python manage.py owasp_sync_posts" $(MAKE) backend-exec-command diff --git a/backend/src/apps/common/open_ai.py b/backend/src/apps/common/open_ai.py index 772d105532..0a2827e13d 100644 --- a/backend/src/apps/common/open_ai.py +++ b/backend/src/apps/common/open_ai.py @@ -3,18 +3,28 @@ from __future__ import annotations import logging +from typing import TYPE_CHECKING, TypeVar import openai from django.conf import settings +if TYPE_CHECKING: + from pydantic import BaseModel + logger: logging.Logger = logging.getLogger(__name__) +T = TypeVar("T", bound="BaseModel") + class OpenAi: """Open AI communication class.""" def __init__( - self, model: str = "gpt-4o-mini", max_tokens: int = 1000, temperature: float = 0.7 + self, + model: str = "gpt-4o-mini", + max_tokens: int = 1000, + temperature: float = 0.7, + timeout: int = 30, ) -> None: """OpenAi constructor. @@ -22,11 +32,12 @@ def __init__( model (str, optional): The model to use. max_tokens (int, optional): Maximum tokens for the response. temperature (float, optional): Sampling temperature. + timeout (int, optional): Request timeout in seconds. Defaults to 30. """ self.client = openai.OpenAI( api_key=settings.OPEN_AI_SECRET_KEY, - timeout=30, # In seconds. + timeout=timeout, ) self.max_tokens = max_tokens @@ -75,6 +86,50 @@ def set_prompt(self, content: str) -> OpenAi: return self + def parse(self, schema: type[T]) -> T | None: + """Get a structured response validated against a pydantic schema. + + Args: + schema (type[T]): A pydantic model class. + + Returns: + T | None: A validated instance of the schema, or None on error. + + """ + try: + response = self.client.beta.chat.completions.parse( + max_tokens=self.max_tokens, + messages=[ + {"role": "system", "content": self.prompt}, + {"role": "user", "content": self.input}, + ], + model=self.model, + response_format=schema, + temperature=self.temperature, + ) + return response.choices[0].message.parsed + except openai.AuthenticationError: + logger.exception("OpenAI authentication failed: invalid or missing API key. ") + except openai.RateLimitError as e: + logger.warning( + "OpenAI rate limit exceeded: %s. Request may be retried with backoff.", + e, + ) + except openai.BadRequestError: + logger.exception( + "OpenAI invalid request. Check model name, message format, and input size." + ) + except openai.APIConnectionError: + logger.exception( + "OpenAI connection failed. Check network connectivity and firewall/proxy settings." + ) + except Exception as e: + logger.exception( + "Unexpected OpenAI API error: %s", + type(e).__name__, + ) + return None + def complete(self) -> str | None: """Get API response. diff --git a/backend/src/apps/owasp/admin/__init__.py b/backend/src/apps/owasp/admin/__init__.py index 261225145a..307d7ecda5 100644 --- a/backend/src/apps/owasp/admin/__init__.py +++ b/backend/src/apps/owasp/admin/__init__.py @@ -4,7 +4,13 @@ from apps.owasp.models.project_health_requirements import ProjectHealthRequirements +from .board_discussion import BoardDiscussionAdmin +from .board_meeting import BoardMeetingAdmin +from .board_meeting_action import BoardMeetingActionAdmin +from .board_motion import BoardMotionAdmin from .board_of_directors import BoardOfDirectorsAdmin +from .board_outcome import BoardOutcomeAdmin +from .board_vote import BoardVoteAdmin from .chapter import ChapterAdmin from .committee import CommitteeAdmin from .entity_channel import EntityChannelAdmin diff --git a/backend/src/apps/owasp/admin/board_discussion.py b/backend/src/apps/owasp/admin/board_discussion.py new file mode 100644 index 0000000000..8de8facc3e --- /dev/null +++ b/backend/src/apps/owasp/admin/board_discussion.py @@ -0,0 +1,16 @@ +"""Board discussion admin configuration.""" + +from django.contrib import admin + +from apps.owasp.models.board_discussion import BoardDiscussion + + +class BoardDiscussionAdmin(admin.ModelAdmin): + """Admin for BoardDiscussion model.""" + + autocomplete_fields = ("participants",) + list_display = ("topic",) + search_fields = ("topic", "description") + + +admin.site.register(BoardDiscussion, BoardDiscussionAdmin) diff --git a/backend/src/apps/owasp/admin/board_meeting.py b/backend/src/apps/owasp/admin/board_meeting.py new file mode 100644 index 0000000000..51d61f0415 --- /dev/null +++ b/backend/src/apps/owasp/admin/board_meeting.py @@ -0,0 +1,18 @@ +"""Board meeting admin configuration.""" + +from django.contrib import admin + +from apps.owasp.models.board_meeting import BoardMeeting + + +class BoardMeetingAdmin(admin.ModelAdmin): + """Admin for BoardMeeting model.""" + + autocomplete_fields = ("board", "attendees", "absentees") + list_display = ("title", "date", "board", "type", "quorum_present") + list_filter = ("type", "quorum_present", "board__year") + ordering = ("-date",) + search_fields = ("title", "location", "source_path") + + +admin.site.register(BoardMeeting, BoardMeetingAdmin) diff --git a/backend/src/apps/owasp/admin/board_meeting_action.py b/backend/src/apps/owasp/admin/board_meeting_action.py new file mode 100644 index 0000000000..6f03ca7852 --- /dev/null +++ b/backend/src/apps/owasp/admin/board_meeting_action.py @@ -0,0 +1,23 @@ +"""Board meeting action admin configuration.""" + +from django.contrib import admin + +from apps.owasp.models.board_meeting_action import BoardMeetingAction + + +class BoardMeetingActionAdmin(admin.ModelAdmin): + """Admin for BoardMeetingAction model.""" + + list_display = ("meeting", "order", "discussion", "motion", "outcome") + list_filter = ("meeting__type",) + ordering = ("meeting", "order") + raw_id_fields = ("meeting", "discussion", "motion", "outcome") + search_fields = ( + "meeting__title", + "discussion__topic", + "motion__title", + "outcome__description", + ) + + +admin.site.register(BoardMeetingAction, BoardMeetingActionAdmin) diff --git a/backend/src/apps/owasp/admin/board_motion.py b/backend/src/apps/owasp/admin/board_motion.py new file mode 100644 index 0000000000..6c6dcf112b --- /dev/null +++ b/backend/src/apps/owasp/admin/board_motion.py @@ -0,0 +1,16 @@ +"""Board motion admin configuration.""" + +from django.contrib import admin + +from apps.owasp.models.board_motion import BoardMotion + + +class BoardMotionAdmin(admin.ModelAdmin): + """Admin for BoardMotion model.""" + + list_display = ("title", "sponsor", "second") + raw_id_fields = ("sponsor", "second", "amends_motion") + search_fields = ("title", "description", "background") + + +admin.site.register(BoardMotion, BoardMotionAdmin) diff --git a/backend/src/apps/owasp/admin/board_outcome.py b/backend/src/apps/owasp/admin/board_outcome.py new file mode 100644 index 0000000000..5adb87f355 --- /dev/null +++ b/backend/src/apps/owasp/admin/board_outcome.py @@ -0,0 +1,18 @@ +"""Board outcome admin configuration.""" + +from django.contrib import admin + +from apps.owasp.models.board_outcome import BoardOutcome + + +class BoardOutcomeAdmin(admin.ModelAdmin): + """Admin for BoardOutcome model.""" + + autocomplete_fields = ("assignees",) + list_display = ("description", "status", "due_date") + list_filter = ("status",) + ordering = ("-due_date",) + search_fields = ("description",) + + +admin.site.register(BoardOutcome, BoardOutcomeAdmin) diff --git a/backend/src/apps/owasp/admin/board_vote.py b/backend/src/apps/owasp/admin/board_vote.py new file mode 100644 index 0000000000..9690f6a81d --- /dev/null +++ b/backend/src/apps/owasp/admin/board_vote.py @@ -0,0 +1,18 @@ +"""Board vote admin configuration.""" + +from django.contrib import admin + +from apps.owasp.models.board_vote import BoardVote + + +class BoardVoteAdmin(admin.ModelAdmin): + """Admin for BoardVote model.""" + + autocomplete_fields = ("in_favor", "against", "abstain", "recused") + list_display = ("motion", "result", "type", "tally") + list_filter = ("result", "type") + raw_id_fields = ("motion",) + search_fields = ("motion__title", "tally") + + +admin.site.register(BoardVote, BoardVoteAdmin) diff --git a/backend/src/apps/owasp/management/commands/owasp_sync_board_activity.py b/backend/src/apps/owasp/management/commands/owasp_sync_board_activity.py new file mode 100644 index 0000000000..678d35465d --- /dev/null +++ b/backend/src/apps/owasp/management/commands/owasp_sync_board_activity.py @@ -0,0 +1,86 @@ +"""Sync board activity from OWASP/www-board into Django models.""" + +from django.core.management.base import BaseCommand, CommandError + +from apps.owasp.parsers.board_activity import sync +from apps.owasp.parsers.board_activity.sync import MAX_YEAR, MIN_YEAR, SyncStatus + +MIN_MONTH = 1 +MAX_MONTH = 12 + + +class Command(BaseCommand): + help = "Sync OWASP board meeting activity from the www-board repository." + + def add_arguments(self, parser): + """Add command-line arguments. + + Args: + parser (argparse.ArgumentParser): The argument parser. + + """ + parser.add_argument( + "--year", + type=int, + help="Only sync files whose filename begins with this 4-digit year.", + ) + parser.add_argument( + "--month", + type=int, + help="Further restrict to a specific month (1-12). Requires --year.", + ) + parser.add_argument( + "--path", + type=str, + help="Sync only a single repo-relative file path (ignores --year/--month).", + ) + parser.add_argument( + "--force", + action="store_true", + help="Re-parse even when the stored git blob SHA matches.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Parse and log intended writes without persisting.", + ) + + def handle(self, *args, **options): + """Run the board activity sync. + + Raises: + CommandError: If --month or --year is invalid, or if any file failed to sync. + + """ + year = options.get("year") + month = options.get("month") + path = options.get("path") + + if not path: + if month is not None and year is None: + message = "--month requires --year." + raise CommandError(message) + + if month is not None and not (MIN_MONTH <= month <= MAX_MONTH): + message = f"--month must be between {MIN_MONTH} and {MAX_MONTH}." + raise CommandError(message) + + if year is not None and not (MIN_YEAR <= year <= MAX_YEAR): + message = f"--year must be a 4-digit value between {MIN_YEAR} and {MAX_YEAR}." + raise CommandError(message) + + stats = sync.run( + year=year, + month=month, + path=path, + force=options.get("force", False), + dry_run=options.get("dry_run", False), + ) + + summary = ", ".join(f"{k}={v}" for k, v in sorted(stats.counts.items())) or "no files" + self.stdout.write(self.style.SUCCESS(f"Board activity sync: {summary}")) + + errored = stats.counts.get(SyncStatus.ERRORED, 0) + if errored: + message = f"Board activity sync had {errored} errored file(s)." + raise CommandError(message) diff --git a/backend/src/apps/owasp/migrations/0073_boarddiscussion_boardmeeting_boardmotion_and_more.py b/backend/src/apps/owasp/migrations/0073_boarddiscussion_boardmeeting_boardmotion_and_more.py new file mode 100644 index 0000000000..036d8bf306 --- /dev/null +++ b/backend/src/apps/owasp/migrations/0073_boarddiscussion_boardmeeting_boardmotion_and_more.py @@ -0,0 +1,330 @@ +# Generated by Django 6.0.8 on 2026-08-31 14:56 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("owasp", "0072_project_project_name_gin_idx_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="BoardDiscussion", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ("description", models.TextField()), + ("metadata", models.JSONField(blank=True, default=dict)), + ("topic", models.CharField(max_length=500)), + ( + "participants", + models.ManyToManyField(blank=True, related_name="+", to="owasp.entitymember"), + ), + ], + options={ + "verbose_name_plural": "Board Discussions", + "db_table": "owasp_board_discussions", + }, + ), + migrations.CreateModel( + name="BoardMeeting", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ("attachments", models.JSONField(blank=True, default=list)), + ("call_in_url", models.URLField(blank=True, default="")), + ("date", models.DateTimeField()), + ("guests", models.JSONField(blank=True, default=list)), + ("location", models.CharField(blank=True, default="", max_length=255)), + ("metadata", models.JSONField(blank=True, default=dict)), + ("quorum_present", models.BooleanField(blank=True, null=True)), + ("recording_url", models.URLField(blank=True, default="")), + ("source_checksum", models.CharField(blank=True, default="", max_length=64)), + ("source_path", models.CharField(max_length=500, unique=True)), + ("title", models.CharField(blank=True, default="", max_length=255)), + ( + "type", + models.CharField( + choices=[ + ("private", "Private"), + ("public", "Public"), + ("special", "Special"), + ("summit", "Summit"), + ], + default="public", + max_length=7, + ), + ), + ( + "absentees", + models.ManyToManyField(blank=True, related_name="+", to="owasp.entitymember"), + ), + ( + "attendees", + models.ManyToManyField(blank=True, related_name="+", to="owasp.entitymember"), + ), + ( + "board", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="meetings", + to="owasp.boardofdirectors", + ), + ), + ], + options={ + "verbose_name_plural": "Board Meetings", + "db_table": "owasp_board_meetings", + }, + ), + migrations.CreateModel( + name="BoardMotion", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ("background", models.TextField(blank=True, default="")), + ("description", models.TextField()), + ("metadata", models.JSONField(blank=True, default=dict)), + ("references", models.JSONField(blank=True, default=list)), + ("title", models.CharField(max_length=500)), + ( + "amends_motion", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="owasp.boardmotion", + ), + ), + ( + "second", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="owasp.entitymember", + ), + ), + ( + "sponsor", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to="owasp.entitymember", + ), + ), + ], + options={ + "verbose_name_plural": "Board Motions", + "db_table": "owasp_board_motions", + }, + ), + migrations.CreateModel( + name="BoardOutcome", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ("description", models.TextField()), + ("due_date", models.DateField(blank=True, null=True)), + ("metadata", models.JSONField(blank=True, default=dict)), + ( + "status", + models.CharField( + choices=[ + ("cancelled", "Cancelled"), + ("completed", "Completed"), + ("in_progress", "In Progress"), + ("pending", "Pending"), + ], + default="pending", + max_length=11, + ), + ), + ( + "assignees", + models.ManyToManyField(blank=True, related_name="+", to="owasp.entitymember"), + ), + ], + options={ + "verbose_name_plural": "Board Outcomes", + "db_table": "owasp_board_outcomes", + }, + ), + migrations.CreateModel( + name="BoardVote", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ("metadata", models.JSONField(blank=True, default=dict)), + ( + "result", + models.CharField( + choices=[ + ("deferred", "Deferred"), + ("failed", "Failed"), + ("passed", "Passed"), + ("tabled", "Tabled"), + ("withdrawn", "Withdrawn"), + ], + max_length=9, + ), + ), + ("tally", models.CharField(blank=True, default="", max_length=255)), + ( + "type", + models.CharField( + choices=[("e_vote", "E-Vote"), ("vote", "Vote")], + default="vote", + max_length=6, + ), + ), + ( + "abstain", + models.ManyToManyField(blank=True, related_name="+", to="owasp.entitymember"), + ), + ( + "against", + models.ManyToManyField(blank=True, related_name="+", to="owasp.entitymember"), + ), + ( + "in_favor", + models.ManyToManyField(blank=True, related_name="+", to="owasp.entitymember"), + ), + ( + "motion", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="votes", + to="owasp.boardmotion", + ), + ), + ( + "recused", + models.ManyToManyField(blank=True, related_name="+", to="owasp.entitymember"), + ), + ], + options={ + "verbose_name_plural": "Board Votes", + "db_table": "owasp_board_votes", + }, + ), + migrations.CreateModel( + name="BoardMeetingAction", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ("order", models.PositiveIntegerField()), + ( + "discussion", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="owasp.boarddiscussion", + ), + ), + ( + "meeting", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="actions", + to="owasp.boardmeeting", + ), + ), + ( + "motion", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="owasp.boardmotion", + ), + ), + ( + "outcome", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to="owasp.boardoutcome", + ), + ), + ], + options={ + "verbose_name_plural": "Board Meeting Actions", + "db_table": "owasp_board_meeting_actions", + "constraints": [ + models.UniqueConstraint( + fields=("meeting", "order"), + name="board_meeting_action_unique_meeting_order", + ), + models.CheckConstraint( + condition=models.Q( + models.Q( + ("discussion__isnull", False), + ("motion__isnull", True), + ("outcome__isnull", True), + ), + models.Q( + ("discussion__isnull", True), + ("motion__isnull", False), + ("outcome__isnull", True), + ), + models.Q( + ("discussion__isnull", True), + ("motion__isnull", True), + ("outcome__isnull", False), + ), + _connector="OR", + ), + name="board_meeting_action_exactly_one_target", + ), + ], + }, + ), + ] diff --git a/backend/src/apps/owasp/models/__init__.py b/backend/src/apps/owasp/models/__init__.py index 3cbb120b8b..ddcc1bf9e9 100644 --- a/backend/src/apps/owasp/models/__init__.py +++ b/backend/src/apps/owasp/models/__init__.py @@ -1,4 +1,10 @@ +from .board_discussion import BoardDiscussion +from .board_meeting import BoardMeeting +from .board_meeting_action import BoardMeetingAction +from .board_motion import BoardMotion from .board_of_directors import BoardOfDirectors +from .board_outcome import BoardOutcome +from .board_vote import BoardVote from .chapter import Chapter from .committee import Committee from .entity_channel import EntityChannel diff --git a/backend/src/apps/owasp/models/board_discussion.py b/backend/src/apps/owasp/models/board_discussion.py new file mode 100644 index 0000000000..c79f349d60 --- /dev/null +++ b/backend/src/apps/owasp/models/board_discussion.py @@ -0,0 +1,30 @@ +"""OWASP app board discussion model.""" + +from django.db import models + +from apps.common.models import TimestampedModel +from apps.owasp.models.entity_member import EntityMember + + +class BoardDiscussion(TimestampedModel): + """Board discussion model.""" + + class Meta: + """Model options.""" + + db_table = "owasp_board_discussions" + verbose_name_plural = "Board Discussions" + + description = models.TextField() + metadata = models.JSONField(default=dict, blank=True) + topic = models.CharField(max_length=500) + + participants = models.ManyToManyField( + EntityMember, + blank=True, + related_name="+", + ) + + def __str__(self) -> str: + """Return the board discussion human-readable representation.""" + return f"Discussion: {self.topic}" diff --git a/backend/src/apps/owasp/models/board_meeting.py b/backend/src/apps/owasp/models/board_meeting.py new file mode 100644 index 0000000000..43e42d8244 --- /dev/null +++ b/backend/src/apps/owasp/models/board_meeting.py @@ -0,0 +1,52 @@ +"""OWASP app board meeting model.""" + +from django.db import models + +from apps.common.models import TimestampedModel +from apps.owasp.models.board_of_directors import BoardOfDirectors +from apps.owasp.models.entity_member import EntityMember + + +class BoardMeeting(TimestampedModel): + """Board meeting model.""" + + class Type(models.TextChoices): + """Board meeting type choices.""" + + PRIVATE = "private", "Private" + PUBLIC = "public", "Public" + SPECIAL = "special", "Special" + SUMMIT = "summit", "Summit" + + class Meta: + """Model options.""" + + db_table = "owasp_board_meetings" + verbose_name_plural = "Board Meetings" + + attachments = models.JSONField(default=list, blank=True) + call_in_url = models.URLField(blank=True, default="") + date = models.DateTimeField() + guests = models.JSONField(default=list, blank=True) + location = models.CharField(max_length=255, blank=True, default="") + metadata = models.JSONField(default=dict, blank=True) + quorum_present = models.BooleanField(blank=True, null=True) + recording_url = models.URLField(blank=True, default="") + source_checksum = models.CharField(max_length=64, blank=True, default="") + source_path = models.CharField(max_length=500, unique=True) + title = models.CharField(max_length=255, blank=True, default="") + type = models.CharField(max_length=7, choices=Type.choices, default=Type.PUBLIC) + + absentees = models.ManyToManyField(EntityMember, blank=True, related_name="+") + attendees = models.ManyToManyField(EntityMember, blank=True, related_name="+") + + board = models.ForeignKey( + BoardOfDirectors, + on_delete=models.CASCADE, + related_name="meetings", + ) + + def __str__(self) -> str: + """Return the board meeting human-readable representation.""" + label = self.title or self.date.isoformat() + return f"Board Meeting: {label}" diff --git a/backend/src/apps/owasp/models/board_meeting_action.py b/backend/src/apps/owasp/models/board_meeting_action.py new file mode 100644 index 0000000000..c9d8663265 --- /dev/null +++ b/backend/src/apps/owasp/models/board_meeting_action.py @@ -0,0 +1,78 @@ +"""OWASP app board meeting action model.""" + +from django.db import models + +from apps.common.models import TimestampedModel +from apps.owasp.models.board_discussion import BoardDiscussion +from apps.owasp.models.board_meeting import BoardMeeting +from apps.owasp.models.board_motion import BoardMotion +from apps.owasp.models.board_outcome import BoardOutcome + + +class BoardMeetingAction(TimestampedModel): + """Board meeting action model.""" + + class Meta: + """Model options.""" + + db_table = "owasp_board_meeting_actions" + verbose_name_plural = "Board Meeting Actions" + constraints = [ + models.UniqueConstraint( + fields=["meeting", "order"], + name="board_meeting_action_unique_meeting_order", + ), + models.CheckConstraint( + condition=( + models.Q( + discussion__isnull=False, + motion__isnull=True, + outcome__isnull=True, + ) + | models.Q( + discussion__isnull=True, + motion__isnull=False, + outcome__isnull=True, + ) + | models.Q( + discussion__isnull=True, + motion__isnull=True, + outcome__isnull=False, + ) + ), + name="board_meeting_action_exactly_one_target", + ), + ] + + order = models.PositiveIntegerField() + + discussion = models.ForeignKey( + BoardDiscussion, + blank=True, + null=True, + on_delete=models.CASCADE, + related_name="+", + ) + meeting = models.ForeignKey( + BoardMeeting, + on_delete=models.CASCADE, + related_name="actions", + ) + motion = models.ForeignKey( + BoardMotion, + blank=True, + null=True, + on_delete=models.CASCADE, + related_name="+", + ) + outcome = models.ForeignKey( + BoardOutcome, + blank=True, + null=True, + on_delete=models.CASCADE, + related_name="+", + ) + + def __str__(self) -> str: + """Return the board meeting action human-readable representation.""" + return f"Meeting Action #{self.order} in meeting {self.meeting_id}" diff --git a/backend/src/apps/owasp/models/board_motion.py b/backend/src/apps/owasp/models/board_motion.py new file mode 100644 index 0000000000..4c134247a7 --- /dev/null +++ b/backend/src/apps/owasp/models/board_motion.py @@ -0,0 +1,48 @@ +"""OWASP app board motion model.""" + +from django.db import models + +from apps.common.models import TimestampedModel +from apps.owasp.models.entity_member import EntityMember + + +class BoardMotion(TimestampedModel): + """Board motion model.""" + + class Meta: + """Model options.""" + + db_table = "owasp_board_motions" + verbose_name_plural = "Board Motions" + + background = models.TextField(blank=True, default="") + description = models.TextField() + metadata = models.JSONField(default=dict, blank=True) + references = models.JSONField(default=list, blank=True) + title = models.CharField(max_length=500) + + amends_motion = models.ForeignKey( + "self", + blank=True, + null=True, + on_delete=models.SET_NULL, + related_name="+", + ) + second = models.ForeignKey( + EntityMember, + blank=True, + null=True, + on_delete=models.SET_NULL, + related_name="+", + ) + sponsor = models.ForeignKey( + EntityMember, + blank=True, + null=True, + on_delete=models.SET_NULL, + related_name="+", + ) + + def __str__(self) -> str: + """Return the board motion human-readable representation.""" + return f"Motion: {self.title}" diff --git a/backend/src/apps/owasp/models/board_outcome.py b/backend/src/apps/owasp/models/board_outcome.py new file mode 100644 index 0000000000..e729a32286 --- /dev/null +++ b/backend/src/apps/owasp/models/board_outcome.py @@ -0,0 +1,43 @@ +"""OWASP app board outcome model.""" + +from django.db import models + +from apps.common.models import TimestampedModel +from apps.owasp.models.entity_member import EntityMember + + +class BoardOutcome(TimestampedModel): + """Board outcome model.""" + + class Status(models.TextChoices): + """Board outcome status choices.""" + + CANCELLED = "cancelled", "Cancelled" + COMPLETED = "completed", "Completed" + IN_PROGRESS = "in_progress", "In Progress" + PENDING = "pending", "Pending" + + class Meta: + """Model options.""" + + db_table = "owasp_board_outcomes" + verbose_name_plural = "Board Outcomes" + + description = models.TextField() + due_date = models.DateField(blank=True, null=True) + metadata = models.JSONField(default=dict, blank=True) + status = models.CharField( + max_length=11, + choices=Status.choices, + default=Status.PENDING, + ) + + assignees = models.ManyToManyField( + EntityMember, + blank=True, + related_name="+", + ) + + def __str__(self) -> str: + """Return the board outcome human-readable representation.""" + return f"Outcome ({self.get_status_display()}): {self.description[:60]}" diff --git a/backend/src/apps/owasp/models/board_vote.py b/backend/src/apps/owasp/models/board_vote.py new file mode 100644 index 0000000000..296e4d4391 --- /dev/null +++ b/backend/src/apps/owasp/models/board_vote.py @@ -0,0 +1,52 @@ +"""OWASP app board vote model.""" + +from django.db import models + +from apps.common.models import TimestampedModel +from apps.owasp.models.board_motion import BoardMotion +from apps.owasp.models.entity_member import EntityMember + + +class BoardVote(TimestampedModel): + """Board vote model.""" + + class Result(models.TextChoices): + """Board vote result choices.""" + + DEFERRED = "deferred", "Deferred" + FAILED = "failed", "Failed" + PASSED = "passed", "Passed" + TABLED = "tabled", "Tabled" + WITHDRAWN = "withdrawn", "Withdrawn" + + class Type(models.TextChoices): + """Board vote type choices.""" + + E_VOTE = "e_vote", "E-Vote" + VOTE = "vote", "Vote" + + class Meta: + """Model options.""" + + db_table = "owasp_board_votes" + verbose_name_plural = "Board Votes" + + metadata = models.JSONField(default=dict, blank=True) + result = models.CharField(max_length=9, choices=Result.choices) + tally = models.CharField(max_length=255, blank=True, default="") + type = models.CharField(max_length=6, choices=Type.choices, default=Type.VOTE) + + abstain = models.ManyToManyField(EntityMember, blank=True, related_name="+") + against = models.ManyToManyField(EntityMember, blank=True, related_name="+") + in_favor = models.ManyToManyField(EntityMember, blank=True, related_name="+") + recused = models.ManyToManyField(EntityMember, blank=True, related_name="+") + + motion = models.ForeignKey( + BoardMotion, + on_delete=models.CASCADE, + related_name="votes", + ) + + def __str__(self) -> str: + """Return the board vote human-readable representation.""" + return f"Vote ({self.get_result_display()}): {self.tally or 'n/a'}" diff --git a/backend/src/apps/owasp/parsers/__init__.py b/backend/src/apps/owasp/parsers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/apps/owasp/parsers/board_activity/__init__.py b/backend/src/apps/owasp/parsers/board_activity/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/apps/owasp/parsers/board_activity/schemas.py b/backend/src/apps/owasp/parsers/board_activity/schemas.py new file mode 100644 index 0000000000..6ee39f4c08 --- /dev/null +++ b/backend/src/apps/owasp/parsers/board_activity/schemas.py @@ -0,0 +1,131 @@ +"""Pydantic extraction schema for board meeting markdown parsing.""" + +from __future__ import annotations + +from datetime import datetime # noqa: TC003 +from enum import StrEnum + +from pydantic import BaseModel, Field + + +class ActionKind(StrEnum): + """Discriminator for the payload carried by a ParsedAction.""" + + DISCUSSION = "discussion" + MOTION = "motion" + OUTCOME = "outcome" + + +class MeetingType(StrEnum): + """Board meeting type.""" + + PRIVATE = "private" + PUBLIC = "public" + SPECIAL = "special" + SUMMIT = "summit" + + +class OutcomeStatus(StrEnum): + """Board outcome status.""" + + CANCELLED = "cancelled" + COMPLETED = "completed" + IN_PROGRESS = "in_progress" + PENDING = "pending" + + +class VoteResult(StrEnum): + """Result of a recorded vote.""" + + DEFERRED = "deferred" + FAILED = "failed" + PASSED = "passed" + TABLED = "tabled" + WITHDRAWN = "withdrawn" + + +class VoteType(StrEnum): + """Vote type.""" + + E_VOTE = "e_vote" + VOTE = "vote" + + +class LabeledUrl(BaseModel): + """A titled link that appears in meeting minutes.""" + + label: str + url: str + + +class ParsedPersonRef(BaseModel): + """A person mentioned in the meeting.""" + + name: str + + +class ParsedVote(BaseModel): + """A recorded vote on a motion.""" + + result: VoteResult + type: VoteType + tally: str = "" + in_favor: list[ParsedPersonRef] = Field(default_factory=list) + against: list[ParsedPersonRef] = Field(default_factory=list) + abstain: list[ParsedPersonRef] = Field(default_factory=list) + recused: list[ParsedPersonRef] = Field(default_factory=list) + + +class ParsedMotion(BaseModel): + """A motion put forward at a meeting.""" + + title: str + description: str + background: str = "" + references: list[LabeledUrl] = Field(default_factory=list) + sponsor: ParsedPersonRef | None = None + second: ParsedPersonRef | None = None + vote: ParsedVote | None = None + + +class ParsedDiscussion(BaseModel): + """A discussion item on the meeting agenda.""" + + topic: str + description: str + participants: list[ParsedPersonRef] = Field(default_factory=list) + + +class ParsedOutcome(BaseModel): + """An action item / outcome produced by a meeting.""" + + description: str + status: OutcomeStatus = OutcomeStatus.PENDING + due_date: str | None = None + assignees: list[ParsedPersonRef] = Field(default_factory=list) + + +class ParsedAction(BaseModel): + """A single agenda item.""" + + kind: ActionKind + motion: ParsedMotion | None = None + discussion: ParsedDiscussion | None = None + outcome: ParsedOutcome | None = None + + +class ParsedMeeting(BaseModel): + """The full extraction of a single meeting markdown file.""" + + title: str + date: datetime + type: MeetingType = MeetingType.PUBLIC + location: str = "" + call_in_url: str = "" + recording_url: str = "" + quorum_present: bool | None = None + attachments: list[LabeledUrl] = Field(default_factory=list) + attendees: list[ParsedPersonRef] = Field(default_factory=list) + absentees: list[ParsedPersonRef] = Field(default_factory=list) + guests: list[str] = Field(default_factory=list) + actions: list[ParsedAction] = Field(default_factory=list) diff --git a/backend/src/apps/owasp/parsers/board_activity/sync.py b/backend/src/apps/owasp/parsers/board_activity/sync.py new file mode 100644 index 0000000000..50d3c53330 --- /dev/null +++ b/backend/src/apps/owasp/parsers/board_activity/sync.py @@ -0,0 +1,488 @@ +"""Board activity sync orchestration.""" + +from __future__ import annotations + +import logging +import os +import re +import sys +from dataclasses import dataclass, field +from enum import StrEnum +from typing import TYPE_CHECKING + +import requests +from requests.exceptions import RequestException + +from apps.common.open_ai import OpenAi +from apps.owasp.models.board_meeting import BoardMeeting +from apps.owasp.parsers.board_activity import translator +from apps.owasp.parsers.board_activity.schemas import ParsedMeeting + +if TYPE_CHECKING: + from collections.abc import Iterable + +logger = logging.getLogger(__name__) + +DEFAULT_MODEL = "gpt-4o" +DEFAULT_MAX_TOKENS = 16000 +DEFAULT_TEMPERATURE = 0.1 +DEFAULT_TIMEOUT = 60 + +REPO_OWNER = "OWASP" +REPO_NAME = "www-board" +REPO_BRANCH = "master" + +TREE_URL = ( + f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/git/trees/{REPO_BRANCH}?recursive=1" +) +RAW_FILE_URL_TEMPLATE = ( + f"https://raw.githubusercontent.com/{REPO_OWNER}/{REPO_NAME}/{REPO_BRANCH}/{{path}}" +) + +MEETING_DIRS: tuple[str, ...] = ( + "meetings-historical/", + "minutes-deprecated/", +) +YEARMONTH_PREFIX_LEN = 6 # YYYY + MM +FILENAME_PREFIX_RE = re.compile(rf"^\d{{{YEARMONTH_PREFIX_LEN}}}") +MIN_YEAR = 1000 +MAX_YEAR = 9999 + +SYSTEM_PROMPT = """\ +You extract structured data from OWASP Board of Directors meeting minutes (unstructured Markdown) +into the provided schema. Board minutes are a legal record; the goal is to preserve every agenda +item and every decision faithfully. + +## Section boundaries + +Every markdown heading (`##`, `###`, `####`) introduces a new section. Each substantive heading +maps to its own action. NEVER merge content from different headings into a single description, +even if they share a parent heading (e.g. six H4 sub-sections under "Executive Director Report" +are SIX separate discussions, not one big one). When in doubt, split. + +## Meeting structure + +Meetings follow a fairly consistent shape. Typical top-level sections (H2/H3): +- CALL TO ORDER - a roster of board members present and guests. Extract into `attendees` and + `guests`. Board members explicitly listed as absent (or implied by omission with an "Absent:" + note) go in `absentees`. Does not produce any action - do not emit as a discussion. + +- CONFLICT OF INTEREST / ANTI-TRUST STATEMENT - boilerplate. Skip. The typical text starts with: + "As the Board consists of individuals from many competing organizations, OWASP and its Board + shall abide by all applicable anti-trust and competition laws..." Any body matching this shape + (or wholly consisting of policy links + the phrase "must recuse themselves") produces zero + actions. If a specific disclosure IS made in the section ("Andrew and Dawn to review the + conflict"), extract only that as its own outcome; do not also emit the boilerplate as a + discussion. + +- CHANGES TO THE AGENDA - if there's real content (e.g. "add discussion on X"), emit as a + discussion. If the body is only the standard policy paragraph starting with "Changes to the + agenda - unless otherwise prohibited by anti-trust or competition laws - including adding, + altering, or tabling of motions is permitted by following Roberts Rules of Order", skip entirely. + +- APPROVAL OF MINUTES - a vote on the previous meeting's minutes. Synthesize a stub motion + (see "Votes" below). + +- PRE-READING MATERIAL - a list of links. Add each link to `attachments`, do NOT emit as a + discussion/motion/outcome unless the section also has narrative content (multi-sentence prose + beyond a link caption). A bare bulleted link list like + - [Executive Director Board Summary](https://...) + - [Finance Board Narrative](/attachments/...) + produces zero actions - only attachments. + +- REPORTS - subsections (usually H4) like "Executive Director report", "Finance report", + "Committee reports". Each substantive subsection is a `discussion`. Don't collapse them into one. + +- NEW BUSINESS - mix of motions, discussions, and status updates. Each H3/H4 subsection + is its own action. + +- COMMENTS, ANNOUNCEMENTS - usually thin. If empty, skip. + +- ADJOURNMENT - a motion, usually with sponsor+second but often no formal vote tally. + Emit as a motion; vote is null if no tally is given. + +## Action kinds + +Every substantive subsection maps to exactly one action: + +- `discussion`: any narrative topic without a formal vote - status updates, ED/finance reports, + working group updates, executive session summaries, policy deliberations, etc. Populate `topic + (the heading) and `description` (the paragraph text). + + Populate `participants` from ALL of: + - names in the section heading (e.g. "#### Andrew van der Stock - + Executive Director" -> Andrew van der Stock) + - names in the body prose ("Andrew presented...", "Sam highlighted...") + - role references that can be resolved against the meeting's known people + ("The Executive Director" -> Andrew van der Stock if Andrew is listed as ED) + - explicit sponsors of a discussion (e.g. "Discussion on X (sponsored by Ricardo)" -> Ricardo) + If no specific person is named or resolvable, leave empty. + + Do NOT enumerate all attendees when the source refers to the group as a whole. Treat these + phrases as "the whole group" and leave the corresponding participant/assignee list empty: + - "the Board", "The Board of Directors" + - "All Board Members", "all directors", "every board director" + - "the Directors" (when unqualified) + - "the Board members present", "attending directors" + Only enumerate individuals when the source itself names them. + +- `motion`: a formal proposal, usually phrased "Motion:" or "Resolved, that...". Almost always has + a sponsor + second and (usually) a vote. Populate `title` (a short label), `description` (the + resolved-that clause), `background` (any explanatory prose before the motion), + `sponsor`, `second`, and `vote`. + +- `outcome`: an action item - an explicit commitment for someone (or a group) to do something after + the meeting. Populate `description` with the action text and `assignees` with the named person(s) + `status` defaults to "pending". + + Action items appear in TWO places: + a) explicit "Action Items:" / "Board Comments & Actions" bullet blocks. + b) INLINE within discussion prose, e.g. "Andrew to confirm SLA with Belgian authorities", + "directors Diego and Illia to provide ID by 9 September", "Sam and Andrew to consolidate + proposed edits". + + Extract BOTH. For (b), scan every discussion description for sentences of the form + " to " (future commitment) or " will ", + and emit each as a separate `outcome` action that follows the parent discussion in agenda order. + + `due_date`: parse deadlines from phrases like "by ", "before ", + "no later than ". Populate as ISO 8601 date (YYYY-MM-DD). Rules: + - "by 9 September" in a 2025 meeting -> "2025-09-09" (infer year from + the meeting date, month always closer future). + - "by February" (bare month, no day) -> null. Do not guess a day. + - "by end of Q1" / "by next meeting" / vague relative dates -> null. + - Absolute dates ("by March 15, 2026") -> "2026-03-15". + +Emit actions in the order they appear in the source. Be exhaustive: every substantive subsection +should produce an action, and every embedded commitment should produce its own outcome. + +## Structured content in descriptions + +When a section body contains structured data - bulleted stat lists (KPIs, finance figures, dollar +amounts, percentages, counts), tables, or numeric enumerations - keep the raw list/table content +in the `description` as-is rather than summarizing to prose. Join bullets with newlines. +This applies only within a single section; do not merge multiple sections into one description. + +## Votes + +Every vote must be attached to a motion (`ParsedVote` is nested under `ParsedMotion`). If a vote +appears without a formal motion - most commonly for "APPROVAL OF MINUTES" - +synthesize a stub motion: + title: "Approve meeting minutes" + description: "Resolved, that the meeting minutes are approved." + sponsor / second: null + +For each vote, populate the four cast lists (`in_favor`, `against`, `abstain`, `recused`) +from the individual member votes in the source. + +`tally`: dash-separated counts in the order in_favor-against-abstain(-recused). Always include the +abstain and recused counts if any member abstained or recused +- e.g. 6 yes + 0 no + 1 abstain -> "6-0-1", NOT "6-0". Only omit trailing zeros +(so "7-0" is fine when nobody voted against, abstained, or recused). + +## Names + +Preserve people's names exactly as they appear in the source, with one exception: when the source +uses a first-name-only reference (e.g. "Dave", "Ricardo") inside a vote record, discussion, +or outcome, resolve it to the corresponding full name from the meeting's attendees/guests roster +IF the mapping is unambiguous. Example: + Roster: "Dave Balbin", "Kerlyn Wandji", "Matt Stegen" + Source: "(Approve: Dave, Kerlyn, Matt)" + -> in_favor: ["Dave Balbin", "Kerlyn Wandji", "Matt Stegen"] + +If a short-form name matches multiple people in the roster (e.g. two "Daves"), keep it as-is rather +than guessing. Do not normalize casing or fix typos. + +The same person may be referenced multiple times in one meeting - use identical strings each time +so downstream deduplication works. + +## Attachments and references + +Links appear throughout: pre-reading material, motion supporting docs, next-meeting pointers, +video recordings. Extract every labeled Markdown link `[label](url)` as `{label, url}`. Meeting +level links go in `attachments`, motion-specific supporting docs go in the motion's `references`. + +## Metadata + +- `date`: ISO 8601 UTC (e.g. "2025-08-26T13:00:00+00:00"). If only the date + is known, use 00:00:00 UTC for time. +- `type`: public (default), private, special, or summit - infer from the + title or filename hints (e.g. "-special" suffix). +- `quorum_present`: null unless explicitly stated. +- `location`, `call_in_url`, `recording_url`: populate from the "Meeting Details" block. +If the only URL is a YouTube recording, put it in `recording_url`. + +## Editor annotations + +Wiki-style editor notes such as `[ ajv edit 2021-01-27: adding financial packages ]`, +`[TODO: fill in]`, or `` HTML comments are housekeeping, not meeting content. +Skip them entirely; do not emit as discussions and do not include in any description. + +## Unknown fields + +If a field is unknown, use the schema default (empty string, null, or empty list). +Do not fabricate. + +## Example - outcome extraction from a discussion section + +Input: + #### EU Entity Status + **Background** The ED will update on the new EU entity. + - Belgian Government approval pending. + - Directors Diego and Illia to provide ID by 9 September for office access. + - Andrew to confirm SLA with Belgian authorities. + - Board to review draft charter by November - Board Meeting. + - Sam to send meeting invites on October 25. + +Meeting date: 2025-08-26 + +Output (in agenda order): + discussion: + topic: "EU Entity Status" + description: "The ED will update on the new EU entity. + Belgian Government approval pending." + participants: [] + outcome: + description: "Diego and Illia to provide ID for office access." + assignees: [Diego Silva Martins, Illia Oleksiuk] + due_date: "2025-09-09" # "by 9 September" -> specific day, year inferred + outcome: + description: "Andrew to confirm SLA with Belgian authorities." + assignees: [Andrew van der Stock] + due_date: null # no deadline stated + outcome: + description: "Review draft charter." + assignees: [] # "Board" is a group reference; keep empty + due_date: null # "November - Board Meeting" is bare month; NEVER guess a day + outcome: + description: "Sam to send meeting invites." + assignees: [Sam Stepanyan] + due_date: "2025-10-25" # "on " is a deadline +""" + + +class SyncStatus(StrEnum): + """Terminal status codes for a per-file sync.""" + + CREATED = "created" + ERRORED = "errored" + SKIPPED = "skipped" + UNCHANGED = "unchanged" + UPDATED = "updated" + WOULD_UPDATE = "would_update" + + +@dataclass +class SyncStats: + """Aggregate status counters for a sync run.""" + + counts: dict[str, int] = field(default_factory=dict) + + def record(self, status: str) -> None: + """Increment the counter for the given status. + + Args: + status (str): A SyncStatus value. + + """ + self.counts[status] = self.counts.get(status, 0) + 1 + + +def fetch_tree() -> dict[str, str]: + """Fetch the recursive git tree for the www-board repository. + + Returns: + dict[str, str]: Map of file path to git blob SHA. + + Raises: + RequestException: If the GitHub API call fails. + + """ + headers = {"Accept": "application/vnd.github+json"} + if token := os.getenv("GITHUB_TOKEN"): + headers["Authorization"] = f"Bearer {token}" + + response = requests.get(TREE_URL, headers=headers, timeout=30) + response.raise_for_status() + payload = response.json() + + return { + item["path"]: item["sha"] for item in payload.get("tree", []) if item["type"] == "blob" + } + + +def target_paths( + tree: dict[str, str], + *, + year: int | None = None, + month: int | None = None, + path: str | None = None, +) -> Iterable[str]: + """Yield meeting markdown file paths matching the given filters. + + Args: + tree (dict[str, str]): Map of path to blob SHA. + year (int, optional): Restrict to files whose filename begins with the given 4-digit year. + month (int, optional): Restrict to files whose filename begins with + year and month. Requires year. + path (str, optional): Return only the given path if present in the tree. + + Yields: + str: A matching file path. + + Raises: + ValueError: If year is not a 4-digit value. + + """ + if path: + if path in tree: + yield path + return + + prefix = "" + if year is not None: + if not MIN_YEAR <= year <= MAX_YEAR: + message = f"year must be a 4-digit value between {MIN_YEAR} and {MAX_YEAR}." + raise ValueError(message) + prefix = str(year) + if month is not None: + prefix = f"{year}{month:02d}" + + for tree_path in tree: + if not tree_path.endswith(".md"): + continue + if not any(tree_path.startswith(d) for d in MEETING_DIRS): + continue + + filename = tree_path.rsplit("/", 1)[-1] + if filename.startswith("_"): + continue + if not FILENAME_PREFIX_RE.match(filename): + continue + if prefix and not filename.startswith(prefix): + continue + + yield tree_path + + +def fetch_file_content(path: str) -> str: + """Fetch the raw content of a file from the www-board repository. + + Args: + path (str): Repo-relative path of the file. + + Returns: + str: The file content, or empty string if the fetch failed or returned non-2xx. + + """ + url = RAW_FILE_URL_TEMPLATE.format(path=path) + try: + response = requests.get(url, timeout=30) + except RequestException: + logger.exception("Failed to fetch %s", url) + return "" + + if not response.ok: + logger.warning("Non-OK status %s for %s", response.status_code, url) + return "" + + return response.text + + +def sync_file( + path: str, + blob_sha: str, + *, + force: bool = False, + dry_run: bool = False, +) -> SyncStatus: + """Sync a single meeting file into the database. + + Args: + path (str): Repo-relative path of the file. + blob_sha (str): Current git blob SHA for the file. + force (bool, optional): Re-parse even when the stored checksum matches. Defaults to False. + dry_run (bool, optional): Skip DB writes. Defaults to False. + + Returns: + SyncStatus: A SyncStatus value. + + """ + existing = BoardMeeting.objects.filter(source_path=path).first() + if existing and existing.source_checksum == blob_sha and not force: + return SyncStatus.UNCHANGED + + content = fetch_file_content(path) + if not content: + logger.warning("Empty content for %s; skipping", path) + return SyncStatus.SKIPPED + + client = OpenAi( + model=DEFAULT_MODEL, + max_tokens=DEFAULT_MAX_TOKENS, + temperature=DEFAULT_TEMPERATURE, + timeout=DEFAULT_TIMEOUT, + ) + parsed = client.set_prompt(SYSTEM_PROMPT).set_input(content).parse(ParsedMeeting) + + if parsed is None: + logger.error("LLM parse failed for %s", path) + return SyncStatus.ERRORED + + if dry_run: + sys.stdout.write(f"\n=== Parsed {path} ===\n") + sys.stdout.write(parsed.model_dump_json(indent=2)) + sys.stdout.write("\n") + return SyncStatus.WOULD_UPDATE + + translator.upsert(parsed, source_path=path, source_checksum=blob_sha) + return SyncStatus.UPDATED if existing else SyncStatus.CREATED + + +def run( + *, + year: int | None = None, + month: int | None = None, + path: str | None = None, + force: bool = False, + dry_run: bool = False, +) -> SyncStats: + """Run a board activity sync against the www-board repository. + + Args: + year (int, optional): Restrict to files matching a 4-digit year prefix. + month (int, optional): Further restrict to a month. Requires year. + path (str, optional): Sync only a single repo-relative file path. + force (bool, optional): Re-parse even when checksums match. Defaults to False. + dry_run (bool, optional): Skip DB writes. Defaults to False. + + Returns: + SyncStats: Aggregate status counters. + + """ + stats = SyncStats() + + try: + tree = fetch_tree() + except RequestException: + logger.exception("Failed to fetch git tree for %s/%s", REPO_OWNER, REPO_NAME) + stats.record(SyncStatus.ERRORED) + return stats + + for target_path in target_paths(tree, year=year, month=month, path=path): + blob_sha = tree[target_path] + try: + status = sync_file( + target_path, + blob_sha, + force=force, + dry_run=dry_run, + ) + except Exception: + logger.exception("Sync failed for %s", target_path) + status = SyncStatus.ERRORED + + stats.record(status) + logger.info("board-activity: %s -> %s", target_path, status) + + return stats diff --git a/backend/src/apps/owasp/parsers/board_activity/translator.py b/backend/src/apps/owasp/parsers/board_activity/translator.py new file mode 100644 index 0000000000..1aa27a5816 --- /dev/null +++ b/backend/src/apps/owasp/parsers/board_activity/translator.py @@ -0,0 +1,286 @@ +"""Translator of pydantic to Django board-activity rows.""" + +from __future__ import annotations + +from datetime import date +from typing import TYPE_CHECKING + +from django.contrib.contenttypes.models import ContentType +from django.db import transaction + +from apps.owasp.models.board_discussion import BoardDiscussion +from apps.owasp.models.board_meeting import BoardMeeting +from apps.owasp.models.board_meeting_action import BoardMeetingAction +from apps.owasp.models.board_motion import BoardMotion +from apps.owasp.models.board_of_directors import BoardOfDirectors +from apps.owasp.models.board_outcome import BoardOutcome +from apps.owasp.models.board_vote import BoardVote +from apps.owasp.models.entity_member import EntityMember +from apps.owasp.parsers.board_activity.schemas import ( + ActionKind, + ParsedAction, + ParsedDiscussion, + ParsedMeeting, + ParsedMotion, + ParsedOutcome, + ParsedPersonRef, + ParsedVote, +) + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable + + PersonResolver = Callable[..., list[EntityMember]] + + +def upsert(parsed: ParsedMeeting, *, source_path: str, source_checksum: str) -> BoardMeeting: + """Upsert a parsed meeting and its child rows. + + Args: + parsed (ParsedMeeting): The parsed meeting payload. + source_path (str): Repo-relative path of the source markdown file. + source_checksum (str): Git blob SHA of the source file. + + Returns: + BoardMeeting: The created or updated meeting row. + + """ + with transaction.atomic(): + board, _ = BoardOfDirectors.objects.get_or_create(year=parsed.date.year) + + meeting, _ = BoardMeeting.objects.update_or_create( + source_path=source_path, + defaults={ + "attachments": [a.model_dump() for a in parsed.attachments], + "board": board, + "call_in_url": parsed.call_in_url, + "date": parsed.date, + "guests": list(parsed.guests), + "location": parsed.location, + "quorum_present": parsed.quorum_present, + "recording_url": parsed.recording_url, + "source_checksum": source_checksum, + "title": parsed.title, + "type": parsed.type, + }, + ) + + delete_meeting_children(meeting) + + board_content_type = ContentType.objects.get_for_model(BoardOfDirectors) + resolve = build_person_resolver(board, board_content_type) + + meeting.attendees.set(resolve(parsed.attendees, EntityMember.Role.MEMBER)) + meeting.absentees.set(resolve(parsed.absentees, EntityMember.Role.MEMBER)) + + for order, action in enumerate(parsed.actions, start=1): + create_action(meeting, order, action, resolve) + + return meeting + + +def delete_meeting_children(meeting: BoardMeeting) -> None: + """Delete all action rows and their referenced children for a meeting. + + Args: + meeting (BoardMeeting): The meeting whose agenda should be cleared. + + """ + actions = meeting.actions.all() + discussion_ids = list(actions.exclude(discussion=None).values_list("discussion_id", flat=True)) + motion_ids = list(actions.exclude(motion=None).values_list("motion_id", flat=True)) + outcome_ids = list(actions.exclude(outcome=None).values_list("outcome_id", flat=True)) + + BoardDiscussion.objects.filter(id__in=discussion_ids).delete() + BoardMotion.objects.filter(id__in=motion_ids).delete() + BoardOutcome.objects.filter(id__in=outcome_ids).delete() + + +def build_person_resolver( + board: BoardOfDirectors, board_content_type: ContentType +) -> PersonResolver: + """Build a person to EntityMember resolver bound to a board year. + + Args: + board (BoardOfDirectors): The board year to bind the resolver to. + board_content_type (ContentType): ContentType for BoardOfDirectors. + + Returns: + PersonResolver: A resolver callable. + + """ + + def resolver(people: Iterable[ParsedPersonRef], role) -> list[EntityMember]: + return [resolve_person(board, board_content_type, person.name, role) for person in people] + + return resolver + + +def resolve_person( + board: BoardOfDirectors, + board_content_type: ContentType, + name: str, + role, +) -> EntityMember: + """Get or create an EntityMember for a named person on a board. + + Args: + board (BoardOfDirectors): The board year to attach the person to. + board_content_type (ContentType): ContentType for BoardOfDirectors. + name (str): The person's source-text name. + role: An EntityMember.Role value assigned when a new row is created. + + Returns: + EntityMember: The resolved entity member row. + + """ + existing = EntityMember.objects.filter( + entity_id=board.id, + entity_type=board_content_type, + member_name=name, + ).first() + if existing: + return existing + + return EntityMember.objects.create( + entity_id=board.id, + entity_type=board_content_type, + member_name=name, + role=role, + ) + + +def create_discussion(discussion: ParsedDiscussion, resolve: PersonResolver) -> BoardDiscussion: + """Create a BoardDiscussion row from parsed input. + + Args: + discussion (ParsedDiscussion): The parsed discussion payload. + resolve (PersonResolver): Person resolver from build_person_resolver. + + Returns: + BoardDiscussion: The created row. + + """ + row = BoardDiscussion.objects.create( + description=discussion.description, + topic=discussion.topic, + ) + row.participants.set(resolve(discussion.participants, EntityMember.Role.MEMBER)) + return row + + +def create_motion(motion: ParsedMotion, resolve: PersonResolver) -> BoardMotion: + """Create a BoardMotion row and any attached vote from parsed input. + + Args: + motion (ParsedMotion): The parsed motion payload. + resolve (PersonResolver): Person resolver from build_person_resolver. + + Returns: + BoardMotion: The created row. + + """ + sponsor = resolve([motion.sponsor], EntityMember.Role.MEMBER)[0] if motion.sponsor else None + second = resolve([motion.second], EntityMember.Role.MEMBER)[0] if motion.second else None + + row = BoardMotion.objects.create( + background=motion.background, + description=motion.description, + references=[r.model_dump() for r in motion.references], + second=second, + sponsor=sponsor, + title=motion.title, + ) + + if motion.vote: + create_vote(row, motion.vote, resolve) + + return row + + +def create_outcome(outcome: ParsedOutcome, resolve: PersonResolver) -> BoardOutcome: + """Create a BoardOutcome row from parsed input. + + Args: + outcome (ParsedOutcome): The parsed outcome payload. + resolve (PersonResolver): Person resolver from build_person_resolver. + + Returns: + BoardOutcome: The created row. + + """ + try: + due_date = date.fromisoformat(outcome.due_date) if outcome.due_date else None + except ValueError: + due_date = None + + row = BoardOutcome.objects.create( + description=outcome.description, + due_date=due_date, + status=outcome.status, + ) + row.assignees.set(resolve(outcome.assignees, EntityMember.Role.MEMBER)) + return row + + +def create_vote(motion: BoardMotion, vote: ParsedVote, resolve: PersonResolver) -> BoardVote: + """Create a BoardVote row attached to the given motion. + + Args: + motion (BoardMotion): The motion the vote applies to. + vote (ParsedVote): The parsed vote payload. + resolve (PersonResolver): Person resolver from build_person_resolver. + + Returns: + BoardVote: The created row. + + """ + row = BoardVote.objects.create( + motion=motion, + result=vote.result, + tally=vote.tally, + type=vote.type, + ) + row.in_favor.set(resolve(vote.in_favor, EntityMember.Role.MEMBER)) + row.against.set(resolve(vote.against, EntityMember.Role.MEMBER)) + row.abstain.set(resolve(vote.abstain, EntityMember.Role.MEMBER)) + row.recused.set(resolve(vote.recused, EntityMember.Role.MEMBER)) + return row + + +ACTION_CREATORS = { + ActionKind.DISCUSSION: create_discussion, + ActionKind.MOTION: create_motion, + ActionKind.OUTCOME: create_outcome, +} + + +def create_action( + meeting: BoardMeeting, + order: int, + action: ParsedAction, + resolve: PersonResolver, +) -> None: + """Create the child row for a parsed action and link it to the meeting. + + Args: + meeting (BoardMeeting): The parent meeting. + order (int): position in the agenda. + action (ParsedAction): The parsed action payload. + resolve (PersonResolver): Person resolver from build_person_resolver. + + Raises: + ValueError: If the action's kind has no matching payload. + + """ + payload = getattr(action, action.kind) + if not payload: + message = f"ParsedAction(kind={action.kind}) has no matching payload." + raise ValueError(message) + + row = ACTION_CREATORS[action.kind](payload, resolve) + BoardMeetingAction.objects.create( + meeting=meeting, + order=order, + **{action.kind: row}, + ) diff --git a/backend/tests/unit/apps/common/open_ai_test.py b/backend/tests/unit/apps/common/open_ai_test.py index 2533c78d32..5c53c359cc 100644 --- a/backend/tests/unit/apps/common/open_ai_test.py +++ b/backend/tests/unit/apps/common/open_ai_test.py @@ -2,9 +2,15 @@ import openai as openai_module import pytest +from pydantic import BaseModel from apps.common.open_ai import OpenAi + +class ParseSchema(BaseModel): + value: str + + # Constants to replace magic values DEFAULT_MAX_TOKENS = 1000 DEFAULT_MODEL = "gpt-4o-mini" @@ -31,6 +37,15 @@ def test_init(self, mock_openai, mock_settings): assert instance.model == DEFAULT_MODEL assert instance.temperature == DEFAULT_TEMPERATURE + @patch("apps.common.open_ai.settings") + @patch("openai.OpenAI") + def test_init_custom_timeout(self, mock_openai, mock_settings): + mock_settings.OPEN_AI_SECRET_KEY = DEFAULT_API_KEY + + OpenAi(timeout=90) + + mock_openai.assert_called_once_with(api_key=DEFAULT_API_KEY, timeout=90) + @pytest.mark.parametrize( ("input_content", "expected_input"), [("Test input content", "Test input content")] ) @@ -192,3 +207,130 @@ def test_complete_generic_exception_includes_type(self, mock_openai, mock_logger "Unexpected OpenAI API error: %s", "ValueError", ) + + @patch("openai.OpenAI") + def test_parse_success(self, mock_openai): + """Test successful parse returns the parsed schema instance.""" + mock_client = MagicMock() + mock_response = MagicMock() + parsed_value = ParseSchema(value="parsed") + mock_response.choices = [MagicMock()] + mock_response.choices[0].message.parsed = parsed_value + mock_client.beta.chat.completions.parse.return_value = mock_response + mock_openai.return_value = mock_client + + openai_instance = OpenAi() + openai_instance.set_prompt("Test prompt").set_input("Test input") + response = openai_instance.parse(ParseSchema) + + assert response is parsed_value + mock_client.beta.chat.completions.parse.assert_called_once() + _, kwargs = mock_client.beta.chat.completions.parse.call_args + assert kwargs["response_format"] is ParseSchema + assert kwargs["model"] == DEFAULT_MODEL + assert kwargs["max_tokens"] == DEFAULT_MAX_TOKENS + assert kwargs["temperature"] == DEFAULT_TEMPERATURE + + @patch("apps.common.open_ai.logger") + @patch("openai.OpenAI") + def test_parse_authentication_error(self, mock_openai, mock_logger): + """Test that AuthenticationError is caught and logged.""" + mock_client = MagicMock() + auth_error = openai_module.AuthenticationError( + "Invalid API key", + response=MagicMock(), + body={}, + ) + mock_client.beta.chat.completions.parse.side_effect = auth_error + mock_openai.return_value = mock_client + + openai_instance = OpenAi() + openai_instance.set_prompt("Test prompt").set_input("Test input") + response = openai_instance.parse(ParseSchema) + + assert response is None + mock_logger.exception.assert_called_once_with( + "OpenAI authentication failed: invalid or missing API key. " + ) + + @patch("apps.common.open_ai.logger") + @patch("openai.OpenAI") + def test_parse_rate_limit_error(self, mock_openai, mock_logger): + """Test that RateLimitError is caught and logged as a warning.""" + mock_client = MagicMock() + rate_limit_error = openai_module.RateLimitError( + "Rate limit exceeded", + response=MagicMock(), + body={}, + ) + mock_client.beta.chat.completions.parse.side_effect = rate_limit_error + mock_openai.return_value = mock_client + + openai_instance = OpenAi() + openai_instance.set_prompt("Test prompt").set_input("Test input") + response = openai_instance.parse(ParseSchema) + + assert response is None + mock_logger.warning.assert_called_once_with( + "OpenAI rate limit exceeded: %s. Request may be retried with backoff.", + rate_limit_error, + ) + mock_logger.exception.assert_not_called() + + @patch("apps.common.open_ai.logger") + @patch("openai.OpenAI") + def test_parse_bad_request_error(self, mock_openai, mock_logger): + """Test that BadRequestError is caught and logged.""" + mock_client = MagicMock() + bad_request_error = openai_module.BadRequestError( + "Invalid request", + response=MagicMock(), + body={}, + ) + mock_client.beta.chat.completions.parse.side_effect = bad_request_error + mock_openai.return_value = mock_client + + openai_instance = OpenAi() + openai_instance.set_prompt("Test prompt").set_input("Test input") + response = openai_instance.parse(ParseSchema) + + assert response is None + mock_logger.exception.assert_called_once_with( + "OpenAI invalid request. Check model name, message format, and input size." + ) + + @patch("apps.common.open_ai.logger") + @patch("openai.OpenAI") + def test_parse_api_connection_error(self, mock_openai, mock_logger): + """Test that APIConnectionError is caught and logged.""" + mock_client = MagicMock() + api_error = openai_module.APIConnectionError(request=MagicMock()) + mock_client.beta.chat.completions.parse.side_effect = api_error + mock_openai.return_value = mock_client + + openai_instance = OpenAi() + openai_instance.set_prompt("Test prompt").set_input("Test input") + response = openai_instance.parse(ParseSchema) + + assert response is None + mock_logger.exception.assert_called_once_with( + "OpenAI connection failed. Check network connectivity and firewall/proxy settings." + ) + + @patch("apps.common.open_ai.logger") + @patch("openai.OpenAI") + def test_parse_generic_exception_includes_type(self, mock_openai, mock_logger): + """Test that generic exceptions are caught with error type in log message.""" + mock_client = MagicMock() + mock_client.beta.chat.completions.parse.side_effect = ValueError("Custom error") + mock_openai.return_value = mock_client + + openai_instance = OpenAi() + openai_instance.set_prompt("Test prompt").set_input("Test input") + response = openai_instance.parse(ParseSchema) + + assert response is None + mock_logger.exception.assert_called_once_with( + "Unexpected OpenAI API error: %s", + "ValueError", + ) diff --git a/backend/tests/unit/apps/owasp/management/commands/owasp_sync_board_activity_test.py b/backend/tests/unit/apps/owasp/management/commands/owasp_sync_board_activity_test.py new file mode 100644 index 0000000000..6fc12f43db --- /dev/null +++ b/backend/tests/unit/apps/owasp/management/commands/owasp_sync_board_activity_test.py @@ -0,0 +1,100 @@ +"""Tests for the owasp_sync_board_activity management command.""" + +from unittest.mock import Mock + +import pytest +from django.core.management.base import CommandError + +from apps.owasp.management.commands.owasp_sync_board_activity import Command +from apps.owasp.parsers.board_activity.sync import SyncStats, SyncStatus + + +class TestSyncBoardActivityCommand: + """Test cases for the owasp_sync_board_activity command.""" + + @pytest.fixture + def command(self): + """Instantiate the command with mocked I/O.""" + cmd = Command() + cmd.stdout = Mock() + cmd.stderr = Mock() + cmd.style = Mock(SUCCESS=lambda s: s) + return cmd + + def test_month_without_year_raises(self, command): + """Passing --month without --year is rejected with a CommandError.""" + with pytest.raises(CommandError, match="--month requires --year"): + command.handle(year=None, month=8, path=None) + + @pytest.mark.parametrize("month", [0, -1, 13, 100]) + def test_month_out_of_range_raises(self, command, month): + """Passing --month outside 1-12 is rejected with a CommandError.""" + with pytest.raises(CommandError, match="--month must be between 1 and 12"): + command.handle(year=2025, month=month, path=None) + + @pytest.mark.parametrize("year", [1, 202, 10000]) + def test_year_out_of_range_raises(self, command, year): + """Passing a non-4-digit --year is rejected with a CommandError.""" + with pytest.raises(CommandError, match="4-digit"): + command.handle(year=year, month=None, path=None) + + def test_path_bypasses_year_month_validation(self, command, mocker): + """--path takes precedence over --year/--month, so range checks are skipped.""" + mocker.patch( + "apps.owasp.management.commands.owasp_sync_board_activity.sync.run", + return_value=SyncStats(), + ) + + command.handle(year=1, month=99, path="meetings-historical/2025/202508.md") + + def test_errored_count_raises_command_error(self, command, mocker): + """A non-zero ERRORED count exits the command with a CommandError.""" + stats = SyncStats() + stats.record(SyncStatus.ERRORED) + mocker.patch( + "apps.owasp.management.commands.owasp_sync_board_activity.sync.run", + return_value=stats, + ) + + with pytest.raises(CommandError, match="errored"): + command.handle(year=None, month=None, path=None, force=False, dry_run=False) + + def test_handle_delegates_to_sync_run(self, command, mocker): + """CLI options are threaded through to sync.run.""" + mock_run = mocker.patch( + "apps.owasp.management.commands.owasp_sync_board_activity.sync.run", + return_value=SyncStats(), + ) + + command.handle( + year=2025, + month=8, + path=None, + force=True, + dry_run=False, + ) + + mock_run.assert_called_once_with( + year=2025, + month=8, + path=None, + force=True, + dry_run=False, + ) + + def test_summary_line_is_written(self, command, mocker): + """The stdout summary reflects the SyncStats.counts contents.""" + stats = SyncStats() + stats.record("created") + stats.record("created") + stats.record("unchanged") + mocker.patch( + "apps.owasp.management.commands.owasp_sync_board_activity.sync.run", + return_value=stats, + ) + + command.handle(year=None, month=None, path=None, force=False, dry_run=False) + + written = command.stdout.write.call_args.args[0] + assert "created=2" in written + assert "unchanged=1" in written diff --git a/backend/tests/unit/apps/owasp/models/board_discussion_test.py b/backend/tests/unit/apps/owasp/models/board_discussion_test.py new file mode 100644 index 0000000000..3970249615 --- /dev/null +++ b/backend/tests/unit/apps/owasp/models/board_discussion_test.py @@ -0,0 +1,38 @@ +"""Tests for BoardDiscussion model.""" + +from apps.owasp.models.board_discussion import BoardDiscussion + + +class TestBoardDiscussionModel: + """Test cases for BoardDiscussion model.""" + + def test_str_representation(self): + """Test string representation.""" + discussion = BoardDiscussion(topic="Marketing strategy for 2026") + + assert str(discussion) == "Discussion: Marketing strategy for 2026" + + def test_meta_options(self): + """Test Meta options for BoardDiscussion.""" + assert BoardDiscussion._meta.db_table == "owasp_board_discussions" + assert BoardDiscussion._meta.verbose_name_plural == "Board Discussions" + + def test_field_defaults(self): + """Test default values on optional fields.""" + assert BoardDiscussion._meta.get_field("metadata").default is dict + + def test_topic_max_length(self): + """Test topic max_length is 500.""" + assert BoardDiscussion._meta.get_field("topic").max_length == 500 + + def test_participants_m2m_targets_entity_member(self): + """Test participants is M2M to EntityMember.""" + field = BoardDiscussion._meta.get_field("participants") + + assert field.many_to_many + assert field.related_model.__name__ == "EntityMember" + + def test_has_timestamp_fields(self): + """Test presence of TimestampedModel fields.""" + assert hasattr(BoardDiscussion, "nest_created_at") + assert hasattr(BoardDiscussion, "nest_updated_at") diff --git a/backend/tests/unit/apps/owasp/models/board_meeting_action_test.py b/backend/tests/unit/apps/owasp/models/board_meeting_action_test.py new file mode 100644 index 0000000000..73446d2890 --- /dev/null +++ b/backend/tests/unit/apps/owasp/models/board_meeting_action_test.py @@ -0,0 +1,59 @@ +"""Tests for BoardMeetingAction model.""" + +from apps.owasp.models.board_meeting_action import BoardMeetingAction + + +class TestBoardMeetingActionModel: + """Test cases for BoardMeetingAction model.""" + + def test_str_representation(self): + """Test string representation includes order and meeting id.""" + action = BoardMeetingAction(order=3, meeting_id=42) + + assert str(action) == "Meeting Action #3 in meeting 42" + + def test_meta_options(self): + """Test Meta options for BoardMeetingAction.""" + assert BoardMeetingAction._meta.db_table == "owasp_board_meeting_actions" + assert BoardMeetingAction._meta.verbose_name_plural == "Board Meeting Actions" + + def test_unique_meeting_order_constraint(self): + """Test unique constraint on (meeting, order) is declared.""" + constraint_names = {c.name for c in BoardMeetingAction._meta.constraints} + + assert "board_meeting_action_unique_meeting_order" in constraint_names + + def test_exactly_one_target_check_constraint(self): + """Test check constraint enforcing exactly one target foreign key is declared.""" + constraint_names = {c.name for c in BoardMeetingAction._meta.constraints} + + assert "board_meeting_action_exactly_one_target" in constraint_names + + def test_target_fields_are_nullable(self): + """Test all three target foreign keys are nullable.""" + for name in ("discussion", "motion", "outcome"): + field = BoardMeetingAction._meta.get_field(name) + assert field.null is True + + def test_meeting_fk_is_required(self): + """Test meeting foreign key is required.""" + field = BoardMeetingAction._meta.get_field("meeting") + + assert field.null is False + assert field.related_model.__name__ == "BoardMeeting" + + def test_target_foreign_keys_point_at_expected_models(self): + """Test target foreign keys point at the expected models.""" + expected = { + "discussion": "BoardDiscussion", + "motion": "BoardMotion", + "outcome": "BoardOutcome", + } + for name, model_name in expected.items(): + field = BoardMeetingAction._meta.get_field(name) + assert field.related_model.__name__ == model_name + + def test_has_timestamp_fields(self): + """Test presence of TimestampedModel fields.""" + assert hasattr(BoardMeetingAction, "nest_created_at") + assert hasattr(BoardMeetingAction, "nest_updated_at") diff --git a/backend/tests/unit/apps/owasp/models/board_meeting_test.py b/backend/tests/unit/apps/owasp/models/board_meeting_test.py new file mode 100644 index 0000000000..284ad943a1 --- /dev/null +++ b/backend/tests/unit/apps/owasp/models/board_meeting_test.py @@ -0,0 +1,102 @@ +"""Tests for BoardMeeting model.""" + +from datetime import UTC, datetime + +from apps.owasp.models.board_meeting import BoardMeeting + + +class TestBoardMeetingModel: + """Test cases for BoardMeeting model.""" + + def test_str_representation_with_title(self): + """Test string representation prefers title.""" + meeting = BoardMeeting( + title="August 2025", + date=datetime(2025, 8, 26, 13, 0, tzinfo=UTC), + ) + + assert str(meeting) == "Board Meeting: August 2025" + + def test_str_representation_without_title(self): + """Test string representation falls back to ISO datetime when title is empty.""" + meeting = BoardMeeting( + title="", + date=datetime(2025, 8, 26, 13, 0, tzinfo=UTC), + ) + + assert str(meeting) == "Board Meeting: 2025-08-26T13:00:00+00:00" + + def test_meta_options(self): + """Test Meta options for BoardMeeting.""" + assert BoardMeeting._meta.db_table == "owasp_board_meetings" + assert BoardMeeting._meta.verbose_name_plural == "Board Meetings" + + def test_source_path_unique(self): + """Test source_path is unique.""" + assert BoardMeeting._meta.get_field("source_path").unique is True + + def test_type_choices(self): + """Test Type choices are correctly defined.""" + assert BoardMeeting.Type.PRIVATE == "private" + assert BoardMeeting.Type.PUBLIC == "public" + assert BoardMeeting.Type.SPECIAL == "special" + assert BoardMeeting.Type.SUMMIT == "summit" + + def test_type_default_is_public(self): + """Test type default is PUBLIC.""" + assert BoardMeeting._meta.get_field("type").default == BoardMeeting.Type.PUBLIC + + def test_field_defaults(self): + """Test default values on optional fields.""" + assert BoardMeeting._meta.get_field("metadata").default is dict + assert BoardMeeting._meta.get_field("attachments").default is list + assert BoardMeeting._meta.get_field("call_in_url").default == "" + assert BoardMeeting._meta.get_field("recording_url").default == "" + assert BoardMeeting._meta.get_field("location").default == "" + assert BoardMeeting._meta.get_field("title").default == "" + assert BoardMeeting._meta.get_field("source_checksum").default == "" + + def test_quorum_present_nullable(self): + """Test quorum_present is nullable.""" + assert BoardMeeting._meta.get_field("quorum_present").null is True + + def test_date_is_datetime_field(self): + """Test date field is a DateTimeField.""" + field = BoardMeeting._meta.get_field("date") + + assert field.get_internal_type() == "DateTimeField" + + def test_board_foreign_key_targets_board_of_directors_cascade(self): + """Test board foreign key targets BoardOfDirectors and cascades.""" + field = BoardMeeting._meta.get_field("board") + + assert field.related_model.__name__ == "BoardOfDirectors" + assert field.null is False + assert field.remote_field.on_delete.__name__ == "CASCADE" + + def test_source_path_max_length(self): + """Test source_path max_length is 500.""" + assert BoardMeeting._meta.get_field("source_path").max_length == 500 + + def test_attendance_m2m_fields_target_entity_member(self): + """Test attendees and absentees M2M fields target EntityMember.""" + for name in ("attendees", "absentees"): + field = BoardMeeting._meta.get_field(name) + assert field.many_to_many + assert field.related_model.__name__ == "EntityMember" + + def test_guests_is_json_list(self): + """Test guests is a JSON field defaulting to an empty list.""" + field = BoardMeeting._meta.get_field("guests") + + assert field.get_internal_type() == "JSONField" + assert field.default is list + + def test_source_checksum_max_length(self): + """Test source_checksum max_length is 64 (SHA-256 hex).""" + assert BoardMeeting._meta.get_field("source_checksum").max_length == 64 + + def test_has_timestamp_fields(self): + """Test presence of TimestampedModel fields.""" + assert hasattr(BoardMeeting, "nest_created_at") + assert hasattr(BoardMeeting, "nest_updated_at") diff --git a/backend/tests/unit/apps/owasp/models/board_motion_test.py b/backend/tests/unit/apps/owasp/models/board_motion_test.py new file mode 100644 index 0000000000..daeae28cba --- /dev/null +++ b/backend/tests/unit/apps/owasp/models/board_motion_test.py @@ -0,0 +1,45 @@ +"""Tests for BoardMotion model.""" + +from apps.owasp.models.board_motion import BoardMotion + + +class TestBoardMotionModel: + """Test cases for BoardMotion model.""" + + def test_str_representation(self): + """Test string representation.""" + motion = BoardMotion(title="Approve 2026 Budget") + + assert str(motion) == "Motion: Approve 2026 Budget" + + def test_meta_options(self): + """Test Meta options for BoardMotion.""" + assert BoardMotion._meta.db_table == "owasp_board_motions" + assert BoardMotion._meta.verbose_name_plural == "Board Motions" + + def test_amends_motion_is_nullable_self_reference_set_null(self): + """Test amends_motion is a nullable self-reference with SET_NULL.""" + field = BoardMotion._meta.get_field("amends_motion") + + assert field.related_model is BoardMotion + assert field.null is True + assert field.remote_field.on_delete.__name__ == "SET_NULL" + + def test_sponsor_and_second_foreign_keys_target_entity_member(self): + """Test sponsor and second foreign keys target EntityMember and SET_NULL.""" + for name in ("sponsor", "second"): + field = BoardMotion._meta.get_field(name) + assert field.related_model.__name__ == "EntityMember" + assert field.null is True + assert field.remote_field.on_delete.__name__ == "SET_NULL" + + def test_field_defaults(self): + """Test default values on optional fields.""" + assert BoardMotion._meta.get_field("metadata").default is dict + assert BoardMotion._meta.get_field("references").default is list + assert BoardMotion._meta.get_field("background").default == "" + + def test_has_timestamp_fields(self): + """Test presence of TimestampedModel fields.""" + assert hasattr(BoardMotion, "nest_created_at") + assert hasattr(BoardMotion, "nest_updated_at") diff --git a/backend/tests/unit/apps/owasp/models/board_outcome_test.py b/backend/tests/unit/apps/owasp/models/board_outcome_test.py new file mode 100644 index 0000000000..07705e22ca --- /dev/null +++ b/backend/tests/unit/apps/owasp/models/board_outcome_test.py @@ -0,0 +1,53 @@ +"""Tests for BoardOutcome model.""" + +from apps.owasp.models.board_outcome import BoardOutcome + + +class TestBoardOutcomeModel: + """Test cases for BoardOutcome model.""" + + def test_str_representation(self): + """Test string representation includes status and truncated description.""" + outcome = BoardOutcome( + description="Finalize the Q1 audit report and distribute to the board.", + status=BoardOutcome.Status.PENDING, + ) + + assert str(outcome).startswith("Outcome (Pending):") + assert "Finalize the Q1 audit report" in str(outcome) + + def test_meta_options(self): + """Test Meta options for BoardOutcome.""" + assert BoardOutcome._meta.db_table == "owasp_board_outcomes" + assert BoardOutcome._meta.verbose_name_plural == "Board Outcomes" + + def test_status_choices(self): + """Test Status choices are correctly defined.""" + assert BoardOutcome.Status.CANCELLED == "cancelled" + assert BoardOutcome.Status.COMPLETED == "completed" + assert BoardOutcome.Status.IN_PROGRESS == "in_progress" + assert BoardOutcome.Status.PENDING == "pending" + + def test_status_default_is_pending(self): + """Test status default is PENDING.""" + assert BoardOutcome._meta.get_field("status").default == BoardOutcome.Status.PENDING + + def test_field_defaults(self): + """Test default values on optional fields.""" + assert BoardOutcome._meta.get_field("metadata").default is dict + + def test_due_date_nullable(self): + """Test due_date is nullable.""" + assert BoardOutcome._meta.get_field("due_date").null is True + + def test_assignees_m2m_targets_entity_member(self): + """Test assignees is M2M to EntityMember.""" + field = BoardOutcome._meta.get_field("assignees") + + assert field.many_to_many + assert field.related_model.__name__ == "EntityMember" + + def test_has_timestamp_fields(self): + """Test presence of TimestampedModel fields.""" + assert hasattr(BoardOutcome, "nest_created_at") + assert hasattr(BoardOutcome, "nest_updated_at") diff --git a/backend/tests/unit/apps/owasp/models/board_vote_test.py b/backend/tests/unit/apps/owasp/models/board_vote_test.py new file mode 100644 index 0000000000..0003fc5327 --- /dev/null +++ b/backend/tests/unit/apps/owasp/models/board_vote_test.py @@ -0,0 +1,66 @@ +"""Tests for BoardVote model.""" + +from apps.owasp.models.board_vote import BoardVote + + +class TestBoardVoteModel: + """Test cases for BoardVote model.""" + + def test_str_representation_with_tally(self): + """Test string representation includes result display and tally.""" + vote = BoardVote(result=BoardVote.Result.PASSED, tally="7-0") + + assert str(vote) == "Vote (Passed): 7-0" + + def test_str_representation_without_tally(self): + """Test string representation when tally is empty falls back to 'n/a'.""" + vote = BoardVote(result=BoardVote.Result.FAILED) + + assert str(vote) == "Vote (Failed): n/a" + + def test_meta_options(self): + """Test Meta options for BoardVote.""" + assert BoardVote._meta.db_table == "owasp_board_votes" + assert BoardVote._meta.verbose_name_plural == "Board Votes" + + def test_result_choices(self): + """Test Result choices are correctly defined.""" + assert BoardVote.Result.DEFERRED == "deferred" + assert BoardVote.Result.FAILED == "failed" + assert BoardVote.Result.PASSED == "passed" + assert BoardVote.Result.TABLED == "tabled" + assert BoardVote.Result.WITHDRAWN == "withdrawn" + + def test_type_choices(self): + """Test Type choices are correctly defined.""" + assert BoardVote.Type.E_VOTE == "e_vote" + assert BoardVote.Type.VOTE == "vote" + + def test_type_default_is_vote(self): + """Test type default is VOTE.""" + assert BoardVote._meta.get_field("type").default == BoardVote.Type.VOTE + + def test_field_defaults(self): + """Test default values on optional fields.""" + assert BoardVote._meta.get_field("metadata").default is dict + assert BoardVote._meta.get_field("tally").default == "" + + def test_vote_cast_m2m_fields_target_entity_member(self): + """Test in_favor, against, abstain, and recused M2M fields target EntityMember.""" + for name in ("in_favor", "against", "abstain", "recused"): + field = BoardVote._meta.get_field(name) + assert field.many_to_many + assert field.related_model.__name__ == "EntityMember" + + def test_motion_foreign_key_required_cascade(self): + """Test motion foreign key is required and uses CASCADE on delete.""" + field = BoardVote._meta.get_field("motion") + + assert field.null is False + assert field.related_model.__name__ == "BoardMotion" + assert field.remote_field.on_delete.__name__ == "CASCADE" + + def test_has_timestamp_fields(self): + """Test presence of TimestampedModel fields.""" + assert hasattr(BoardVote, "nest_created_at") + assert hasattr(BoardVote, "nest_updated_at") diff --git a/backend/tests/unit/apps/owasp/parsers/__init__.py b/backend/tests/unit/apps/owasp/parsers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/tests/unit/apps/owasp/parsers/board_activity/__init__.py b/backend/tests/unit/apps/owasp/parsers/board_activity/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/tests/unit/apps/owasp/parsers/board_activity/sync_test.py b/backend/tests/unit/apps/owasp/parsers/board_activity/sync_test.py new file mode 100644 index 0000000000..ceecc0bf28 --- /dev/null +++ b/backend/tests/unit/apps/owasp/parsers/board_activity/sync_test.py @@ -0,0 +1,360 @@ +"""Tests for board_activity.sync.""" + +from unittest.mock import Mock + +import pytest +from requests.exceptions import RequestException + +from apps.owasp.parsers.board_activity import sync + + +class TestFetchTree: + """Tests for fetch_tree.""" + + def test_returns_blob_paths_only(self, mocker): + """Only tree entries of type=blob are returned in the path->sha map.""" + response = Mock() + response.json.return_value = { + "tree": [ + {"path": "a.md", "sha": "sha_a", "type": "blob"}, + {"path": "some/dir", "sha": "sha_dir", "type": "tree"}, + {"path": "b.md", "sha": "sha_b", "type": "blob"}, + ] + } + response.raise_for_status = Mock() + mock_get = mocker.patch( + "apps.owasp.parsers.board_activity.sync.requests.get", + return_value=response, + ) + + result = sync.fetch_tree() + + assert result == {"a.md": "sha_a", "b.md": "sha_b"} + mock_get.assert_called_once() + + def test_adds_authorization_when_token_set(self, mocker): + """GITHUB_TOKEN env var is threaded through as a Bearer header.""" + response = Mock() + response.json.return_value = {"tree": []} + response.raise_for_status = Mock() + mocker.patch.dict("os.environ", {"GITHUB_TOKEN": "secret123"}) + mock_get = mocker.patch( + "apps.owasp.parsers.board_activity.sync.requests.get", + return_value=response, + ) + + sync.fetch_tree() + + assert mock_get.call_args.kwargs["headers"]["Authorization"] == "Bearer secret123" + + def test_omits_authorization_when_token_absent(self, mocker): + """No GITHUB_TOKEN means no Authorization header.""" + response = Mock() + response.json.return_value = {"tree": []} + response.raise_for_status = Mock() + mocker.patch.dict("os.environ", {}, clear=True) + mock_get = mocker.patch( + "apps.owasp.parsers.board_activity.sync.requests.get", + return_value=response, + ) + + sync.fetch_tree() + + assert "Authorization" not in mock_get.call_args.kwargs["headers"] + + +class TestTargetPaths: + """Tests for target_paths filtering logic.""" + + @pytest.fixture + def tree(self): + return { + "meetings/202601.md": "s1", + "meetings/_template.meeting.md": "s2", + "meetings-historical/2025/202508.md": "s3", + "meetings-historical/2025/202503-18.md": "s4", + "meetings-historical/2025/README.md": "s5", + "meetings-historical/2024/202412.md": "s6", + "minutes-deprecated/2020/202003.md": "s7", + "_data/votes.yml": "s8", + "elections/2025_elections.md": "s9", + "attachments/foo.pdf": "s10", + } + + def test_yields_only_meeting_markdown_files(self, tree): + """Upcoming meetings/, non-meeting files, non-.md, non-YYYYMM names are filtered out.""" + result = set(sync.target_paths(tree)) + + assert result == { + "meetings-historical/2025/202508.md", + "meetings-historical/2025/202503-18.md", + "meetings-historical/2024/202412.md", + "minutes-deprecated/2020/202003.md", + } + + def test_filters_by_year(self, tree): + """--year restricts to files with matching 4-digit prefix.""" + result = set(sync.target_paths(tree, year=2025)) + + assert result == { + "meetings-historical/2025/202508.md", + "meetings-historical/2025/202503-18.md", + } + + def test_filters_by_year_and_month(self, tree): + """--year + --month restricts to YYYYMM prefix.""" + result = set(sync.target_paths(tree, year=2025, month=8)) + + assert result == {"meetings-historical/2025/202508.md"} + + def test_single_path_short_circuits(self, tree): + """Passing --path returns that path only when present in the tree.""" + result = list(sync.target_paths(tree, path="meetings-historical/2025/202508.md")) + + assert result == ["meetings-historical/2025/202508.md"] + + def test_single_path_absent_yields_nothing(self, tree): + """Missing --path yields no results.""" + result = list(sync.target_paths(tree, path="nonexistent.md")) + + assert result == [] + + @pytest.mark.parametrize("year", [1, 202, 10000]) + def test_rejects_non_four_digit_year(self, tree, year): + """Year outside 1000-9999 is rejected so short prefixes don't match multiple years.""" + with pytest.raises(ValueError, match="4-digit"): + list(sync.target_paths(tree, year=year)) + + +class TestFetchFileContent: + """Tests for fetch_file_content status handling.""" + + def test_returns_text_on_2xx(self, mocker): + """A 200 response returns the body text.""" + response = Mock(ok=True, status_code=200, text="hello") + mocker.patch( + "apps.owasp.parsers.board_activity.sync.requests.get", + return_value=response, + ) + + assert sync.fetch_file_content("p.md") == "hello" + + def test_returns_empty_on_non_ok(self, mocker): + """A non-ok response returns empty string so the error body is not fed to the LLM.""" + response = Mock(ok=False, status_code=404, text="error body") + mocker.patch( + "apps.owasp.parsers.board_activity.sync.requests.get", + return_value=response, + ) + + assert sync.fetch_file_content("p.md") == "" + + def test_returns_empty_on_request_exception(self, mocker): + """A connection error returns empty string rather than propagating.""" + mocker.patch( + "apps.owasp.parsers.board_activity.sync.requests.get", + side_effect=RequestException("boom"), + ) + + assert sync.fetch_file_content("p.md") == "" + + +class TestSyncFile: + """Tests for sync_file per-file behavior.""" + + def test_unchanged_when_checksum_matches(self, mocker): + """Existing meeting with matching blob SHA is skipped.""" + existing = Mock(source_checksum="abc") + mock_qs = Mock() + mock_qs.first.return_value = existing + mocker.patch( + "apps.owasp.models.board_meeting.BoardMeeting.objects.filter", + return_value=mock_qs, + ) + mock_fetch = mocker.patch("apps.owasp.parsers.board_activity.sync.fetch_file_content") + + result = sync.sync_file("meetings/202601.md", "abc") + + assert result == sync.SyncStatus.UNCHANGED + mock_fetch.assert_not_called() + + def test_force_reparses_even_when_checksum_matches(self, mocker): + """--force triggers a re-parse regardless of checksum.""" + existing = Mock(source_checksum="abc") + mock_qs = Mock() + mock_qs.first.return_value = existing + mocker.patch( + "apps.owasp.models.board_meeting.BoardMeeting.objects.filter", + return_value=mock_qs, + ) + mocker.patch( + "apps.owasp.parsers.board_activity.sync.fetch_file_content", + return_value="markdown content", + ) + client = Mock() + client.set_prompt.return_value = client + client.set_input.return_value = client + client.parse.return_value = Mock() + mocker.patch( + "apps.owasp.parsers.board_activity.sync.OpenAi", + return_value=client, + ) + mocker.patch("apps.owasp.parsers.board_activity.sync.translator.upsert") + + result = sync.sync_file("p.md", "abc", force=True) + + assert result == sync.SyncStatus.UPDATED + + def test_empty_content_skips(self, mocker): + """Empty fetch response yields SKIPPED without an LLM call.""" + mock_qs = Mock() + mock_qs.first.return_value = None + mocker.patch( + "apps.owasp.models.board_meeting.BoardMeeting.objects.filter", + return_value=mock_qs, + ) + mocker.patch("apps.owasp.parsers.board_activity.sync.fetch_file_content", return_value="") + mock_openai = mocker.patch("apps.owasp.parsers.board_activity.sync.OpenAi") + + result = sync.sync_file("p.md", "sha") + + assert result == sync.SyncStatus.SKIPPED + mock_openai.assert_not_called() + + def test_llm_failure_yields_errored(self, mocker): + """LLM returning None yields ERRORED and skips DB writes.""" + mock_qs = Mock() + mock_qs.first.return_value = None + mocker.patch( + "apps.owasp.models.board_meeting.BoardMeeting.objects.filter", + return_value=mock_qs, + ) + mocker.patch( + "apps.owasp.parsers.board_activity.sync.fetch_file_content", + return_value="content", + ) + client = Mock() + client.set_prompt.return_value = client + client.set_input.return_value = client + client.parse.return_value = None + mocker.patch( + "apps.owasp.parsers.board_activity.sync.OpenAi", + return_value=client, + ) + mock_upsert = mocker.patch("apps.owasp.parsers.board_activity.sync.translator.upsert") + + result = sync.sync_file("p.md", "sha") + + assert result == sync.SyncStatus.ERRORED + mock_upsert.assert_not_called() + + def test_dry_run_parses_but_does_not_upsert(self, mocker): + """--dry-run runs the LLM but skips the DB write.""" + mock_qs = Mock() + mock_qs.first.return_value = None + mocker.patch( + "apps.owasp.models.board_meeting.BoardMeeting.objects.filter", + return_value=mock_qs, + ) + mocker.patch( + "apps.owasp.parsers.board_activity.sync.fetch_file_content", + return_value="content", + ) + parsed = Mock() + parsed.model_dump_json.return_value = "{}" + client = Mock() + client.set_prompt.return_value = client + client.set_input.return_value = client + client.parse.return_value = parsed + mocker.patch( + "apps.owasp.parsers.board_activity.sync.OpenAi", + return_value=client, + ) + mock_upsert = mocker.patch("apps.owasp.parsers.board_activity.sync.translator.upsert") + + result = sync.sync_file("p.md", "sha", dry_run=True) + + assert result == sync.SyncStatus.WOULD_UPDATE + mock_upsert.assert_not_called() + + def test_new_meeting_returns_created(self, mocker): + """Meeting not previously in DB returns CREATED after upsert.""" + mock_qs = Mock() + mock_qs.first.return_value = None + mocker.patch( + "apps.owasp.models.board_meeting.BoardMeeting.objects.filter", + return_value=mock_qs, + ) + mocker.patch( + "apps.owasp.parsers.board_activity.sync.fetch_file_content", + return_value="content", + ) + client = Mock() + client.set_prompt.return_value = client + client.set_input.return_value = client + client.parse.return_value = Mock() + mocker.patch( + "apps.owasp.parsers.board_activity.sync.OpenAi", + return_value=client, + ) + mocker.patch("apps.owasp.parsers.board_activity.sync.translator.upsert") + + result = sync.sync_file("p.md", "sha") + + assert result == sync.SyncStatus.CREATED + + +class TestRun: + """Tests for the top-level run function.""" + + def test_returns_errored_when_tree_fetch_fails(self, mocker): + """Tree fetch RequestException is caught and recorded as ERRORED.""" + mocker.patch( + "apps.owasp.parsers.board_activity.sync.fetch_tree", + side_effect=RequestException("boom"), + ) + + stats = sync.run() + + assert stats.counts == {sync.SyncStatus.ERRORED: 1} + + def test_aggregates_status_counts_across_files(self, mocker): + """Per-file statuses are aggregated in SyncStats.counts.""" + mocker.patch( + "apps.owasp.parsers.board_activity.sync.fetch_tree", + return_value={ + "meetings-historical/2025/202501.md": "s1", + "meetings-historical/2025/202502.md": "s2", + "meetings-historical/2025/202503.md": "s3", + }, + ) + mocker.patch( + "apps.owasp.parsers.board_activity.sync.sync_file", + side_effect=[ + sync.SyncStatus.CREATED, + sync.SyncStatus.UNCHANGED, + sync.SyncStatus.CREATED, + ], + ) + + stats = sync.run(year=2025) + + assert stats.counts == { + sync.SyncStatus.CREATED: 2, + sync.SyncStatus.UNCHANGED: 1, + } + + def test_sync_file_exception_is_recorded_as_errored(self, mocker): + """A per-file exception is caught and recorded as ERRORED without aborting.""" + mocker.patch( + "apps.owasp.parsers.board_activity.sync.fetch_tree", + return_value={"meetings-historical/2025/202501.md": "s1"}, + ) + mocker.patch( + "apps.owasp.parsers.board_activity.sync.sync_file", + side_effect=RuntimeError("bad"), + ) + + stats = sync.run(year=2025) + + assert stats.counts == {sync.SyncStatus.ERRORED: 1} diff --git a/backend/tests/unit/apps/owasp/parsers/board_activity/translator_test.py b/backend/tests/unit/apps/owasp/parsers/board_activity/translator_test.py new file mode 100644 index 0000000000..fcd7518fd4 --- /dev/null +++ b/backend/tests/unit/apps/owasp/parsers/board_activity/translator_test.py @@ -0,0 +1,435 @@ +"""Tests for board_activity.translator.""" + +from contextlib import contextmanager +from datetime import UTC, date, datetime +from unittest.mock import Mock + +import pytest + +from apps.owasp.models.entity_member import EntityMember +from apps.owasp.parsers.board_activity import translator +from apps.owasp.parsers.board_activity.schemas import ( + ActionKind, + LabeledUrl, + ParsedAction, + ParsedDiscussion, + ParsedMeeting, + ParsedMotion, + ParsedOutcome, + ParsedPersonRef, + ParsedVote, +) + + +@contextmanager +def noop_transaction(): + """No-op replacement for django.db.transaction.atomic in unit tests.""" + yield + + +@pytest.fixture(autouse=True) +def stub_transaction_atomic(mocker): + """Stub out transaction.atomic so tests don't open a real DB connection. + + The @transaction.atomic decorator on translator.upsert calls + connection.get_autocommit(), which requires a real DB. Replace it with a + no-op context manager for unit tests. + """ + mocker.patch( + "apps.owasp.parsers.board_activity.translator.transaction.atomic", + side_effect=noop_transaction, + ) + + +class TestResolvePerson: + """Tests for resolve_person.""" + + def test_returns_existing_member(self, mocker): + """Existing EntityMember matched by (board, member_name) is returned as-is.""" + existing = Mock() + mock_qs = Mock() + mock_qs.first.return_value = existing + mock_manager = mocker.patch("apps.owasp.models.entity_member.EntityMember.objects") + mock_manager.filter.return_value = mock_qs + + board = Mock(id=42) + ct = Mock() + + result = translator.resolve_person(board, ct, "Ricardo Griffith", EntityMember.Role.MEMBER) + + assert result is existing + mock_manager.filter.assert_called_once_with( + entity_type=ct, entity_id=42, member_name="Ricardo Griffith" + ) + mock_manager.create.assert_not_called() + + def test_creates_new_member_when_missing(self, mocker): + """Missing person triggers create with the supplied role.""" + mock_qs = Mock() + mock_qs.first.return_value = None + created = Mock() + mock_manager = mocker.patch("apps.owasp.models.entity_member.EntityMember.objects") + mock_manager.filter.return_value = mock_qs + mock_manager.create.return_value = created + + board = Mock(id=7) + ct = Mock() + + result = translator.resolve_person(board, ct, "Guest Name", EntityMember.Role.MEMBER) + + assert result is created + mock_manager.create.assert_called_once_with( + entity_type=ct, + entity_id=7, + member_name="Guest Name", + role=EntityMember.Role.MEMBER, + ) + + +class TestBuildPersonResolver: + """Tests for build_person_resolver.""" + + def test_resolver_calls_resolve_person_per_input(self, mocker): + """The returned callable resolves each person via resolve_person.""" + mock_resolve = mocker.patch("apps.owasp.parsers.board_activity.translator.resolve_person") + mock_resolve.side_effect = ["a", "b", "c"] + + board = Mock() + ct = Mock() + resolver = translator.build_person_resolver(board, ct) + people = [ParsedPersonRef(name=n) for n in ("Alice", "Bob", "Carol")] + + result = resolver(people, EntityMember.Role.MEMBER) + + assert result == ["a", "b", "c"] + assert mock_resolve.call_count == 3 + + def test_resolver_handles_empty_input(self, mocker): + """An empty input list yields an empty output list without calling resolve_person.""" + mock_resolve = mocker.patch("apps.owasp.parsers.board_activity.translator.resolve_person") + + resolver = translator.build_person_resolver(Mock(), Mock()) + assert resolver([], EntityMember.Role.MEMBER) == [] + mock_resolve.assert_not_called() + + +class TestCreateDiscussion: + """Tests for create_discussion.""" + + def test_creates_row_and_sets_participants(self, mocker): + """Discussion row is created and participants M2M is populated.""" + row = Mock() + mocker.patch( + "apps.owasp.models.board_discussion.BoardDiscussion.objects.create", + return_value=row, + ) + resolve = Mock(return_value=["p1", "p2"]) + discussion = ParsedDiscussion( + topic="Marketing", + description="Discuss Q1 marketing plan.", + participants=[ParsedPersonRef(name="Alice"), ParsedPersonRef(name="Bob")], + ) + + result = translator.create_discussion(discussion, resolve) + + assert result is row + row.participants.set.assert_called_once_with(["p1", "p2"]) + resolve.assert_called_once_with(discussion.participants, EntityMember.Role.MEMBER) + + +class TestCreateMotion: + """Tests for create_motion.""" + + def test_creates_motion_without_vote(self, mocker): + """Motion without a vote produces a single BoardMotion row.""" + row = Mock() + mock_create = mocker.patch( + "apps.owasp.models.board_motion.BoardMotion.objects.create", + return_value=row, + ) + mock_vote = mocker.patch("apps.owasp.parsers.board_activity.translator.create_vote") + resolve = Mock(return_value=[Mock(), Mock()]) + + motion = ParsedMotion( + title="Approve Budget", + description="Resolved, that the budget is approved.", + sponsor=ParsedPersonRef(name="Alice"), + second=ParsedPersonRef(name="Bob"), + references=[LabeledUrl(label="Doc", url="https://example.com/doc")], + ) + + result = translator.create_motion(motion, resolve) + + assert result is row + mock_create.assert_called_once() + assert mock_create.call_args.kwargs["title"] == "Approve Budget" + assert mock_create.call_args.kwargs["references"] == [ + {"label": "Doc", "url": "https://example.com/doc"} + ] + mock_vote.assert_not_called() + + def test_creates_motion_with_vote(self, mocker): + """Motion with a vote triggers create_vote.""" + row = Mock() + mocker.patch( + "apps.owasp.models.board_motion.BoardMotion.objects.create", + return_value=row, + ) + mock_vote = mocker.patch("apps.owasp.parsers.board_activity.translator.create_vote") + resolve = Mock(return_value=[]) + + vote = ParsedVote(result="passed", type="vote", tally="7-0") + motion = ParsedMotion(title="M", description="D", vote=vote) + + translator.create_motion(motion, resolve) + + mock_vote.assert_called_once_with(row, vote, resolve) + + def test_null_sponsor_and_second_pass_through(self, mocker): + """Motion with no sponsor/second creates row with None for those FKs.""" + mock_create = mocker.patch("apps.owasp.models.board_motion.BoardMotion.objects.create") + mocker.patch("apps.owasp.parsers.board_activity.translator.create_vote") + resolve = Mock() + + motion = ParsedMotion(title="M", description="D") + + translator.create_motion(motion, resolve) + + assert mock_create.call_args.kwargs["sponsor"] is None + assert mock_create.call_args.kwargs["second"] is None + resolve.assert_not_called() + + +class TestCreateOutcome: + """Tests for create_outcome.""" + + def test_creates_row_and_sets_assignees(self, mocker): + """Outcome row is created and assignees M2M is populated.""" + row = Mock() + mock_create = mocker.patch( + "apps.owasp.models.board_outcome.BoardOutcome.objects.create", + return_value=row, + ) + resolve = Mock(return_value=["a1"]) + + outcome = ParsedOutcome( + description="Ship the audit", + status="in_progress", + due_date="2026-01-15", + assignees=[ParsedPersonRef(name="Alice")], + ) + + result = translator.create_outcome(outcome, resolve) + + assert result is row + assert mock_create.call_args.kwargs["due_date"] == date(2026, 1, 15) + row.assignees.set.assert_called_once_with(["a1"]) + + def test_malformed_due_date_is_dropped_to_none(self, mocker): + """A due_date the LLM couldn't format properly saves as None instead of raising.""" + mock_create = mocker.patch("apps.owasp.models.board_outcome.BoardOutcome.objects.create") + + outcome = ParsedOutcome(description="X", status="pending", due_date="end of Q1") + translator.create_outcome(outcome, Mock(return_value=[])) + + assert mock_create.call_args.kwargs["due_date"] is None + + +class TestDeleteMeetingChildren: + """Tests for delete_meeting_children.""" + + def test_deletes_discussions_motions_and_outcomes(self, mocker): + """Distinct child ids across the meeting's actions are collected and deleted per type.""" + actions = Mock() + actions.exclude.side_effect = lambda **_: actions + actions.values_list.side_effect = [[1, 2], [3], [4, 5]] + meeting = Mock() + meeting.actions.all.return_value = actions + + mock_discussion = mocker.patch( + "apps.owasp.models.board_discussion.BoardDiscussion.objects.filter" + ) + mock_motion = mocker.patch("apps.owasp.models.board_motion.BoardMotion.objects.filter") + mock_outcome = mocker.patch("apps.owasp.models.board_outcome.BoardOutcome.objects.filter") + + translator.delete_meeting_children(meeting) + + mock_discussion.assert_called_once_with(id__in=[1, 2]) + mock_motion.assert_called_once_with(id__in=[3]) + mock_outcome.assert_called_once_with(id__in=[4, 5]) + mock_discussion.return_value.delete.assert_called_once() + mock_motion.return_value.delete.assert_called_once() + mock_outcome.return_value.delete.assert_called_once() + + +class TestCreateVote: + """Tests for create_vote.""" + + def test_creates_vote_and_sets_all_four_m2ms(self, mocker): + """Vote row is created and all four cast M2Ms are populated.""" + row = Mock() + mocker.patch( + "apps.owasp.models.board_vote.BoardVote.objects.create", + return_value=row, + ) + resolve = Mock(side_effect=[["f"], ["a"], ["ab"], ["r"]]) + motion = Mock() + + vote = ParsedVote( + result="passed", + type="vote", + tally="6-1", + in_favor=[ParsedPersonRef(name="A")], + against=[ParsedPersonRef(name="B")], + abstain=[ParsedPersonRef(name="C")], + recused=[ParsedPersonRef(name="D")], + ) + + translator.create_vote(motion, vote, resolve) + + row.in_favor.set.assert_called_once_with(["f"]) + row.against.set.assert_called_once_with(["a"]) + row.abstain.set.assert_called_once_with(["ab"]) + row.recused.set.assert_called_once_with(["r"]) + + +class TestCreateAction: + """Tests for create_action dispatch.""" + + @pytest.fixture + def mock_action_create(self, mocker): + """Patch BoardMeetingAction.objects.create.""" + return mocker.patch( + "apps.owasp.models.board_meeting_action.BoardMeetingAction.objects.create" + ) + + def test_dispatches_to_discussion(self, mocker, mock_action_create): + """Discussion kind creates a discussion and creates a linked action.""" + creator = Mock(return_value="d_row") + mocker.patch.dict( + "apps.owasp.parsers.board_activity.translator.ACTION_CREATORS", + {ActionKind.DISCUSSION: creator}, + ) + meeting = Mock() + action = ParsedAction( + kind=ActionKind.DISCUSSION, + discussion=ParsedDiscussion(topic="t", description="d"), + ) + + translator.create_action(meeting, 1, action, Mock()) + + creator.assert_called_once() + mock_action_create.assert_called_once_with(meeting=meeting, order=1, discussion="d_row") + + def test_dispatches_to_motion(self, mocker, mock_action_create): + """Motion kind creates a motion and creates a linked action.""" + creator = Mock(return_value="m_row") + mocker.patch.dict( + "apps.owasp.parsers.board_activity.translator.ACTION_CREATORS", + {ActionKind.MOTION: creator}, + ) + meeting = Mock() + action = ParsedAction( + kind=ActionKind.MOTION, + motion=ParsedMotion(title="t", description="d"), + ) + + translator.create_action(meeting, 2, action, Mock()) + + creator.assert_called_once() + mock_action_create.assert_called_once_with(meeting=meeting, order=2, motion="m_row") + + def test_dispatches_to_outcome(self, mocker, mock_action_create): + """Outcome kind creates an outcome and creates a linked action.""" + creator = Mock(return_value="o_row") + mocker.patch.dict( + "apps.owasp.parsers.board_activity.translator.ACTION_CREATORS", + {ActionKind.OUTCOME: creator}, + ) + meeting = Mock() + action = ParsedAction( + kind=ActionKind.OUTCOME, + outcome=ParsedOutcome(description="do it"), + ) + + translator.create_action(meeting, 3, action, Mock()) + + creator.assert_called_once() + mock_action_create.assert_called_once_with(meeting=meeting, order=3, outcome="o_row") + + def test_raises_when_payload_missing(self, mock_action_create): + """Malformed action (kind set but payload None) raises so upsert rolls back.""" + action = ParsedAction(kind=ActionKind.MOTION, motion=None) + meeting = Mock() + resolve = Mock() + + with pytest.raises(ValueError, match="has no matching payload"): + translator.create_action(meeting, 1, action, resolve) + + mock_action_create.assert_not_called() + + +class TestUpsert: + """Tests for the top-level upsert function.""" + + def test_upsert_creates_meeting_and_dispatches_actions(self, mocker): + """Upsert wires the board, meeting, attendance M2Ms, and actions correctly.""" + board = Mock(id=1) + mock_board_get_or_create = mocker.patch( + "apps.owasp.models.board_of_directors.BoardOfDirectors.objects.get_or_create", + return_value=(board, True), + ) + meeting = Mock() + mock_delete_children = mocker.patch( + "apps.owasp.parsers.board_activity.translator.delete_meeting_children" + ) + mock_update_or_create = mocker.patch( + "apps.owasp.models.board_meeting.BoardMeeting.objects.update_or_create", + return_value=(meeting, True), + ) + ct = Mock() + mocker.patch( + "django.contrib.contenttypes.models.ContentType.objects.get_for_model", + return_value=ct, + ) + resolver = Mock(side_effect=lambda people, role: [f"{p.name}:{role}" for p in people]) + mocker.patch( + "apps.owasp.parsers.board_activity.translator.build_person_resolver", + return_value=resolver, + ) + mock_action = mocker.patch("apps.owasp.parsers.board_activity.translator.create_action") + + parsed = ParsedMeeting( + title="August 2025", + date="2025-08-26T13:00:00+00:00", + type="public", + attendees=[ParsedPersonRef(name="Alice")], + absentees=[ParsedPersonRef(name="Bob")], + guests=["Guest"], + actions=[ + ParsedAction( + kind=ActionKind.MOTION, motion=ParsedMotion(title="M", description="D") + ), + ParsedAction( + kind=ActionKind.DISCUSSION, + discussion=ParsedDiscussion(topic="t", description="d"), + ), + ], + ) + + result = translator.upsert( + parsed, source_path="meetings-historical/2025/202508.md", source_checksum="abc123" + ) + + assert result is meeting + assert mock_update_or_create.call_args.kwargs["defaults"]["guests"] == ["Guest"] + assert mock_update_or_create.call_args.kwargs["defaults"]["date"] == datetime( + 2025, 8, 26, 13, 0, 0, tzinfo=UTC + ) + mock_board_get_or_create.assert_called_once_with(year=2025) + mock_delete_children.assert_called_once_with(meeting) + meeting.attendees.set.assert_called_once() + meeting.absentees.set.assert_called_once() + assert mock_action.call_count == 2 + assert mock_action.call_args_list[0].args[1] == 1 + assert mock_action.call_args_list[1].args[1] == 2 diff --git a/docker-compose/local/compose.yaml b/docker-compose/local/compose.yaml index b41dbdf921..0f2f7d837d 100644 --- a/docker-compose/local/compose.yaml +++ b/docker-compose/local/compose.yaml @@ -23,7 +23,7 @@ services: - 8000:8000 volumes: - ../../backend:/home/owasp - - backend-venv:/home/owasp/.venv + - bod-activity-backend-venv:/home/owasp/.venv cache: command: > @@ -41,7 +41,7 @@ services: networks: - nest-network volumes: - - cache-data:/data + - bod-activity-cache-data:/data db: container_name: nest-db @@ -55,7 +55,7 @@ services: networks: - nest-network volumes: - - db-data:/var/lib/postgresql/data + - bod-activity-db-data:/var/lib/postgresql/data docs: container_name: nest-docs @@ -75,7 +75,7 @@ services: - ../../docs:/home/owasp/docs - ../../frontend:/home/owasp/frontend:ro - ../../README.md:/home/owasp/README.md:ro - - docs-venv:/home/owasp/.venv + - bod-activity-docs-venv:/home/owasp/.venv frontend: container_name: nest-frontend @@ -97,8 +97,8 @@ services: - 3000:3000 volumes: - ../../frontend:/home/owasp - - frontend-next:/home/owasp/.next - - frontend-node-modules:/home/owasp/node_modules + - bod-activity-frontend-next:/home/owasp/.next + - bod-activity-frontend-node-modules:/home/owasp/node_modules worker: container_name: nest-worker @@ -119,15 +119,15 @@ services: - nest-network volumes: - ../../backend:/home/owasp - - backend-venv:/home/owasp/.venv + - bod-activity-backend-venv:/home/owasp/.venv networks: nest-network: volumes: - backend-venv: - cache-data: - db-data: - docs-venv: - frontend-next: - frontend-node-modules: + bod-activity-backend-venv: + bod-activity-cache-data: + bod-activity-db-data: + bod-activity-docs-venv: + bod-activity-frontend-next: + bod-activity-frontend-node-modules: