From b10a3f3da64ff2facdddc5382f258f90ec9dcb19 Mon Sep 17 00:00:00 2001 From: Anurag Yadav Date: Sat, 18 Jul 2026 22:08:21 +0530 Subject: [PATCH 1/8] Implemented data model --- backend/src/apps/github/admin/__init__.py | 1 + .../src/apps/github/admin/activity_event.py | 33 ++++++++ .../github/migrations/0045_activityevent.py | 35 +++++++++ backend/src/apps/github/models/__init__.py | 1 + .../src/apps/github/models/activity_event.py | 78 +++++++++++++++++++ .../src/apps/github/models/enums/__init__.py | 0 .../github/models/enums/activity_event.py | 18 +++++ docker-compose/local/compose.yaml | 26 +++---- 8 files changed, 179 insertions(+), 13 deletions(-) create mode 100644 backend/src/apps/github/admin/activity_event.py create mode 100644 backend/src/apps/github/migrations/0045_activityevent.py create mode 100644 backend/src/apps/github/models/activity_event.py create mode 100644 backend/src/apps/github/models/enums/__init__.py create mode 100644 backend/src/apps/github/models/enums/activity_event.py diff --git a/backend/src/apps/github/admin/__init__.py b/backend/src/apps/github/admin/__init__.py index 60ae27406f..eb7170e91f 100644 --- a/backend/src/apps/github/admin/__init__.py +++ b/backend/src/apps/github/admin/__init__.py @@ -1,5 +1,6 @@ """Github app admin.""" +from .activity_event import ActivityEventAdmin from .comment import CommentAdmin from .commit import CommitAdmin from .issue import IssueAdmin diff --git a/backend/src/apps/github/admin/activity_event.py b/backend/src/apps/github/admin/activity_event.py new file mode 100644 index 0000000000..715aba9cd5 --- /dev/null +++ b/backend/src/apps/github/admin/activity_event.py @@ -0,0 +1,33 @@ +"""GitHub app ActivityEvent model admin.""" + +from django.contrib import admin + +from apps.github.models.activity_event import ActivityEvent + + +class ActivityEventAdmin(admin.ModelAdmin): + """Admin for ActivityEvent model.""" + + autocomplete_fields = ( + "actor", + "repository", + ) + list_display = ( + "activity_type", + "actor", + "nest_created_at", + "occurred_at", + "repository", + ) + list_filter = ( + "activity_type", + "occurred_at", + ) + search_fields = ( + "activity_type", + "actor__login", + "repository__name", + ) + + +admin.site.register(ActivityEvent, ActivityEventAdmin) diff --git a/backend/src/apps/github/migrations/0045_activityevent.py b/backend/src/apps/github/migrations/0045_activityevent.py new file mode 100644 index 0000000000..046aa38f51 --- /dev/null +++ b/backend/src/apps/github/migrations/0045_activityevent.py @@ -0,0 +1,35 @@ +# Generated by Django 6.0.7 on 2026-07-17 08:13 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('github', '0044_user_indexes'), + ] + + operations = [ + migrations.CreateModel( + name='ActivityEvent', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('nest_created_at', models.DateTimeField(auto_now_add=True)), + ('nest_updated_at', models.DateTimeField(auto_now=True)), + ('activity_type', models.CharField(choices=[('pr_opened', 'PR Opened'), ('pr_closed', 'PR Closed'), ('pr_merged', 'PR Merged'), ('issue_opened', 'Issue Opened'), ('issue_closed', 'Issue Closed'), ('commit_pushed', 'Commit Pushed'), ('release_published', 'Release Published')], max_length=32, verbose_name='Activity Type')), + ('occurred_at', models.DateTimeField(help_text='Timestamp when the activity event occurred on GitHub', verbose_name='Occurred at')), + ('object_id', models.PositiveBigIntegerField()), + ('actor', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='activity_events', to='github.user', verbose_name='Actor')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), + ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='activity_events', to='github.repository', verbose_name='Repository')), + ], + options={ + 'verbose_name_plural': 'Activity Events', + 'db_table': 'github_activity_events', + 'indexes': [models.Index(fields=['occurred_at'], name='activity_event_occurred_at_idx'), models.Index(fields=['activity_type'], name='activity_event_type_idx'), models.Index(fields=['repository'], name='activity_event_repository_idx'), models.Index(fields=['actor'], name='activity_event_actor_idx'), models.Index(fields=['content_type', 'object_id'], name='activity_event_source_idx')], + 'constraints': [models.UniqueConstraint(fields=('activity_type', 'content_type', 'object_id'), name='unique_activity_event')], + }, + ), + ] diff --git a/backend/src/apps/github/models/__init__.py b/backend/src/apps/github/models/__init__.py index 5b4113b60b..d25fed562a 100644 --- a/backend/src/apps/github/models/__init__.py +++ b/backend/src/apps/github/models/__init__.py @@ -1,5 +1,6 @@ """Github app.""" +from .activity_event import ActivityEvent from .comment import Comment from .commit import Commit from .issue import Issue diff --git a/backend/src/apps/github/models/activity_event.py b/backend/src/apps/github/models/activity_event.py new file mode 100644 index 0000000000..19bcc14f03 --- /dev/null +++ b/backend/src/apps/github/models/activity_event.py @@ -0,0 +1,78 @@ +"""Github app activity event model.""" + +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.db import models + +from apps.common.models import BulkSaveModel, TimestampedModel + + +from apps.github.models.enums.activity_event import ActivityType + + +class ActivityEvent(BulkSaveModel, TimestampedModel): + """Represents a discrete GitHub activity event linked to a single source object via a polymorphic GenericForeignKey.""" + + class Meta: + """Model options.""" + + db_table = "github_activity_events" + verbose_name_plural = "Activity Events" + + constraints = [ + models.UniqueConstraint( + fields=[ + "activity_type", + "content_type", + "object_id", + ], + name="unique_activity_event", + ), + ] + + indexes = [ + models.Index(fields=["activity_type"], name="activity_event_type_idx"), + models.Index(fields=["actor"], name="activity_event_actor_idx"), + models.Index( + fields=["content_type", "object_id"], + name="activity_event_source_idx", + ), + models.Index(fields=["occurred_at"], name="activity_event_occurred_at_idx"), + models.Index(fields=["repository"], name="activity_event_repository_idx"), + ] + + activity_type = models.CharField( + verbose_name="Activity Type", + max_length=32, + choices=ActivityType.choices, + ) + actor = models.ForeignKey( + "github.User", + verbose_name="Actor", + on_delete=models.SET_NULL, + blank=True, + null=True, + related_name="activity_events", + ) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) + object_id = models.PositiveBigIntegerField() + occurred_at = models.DateTimeField( + verbose_name="Occurred at", + help_text="Timestamp when the activity event occurred on GitHub", + ) + repository = models.ForeignKey( + "github.Repository", + verbose_name="Repository", + on_delete=models.CASCADE, + related_name="activity_events", + ) + source_object = GenericForeignKey("content_type", "object_id") + + def __str__(self) -> str: + """Return human-readable representation.""" + return f"{self.activity_type} by {self.actor} in {self.repository}" + + @staticmethod + def bulk_save(activity_events, fields=None) -> None: # type: ignore[override] + """Bulk save activity events.""" + BulkSaveModel.bulk_save(ActivityEvent, activity_events, fields=fields) diff --git a/backend/src/apps/github/models/enums/__init__.py b/backend/src/apps/github/models/enums/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/src/apps/github/models/enums/activity_event.py b/backend/src/apps/github/models/enums/activity_event.py new file mode 100644 index 0000000000..caffcd055a --- /dev/null +++ b/backend/src/apps/github/models/enums/activity_event.py @@ -0,0 +1,18 @@ +"""Enums for GitHub activity events.""" + +from django.db import models + + +class ActivityType(models.TextChoices): + """Activity type choices.""" + + COMMIT_PUSHED = "commit_pushed", "Commit Pushed" + + ISSUE_CLOSED = "issue_closed", "Issue Closed" + ISSUE_OPENED = "issue_opened", "Issue Opened" + + PR_CLOSED = "pr_closed", "PR Closed" + PR_MERGED = "pr_merged", "PR Merged" + PR_OPENED = "pr_opened", "PR Opened" + + RELEASE_PUBLISHED = "release_published", "Release Published" diff --git a/docker-compose/local/compose.yaml b/docker-compose/local/compose.yaml index 8b04f7b329..f9a8a056d0 100644 --- a/docker-compose/local/compose.yaml +++ b/docker-compose/local/compose.yaml @@ -23,7 +23,7 @@ services: - 8000:8000 volumes: - ../../backend:/home/owasp - - backend-venv:/home/owasp/.venv + - backend-venv-pulse:/home/owasp/.venv cache: command: > @@ -41,7 +41,7 @@ services: networks: - nest-network volumes: - - cache-data:/data + - cache-data-pulse:/data db: container_name: nest-db @@ -55,7 +55,7 @@ services: networks: - nest-network volumes: - - db-data-5079:/var/lib/postgresql/data + - db-data-pulse:/var/lib/postgresql/data docs: container_name: nest-docs @@ -77,7 +77,7 @@ services: - ../../README.md:/home/owasp/README.md:ro - ../../CODE_OF_CONDUCT.md:/home/owasp/CODE_OF_CONDUCT.md:ro - ../../CONTRIBUTING.md:/home/owasp/CONTRIBUTING.md:ro - - docs-venv:/home/owasp/.venv + - docs-venv-pulse:/home/owasp/.venv frontend: container_name: nest-frontend @@ -99,8 +99,8 @@ services: - 3000:3000 volumes: - ../../frontend:/home/owasp - - frontend-next:/home/owasp/.next - - frontend-node-modules:/home/owasp/node_modules + - frontend-next-pulse:/home/owasp/.next + - frontend-node-modules-pulse:/home/owasp/node_modules worker: container_name: nest-worker @@ -121,15 +121,15 @@ services: - nest-network volumes: - ../../backend:/home/owasp - - backend-venv:/home/owasp/.venv + - backend-venv-pulse:/home/owasp/.venv networks: nest-network: volumes: - backend-venv: - cache-data: - db-data-5079: - docs-venv: - frontend-next: - frontend-node-modules: + backend-venv-pulse: + cache-data-pulse: + db-data-pulse: + docs-venv-pulse: + frontend-next-pulse: + frontend-node-modules-pulse: From dc695f521927bd768bf9c9508c2458c6d43623d9 Mon Sep 17 00:00:00 2001 From: Anurag Yadav Date: Sat, 18 Jul 2026 22:54:27 +0530 Subject: [PATCH 2/8] adress review --- .../github/migrations/0045_activityevent.py | 97 +++++++++++++++---- .../src/apps/github/models/activity_event.py | 8 +- .../github/models/enums/activity_event.py | 3 - 3 files changed, 84 insertions(+), 24 deletions(-) diff --git a/backend/src/apps/github/migrations/0045_activityevent.py b/backend/src/apps/github/migrations/0045_activityevent.py index 046aa38f51..cba54f7b0c 100644 --- a/backend/src/apps/github/migrations/0045_activityevent.py +++ b/backend/src/apps/github/migrations/0045_activityevent.py @@ -1,35 +1,96 @@ -# Generated by Django 6.0.7 on 2026-07-17 08:13 +# Generated by Django 6.0.7 on 2026-07-18 17:15 import django.db.models.deletion from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('github', '0044_user_indexes'), + ("contenttypes", "0002_remove_content_type_name"), + ("github", "0044_user_indexes"), ] operations = [ migrations.CreateModel( - name='ActivityEvent', + name="ActivityEvent", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('nest_created_at', models.DateTimeField(auto_now_add=True)), - ('nest_updated_at', models.DateTimeField(auto_now=True)), - ('activity_type', models.CharField(choices=[('pr_opened', 'PR Opened'), ('pr_closed', 'PR Closed'), ('pr_merged', 'PR Merged'), ('issue_opened', 'Issue Opened'), ('issue_closed', 'Issue Closed'), ('commit_pushed', 'Commit Pushed'), ('release_published', 'Release Published')], max_length=32, verbose_name='Activity Type')), - ('occurred_at', models.DateTimeField(help_text='Timestamp when the activity event occurred on GitHub', verbose_name='Occurred at')), - ('object_id', models.PositiveBigIntegerField()), - ('actor', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='activity_events', to='github.user', verbose_name='Actor')), - ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), - ('repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='activity_events', to='github.repository', verbose_name='Repository')), + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ( + "activity_type", + models.CharField( + choices=[ + ("commit_pushed", "Commit Pushed"), + ("issue_closed", "Issue Closed"), + ("issue_opened", "Issue Opened"), + ("pr_closed", "PR Closed"), + ("pr_merged", "PR Merged"), + ("pr_opened", "PR Opened"), + ("release_published", "Release Published"), + ], + max_length=32, + verbose_name="Activity Type", + ), + ), + ("object_id", models.PositiveBigIntegerField()), + ( + "occurred_at", + models.DateTimeField( + help_text="Timestamp when the activity event occurred on GitHub", + verbose_name="Occurred at", + ), + ), + ( + "actor", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="activity_events", + to="github.user", + verbose_name="Actor", + ), + ), + ( + "content_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="contenttypes.contenttype" + ), + ), + ( + "repository", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="activity_events", + to="github.repository", + verbose_name="Repository", + ), + ), ], options={ - 'verbose_name_plural': 'Activity Events', - 'db_table': 'github_activity_events', - 'indexes': [models.Index(fields=['occurred_at'], name='activity_event_occurred_at_idx'), models.Index(fields=['activity_type'], name='activity_event_type_idx'), models.Index(fields=['repository'], name='activity_event_repository_idx'), models.Index(fields=['actor'], name='activity_event_actor_idx'), models.Index(fields=['content_type', 'object_id'], name='activity_event_source_idx')], - 'constraints': [models.UniqueConstraint(fields=('activity_type', 'content_type', 'object_id'), name='unique_activity_event')], + "verbose_name_plural": "Activity Events", + "db_table": "github_activity_events", + "indexes": [ + models.Index(fields=["activity_type"], name="activity_event_type_idx"), + models.Index(fields=["actor"], name="activity_event_actor_idx"), + models.Index( + fields=["content_type", "object_id"], name="activity_event_source_idx" + ), + models.Index(fields=["occurred_at"], name="activity_event_occurred_at_idx"), + models.Index(fields=["repository"], name="activity_event_repository_idx"), + ], + "constraints": [ + models.UniqueConstraint( + fields=("activity_type", "content_type", "object_id", "occurred_at"), + name="unique_activity_event", + ) + ], }, ), ] diff --git a/backend/src/apps/github/models/activity_event.py b/backend/src/apps/github/models/activity_event.py index 19bcc14f03..586eacac1b 100644 --- a/backend/src/apps/github/models/activity_event.py +++ b/backend/src/apps/github/models/activity_event.py @@ -5,13 +5,14 @@ from django.db import models from apps.common.models import BulkSaveModel, TimestampedModel - - from apps.github.models.enums.activity_event import ActivityType class ActivityEvent(BulkSaveModel, TimestampedModel): - """Represents a discrete GitHub activity event linked to a single source object via a polymorphic GenericForeignKey.""" + """Represents a discrete GitHub activity event linked to a single source object. + + Uses a polymorphic GenericForeignKey to reference the source object. + """ class Meta: """Model options.""" @@ -25,6 +26,7 @@ class Meta: "activity_type", "content_type", "object_id", + "occurred_at", ], name="unique_activity_event", ), diff --git a/backend/src/apps/github/models/enums/activity_event.py b/backend/src/apps/github/models/enums/activity_event.py index caffcd055a..ea998da463 100644 --- a/backend/src/apps/github/models/enums/activity_event.py +++ b/backend/src/apps/github/models/enums/activity_event.py @@ -7,12 +7,9 @@ class ActivityType(models.TextChoices): """Activity type choices.""" COMMIT_PUSHED = "commit_pushed", "Commit Pushed" - ISSUE_CLOSED = "issue_closed", "Issue Closed" ISSUE_OPENED = "issue_opened", "Issue Opened" - PR_CLOSED = "pr_closed", "PR Closed" PR_MERGED = "pr_merged", "PR Merged" PR_OPENED = "pr_opened", "PR Opened" - RELEASE_PUBLISHED = "release_published", "Release Published" From 61931da99153ffb3e355c2e9633bb361598e5a8c Mon Sep 17 00:00:00 2001 From: Anurag Yadav Date: Wed, 29 Jul 2026 15:41:40 +0530 Subject: [PATCH 3/8] added actitivty builder --- backend/src/apps/github/admin/__init__.py | 1 - backend/src/apps/github/common.py | 5 + .../github/migrations/0045_activityevent.py | 96 ----------- backend/src/apps/github/models/__init__.py | 1 - .../src/apps/github/models/activity_event.py | 80 --------- .../src/apps/github/models/enums/__init__.py | 0 .../github/models/enums/activity_event.py | 15 -- backend/src/apps/owasp/admin/__init__.py | 1 + .../{github => owasp}/admin/activity_event.py | 17 +- .../owasp/migrations/0073_activityevent.py | 36 +++++ backend/src/apps/owasp/models/__init__.py | 1 + .../src/apps/owasp/models/activity_event.py | 153 ++++++++++++++++++ backend/tests/unit/apps/github/common_test.py | 1 + 13 files changed, 205 insertions(+), 202 deletions(-) delete mode 100644 backend/src/apps/github/migrations/0045_activityevent.py delete mode 100644 backend/src/apps/github/models/activity_event.py delete mode 100644 backend/src/apps/github/models/enums/__init__.py delete mode 100644 backend/src/apps/github/models/enums/activity_event.py rename backend/src/apps/{github => owasp}/admin/activity_event.py (60%) create mode 100644 backend/src/apps/owasp/migrations/0073_activityevent.py create mode 100644 backend/src/apps/owasp/models/activity_event.py diff --git a/backend/src/apps/github/admin/__init__.py b/backend/src/apps/github/admin/__init__.py index eb7170e91f..60ae27406f 100644 --- a/backend/src/apps/github/admin/__init__.py +++ b/backend/src/apps/github/admin/__init__.py @@ -1,6 +1,5 @@ """Github app admin.""" -from .activity_event import ActivityEventAdmin from .comment import CommentAdmin from .commit import CommitAdmin from .issue import IssueAdmin diff --git a/backend/src/apps/github/common.py b/backend/src/apps/github/common.py index 1371fcf782..eb0abfe3d7 100644 --- a/backend/src/apps/github/common.py +++ b/backend/src/apps/github/common.py @@ -23,6 +23,7 @@ from apps.github.models.repository_contributor import RepositoryContributor from apps.github.models.user import User from apps.github.utils import check_owasp_site_repository +from apps.owasp.models.activity_event import ActivityEvent logger: logging.Logger = logging.getLogger(__name__) @@ -138,6 +139,7 @@ def sync_repository( milestone=milestone, repository=repository, ) + ActivityEvent.update_data(issue) # Assignees. issue.assignees.clear() @@ -186,6 +188,7 @@ def sync_repository( milestone=milestone, repository=repository, ) + ActivityEvent.update_data(pull_request) # Assignees. pull_request.assignees.clear() @@ -217,6 +220,8 @@ def sync_repository( author = User.update_data(gh_release.author) releases.append(Release.update_data(gh_release, author=author, repository=repository)) Release.bulk_save(releases) + for release in releases: + ActivityEvent.update_data(release) # GitHub repository contributors. RepositoryContributor.bulk_save( diff --git a/backend/src/apps/github/migrations/0045_activityevent.py b/backend/src/apps/github/migrations/0045_activityevent.py deleted file mode 100644 index cba54f7b0c..0000000000 --- a/backend/src/apps/github/migrations/0045_activityevent.py +++ /dev/null @@ -1,96 +0,0 @@ -# Generated by Django 6.0.7 on 2026-07-18 17:15 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("contenttypes", "0002_remove_content_type_name"), - ("github", "0044_user_indexes"), - ] - - operations = [ - migrations.CreateModel( - name="ActivityEvent", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, primary_key=True, serialize=False, verbose_name="ID" - ), - ), - ("nest_created_at", models.DateTimeField(auto_now_add=True)), - ("nest_updated_at", models.DateTimeField(auto_now=True)), - ( - "activity_type", - models.CharField( - choices=[ - ("commit_pushed", "Commit Pushed"), - ("issue_closed", "Issue Closed"), - ("issue_opened", "Issue Opened"), - ("pr_closed", "PR Closed"), - ("pr_merged", "PR Merged"), - ("pr_opened", "PR Opened"), - ("release_published", "Release Published"), - ], - max_length=32, - verbose_name="Activity Type", - ), - ), - ("object_id", models.PositiveBigIntegerField()), - ( - "occurred_at", - models.DateTimeField( - help_text="Timestamp when the activity event occurred on GitHub", - verbose_name="Occurred at", - ), - ), - ( - "actor", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name="activity_events", - to="github.user", - verbose_name="Actor", - ), - ), - ( - "content_type", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, to="contenttypes.contenttype" - ), - ), - ( - "repository", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="activity_events", - to="github.repository", - verbose_name="Repository", - ), - ), - ], - options={ - "verbose_name_plural": "Activity Events", - "db_table": "github_activity_events", - "indexes": [ - models.Index(fields=["activity_type"], name="activity_event_type_idx"), - models.Index(fields=["actor"], name="activity_event_actor_idx"), - models.Index( - fields=["content_type", "object_id"], name="activity_event_source_idx" - ), - models.Index(fields=["occurred_at"], name="activity_event_occurred_at_idx"), - models.Index(fields=["repository"], name="activity_event_repository_idx"), - ], - "constraints": [ - models.UniqueConstraint( - fields=("activity_type", "content_type", "object_id", "occurred_at"), - name="unique_activity_event", - ) - ], - }, - ), - ] diff --git a/backend/src/apps/github/models/__init__.py b/backend/src/apps/github/models/__init__.py index d25fed562a..5b4113b60b 100644 --- a/backend/src/apps/github/models/__init__.py +++ b/backend/src/apps/github/models/__init__.py @@ -1,6 +1,5 @@ """Github app.""" -from .activity_event import ActivityEvent from .comment import Comment from .commit import Commit from .issue import Issue diff --git a/backend/src/apps/github/models/activity_event.py b/backend/src/apps/github/models/activity_event.py deleted file mode 100644 index 586eacac1b..0000000000 --- a/backend/src/apps/github/models/activity_event.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Github app activity event model.""" - -from django.contrib.contenttypes.fields import GenericForeignKey -from django.contrib.contenttypes.models import ContentType -from django.db import models - -from apps.common.models import BulkSaveModel, TimestampedModel -from apps.github.models.enums.activity_event import ActivityType - - -class ActivityEvent(BulkSaveModel, TimestampedModel): - """Represents a discrete GitHub activity event linked to a single source object. - - Uses a polymorphic GenericForeignKey to reference the source object. - """ - - class Meta: - """Model options.""" - - db_table = "github_activity_events" - verbose_name_plural = "Activity Events" - - constraints = [ - models.UniqueConstraint( - fields=[ - "activity_type", - "content_type", - "object_id", - "occurred_at", - ], - name="unique_activity_event", - ), - ] - - indexes = [ - models.Index(fields=["activity_type"], name="activity_event_type_idx"), - models.Index(fields=["actor"], name="activity_event_actor_idx"), - models.Index( - fields=["content_type", "object_id"], - name="activity_event_source_idx", - ), - models.Index(fields=["occurred_at"], name="activity_event_occurred_at_idx"), - models.Index(fields=["repository"], name="activity_event_repository_idx"), - ] - - activity_type = models.CharField( - verbose_name="Activity Type", - max_length=32, - choices=ActivityType.choices, - ) - actor = models.ForeignKey( - "github.User", - verbose_name="Actor", - on_delete=models.SET_NULL, - blank=True, - null=True, - related_name="activity_events", - ) - content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) - object_id = models.PositiveBigIntegerField() - occurred_at = models.DateTimeField( - verbose_name="Occurred at", - help_text="Timestamp when the activity event occurred on GitHub", - ) - repository = models.ForeignKey( - "github.Repository", - verbose_name="Repository", - on_delete=models.CASCADE, - related_name="activity_events", - ) - source_object = GenericForeignKey("content_type", "object_id") - - def __str__(self) -> str: - """Return human-readable representation.""" - return f"{self.activity_type} by {self.actor} in {self.repository}" - - @staticmethod - def bulk_save(activity_events, fields=None) -> None: # type: ignore[override] - """Bulk save activity events.""" - BulkSaveModel.bulk_save(ActivityEvent, activity_events, fields=fields) diff --git a/backend/src/apps/github/models/enums/__init__.py b/backend/src/apps/github/models/enums/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/src/apps/github/models/enums/activity_event.py b/backend/src/apps/github/models/enums/activity_event.py deleted file mode 100644 index ea998da463..0000000000 --- a/backend/src/apps/github/models/enums/activity_event.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Enums for GitHub activity events.""" - -from django.db import models - - -class ActivityType(models.TextChoices): - """Activity type choices.""" - - COMMIT_PUSHED = "commit_pushed", "Commit Pushed" - ISSUE_CLOSED = "issue_closed", "Issue Closed" - ISSUE_OPENED = "issue_opened", "Issue Opened" - PR_CLOSED = "pr_closed", "PR Closed" - PR_MERGED = "pr_merged", "PR Merged" - PR_OPENED = "pr_opened", "PR Opened" - RELEASE_PUBLISHED = "release_published", "Release Published" diff --git a/backend/src/apps/owasp/admin/__init__.py b/backend/src/apps/owasp/admin/__init__.py index 261225145a..524c70a757 100644 --- a/backend/src/apps/owasp/admin/__init__.py +++ b/backend/src/apps/owasp/admin/__init__.py @@ -4,6 +4,7 @@ from apps.owasp.models.project_health_requirements import ProjectHealthRequirements +from .activity_event import ActivityEventAdmin from .board_of_directors import BoardOfDirectorsAdmin from .chapter import ChapterAdmin from .committee import CommitteeAdmin diff --git a/backend/src/apps/github/admin/activity_event.py b/backend/src/apps/owasp/admin/activity_event.py similarity index 60% rename from backend/src/apps/github/admin/activity_event.py rename to backend/src/apps/owasp/admin/activity_event.py index 715aba9cd5..72da546259 100644 --- a/backend/src/apps/github/admin/activity_event.py +++ b/backend/src/apps/owasp/admin/activity_event.py @@ -1,23 +1,22 @@ -"""GitHub app ActivityEvent model admin.""" +"""OWASP app ActivityEvent model admin.""" from django.contrib import admin -from apps.github.models.activity_event import ActivityEvent +from apps.owasp.models.activity_event import ActivityEvent class ActivityEventAdmin(admin.ModelAdmin): """Admin for ActivityEvent model.""" autocomplete_fields = ( - "actor", - "repository", + "github_user", + "github_repository", ) list_display = ( "activity_type", - "actor", - "nest_created_at", + "github_user", "occurred_at", - "repository", + "github_repository", ) list_filter = ( "activity_type", @@ -25,8 +24,8 @@ class ActivityEventAdmin(admin.ModelAdmin): ) search_fields = ( "activity_type", - "actor__login", - "repository__name", + "github_user__login", + "github_repository__name", ) diff --git a/backend/src/apps/owasp/migrations/0073_activityevent.py b/backend/src/apps/owasp/migrations/0073_activityevent.py new file mode 100644 index 0000000000..f5cf1fbc03 --- /dev/null +++ b/backend/src/apps/owasp/migrations/0073_activityevent.py @@ -0,0 +1,36 @@ +# Generated by Django 6.0.7 on 2026-07-28 07:54 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('github', '0044_user_indexes'), + ('owasp', '0072_project_project_name_gin_idx_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='ActivityEvent', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('nest_created_at', models.DateTimeField(auto_now_add=True)), + ('nest_updated_at', models.DateTimeField(auto_now=True)), + ('activity_type', models.CharField(choices=[('issue_closed', 'Issue Closed'), ('issue_opened', 'Issue Opened'), ('pr_closed', 'PR Closed'), ('pr_merged', 'PR Merged'), ('pr_opened', 'PR Opened'), ('release_published', 'Release Published')], max_length=32, verbose_name='Activity Type')), + ('object_id', models.PositiveBigIntegerField()), + ('occurred_at', models.DateTimeField(help_text='Timestamp when the activity event occurred on GitHub', verbose_name='Occurred at')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), + ('github_repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='activity_events', to='github.repository', verbose_name='GitHub Repository')), + ('github_user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='activity_events', to='github.user', verbose_name='GitHub User')), + ], + options={ + 'verbose_name_plural': 'Activity Events', + 'db_table': 'github_activity_events', + 'indexes': [models.Index(fields=['activity_type'], name='activity_event_type_idx'), models.Index(fields=['github_user'], name='activity_event_github_user_idx'), models.Index(fields=['content_type', 'object_id'], name='activity_event_source_idx'), models.Index(fields=['occurred_at'], name='activity_event_occurred_at_idx'), models.Index(fields=['github_repository'], name='activity_event_github_repo_idx')], + 'constraints': [models.UniqueConstraint(fields=('activity_type', 'content_type', 'object_id', 'occurred_at'), name='unique_activity_event')], + }, + ), + ] diff --git a/backend/src/apps/owasp/models/__init__.py b/backend/src/apps/owasp/models/__init__.py index 3cbb120b8b..3bbb4a7832 100644 --- a/backend/src/apps/owasp/models/__init__.py +++ b/backend/src/apps/owasp/models/__init__.py @@ -1,3 +1,4 @@ +from .activity_event import ActivityEvent from .board_of_directors import BoardOfDirectors from .chapter import Chapter from .committee import Committee diff --git a/backend/src/apps/owasp/models/activity_event.py b/backend/src/apps/owasp/models/activity_event.py new file mode 100644 index 0000000000..79b2194ffd --- /dev/null +++ b/backend/src/apps/owasp/models/activity_event.py @@ -0,0 +1,153 @@ +"""OWASP app activity event model.""" + +import logging + +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.db import models + +from apps.common.models import BulkSaveModel, TimestampedModel + +logger = logging.getLogger(__name__) + + +class ActivityEvent(BulkSaveModel, TimestampedModel): + """Represents a discrete GitHub activity event linked to a single source object. + + Uses a polymorphic GenericForeignKey to reference the source object. + """ + + class Meta: + """Model options.""" + + db_table = "github_activity_events" + verbose_name_plural = "Activity Events" + + constraints = [ + models.UniqueConstraint( + fields=[ + "activity_type", + "content_type", + "object_id", + "occurred_at", + ], + name="unique_activity_event", + ), + ] + + indexes = [ + models.Index(fields=["activity_type"], name="activity_event_type_idx"), + models.Index(fields=["github_user"], name="activity_event_github_user_idx"), + models.Index( + fields=["content_type", "object_id"], + name="activity_event_source_idx", + ), + models.Index(fields=["occurred_at"], name="activity_event_occurred_at_idx"), + models.Index(fields=["github_repository"], name="activity_event_github_repo_idx"), + ] + + class ActivityType(models.TextChoices): + """Activity type choices.""" + + ISSUE_CLOSED = "issue_closed", "Issue Closed" + ISSUE_OPENED = "issue_opened", "Issue Opened" + PR_CLOSED = "pr_closed", "PR Closed" + PR_MERGED = "pr_merged", "PR Merged" + PR_OPENED = "pr_opened", "PR Opened" + RELEASE_PUBLISHED = "release_published", "Release Published" + + activity_type = models.CharField( + verbose_name="Activity Type", + max_length=32, + choices=ActivityType.choices, + ) + github_user = models.ForeignKey( + "github.User", + verbose_name="GitHub User", + on_delete=models.SET_NULL, + blank=True, + null=True, + related_name="activity_events", + ) + content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE) + object_id = models.PositiveBigIntegerField() + occurred_at = models.DateTimeField( + verbose_name="Occurred at", + help_text="Timestamp when the activity event occurred on GitHub", + ) + github_repository = models.ForeignKey( + "github.Repository", + verbose_name="GitHub Repository", + on_delete=models.CASCADE, + related_name="activity_events", + ) + source_object = GenericForeignKey("content_type", "object_id") + + HANDLERS: dict[str, str] = { + "Issue": "build_for_issue", + "PullRequest": "build_for_pull_request", + "Release": "build_for_release", + } + + def __str__(self) -> str: + """Return human-readable representation.""" + return f"{self.activity_type} by {self.github_user} in {self.github_repository}" + + @staticmethod + def bulk_save(activity_events, fields=None) -> None: # type: ignore[override] + """Bulk save activity events.""" + BulkSaveModel.bulk_save(ActivityEvent, activity_events, fields=fields) + + @staticmethod + def build_for_issue(issue) -> list[tuple]: + """Return event tuples for an Issue.""" + events = [(ActivityEvent.ActivityType.ISSUE_OPENED, issue.created_at, issue.author)] + if issue.state == "closed" and issue.closed_at: + events.append((ActivityEvent.ActivityType.ISSUE_CLOSED, issue.closed_at, issue.author)) + return events + + @staticmethod + def build_for_pull_request(pr) -> list[tuple]: + """Return event tuples for a PullRequest.""" + events = [(ActivityEvent.ActivityType.PR_OPENED, pr.created_at, pr.author)] + if pr.merged_at: + events.append((ActivityEvent.ActivityType.PR_MERGED, pr.merged_at, pr.author)) + elif pr.state == "closed" and pr.closed_at: + events.append((ActivityEvent.ActivityType.PR_CLOSED, pr.closed_at, pr.author)) + return events + + @staticmethod + def build_for_release(release) -> list[tuple]: + """Return event tuples for a Release.""" + occurred_at = release.published_at or release.created_at + return [(ActivityEvent.ActivityType.RELEASE_PUBLISHED, occurred_at, release.author)] + + @staticmethod + def update_data(obj) -> None: + """Create ActivityEvent row(s) for a saved GitHub model instance if they do not exist.""" + handler_name = ActivityEvent.HANDLERS.get(type(obj).__name__) + if handler_name is None: + logger.error( + "ActivityEvent.update_data received unsupported model type: %s", + type(obj).__name__, + ) + raise TypeError(f"Unsupported model type: {type(obj)}") + + handler = getattr(ActivityEvent, handler_name) + events = handler(obj) + content_type = ContentType.objects.get_for_model(obj) + + for activity_type, occurred_at, github_user in events: + if occurred_at is None: + continue + + ActivityEvent.objects.get_or_create( + activity_type=activity_type, + content_type=content_type, + object_id=obj.pk, + occurred_at=occurred_at, + defaults={ + "github_user": github_user, + "github_repository": obj.repository, + }, + ) diff --git a/backend/tests/unit/apps/github/common_test.py b/backend/tests/unit/apps/github/common_test.py index a5501eed1c..8106f85584 100644 --- a/backend/tests/unit/apps/github/common_test.py +++ b/backend/tests/unit/apps/github/common_test.py @@ -21,6 +21,7 @@ def mock_common_deps(mocker): "Label": mocker.patch("apps.github.common.Label"), "Release": mocker.patch("apps.github.common.Release"), "RepositoryContributor": mocker.patch("apps.github.common.RepositoryContributor"), + "ActivityEvent": mocker.patch("apps.github.common.ActivityEvent"), "check_owasp": mocker.patch( "apps.github.common.check_owasp_site_repository", return_value=False ), From ae10e4fb532817b9f5047e6e81f9e8afab8405e4 Mon Sep 17 00:00:00 2001 From: Anurag Yadav Date: Thu, 30 Jul 2026 00:37:26 +0530 Subject: [PATCH 4/8] Address review --- .../owasp/migrations/0073_activityevent.py | 98 +++++++++++++++---- .../src/apps/owasp/models/activity_event.py | 3 +- docker-compose/local/compose.override.yaml | 13 +++ docker-compose/local/compose.yaml | 10 +- 4 files changed, 100 insertions(+), 24 deletions(-) diff --git a/backend/src/apps/owasp/migrations/0073_activityevent.py b/backend/src/apps/owasp/migrations/0073_activityevent.py index f5cf1fbc03..ba75297650 100644 --- a/backend/src/apps/owasp/migrations/0073_activityevent.py +++ b/backend/src/apps/owasp/migrations/0073_activityevent.py @@ -5,32 +5,94 @@ class Migration(migrations.Migration): - dependencies = [ - ('contenttypes', '0002_remove_content_type_name'), - ('github', '0044_user_indexes'), - ('owasp', '0072_project_project_name_gin_idx_and_more'), + ("contenttypes", "0002_remove_content_type_name"), + ("github", "0044_user_indexes"), + ("owasp", "0072_project_project_name_gin_idx_and_more"), ] operations = [ migrations.CreateModel( - name='ActivityEvent', + name="ActivityEvent", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('nest_created_at', models.DateTimeField(auto_now_add=True)), - ('nest_updated_at', models.DateTimeField(auto_now=True)), - ('activity_type', models.CharField(choices=[('issue_closed', 'Issue Closed'), ('issue_opened', 'Issue Opened'), ('pr_closed', 'PR Closed'), ('pr_merged', 'PR Merged'), ('pr_opened', 'PR Opened'), ('release_published', 'Release Published')], max_length=32, verbose_name='Activity Type')), - ('object_id', models.PositiveBigIntegerField()), - ('occurred_at', models.DateTimeField(help_text='Timestamp when the activity event occurred on GitHub', verbose_name='Occurred at')), - ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.contenttype')), - ('github_repository', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='activity_events', to='github.repository', verbose_name='GitHub Repository')), - ('github_user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='activity_events', to='github.user', verbose_name='GitHub User')), + ( + "id", + models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("nest_created_at", models.DateTimeField(auto_now_add=True)), + ("nest_updated_at", models.DateTimeField(auto_now=True)), + ( + "activity_type", + models.CharField( + choices=[ + ("issue_closed", "Issue Closed"), + ("issue_opened", "Issue Opened"), + ("pr_closed", "PR Closed"), + ("pr_merged", "PR Merged"), + ("pr_opened", "PR Opened"), + ("release_published", "Release Published"), + ], + max_length=32, + verbose_name="Activity Type", + ), + ), + ("object_id", models.PositiveBigIntegerField()), + ( + "occurred_at", + models.DateTimeField( + help_text="Timestamp when the activity event occurred on GitHub", + verbose_name="Occurred at", + ), + ), + ( + "content_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="contenttypes.contenttype" + ), + ), + ( + "github_repository", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="activity_events", + to="github.repository", + verbose_name="GitHub Repository", + ), + ), + ( + "github_user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="activity_events", + to="github.user", + verbose_name="GitHub User", + ), + ), ], options={ - 'verbose_name_plural': 'Activity Events', - 'db_table': 'github_activity_events', - 'indexes': [models.Index(fields=['activity_type'], name='activity_event_type_idx'), models.Index(fields=['github_user'], name='activity_event_github_user_idx'), models.Index(fields=['content_type', 'object_id'], name='activity_event_source_idx'), models.Index(fields=['occurred_at'], name='activity_event_occurred_at_idx'), models.Index(fields=['github_repository'], name='activity_event_github_repo_idx')], - 'constraints': [models.UniqueConstraint(fields=('activity_type', 'content_type', 'object_id', 'occurred_at'), name='unique_activity_event')], + "verbose_name_plural": "Activity Events", + "db_table": "github_activity_events", + "indexes": [ + models.Index(fields=["activity_type"], name="activity_event_type_idx"), + models.Index(fields=["github_user"], name="activity_event_github_user_idx"), + models.Index( + fields=["content_type", "object_id"], name="activity_event_source_idx" + ), + models.Index(fields=["occurred_at"], name="activity_event_occurred_at_idx"), + models.Index( + fields=["github_repository"], name="activity_event_github_repo_idx" + ), + ], + "constraints": [ + models.UniqueConstraint( + fields=("activity_type", "content_type", "object_id", "occurred_at"), + name="unique_activity_event", + ) + ], }, ), ] diff --git a/backend/src/apps/owasp/models/activity_event.py b/backend/src/apps/owasp/models/activity_event.py index 79b2194ffd..3db60269c3 100644 --- a/backend/src/apps/owasp/models/activity_event.py +++ b/backend/src/apps/owasp/models/activity_event.py @@ -131,7 +131,8 @@ def update_data(obj) -> None: "ActivityEvent.update_data received unsupported model type: %s", type(obj).__name__, ) - raise TypeError(f"Unsupported model type: {type(obj)}") + message = f"Unsupported model type: {type(obj)}" + raise TypeError(message) handler = getattr(ActivityEvent, handler_name) events = handler(obj) diff --git a/docker-compose/local/compose.override.yaml b/docker-compose/local/compose.override.yaml index e69de29bb2..25ce0eb8cb 100644 --- a/docker-compose/local/compose.override.yaml +++ b/docker-compose/local/compose.override.yaml @@ -0,0 +1,13 @@ +volumes: + backend-venv: + name: backend-venv-pulse + cache-data: + name: cache-data-pulse + db-data: + name: db-data-pulse + docs-venv: + name: docs-venv-pulse + frontend-next: + name: frontend-next-pulse + frontend-node-modules: + name: frontend-node-modules-pulse diff --git a/docker-compose/local/compose.yaml b/docker-compose/local/compose.yaml index e664a4b493..a76bb34021 100644 --- a/docker-compose/local/compose.yaml +++ b/docker-compose/local/compose.yaml @@ -23,7 +23,7 @@ services: - 8000:8000 volumes: - ../../backend:/home/owasp - - backend-venv-pulse:/home/owasp/.venv + - backend-venv:/home/owasp/.venv cache: command: > @@ -41,7 +41,7 @@ services: networks: - nest-network volumes: - - cache-data-pulse:/data + - cache-data:/data db: container_name: nest-db @@ -97,8 +97,8 @@ services: - 3000:3000 volumes: - ../../frontend:/home/owasp - - frontend-next-pulse:/home/owasp/.next - - frontend-node-modules-pulse:/home/owasp/node_modules + - frontend-next:/home/owasp/.next + - frontend-node-modules:/home/owasp/node_modules worker: container_name: nest-worker @@ -119,7 +119,7 @@ services: - nest-network volumes: - ../../backend:/home/owasp - - backend-venv-pulse:/home/owasp/.venv + - backend-venv:/home/owasp/.venv networks: nest-network: From cc0f1dea1a6a8be17f9203cc1fafe4f95505573d Mon Sep 17 00:00:00 2001 From: Anurag Yadav Date: Thu, 30 Jul 2026 01:07:13 +0530 Subject: [PATCH 5/8] updated order --- backend/src/apps/owasp/admin/activity_event.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/apps/owasp/admin/activity_event.py b/backend/src/apps/owasp/admin/activity_event.py index 72da546259..6a16479878 100644 --- a/backend/src/apps/owasp/admin/activity_event.py +++ b/backend/src/apps/owasp/admin/activity_event.py @@ -14,9 +14,9 @@ class ActivityEventAdmin(admin.ModelAdmin): ) list_display = ( "activity_type", + "github_repository", "github_user", "occurred_at", - "github_repository", ) list_filter = ( "activity_type", @@ -24,8 +24,8 @@ class ActivityEventAdmin(admin.ModelAdmin): ) search_fields = ( "activity_type", - "github_user__login", "github_repository__name", + "github_user__login", ) From 101805cb9d1c7aa3d90f1e0e1a4fdef8cac9c862 Mon Sep 17 00:00:00 2001 From: Anurag Yadav Date: Thu, 30 Jul 2026 22:46:20 +0530 Subject: [PATCH 6/8] Added a command --- backend/make/apps/owasp.mk | 4 + .../owasp_backfill_activity_events.py | 121 +++++++++ .../owasp_backfill_activity_events_test.py | 253 ++++++++++++++++++ 3 files changed, 378 insertions(+) create mode 100644 backend/src/apps/owasp/management/commands/owasp_backfill_activity_events.py create mode 100644 backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py diff --git a/backend/make/apps/owasp.mk b/backend/make/apps/owasp.mk index 22e15ca8a0..7a9179acb4 100644 --- a/backend/make/apps/owasp.mk +++ b/backend/make/apps/owasp.mk @@ -23,6 +23,10 @@ owasp-aggregate-projects: @echo "Aggregating OWASP projects" @CMD="python manage.py owasp_aggregate_projects" $(MAKE) backend-exec-command +owasp-backfill-activity-events: + @echo "Backfilling activity events for existing pull requests, issues, and releases" + @CMD="python manage.py owasp_backfill_activity_events" $(MAKE) backend-exec-command + owasp-create-project-metadata-file: @echo "Generating metadata" @CMD="python manage.py owasp_create_project_metadata_file $(entity_key)" $(MAKE) backend-exec-command diff --git a/backend/src/apps/owasp/management/commands/owasp_backfill_activity_events.py b/backend/src/apps/owasp/management/commands/owasp_backfill_activity_events.py new file mode 100644 index 0000000000..d0d888b2c5 --- /dev/null +++ b/backend/src/apps/owasp/management/commands/owasp_backfill_activity_events.py @@ -0,0 +1,121 @@ +"""A command to backfill activity events for existing pull requests, issues, and releases.""" + +import logging + +from django.core.management.base import BaseCommand + +from apps.github.models.issue import Issue +from apps.github.models.pull_request import PullRequest +from apps.github.models.release import Release +from apps.owasp.models.activity_event import ActivityEvent + +logger: logging.Logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Backfill ActivityEvent records for existing pull requests, issues, and releases." + + def add_arguments(self, parser) -> None: + """Add command-line arguments to the parser.""" + parser.add_argument( + "--offset", + default=0, + required=False, + type=int, + help="Number of records to skip before starting backfill.", + ) + parser.add_argument( + "--model", + default="all", + required=False, + choices=["all", "issue", "pull_request", "release"], + help="Which model type to backfill. Defaults to 'all'.", + ) + + def handle(self, *args, **options) -> None: + """Handle the command execution.""" + offset = options["offset"] + model = options["model"] + + if model in ("all", "issue"): + self.backfill_issues(offset) + + if model in ("all", "pull_request"): + self.backfill_pull_requests(offset) + + if model in ("all", "release"): + self.backfill_releases(offset) + + def backfill_issues(self, offset: int) -> None: + """Backfill ActivityEvent records for existing issues.""" + issues = Issue.objects.select_related("author", "repository").order_by("created_at") + issues_count = issues.count() + self.stdout.write(f"Backfilling activity events for {issues_count} issues...\n") + + created_count = 0 + for issue in issues[offset:]: + + if not issue.repository: + logger.warning("Skipping issue #%s: no repository", issue.number) + continue + + try: + ActivityEvent.update_data(issue) + created_count += 1 + except Exception: + logger.exception("Error backfilling activity events for issue #%s", issue.number) + + self.stdout.write(f"Issues processed: {created_count}\n") + + def backfill_pull_requests(self, offset: int) -> None: + """Backfill ActivityEvent records for existing pull requests.""" + pull_requests = PullRequest.objects.select_related("author", "repository").order_by( + "created_at" + ) + pull_requests_count = pull_requests.count() + self.stdout.write( + f"Backfilling activity events for {pull_requests_count} pull requests...\n" + ) + + created_count = 0 + for pull_request in pull_requests[offset:]: + + if not pull_request.repository: + logger.warning( + "Skipping pull request #%s: no repository", pull_request.number + ) + continue + + try: + ActivityEvent.update_data(pull_request) + created_count += 1 + except Exception: + logger.exception( + "Error backfilling activity events for pull request #%s", + pull_request.number, + ) + + self.stdout.write(f"Pull requests processed: {created_count}\n") + + def backfill_releases(self, offset: int) -> None: + """Backfill ActivityEvent records for existing releases.""" + releases = Release.objects.select_related("author", "repository").order_by("created_at") + releases_count = releases.count() + self.stdout.write(f"Backfilling activity events for {releases_count} releases...\n") + + created_count = 0 + for release in releases[offset:]: + + if not release.repository: + logger.warning("Skipping release %s: no repository", release.tag_name) + continue + + try: + ActivityEvent.update_data(release) + created_count += 1 + except Exception: + logger.exception( + "Error backfilling activity events for release %s", release.tag_name + ) + + self.stdout.write(f"Releases processed: {created_count}\n") diff --git a/backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py b/backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py new file mode 100644 index 0000000000..489587d661 --- /dev/null +++ b/backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py @@ -0,0 +1,253 @@ +"""Tests for the owasp_backfill_activity_events Django management command.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from django.core.management.base import BaseCommand + +from apps.owasp.management.commands.owasp_backfill_activity_events import Command + + +class TestOwaspBackfillActivityEventsCommand: + def test_command_help_text(self): + """Test that the command has the correct help text.""" + command = Command() + assert command.help == ( + "Backfill ActivityEvent records for existing pull requests, issues, and releases." + ) + + def test_command_inheritance(self): + """Test that the command inherits from BaseCommand.""" + assert issubclass(Command, BaseCommand) + + @pytest.mark.parametrize( + ("argument_name", "expected_properties"), + [ + ( + "--offset", + { + "default": 0, + "required": False, + "type": int, + "help": "Number of records to skip before starting backfill.", + }, + ), + ( + "--model", + { + "default": "all", + "required": False, + "choices": ["all", "issue", "pull_request", "release"], + "help": "Which model type to backfill. Defaults to 'all'.", + }, + ), + ], + ) + def test_add_arguments(self, argument_name, expected_properties): + """Test that the command adds the correct arguments.""" + mock_parser = MagicMock() + command = Command() + command.add_arguments(mock_parser) + mock_parser.add_argument.assert_any_call(argument_name, **expected_properties) + + @pytest.mark.parametrize( + "model_option", + ["all", "issue", "pull_request", "release"], + ) + def test_handle_calls_correct_backfill_methods(self, mocker, model_option): + """Test that handle() delegates to the correct backfill methods based on --model.""" + command = Command() + command.stdout = MagicMock() + + mock_backfill_issues = mocker.patch.object(command, "backfill_issues") + mock_backfill_pull_requests = mocker.patch.object(command, "backfill_pull_requests") + mock_backfill_releases = mocker.patch.object(command, "backfill_releases") + + command.handle(offset=0, model=model_option) + + if model_option in ("all", "issue"): + mock_backfill_issues.assert_called_once_with(0) + else: + mock_backfill_issues.assert_not_called() + + if model_option in ("all", "pull_request"): + mock_backfill_pull_requests.assert_called_once_with(0) + else: + mock_backfill_pull_requests.assert_not_called() + + if model_option in ("all", "release"): + mock_backfill_releases.assert_called_once_with(0) + else: + mock_backfill_releases.assert_not_called() + + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Issue") + def test_backfill_issues_processes_all(self, mock_issue_class, mock_activity_event_class): + """Test that backfill_issues processes each issue and calls update_data.""" + mock_issue1 = MagicMock(id=1, number=1, title="Issue 1") + mock_issue1.repository = MagicMock() + mock_issue2 = MagicMock(id=2, number=2, title="Issue 2") + mock_issue2.repository = MagicMock() + + mock_qs = MagicMock() + mock_qs.count.return_value = 2 + mock_qs.__getitem__.return_value = [mock_issue1, mock_issue2] + mock_issue_class.objects.select_related.return_value.order_by.return_value = mock_qs + + command = Command() + command.stdout = MagicMock() + command.backfill_issues(offset=0) + + mock_activity_event_class.update_data.assert_any_call(mock_issue1) + mock_activity_event_class.update_data.assert_any_call(mock_issue2) + assert mock_activity_event_class.update_data.call_count == 2 + + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Issue") + def test_backfill_issues_skips_without_repository( + self, mock_issue_class, mock_activity_event_class + ): + """Test that backfill_issues skips issues that have no repository.""" + mock_issue = MagicMock(id=1, number=1, title="Issue No Repo") + mock_issue.repository = None + + mock_qs = MagicMock() + mock_qs.count.return_value = 1 + mock_qs.__getitem__.return_value = [mock_issue] + mock_issue_class.objects.select_related.return_value.order_by.return_value = mock_qs + + command = Command() + command.stdout = MagicMock() + command.backfill_issues(offset=0) + + mock_activity_event_class.update_data.assert_not_called() + + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Issue") + def test_backfill_issues_continues_on_error( + self, mock_issue_class, mock_activity_event_class + ): + """Test that backfill_issues continues processing when one issue raises an exception.""" + mock_issue1 = MagicMock(id=1, number=1, title="Issue 1") + mock_issue1.repository = MagicMock() + mock_issue2 = MagicMock(id=2, number=2, title="Issue 2") + mock_issue2.repository = MagicMock() + + mock_activity_event_class.update_data.side_effect = [Exception("DB error"), None] + + mock_qs = MagicMock() + mock_qs.count.return_value = 2 + mock_qs.__getitem__.return_value = [mock_issue1, mock_issue2] + mock_issue_class.objects.select_related.return_value.order_by.return_value = mock_qs + + command = Command() + command.stdout = MagicMock() + command.backfill_issues(offset=0) + + assert mock_activity_event_class.update_data.call_count == 2 + + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.PullRequest") + def test_backfill_pull_requests_processes_all( + self, mock_pr_class, mock_activity_event_class + ): + """Test that backfill_pull_requests processes each PR and calls update_data.""" + mock_pr1 = MagicMock(id=1, number=1, title="PR 1") + mock_pr1.repository = MagicMock() + mock_pr2 = MagicMock(id=2, number=2, title="PR 2") + mock_pr2.repository = MagicMock() + + mock_qs = MagicMock() + mock_qs.count.return_value = 2 + mock_qs.__getitem__.return_value = [mock_pr1, mock_pr2] + mock_pr_class.objects.select_related.return_value.order_by.return_value = mock_qs + + command = Command() + command.stdout = MagicMock() + command.backfill_pull_requests(offset=0) + + mock_activity_event_class.update_data.assert_any_call(mock_pr1) + mock_activity_event_class.update_data.assert_any_call(mock_pr2) + assert mock_activity_event_class.update_data.call_count == 2 + + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.PullRequest") + def test_backfill_pull_requests_skips_without_repository( + self, mock_pr_class, mock_activity_event_class + ): + """Test that backfill_pull_requests skips PRs that have no repository.""" + mock_pr = MagicMock(id=1, number=1, title="PR No Repo") + mock_pr.repository = None + + mock_qs = MagicMock() + mock_qs.count.return_value = 1 + mock_qs.__getitem__.return_value = [mock_pr] + mock_pr_class.objects.select_related.return_value.order_by.return_value = mock_qs + + command = Command() + command.stdout = MagicMock() + command.backfill_pull_requests(offset=0) + + mock_activity_event_class.update_data.assert_not_called() + + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Release") + def test_backfill_releases_processes_all(self, mock_release_class, mock_activity_event_class): + """Test that backfill_releases processes each release and calls update_data.""" + mock_release1 = MagicMock(id=1, tag_name="v1.0.0", name="Release 1.0.0") + mock_release1.repository = MagicMock() + mock_release2 = MagicMock(id=2, tag_name="v2.0.0", name="Release 2.0.0") + mock_release2.repository = MagicMock() + + mock_qs = MagicMock() + mock_qs.count.return_value = 2 + mock_qs.__getitem__.return_value = [mock_release1, mock_release2] + mock_release_class.objects.select_related.return_value.order_by.return_value = mock_qs + + command = Command() + command.stdout = MagicMock() + command.backfill_releases(offset=0) + + mock_activity_event_class.update_data.assert_any_call(mock_release1) + mock_activity_event_class.update_data.assert_any_call(mock_release2) + assert mock_activity_event_class.update_data.call_count == 2 + + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Release") + def test_backfill_releases_skips_without_repository( + self, mock_release_class, mock_activity_event_class + ): + """Test that backfill_releases skips releases that have no repository.""" + mock_release = MagicMock(id=1, tag_name="v1.0.0", name="Release 1.0.0") + mock_release.repository = None + + mock_qs = MagicMock() + mock_qs.count.return_value = 1 + mock_qs.__getitem__.return_value = [mock_release] + mock_release_class.objects.select_related.return_value.order_by.return_value = mock_qs + + command = Command() + command.stdout = MagicMock() + command.backfill_releases(offset=0) + + mock_activity_event_class.update_data.assert_not_called() + + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Issue") + def test_backfill_issues_respects_offset(self, mock_issue_class, mock_activity_event_class): + """Test that backfill_issues respects the offset argument.""" + mock_issue1 = MagicMock(id=3, number=3, title="Issue 3") + mock_issue1.repository = MagicMock() + + mock_qs = MagicMock() + mock_qs.count.return_value = 3 + mock_qs.__getitem__.return_value = [mock_issue1] + mock_issue_class.objects.select_related.return_value.order_by.return_value = mock_qs + + command = Command() + command.stdout = MagicMock() + command.backfill_issues(offset=2) + + mock_qs.__getitem__.assert_called_once_with(slice(2, None)) + mock_activity_event_class.update_data.assert_called_once_with(mock_issue1) From 279d99a130bd13ecb56955e2f356b691e29ad3de Mon Sep 17 00:00:00 2001 From: Anurag Yadav Date: Thu, 30 Jul 2026 23:38:56 +0530 Subject: [PATCH 7/8] adress review --- backend/make/apps/owasp.mk | 2 +- .../owasp_backfill_activity_events.py | 95 ++++----- .../owasp_backfill_activity_events_test.py | 194 ++++++++---------- 3 files changed, 127 insertions(+), 164 deletions(-) diff --git a/backend/make/apps/owasp.mk b/backend/make/apps/owasp.mk index 7a9179acb4..995f2498b1 100644 --- a/backend/make/apps/owasp.mk +++ b/backend/make/apps/owasp.mk @@ -1,5 +1,5 @@ .PHONY: owasp-add-project-custom-tags owasp-aggregate-entity-contributions \ - owasp-aggregate-member-contributions owasp-aggregate-projects owasp-create-project-metadata-file \ + owasp-aggregate-member-contributions owasp-aggregate-projects owasp-backfill-activity-events owasp-create-project-metadata-file \ owasp-enrich-chapters owasp-enrich-committees owasp-enrich-events owasp-enrich-projects \ owasp-generate-community-snapshot-video owasp-process-snapshots owasp-scrape-chapters \ owasp-scrape-committees owasp-scrape-projects owasp-sync-posts owasp-update-events \ diff --git a/backend/src/apps/owasp/management/commands/owasp_backfill_activity_events.py b/backend/src/apps/owasp/management/commands/owasp_backfill_activity_events.py index d0d888b2c5..b9175abeb5 100644 --- a/backend/src/apps/owasp/management/commands/owasp_backfill_activity_events.py +++ b/backend/src/apps/owasp/management/commands/owasp_backfill_activity_events.py @@ -1,8 +1,11 @@ """A command to backfill activity events for existing pull requests, issues, and releases.""" import logging +from collections.abc import Callable +from typing import Any from django.core.management.base import BaseCommand +from django.db.models import QuerySet from apps.github.models.issue import Issue from apps.github.models.pull_request import PullRequest @@ -46,76 +49,52 @@ def handle(self, *args, **options) -> None: if model in ("all", "release"): self.backfill_releases(offset) - def backfill_issues(self, offset: int) -> None: - """Backfill ActivityEvent records for existing issues.""" - issues = Issue.objects.select_related("author", "repository").order_by("created_at") - issues_count = issues.count() - self.stdout.write(f"Backfilling activity events for {issues_count} issues...\n") + def backfill_objects( + self, + queryset: QuerySet, + offset: int, + noun: str, + get_label: Callable[[Any], str], + ) -> None: + """Backfill ActivityEvent records for a queryset of GitHub objects.""" + count = queryset.count() + self.stdout.write(f"Backfilling activity events for {count} {noun}...\n") created_count = 0 - for issue in issues[offset:]: - - if not issue.repository: - logger.warning("Skipping issue #%s: no repository", issue.number) + for obj in queryset[offset:].iterator(chunk_size=2000): + if not obj.repository: + logger.warning("Skipping %s %s: no repository", noun.rstrip("s"), get_label(obj)) continue try: - ActivityEvent.update_data(issue) + ActivityEvent.update_data(obj) created_count += 1 except Exception: - logger.exception("Error backfilling activity events for issue #%s", issue.number) + logger.exception( + "Error backfilling activity events for %s %s", + noun.rstrip("s"), + get_label(obj), + ) - self.stdout.write(f"Issues processed: {created_count}\n") + self.stdout.write(f"{noun.capitalize()} processed: {created_count}\n") + + def backfill_issues(self, offset: int) -> None: + """Backfill ActivityEvent records for existing issues.""" + queryset = Issue.objects.select_related("author", "repository").order_by( + "created_at", "pk" + ) + self.backfill_objects(queryset, offset, "issues", lambda obj: f"#{obj.number}") def backfill_pull_requests(self, offset: int) -> None: """Backfill ActivityEvent records for existing pull requests.""" - pull_requests = PullRequest.objects.select_related("author", "repository").order_by( - "created_at" + queryset = PullRequest.objects.select_related("author", "repository").order_by( + "created_at", "pk" ) - pull_requests_count = pull_requests.count() - self.stdout.write( - f"Backfilling activity events for {pull_requests_count} pull requests...\n" - ) - - created_count = 0 - for pull_request in pull_requests[offset:]: - - if not pull_request.repository: - logger.warning( - "Skipping pull request #%s: no repository", pull_request.number - ) - continue - - try: - ActivityEvent.update_data(pull_request) - created_count += 1 - except Exception: - logger.exception( - "Error backfilling activity events for pull request #%s", - pull_request.number, - ) - - self.stdout.write(f"Pull requests processed: {created_count}\n") + self.backfill_objects(queryset, offset, "pull requests", lambda obj: f"#{obj.number}") def backfill_releases(self, offset: int) -> None: """Backfill ActivityEvent records for existing releases.""" - releases = Release.objects.select_related("author", "repository").order_by("created_at") - releases_count = releases.count() - self.stdout.write(f"Backfilling activity events for {releases_count} releases...\n") - - created_count = 0 - for release in releases[offset:]: - - if not release.repository: - logger.warning("Skipping release %s: no repository", release.tag_name) - continue - - try: - ActivityEvent.update_data(release) - created_count += 1 - except Exception: - logger.exception( - "Error backfilling activity events for release %s", release.tag_name - ) - - self.stdout.write(f"Releases processed: {created_count}\n") + queryset = Release.objects.select_related("author", "repository").order_by( + "created_at", "pk" + ) + self.backfill_objects(queryset, offset, "releases", lambda obj: obj.tag_name) diff --git a/backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py b/backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py index 489587d661..c1e175d771 100644 --- a/backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py +++ b/backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py @@ -3,7 +3,6 @@ from unittest.mock import MagicMock, patch import pytest - from django.core.management.base import BaseCommand from apps.owasp.management.commands.owasp_backfill_activity_events import Command @@ -82,172 +81,157 @@ def test_handle_calls_correct_backfill_methods(self, mocker, model_option): mock_backfill_releases.assert_not_called() @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Issue") - def test_backfill_issues_processes_all(self, mock_issue_class, mock_activity_event_class): - """Test that backfill_issues processes each issue and calls update_data.""" - mock_issue1 = MagicMock(id=1, number=1, title="Issue 1") - mock_issue1.repository = MagicMock() - mock_issue2 = MagicMock(id=2, number=2, title="Issue 2") - mock_issue2.repository = MagicMock() + def test_backfill_objects_processes_all(self, mock_activity_event_class): + """Test that backfill_objects calls update_data for every object.""" + mock_obj1 = MagicMock(repository=MagicMock()) + mock_obj2 = MagicMock(repository=MagicMock()) mock_qs = MagicMock() mock_qs.count.return_value = 2 - mock_qs.__getitem__.return_value = [mock_issue1, mock_issue2] - mock_issue_class.objects.select_related.return_value.order_by.return_value = mock_qs + mock_qs.__getitem__.return_value.iterator.return_value = iter([mock_obj1, mock_obj2]) command = Command() command.stdout = MagicMock() - command.backfill_issues(offset=0) + command.backfill_objects(mock_qs, 0, "issues", str) - mock_activity_event_class.update_data.assert_any_call(mock_issue1) - mock_activity_event_class.update_data.assert_any_call(mock_issue2) + mock_activity_event_class.update_data.assert_any_call(mock_obj1) + mock_activity_event_class.update_data.assert_any_call(mock_obj2) assert mock_activity_event_class.update_data.call_count == 2 @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Issue") - def test_backfill_issues_skips_without_repository( - self, mock_issue_class, mock_activity_event_class - ): - """Test that backfill_issues skips issues that have no repository.""" - mock_issue = MagicMock(id=1, number=1, title="Issue No Repo") - mock_issue.repository = None + def test_backfill_objects_skips_without_repository(self, mock_activity_event_class): + """Test that backfill_objects skips objects that have no repository.""" + mock_obj = MagicMock(repository=None) mock_qs = MagicMock() mock_qs.count.return_value = 1 - mock_qs.__getitem__.return_value = [mock_issue] - mock_issue_class.objects.select_related.return_value.order_by.return_value = mock_qs + mock_qs.__getitem__.return_value.iterator.return_value = iter([mock_obj]) command = Command() command.stdout = MagicMock() - command.backfill_issues(offset=0) + command.backfill_objects(mock_qs, 0, "issues", str) mock_activity_event_class.update_data.assert_not_called() @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Issue") - def test_backfill_issues_continues_on_error( - self, mock_issue_class, mock_activity_event_class - ): - """Test that backfill_issues continues processing when one issue raises an exception.""" - mock_issue1 = MagicMock(id=1, number=1, title="Issue 1") - mock_issue1.repository = MagicMock() - mock_issue2 = MagicMock(id=2, number=2, title="Issue 2") - mock_issue2.repository = MagicMock() + def test_backfill_objects_continues_on_error(self, mock_activity_event_class): + """Test that backfill_objects continues when one object raises an exception.""" + mock_obj1 = MagicMock(repository=MagicMock()) + mock_obj2 = MagicMock(repository=MagicMock()) mock_activity_event_class.update_data.side_effect = [Exception("DB error"), None] mock_qs = MagicMock() mock_qs.count.return_value = 2 - mock_qs.__getitem__.return_value = [mock_issue1, mock_issue2] - mock_issue_class.objects.select_related.return_value.order_by.return_value = mock_qs + mock_qs.__getitem__.return_value.iterator.return_value = iter([mock_obj1, mock_obj2]) command = Command() command.stdout = MagicMock() - command.backfill_issues(offset=0) + command.backfill_objects(mock_qs, 0, "issues", str) assert mock_activity_event_class.update_data.call_count == 2 @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.PullRequest") - def test_backfill_pull_requests_processes_all( - self, mock_pr_class, mock_activity_event_class - ): - """Test that backfill_pull_requests processes each PR and calls update_data.""" - mock_pr1 = MagicMock(id=1, number=1, title="PR 1") - mock_pr1.repository = MagicMock() - mock_pr2 = MagicMock(id=2, number=2, title="PR 2") - mock_pr2.repository = MagicMock() - + def test_backfill_objects_respects_offset(self, mock_activity_event_class): + """Test that backfill_objects slices the queryset with the given offset.""" mock_qs = MagicMock() - mock_qs.count.return_value = 2 - mock_qs.__getitem__.return_value = [mock_pr1, mock_pr2] - mock_pr_class.objects.select_related.return_value.order_by.return_value = mock_qs + mock_qs.count.return_value = 5 + mock_qs.__getitem__.return_value.iterator.return_value = iter([]) command = Command() command.stdout = MagicMock() - command.backfill_pull_requests(offset=0) + command.backfill_objects(mock_qs, 3, "issues", str) - mock_activity_event_class.update_data.assert_any_call(mock_pr1) - mock_activity_event_class.update_data.assert_any_call(mock_pr2) - assert mock_activity_event_class.update_data.call_count == 2 - - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.PullRequest") - def test_backfill_pull_requests_skips_without_repository( - self, mock_pr_class, mock_activity_event_class - ): - """Test that backfill_pull_requests skips PRs that have no repository.""" - mock_pr = MagicMock(id=1, number=1, title="PR No Repo") - mock_pr.repository = None + mock_qs.__getitem__.assert_called_once_with(slice(3, None)) + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Issue") + def test_backfill_issues_passes_correct_queryset(self, mock_issue_class, mocker): + """Test that backfill_issues builds the right queryset and delegates.""" mock_qs = MagicMock() - mock_qs.count.return_value = 1 - mock_qs.__getitem__.return_value = [mock_pr] - mock_pr_class.objects.select_related.return_value.order_by.return_value = mock_qs + mock_qs.__getitem__.return_value = mock_qs + mock_issue_class.objects.select_related.return_value.order_by.return_value = mock_qs command = Command() command.stdout = MagicMock() - command.backfill_pull_requests(offset=0) + mock_backfill_objects = mocker.patch.object(command, "backfill_objects") - mock_activity_event_class.update_data.assert_not_called() + command.backfill_issues(offset=0) - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Release") - def test_backfill_releases_processes_all(self, mock_release_class, mock_activity_event_class): - """Test that backfill_releases processes each release and calls update_data.""" - mock_release1 = MagicMock(id=1, tag_name="v1.0.0", name="Release 1.0.0") - mock_release1.repository = MagicMock() - mock_release2 = MagicMock(id=2, tag_name="v2.0.0", name="Release 2.0.0") - mock_release2.repository = MagicMock() + mock_issue_class.objects.select_related.assert_called_once_with("author", "repository") + mock_issue_class.objects.select_related.return_value.order_by.assert_called_once_with( + "created_at", "pk" + ) + assert mock_backfill_objects.call_count == 1 + assert mock_backfill_objects.call_args[0][2] == "issues" + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.PullRequest") + def test_backfill_pull_requests_passes_correct_queryset(self, mock_pr_class, mocker): + """Test that backfill_pull_requests builds the right queryset and delegates.""" mock_qs = MagicMock() - mock_qs.count.return_value = 2 - mock_qs.__getitem__.return_value = [mock_release1, mock_release2] - mock_release_class.objects.select_related.return_value.order_by.return_value = mock_qs + mock_qs.__getitem__.return_value = mock_qs + mock_pr_class.objects.select_related.return_value.order_by.return_value = mock_qs command = Command() command.stdout = MagicMock() - command.backfill_releases(offset=0) + mock_backfill_objects = mocker.patch.object(command, "backfill_objects") - mock_activity_event_class.update_data.assert_any_call(mock_release1) - mock_activity_event_class.update_data.assert_any_call(mock_release2) - assert mock_activity_event_class.update_data.call_count == 2 + command.backfill_pull_requests(offset=0) - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Release") - def test_backfill_releases_skips_without_repository( - self, mock_release_class, mock_activity_event_class - ): - """Test that backfill_releases skips releases that have no repository.""" - mock_release = MagicMock(id=1, tag_name="v1.0.0", name="Release 1.0.0") - mock_release.repository = None + mock_pr_class.objects.select_related.assert_called_once_with("author", "repository") + mock_pr_class.objects.select_related.return_value.order_by.assert_called_once_with( + "created_at", "pk" + ) + assert mock_backfill_objects.call_count == 1 + assert mock_backfill_objects.call_args[0][2] == "pull requests" + @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Release") + def test_backfill_releases_passes_correct_queryset(self, mock_release_class, mocker): + """Test that backfill_releases builds the right queryset and delegates.""" mock_qs = MagicMock() - mock_qs.count.return_value = 1 - mock_qs.__getitem__.return_value = [mock_release] + mock_qs.__getitem__.return_value = mock_qs mock_release_class.objects.select_related.return_value.order_by.return_value = mock_qs command = Command() command.stdout = MagicMock() - command.backfill_releases(offset=0) + mock_backfill_objects = mocker.patch.object(command, "backfill_objects") - mock_activity_event_class.update_data.assert_not_called() + command.backfill_releases(offset=0) - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.Issue") - def test_backfill_issues_respects_offset(self, mock_issue_class, mock_activity_event_class): - """Test that backfill_issues respects the offset argument.""" - mock_issue1 = MagicMock(id=3, number=3, title="Issue 3") - mock_issue1.repository = MagicMock() + mock_release_class.objects.select_related.assert_called_once_with("author", "repository") + mock_release_class.objects.select_related.return_value.order_by.assert_called_once_with( + "created_at", "pk" + ) + assert mock_backfill_objects.call_count == 1 + assert mock_backfill_objects.call_args[0][2] == "releases" + @pytest.mark.parametrize( + ("method_name", "model_patch_path"), + [ + ( + "backfill_issues", + "apps.owasp.management.commands.owasp_backfill_activity_events.Issue", + ), + ( + "backfill_pull_requests", + "apps.owasp.management.commands.owasp_backfill_activity_events.PullRequest", + ), + ( + "backfill_releases", + "apps.owasp.management.commands.owasp_backfill_activity_events.Release", + ), + ], + ) + def test_backfill_wrapper_methods_pass_offset(self, method_name, model_patch_path, mocker): + """Test that wrapper methods forward the offset argument to backfill_objects.""" + mock_model_class = mocker.patch(model_patch_path) mock_qs = MagicMock() - mock_qs.count.return_value = 3 - mock_qs.__getitem__.return_value = [mock_issue1] - mock_issue_class.objects.select_related.return_value.order_by.return_value = mock_qs + mock_model_class.objects.select_related.return_value.order_by.return_value = mock_qs command = Command() command.stdout = MagicMock() - command.backfill_issues(offset=2) + mock_backfill_objects = mocker.patch.object(command, "backfill_objects") + + getattr(command, method_name)(offset=7) - mock_qs.__getitem__.assert_called_once_with(slice(2, None)) - mock_activity_event_class.update_data.assert_called_once_with(mock_issue1) + mock_backfill_objects.assert_called_once() + assert mock_backfill_objects.call_args[0][1] == 7 From e716bd0b0784bcd8bf168d2021e1673989677205 Mon Sep 17 00:00:00 2001 From: Anurag Yadav Date: Thu, 30 Jul 2026 23:48:07 +0530 Subject: [PATCH 8/8] fixed --- .../management/commands/owasp_backfill_activity_events_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py b/backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py index c1e175d771..0e02b678d2 100644 --- a/backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py +++ b/backend/tests/unit/apps/owasp/management/commands/owasp_backfill_activity_events_test.py @@ -131,8 +131,7 @@ def test_backfill_objects_continues_on_error(self, mock_activity_event_class): assert mock_activity_event_class.update_data.call_count == 2 - @patch("apps.owasp.management.commands.owasp_backfill_activity_events.ActivityEvent") - def test_backfill_objects_respects_offset(self, mock_activity_event_class): + def test_backfill_objects_respects_offset(self): """Test that backfill_objects slices the queryset with the given offset.""" mock_qs = MagicMock() mock_qs.count.return_value = 5