diff --git a/Dockerfile b/Dockerfile index 933ce9d8..6431650a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,11 +19,11 @@ RUN apt-get update && apt-get install -y gettext ############################################################################### FROM base AS dev -ARG USER_ID -ARG GROUP_ID +ARG USER_ID=1000 +ARG GROUP_ID=1000 -RUN groupadd -o -g $GROUP_ID -r usergrp -RUN useradd -o -m -u $USER_ID -g $GROUP_ID user +RUN groupadd -r usergrp -g $GROUP_ID && \ + useradd -r -u $USER_ID -g usergrp user RUN chown user /code COPY requirements-dev.txt /code/ diff --git a/celerybeat-schedule b/celerybeat-schedule new file mode 100644 index 00000000..933d5870 Binary files /dev/null and b/celerybeat-schedule differ diff --git a/celerybeat-schedule-shm b/celerybeat-schedule-shm new file mode 100644 index 00000000..c9977afb Binary files /dev/null and b/celerybeat-schedule-shm differ diff --git a/celerybeat-schedule-wal b/celerybeat-schedule-wal new file mode 100644 index 00000000..3b29d5d9 Binary files /dev/null and b/celerybeat-schedule-wal differ diff --git a/compose.yml b/compose.yml index 3dcde70f..0073063f 100644 --- a/compose.yml +++ b/compose.yml @@ -8,7 +8,7 @@ services: POSTGRES_USER: pyladiescon POSTGRES_PASSWORD: pyladiescon POSTGRES_DB: pyladiescon - POSTGRES_HOST_AUTH_METHOD: trust # never do this in production! + POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_FSYNC: null healthcheck: test: ["CMD", "pg_isready", "-U", "pyladiescon", "-d", "pyladiescon"] @@ -22,22 +22,25 @@ services: ports: - "6379:6379" healthcheck: - test: ["CMD", "redis-cli","ping"] - interval: 1s + test: ["CMD", "redis-cli", "ping"] + interval: 1s + volumes: + - redisdata:/data web: build: target: dev image: pyladiescon-portal-web:docker-compose - command: python manage.py runserver 0.0.0.0:8000 + command: ["./wait-for-postgres.sh", "postgres", "python", "manage.py", "runserver", "0.0.0.0:8000"] volumes: - .:/code - ./media:/code/media + - ./wait-for-postgres.sh:/wait-for-postgres.sh ports: - "8000:8000" environment: - DEBUG: True - DJANGO_ALLOWED_HOSTS: localhost,127.0.0.1,[::1] + DEBUG: "1" + DJANGO_ALLOWED_HOSTS: "localhost,127.0.0.1,[::1]" SECRET_KEY: verysecure DATABASE_URL: postgresql://pyladiescon:pyladiescon@postgres:5432/pyladiescon DJANGO_DEFAULT_FROM_EMAIL: PyLadiesCon @@ -49,6 +52,45 @@ services: postgres: condition: service_healthy + celery: + image: pyladiescon-portal-web:docker-compose + command: celery -A portal worker --loglevel=info + volumes: + - .:/code + environment: + DEBUG: "1" + SECRET_KEY: verysecure + DATABASE_URL: postgresql://pyladiescon:pyladiescon@postgres:5432/pyladiescon + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + DJANGO_DEFAULT_FROM_EMAIL: PyLadiesCon + DJANGO_EMAIL_HOST: maildev + DJANGO_EMAIL_PORT: 1025 + DJANGO_ALLOWED_HOSTS: "localhost,127.0.0.1,[::1]" + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + + celery-beat: + image: pyladiescon-portal-web:docker-compose + command: celery -A portal beat --loglevel=info + volumes: + - .:/code + environment: + DEBUG: "1" + SECRET_KEY: verysecure + DATABASE_URL: postgresql://pyladiescon:pyladiescon@postgres:5432/pyladiescon + CELERY_BROKER_URL: redis://redis:6379/0 + CELERY_RESULT_BACKEND: redis://redis:6379/0 + DJANGO_ALLOWED_HOSTS: "localhost,127.0.0.1,[::1]" + depends_on: + redis: + condition: service_healthy + postgres: + condition: service_healthy + maildev: image: maildev/maildev:2.2.1 ports: @@ -57,3 +99,4 @@ services: volumes: pgdata: + redisdata: diff --git a/portal/__init__.py b/portal/__init__.py index e69de29b..9e0d95fd 100644 --- a/portal/__init__.py +++ b/portal/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ('celery_app',) \ No newline at end of file diff --git a/portal/celery.py b/portal/celery.py new file mode 100644 index 00000000..ae77df9d --- /dev/null +++ b/portal/celery.py @@ -0,0 +1,11 @@ +import os +from celery import Celery + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'portal.settings') +app = Celery('portal') +app.config_from_object('django.conf:settings', namespace='CELERY') +app.autodiscover_tasks() + +@app.task(bind=True, ignore_result=True) +def debug_task(self): + print(f'Request: {self.request!r}') \ No newline at end of file diff --git a/portal/settings.py b/portal/settings.py index 0df1f09f..2f224303 100644 --- a/portal/settings.py +++ b/portal/settings.py @@ -52,9 +52,9 @@ ), ], ) -ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS") -if ALLOWED_HOSTS: - ALLOWED_HOSTS = ALLOWED_HOSTS.split(",") +ALLOWED_HOSTS = os.environ.get("DJANGO_ALLOWED_HOSTS", "localhost") +ALLOWED_HOSTS = [host.strip() for host in ALLOWED_HOSTS.split(",")] + # Application definition @@ -335,3 +335,24 @@ PRETIX_API_TOKEN = os.getenv("PRETIX_API_TOKEN") PRETIX_WEBHOOK_SECRET = os.getenv("PRETIX_WEBHOOK_SECRET") + + +CELERY_BROKER_URL = os.environ.get('CELERY_BROKER_URL', 'redis://redis:6379/0') +CELERY_RESULT_BACKEND = os.environ.get('CELERY_RESULT_BACKEND', 'redis://redis:6379/0') + +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' +CELERY_TIMEZONE = 'UTC' + +CELERY_TASK_ROUTES = { + 'sponsorship.tasks.*': {'queue': 'emails'}, + 'volunteer.tasks.*': {'queue': 'emails'}, +} + +CELERY_TASK_TRACK_STARTED = True +CELERY_TASK_TIME_LIMIT = 30 * 60 +CELERY_TASK_SOFT_TIME_LIMIT = 25 * 60 + +CELERY_WORKER_LOG_FORMAT = '[%(asctime)s: %(levelname)s/%(processName)s] %(message)s' +CELERY_WORKER_TASK_LOG_FORMAT = '[%(asctime)s: %(levelname)s/%(processName)s][%(task_name)s(%(task_id)s)] %(message)s' \ No newline at end of file diff --git a/requirements-app.txt b/requirements-app.txt index e74f9e78..96e355ac 100644 --- a/requirements-app.txt +++ b/requirements-app.txt @@ -18,4 +18,6 @@ django-import-export[all]==4.3.12 markdown==3.7 bleach==6.3.0 sentry-sdk[django]==2.43.0 -requests==2.32.5 \ No newline at end of file +requests==2.32.5 +celery==5.4.0 +redis==5.2.1 \ No newline at end of file diff --git a/sponsorship/signals.py b/sponsorship/signals.py index 8dd2e0c4..f504ccf3 100644 --- a/sponsorship/signals.py +++ b/sponsorship/signals.py @@ -5,83 +5,11 @@ from django.dispatch import receiver from common.send_emails import send_email -from volunteer.models import RoleTypes, VolunteerProfile +from volunteer.constants import RoleTypes +from volunteer.models import VolunteerProfile from .models import SponsorshipProfile - -def _send_internal_email( - subject, - *, - markdown_template, - context=None, -): - """Helper function to send an internal email. - - Lookup who the internal team members who should receive the email and then send the emails individually. - Send the email to staff, admin, and sponsorship team members - - Only supports Markdown templates going forward. - """ - - recipients = User.objects.filter( - Q( - id__in=VolunteerProfile.objects.prefetch_related("roles") - .filter(roles__short_name__in=[RoleTypes.ADMIN, RoleTypes.STAFF]) - .values_list("id", flat=True) - ) - | Q(is_superuser=True) - | Q(is_staff=True) - ).distinct() - - if not recipients.exists(): - return - - # send each email individually to each recipient, for privacy reasons - for recipient in recipients: - context["recipient_name"] = recipient.get_full_name() or recipient.username - - send_email( - subject, - [recipient.email], - markdown_template=markdown_template, - context=context, - ) - - -def send_internal_sponsor_onboarding_email(instance): - """Send email to team whenever a new sponsor is created. - - Emails will be sent to team members with the role type Staff or Admin, and to sponsorship team. - Emails will also be sent to users with is_superuser or is_staff set to True. - """ - context = {"profile": instance} - subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} New Sponsorship Tracking: {instance.organization_name}" - - markdown_template = "emails/sponsorship/internal_sponsor_onboarding.md" - - _send_internal_email( - subject, - markdown_template=markdown_template, - context=context, - ) - - -def send_internal_sponsor_progress_update_email(instance): - """Send email to team whenever there is a change in sponsorship progress.""" - - context = {"profile": instance} - subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} Update in Sponsorship Tracking for {instance.organization_name}" - - markdown_template = "emails/sponsorship/internal_sponsor_updated.md" - - _send_internal_email( - subject, - markdown_template=markdown_template, - context=context, - ) - - @receiver(post_save, sender=SponsorshipProfile) def sponsorship_profile_signal(sender, instance, created, **kwargs): """Send emails when sponsorship profile is created or updated. @@ -89,8 +17,12 @@ def sponsorship_profile_signal(sender, instance, created, **kwargs): """ if hasattr(instance, "from_import_export"): return + from sponsorship.tasks import ( + send_internal_sponsor_onboarding_email_task, + send_internal_sponsor_progress_update_email_task + ) + if created: + send_internal_sponsor_onboarding_email_task.delay(instance.id) else: - if created: - send_internal_sponsor_onboarding_email(instance) - else: - send_internal_sponsor_progress_update_email(instance) + send_internal_sponsor_progress_update_email_task.delay(instance.id) + diff --git a/sponsorship/tasks.py b/sponsorship/tasks.py new file mode 100644 index 00000000..82ff5f7b --- /dev/null +++ b/sponsorship/tasks.py @@ -0,0 +1,129 @@ +from celery import shared_task +from django.conf import settings +from django.contrib.auth.models import User +from django.db.models import Q + +from common.send_emails import send_email +from volunteer.constants import RoleTypes +from volunteer.models import VolunteerProfile + + +@shared_task( + name='sponsorship.tasks.send_internal_sponsor_onboarding_email', + bind=True, + max_retries=3, + default_retry_delay=60 # Retry after 60 seconds +) +def send_internal_sponsor_onboarding_email_task(self, profile_id): + """ + Background task to send internal sponsor onboarding email. + + Args: + profile_id: ID of the SponsorshipProfile instance + """ + from sponsorship.models import SponsorshipProfile + + try: + profile = SponsorshipProfile.objects.get(id=profile_id) + + # Get recipients (same logic as before) + recipients = User.objects.filter( + Q( + id__in=VolunteerProfile.objects.prefetch_related("roles") + .filter(roles__short_name__in=[RoleTypes.ADMIN, RoleTypes.STAFF]) + .values_list("id", flat=True) + ) + | Q(is_superuser=True) + | Q(is_staff=True) + ).distinct() + + if not recipients.exists(): + return f"No recipients found for profile {profile_id}" + + subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} New Sponsorship Tracking: {profile.organization_name}" + markdown_template = "emails/sponsorship/internal_sponsor_onboarding.md" + + # Send emails to each recipient + emails_sent = 0 + for recipient in recipients: + context = { + 'profile': profile, + 'recipient_name': recipient.get_full_name() or recipient.username + } + + send_email( + subject, + [recipient.email], + markdown_template=markdown_template, + context=context, + ) + emails_sent += 1 + + return f"Successfully sent {emails_sent} emails for sponsorship {profile.organization_name}" + + except SponsorshipProfile.DoesNotExist: + # Don't retry if profile doesn't exist + return f"SponsorshipProfile {profile_id} not found" + + except Exception as exc: + # Retry on other exceptions + raise self.retry(exc=exc) + + +@shared_task( + name='sponsorship.tasks.send_internal_sponsor_progress_update_email', + bind=True, + max_retries=3, + default_retry_delay=60 +) +def send_internal_sponsor_progress_update_email_task(self, profile_id): + """ + Background task to send internal sponsor progress update email. + + Args: + profile_id: ID of the SponsorshipProfile instance + """ + from sponsorship.models import SponsorshipProfile + + try: + profile = SponsorshipProfile.objects.get(id=profile_id) + + # Get recipients + recipients = User.objects.filter( + Q( + id__in=VolunteerProfile.objects.prefetch_related("roles") + .filter(roles__short_name__in=[RoleTypes.ADMIN, RoleTypes.STAFF]) + .values_list("id", flat=True) + ) + | Q(is_superuser=True) + | Q(is_staff=True) + ).distinct() + + if not recipients.exists(): + return f"No recipients found for profile {profile_id}" + + subject = f"{settings.ACCOUNT_EMAIL_SUBJECT_PREFIX} Update in Sponsorship Tracking for {profile.organization_name}" + markdown_template = "emails/sponsorship/internal_sponsor_updated.md" + + emails_sent = 0 + for recipient in recipients: + context = { + 'profile': profile, + 'recipient_name': recipient.get_full_name() or recipient.username + } + + send_email( + subject, + [recipient.email], + markdown_template=markdown_template, + context=context, + ) + emails_sent += 1 + + return f"Successfully sent {emails_sent} update emails for sponsorship {profile.organization_name}" + + except SponsorshipProfile.DoesNotExist: + return f"SponsorshipProfile {profile_id} not found" + + except Exception as exc: + raise self.retry(exc=exc) \ No newline at end of file