[Feature][ENG-9691] osfmetrics elastic-6-to-8 migration (part 1)

.docker-compose.env

@@ -7,6 +7,8 @@ INTERNAL_DOMAIN=http://192.168.168.167:5000/
 API_DOMAIN=http://localhost:8000/
 ELASTIC_URI=192.168.168.167:9200
 ELASTIC6_URI=192.168.168.167:9201
+ELASTIC8_URI=http://192.168.168.167:9202
+ELASTIC8_USERNAME=elastic
 OSF_DB_HOST=192.168.168.167
 DB_HOST=192.168.168.167
 REDIS_HOST=redis://192.168.168.167:6379

.github/workflows/test-build.yml

@@ -37,7 +37,19 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
+      elasticsearch8: &ES8_SERVICE
+        image: elasticsearch:8.19.14
+        ports:
+          - 9202:9200
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+        options: >-
+          --health-cmd "curl -sf http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=30s"
+          --health-interval 10s
+          --health-timeout 30s
+          --health-retries 5
+      postgres: &POSTGRES_SERVICE
         image: postgres
         env:
           POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
@@ -54,6 +66,8 @@ jobs:
     - uses: ./.github/actions/start-build
     - name: Run tests
       run: poetry run python3 -m invoke test-ci-addons --junit
+      env:
+        ELASTIC8_URI: http://localhost:9202
     - name: Upload report
       if: (success() || failure())    # run this step even if previous step failed
       uses: ./.github/actions/gen-report
@@ -64,18 +78,7 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v6
     - uses: ./.github/actions/start-build
@@ -91,25 +94,17 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      elasticsearch8: *ES8_SERVICE
+      postgres: *POSTGRES_SERVICE
     steps:
       - uses: actions/checkout@v6
       - uses: ./.github/actions/start-build
       - name: NVM & yarn install
         run: poetry run python3 -m invoke assets --dev
       - name: Run test
         run: poetry run python3 -m invoke test-ci-api1-and-js --junit
+        env:
+          ELASTIC8_URI: http://localhost:9202
       - name: Upload report
         if: (success() || failure())    # run this step even if previous step failed
         uses: ./.github/actions/gen-report
@@ -120,23 +115,15 @@ jobs:
     permissions:
       checks: write
     services:
-      postgres:
-        image: postgres
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      elasticsearch8: *ES8_SERVICE
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v6
     - uses: ./.github/actions/start-build
     - name: Run tests
       run: poetry run python3 -m invoke test-ci-api2 --junit
+      env:
+        ELASTIC8_URI: http://localhost:9202
     - name: Upload report
       if: (success() || failure())    # run this step even if previous step failed
       uses: ./.github/actions/gen-report
@@ -147,19 +134,7 @@ jobs:
       checks: write
     needs: build-cache
     services:
-      postgres:
-        image: postgres
-
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v6
     - uses: ./.github/actions/start-build
@@ -175,19 +150,7 @@ jobs:
       checks: write
     needs: build-cache
     services:
-      postgres:
-        image: postgres
-
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
       mailhog:
         image: mailhog/mailhog
         ports:
@@ -208,19 +171,7 @@ jobs:
       checks: write
     needs: build-cache
     services:
-      postgres:
-        image: postgres
-
-        env:
-          POSTGRES_PASSWORD: ${{ env.OSF_DB_PASSWORD }}
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+      postgres: *POSTGRES_SERVICE
     steps:
     - uses: actions/checkout@v6
     - uses: ./.github/actions/start-build

addons/base/views.py

@@ -14,7 +14,7 @@
 import waffle
 from django.db import transaction
 from django.contrib.contenttypes.models import ContentType
-from elasticsearch import exceptions as es_exceptions
+from elasticsearch6 import exceptions as es_exceptions
 from rest_framework import status as http_status
 
 from api.caching.tasks import update_storage_usage_with_size
@@ -34,6 +34,7 @@
 from framework.flask import redirect
 from framework.sentry import log_exception
 from framework.transactions.handlers import no_auto_transaction
+from osf.metrics.es8_metrics import OsfCountedUsageEvent
 from website import settings
 from addons.base import signals as file_signals
 from addons.base.utils import format_last_known_metadata, get_mfr_url
@@ -691,6 +692,18 @@ def osfstoragefile_viewed_update_metrics(self, auth, fileversion, file_node):
                 version=fileversion.identifier,
                 path=file_node.path,
             )
+            OsfCountedUsageEvent.record(
+                user_id=getattr(user, '_id', None),
+                item_osfid=resource._id,
+                action_labels=[
+                    OsfCountedUsageEvent.ActionLabel.VIEW.value,
+                    OsfCountedUsageEvent.ActionLabel.WEB.value,
+                ],
+                # HACK: we don't have the user request, so fabricate a one-off session id
+                # (this means no double-click filtering for anonymous users (same as before)
+                # and potentially inflated "unique" sessionhour view counts)
+                client_session_id=str(uuid.uuid4()),
+            )
         except es_exceptions.ConnectionError:
             log_exception()
 
@@ -718,6 +731,17 @@ def osfstoragefile_downloaded_update_metrics(self, auth, fileversion, file_node)
                 version=fileversion.identifier,
                 path=file_node.path,
             )
+            OsfCountedUsageEvent.record(
+                user_id=getattr(user, '_id', None),
+                item_osfid=resource._id,
+                action_labels=[
+                    OsfCountedUsageEvent.ActionLabel.DOWNLOAD.value,
+                ],
+                # HACK: we don't have the user request, so fabricate a one-off session id
+                # (this means no double-click filtering for anonymous users (same as before)
+                # and potentially inflated "unique" sessionhour view counts)
+                client_session_id=str(uuid.uuid4()),
+            )
         except es_exceptions.ConnectionError:
             log_exception()
 

admin/management/urls.py

@@ -1,4 +1,4 @@
-from django.urls import re_path
+from django.urls import re_path, path
 
 from admin.management import views
 
@@ -21,5 +21,6 @@
     re_path(r'^sync_notification_templates', views.SyncNotificationTemplates.as_view(),
             name='sync_notification_templates'),
     re_path(r'^remove_orcid_from_user_social', views.RemoveOrcidFromUserSocial.as_view(),
-            name='remove_orcid_from_user_social')
+            name='remove_orcid_from_user_social'),
+    path('migrate_osfmetrics_6to8', views.MigrateOsfmetrics6to8.as_view(), name='migrate_osfmetrics_6to8'),
 ]

admin/management/views.py

@@ -1,9 +1,12 @@
+from io import StringIO
+
 from dateutil.parser import isoparse
 from django.views.generic import TemplateView, View
 from django.contrib import messages
 from django.http import HttpResponse
 from django.utils import timezone
 from django.contrib.auth.mixins import PermissionRequiredMixin
+from django.core.management import call_command
 
 from osf.management.commands.manage_switch_flags import manage_waffle
 from osf.management.commands.update_registration_schemas import update_registration_schemas
@@ -190,3 +193,23 @@ def post(self, request):
         remove_orcid_from_user_social()
         messages.success(request, 'Orcid from user social have been successfully removed.')
         return redirect(reverse('management:commands'))
+
+
+class MigrateOsfmetrics6to8(ManagementCommandPermissionView):
+    def post(self, request):
+        _command_kwargs = {
+            'no_setup': True,
+            'no_color': True,
+            'no_counts': request.POST.get('no_counts'),
+            'clear_state': request.POST.get('clear_state'),
+            'clear_es8_data': request.POST.get('clear_es8_data'),
+            'start': request.POST.get('start'),
+            'unchanged': request.POST.get('unchanged'),
+            'usage_reports': request.POST.get('usage_reports'),
+            'usage_events': request.POST.get('usage_events'),
+        }
+        _out_io = StringIO()
+        call_command('migrate_osfmetrics_6to8', **_command_kwargs, stdout=_out_io)
+        for _line in _out_io.getvalue().split('\n'):
+            messages.info(request, _line)
+        return redirect(reverse('management:commands'))

admin/templates/management/commands.html

@@ -178,6 +178,31 @@ <h4><u>Remove existing orcid info from user social</u></h4>
                     </nav>
                 </form>
             </section>
+            <section>
+                <h4><u>migrate osf-metrics 6to8</u></h4>
+                <p>
+                    view progress of the osf-metrics migration from elastic6 to elastic8 (or start it)
+                </p>
+                <form method="post"
+                      action="{% url 'management:migrate_osfmetrics_6to8'%}"
+                      style="display: flex; flex-direction: column;">
+                    {% csrf_token %}
+                    <label><input type="checkbox" name="no_counts"> no counts</label>
+                    <label><input type="checkbox" name="start"> start tasks (caution)</label>
+                    <label><input type="checkbox" name="clear_state"> reset migration start time (caution)</label>
+                    <label><input type="checkbox" name="clear_es8_data"> clear es8 data (big caution)</label>
+                    <fieldset>
+                        (narrow types:
+                        <label><input type="checkbox" name="unchanged"> unchanged events and reports</label>
+                        <label><input type="checkbox" name="usage_events"> usage events</label>
+                        <label><input type="checkbox" name="usage_reports"> usage reports</label>
+                        )
+                    </fieldset>
+                    <nav>
+                        <input class="btn btn-success" type="submit" value="Run" />
+                    </nav>
+                </form>
+            </section>
         </div>
     </section>
 {% endblock %}

api/base/elasticsearch_dsl_views.py

@@ -3,7 +3,7 @@
 import datetime
 import typing
 
-import elasticsearch_dsl as edsl
+import elasticsearch6_dsl as edsl
 from rest_framework import generics, exceptions as drf_exceptions
 from rest_framework.settings import api_settings as drf_settings
 from api.base.settings.defaults import REPORT_FILENAME_FORMAT
@@ -23,7 +23,7 @@
 
 
 class ElasticsearchListView(FilterMixin, JSONAPIBaseView, generics.ListAPIView, abc.ABC):
-    '''abstract view class using `elasticsearch_dsl.Search` as a queryset-analogue
+    '''abstract view class using `elasticsearch6_dsl.Search` as a queryset-analogue
 
     builds a `Search` based on `self.get_default_search()` and the request's
     query parameters for filtering, sorting, and pagination -- fetches only
@@ -36,7 +36,7 @@ class ElasticsearchListView(FilterMixin, JSONAPIBaseView, generics.ListAPIView,
 
     @abc.abstractmethod
     def get_default_search(self) -> edsl.Search | None:
-        '''the base `elasticsearch_dsl.Search` for this list, based on url path
+        '''the base `elasticsearch6_dsl.Search` for this list, based on url path
 
         (common jsonapi query parameters will be considered automatically)
         '''
@@ -95,7 +95,7 @@ def finalize_response(self, request, response, *args, **kwargs):
     # (filtering handled in-view to reuse logic from FilterMixin)
     filter_backends = ()
 
-    # note: because elasticsearch_dsl.Search supports slicing and gives results when iterated on,
+    # note: because elasticsearch6_dsl.Search supports slicing and gives results when iterated on,
     #       it works fine with default pagination
 
     # override rest_framework.generics.GenericAPIView

api/base/settings/defaults.py

@@ -320,10 +320,27 @@
 HASHIDS_SALT = 'pinkhimalayan'
 
 # django-elasticsearch-metrics
-ELASTICSEARCH_DSL = {
-    'default': {
-        'hosts': osf_settings.ELASTIC6_URI,
-        'retry_on_timeout': True,
+DJELME_BACKENDS = {
+    'osfmetrics_es6': {
+        'elasticsearch_metrics.imps.elastic6': {
+            'hosts': osf_settings.ELASTIC6_URI,
+            'retry_on_timeout': True,
+        },
+    },
+    'osfmetrics_es8': {
+        'elasticsearch_metrics.imps.elastic8': {
+            # passthru kwargs to elasticsearch8 connection constructor
+            'hosts': osf_settings.ELASTIC8_URI,
+            'ca_certs': osf_settings.ELASTIC8_CERT_PATH,
+            'basic_auth': (
+                (osf_settings.ELASTIC8_USERNAME, osf_settings.ELASTIC8_SECRET)
+                if osf_settings.ELASTIC8_SECRET is not None
+                else None
+            ),
+            'ssl_assert_hostname': osf_settings.ELASTIC8_ASSERT_HOSTNAME,
+            # djelme-specific kwargs
+            'djelme_default_index_name_prefix': osf_settings.SHARE_PROVIDER_PREPEND,
+        },
     },
 }
 # Store yearly indices for time-series metrics