ENG-648: Update to pymongo 4.2

.github/workflows/tests.yml

@@ -8,7 +8,6 @@ on:
 
 jobs:
   test:
-
     runs-on: ubuntu-20.04
     services:
       mongodb:
@@ -17,7 +16,7 @@ jobs:
           - 27017:27017
     strategy:
       matrix:
-        python-version: [3.5]
+        python-version: [3.7]
         tox-env:
           - pep8
           - common

common/djangoapps/track/backends/mongodb.py

@@ -63,14 +63,14 @@ def __init__(self, **kwargs):
         self.connection = MongoClient(
             host=host,
             port=port,
+            username=user,
+            password=password,
+            authSource=auth_source,
             **extra
         )
 
         database = self.connection[db_name]
 
-        if user or password:
-            database.authenticate(user, password, source=auth_source)
-
         self.collection = database[collection_name]
 
         self._create_indexes()
@@ -84,13 +84,13 @@ def _create_indexes(self):
         # TODO: The creation of indexes can be moved to a Django
         # management command or equivalent. There is also an option to
         # run the indexing on the background, without locking.
-        self.collection.ensure_index([('time', pymongo.DESCENDING)], background=True)
-        self.collection.ensure_index('event_type', background=True)
+        self.collection.create_index([('time', pymongo.DESCENDING)], background=True)
+        self.collection.create_index('event_type', background=True)
 
     def send(self, event):
         """Insert the event in to the Mongo collection"""
         try:
-            self.collection.insert(event, manipulate=False)
+            self.collection.insert_one(event)
         except (PyMongoError, BSONError):
             # The event will be lost in case of a connection error or any error
             # that occurs when trying to insert the event into Mongo.

common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py

@@ -11,7 +11,6 @@
 
 import ddt
 import pytest
-from bson.code import Code
 from path import Path as path
 
 from xmodule.assetstore import AssetMetadata
@@ -172,29 +171,25 @@ def test_asset_sizes(self, source_ms, num_assets):
 
             asset_collection = source_ms.asset_collection()
             # Ensure the asset collection exists.
-            if asset_collection.name in asset_collection.database.collection_names():
-
-                # Map gets the size of each structure.
-                mapper = Code("""
-                    function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) }
-                    """)
-
-                # Reduce finds the largest structure size and returns only it.
-                reducer = Code("""
-                    function(key, values) {
-                        var max_size = 0;
-                        for (var i=0; i < values.length; i++) {
-                            if (values[i] > max_size) {
-                                max_size = values[i];
-                            }
+            if asset_collection.name in asset_collection.database.list_collection_names():
+                # Get the size of each structure, and finds the largest structure size 
+                pipeline = [
+                    {
+                        "$project": {
+                            "size": {"$bsonSize": "$$ROOT"}
+                        }
+                    },
+                    {
+                        "$group": {
+                            "_id": None,
+                            "max_size": {"$max": "$size"}
                         }
-                        return max_size;
                     }
-                """)
+                ]
 
-                results = asset_collection.map_reduce(mapper, reducer, "size_results")
+                results = list(asset_collection.aggregate(pipeline))
                 result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format(
-                    self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()]
+                    self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, results
                 )
                 with open("bson_sizes.txt", "a") as f:
                     f.write(result_str)

common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py

@@ -538,7 +538,8 @@ def delete_course_index(self, course_key):
                 key_attr: getattr(course_key, key_attr)
                 for key_attr in ('org', 'course', 'run')
             }
-            return self.course_index.remove(query)
+            result = self.course_index.delete_many(query)
+            return result.deleted_count
 
     def get_definition(self, key, course_context=None):
         """
@@ -614,9 +615,9 @@ def _drop_database(self, database=True, collections=True, connections=True):
             self.structures.drop()
             self.definitions.drop()
         else:
-            self.course_index.remove({})
-            self.structures.remove({})
-            self.definitions.remove({})
+            self.course_index.delete_many({})
+            self.structures.delete_many({})
+            self.definitions.delete_many({})
 
         if connections:
             connection.close()

common/lib/xmodule/xmodule/mongo_utils.py

@@ -32,18 +32,6 @@ def connect_to_mongodb(
     handles AutoReconnect errors by retrying read operations, since these exceptions
     typically indicate a temporary step-down condition for MongoDB.
     """
-    # The MongoReplicaSetClient class is deprecated in Mongo 3.x, in favor of using
-    # the MongoClient class for all connections. Update/simplify this code when using
-    # PyMongo 3.x.
-    if kwargs.get('replicaSet'):
-        # Enable reading from secondary nodes in the MongoDB replicaset by using the
-        # MongoReplicaSetClient class.
-        # The 'replicaSet' parameter in kwargs is required for secondary reads.
-        # The read_preference should be set to a proper value, like SECONDARY_PREFERRED.
-        mongo_client_class = pymongo.MongoReplicaSetClient
-    else:
-        # No 'replicaSet' in kwargs - so no secondary reads.
-        mongo_client_class = pymongo.MongoClient
 
     # If the MongoDB server uses a separate authentication database that should be specified here
     auth_source = kwargs.get('authsource', '') or None
@@ -67,11 +55,14 @@ def connect_to_mongodb(
             kwargs['read_preference'] = read_preference
 
     mongo_conn = pymongo.database.Database(
-        mongo_client_class(
+        pymongo.MongoClient(
             host=host,
             port=port,
             tz_aware=tz_aware,
             document_class=dict,
+            username=user,
+            password=password,
+            authSource=auth_source, 
             **kwargs
         ),
         db
@@ -82,9 +73,6 @@ def connect_to_mongodb(
             mongo_conn,
             wait_time=retry_wait_time
         )
-    # If credentials were provided, authenticate the user.
-    if user is not None and password is not None:
-        mongo_conn.authenticate(user, password, source=auth_source)
 
     return mongo_conn
 

requirements/edx/base.in

@@ -65,6 +65,7 @@ django-user-tasks
 django-waffle==0.18.0
 django-webpack-loader               # Used to wire webpack bundles into the django asset pipeline
 djangorestframework==3.9.4
+dnspython
 edx-ace
 edx-analytics-data-api-client
 edx-api-doc-tools
@@ -88,7 +89,6 @@ edx-submissions
 edx-user-state-client
 edx-when
 edxval
-event-tracking
 fs==2.0.18
 fs-s3fs==0.1.8
 geoip2                              # Python API for the GeoIP web services and databases

requirements/edx/base.txt

@@ -90,6 +90,7 @@ djangorestframework-xml==2.0.0  # via edx-enterprise
 djangorestframework==3.9.4  # via -r requirements/edx/base.in, django-config-models, django-user-tasks, drf-jwt, drf-yasg, edx-api-doc-tools, edx-completion, edx-drf-extensions, edx-enterprise, edx-organizations, edx-proctoring, edx-submissions, ora2, rest-condition, super-csv
 docopt==0.6.2             # via xmodule
 docutils==0.16            # via botocore
+dnspython==1.16.0         # via -r requirements/edx/base.in
 drf-jwt==1.14.0           # via -c requirements/edx/../constraints.txt, edx-drf-extensions
 drf-yasg==1.17.0          # via -c requirements/edx/../constraints.txt, edx-api-doc-tools
 edx-ace==0.1.15           # via -r requirements/edx/base.in
@@ -121,7 +122,7 @@ edx-when==1.2.3           # via -r requirements/edx/base.in, edx-proctoring
 edxval==1.3.4             # via -r requirements/edx/base.in
 elasticsearch==1.9.0      # via edx-search
 enum34==1.1.10            # via edxval
-event-tracking==0.3.2     # via -r requirements/edx/base.in, edx-proctoring, edx-search
+git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2  # via -r requirements/edx/github.in
 fs-s3fs==0.1.8            # via -r requirements/edx/base.in, django-pyfs
 fs==2.0.18                # via -r requirements/edx/base.in, django-pyfs, fs-s3fs, xblock
 future==0.18.2            # via django-ses, edx-celeryutils, edx-enterprise, pycontracts, pyjwkest
@@ -157,7 +158,7 @@ markey==0.8               # via django-babel-underscore
 markupsafe==1.1.1         # via -r requirements/edx/paver.txt, chem, jinja2, mako, xblock
 maxminddb==1.5.4          # via geoip2
 mock==3.0.5               # via -c requirements/edx/../constraints.txt, -r requirements/edx/paver.txt, xblock-drag-and-drop-v2, xblock-poll
-git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2  # via -r requirements/edx/github.in
+git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2  # via -r requirements/edx/github.in
 mongoengine==0.10.0       # via -r requirements/edx/base.in
 more-itertools==8.3.0     # via -r requirements/edx/paver.txt, zipp
 mpmath==1.1.0             # via sympy
@@ -190,7 +191,7 @@ pycryptodomex==3.9.7      # via -r requirements/edx/base.in, edx-proctoring, pyj
 pygments==2.6.1           # via -r requirements/edx/base.in
 pyjwkest==1.4.2           # via -r requirements/edx/base.in, edx-drf-extensions
 pyjwt==1.5.2              # via -r requirements/edx/base.in, drf-jwt, edx-rest-api-client, social-auth-core
-pymongo==3.9.0            # via -r requirements/edx/base.in, -r requirements/edx/paver.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
+pymongo==4.2.0            # via -r requirements/edx/base.in, -r requirements/edx/paver.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
 pynliner==0.8.0           # via -r requirements/edx/base.in
 pyparsing==2.4.7          # via chem, openedx-calc, packaging, pycontracts
 pysrt==1.1.2              # via -r requirements/edx/base.in, edxval

requirements/edx/development.txt

@@ -135,7 +135,7 @@ edx-when==1.2.3           # via -r requirements/edx/testing.txt, edx-proctoring
 edxval==1.3.4             # via -r requirements/edx/testing.txt
 elasticsearch==1.9.0      # via -r requirements/edx/testing.txt, edx-search
 enum34==1.1.10            # via -r requirements/edx/testing.txt, edxval
-event-tracking==0.3.2     # via -r requirements/edx/testing.txt, edx-proctoring, edx-search
+git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2  # via -r requirements/edx/testing.txt, edx-proctoring, edx-search
 execnet==1.7.1            # via -r requirements/edx/testing.txt, pytest-xdist
 factory-boy==2.8.1        # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt
 faker==4.1.0              # via -r requirements/edx/testing.txt, factory-boy
@@ -190,7 +190,7 @@ maxminddb==1.5.4          # via -r requirements/edx/testing.txt, geoip2
 mccabe==0.6.1             # via -r requirements/edx/testing.txt, flake8, pylint
 mistune==0.8.4            # via m2r
 mock==3.0.5               # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, xblock-drag-and-drop-v2, xblock-poll
-git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2  # via -r requirements/edx/testing.txt
+git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2  # via -r requirements/edx/testing.txt
 mongoengine==0.10.0       # via -r requirements/edx/testing.txt
 more-itertools==8.3.0     # via -r requirements/edx/testing.txt, pytest, zipp
 mpmath==1.1.0             # via -r requirements/edx/testing.txt, sympy

requirements/edx/github.in

@@ -56,7 +56,8 @@ git+https://github.com/edx/openedx-chem.git@ff4e3a03d3c7610e47a9af08eb648d8aabe2
 
 # Third-party:
 -e git+https://github.com/edx/django-wiki.git@0.0.27#egg=django-wiki
-git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2
+git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2
+git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2
 -e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev
 -e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme
 

requirements/edx/paver.in

@@ -18,7 +18,7 @@ mock                                # Stub out code with mock objects and make a
 path                                # Easier manipulation of filesystem paths
 paver                               # Build, distribution and deployment scripting tool
 psutil==1.2.1                       # Library for retrieving information on running processes and system utilization
-pymongo==3.9.0                      # via edx-opaque-keys
+pymongo==4.2.0                      # via edx-opaque-keys
 python-memcached                    # Python interface to the memcached memory cache daemon
 requests                            # Simple interface for making HTTP requests
 stevedore                           # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins

requirements/edx/testing.txt

@@ -131,7 +131,7 @@ edx-when==1.2.3           # via -r requirements/edx/base.txt, edx-proctoring
 edxval==1.3.4             # via -r requirements/edx/base.txt
 elasticsearch==1.9.0      # via -r requirements/edx/base.txt, edx-search
 enum34==1.1.10            # via -r requirements/edx/base.txt, edxval
-event-tracking==0.3.2     # via -r requirements/edx/base.txt, edx-proctoring, edx-search
+git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2  # via -r requirements/edx/base.txt, edx-proctoring, edx-search
 execnet==1.7.1            # via pytest-xdist
 factory-boy==2.8.1        # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.in
 faker==4.1.0              # via factory-boy
@@ -182,7 +182,7 @@ markupsafe==1.1.1         # via -r requirements/edx/base.txt, -r requirements/ed
 maxminddb==1.5.4          # via -r requirements/edx/base.txt, geoip2
 mccabe==0.6.1             # via flake8, pylint
 mock==3.0.5               # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, xblock-drag-and-drop-v2, xblock-poll
-git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2  # via -r requirements/edx/base.txt
+git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2  # via -r requirements/edx/base.txt
 mongoengine==0.10.0       # via -r requirements/edx/base.txt
 more-itertools==8.3.0     # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, pytest, zipp
 mpmath==1.1.0             # via -r requirements/edx/base.txt, sympy
@@ -227,7 +227,7 @@ pylint-celery==0.3        # via edx-lint
 pylint-django==2.0.11     # via edx-lint
 pylint-plugin-utils==0.6  # via pylint-celery, pylint-django
 pylint==2.4.2             # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
-pymongo==3.9.0            # via -r requirements/edx/base.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
+pymongo==4.2.0            # via -r requirements/edx/base.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
 pynliner==0.8.0           # via -r requirements/edx/base.txt
 pyparsing==2.4.7          # via -r requirements/edx/base.txt, chem, openedx-calc, packaging, pycontracts
 pyquery==1.4.1            # via -r requirements/edx/testing.in

tox.ini

@@ -15,7 +15,7 @@ skipsdist=True
 toxworkdir={env:TOX_WORKDIR:{homedir}/edxapp_toxenv}
 
 [testenv]
-basepython=python3.5
+basepython=python3.7
 # TODO: Remove tox-pip-version once we upgrade to Koa+, or whenever we have addressed pip 20.3 strict issues.
 pip_version=pip==20.2.4
 # This ensures "-e ." is installed, so that a link back to the top-level