diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index b96fe22dde49..fd94a505c0e8 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -8,7 +8,6 @@ on:
jobs:
test:
-
runs-on: ubuntu-20.04
services:
mongodb:
@@ -17,7 +16,7 @@ jobs:
- 27017:27017
strategy:
matrix:
- python-version: [3.5]
+ python-version: [3.7]
tox-env:
- pep8
- common
diff --git a/common/djangoapps/pipeline_mako/templates/mako/js.html b/common/djangoapps/pipeline_mako/templates/mako/js.html
index ffc9355675db..d9c180329730 100644
--- a/common/djangoapps/pipeline_mako/templates/mako/js.html
+++ b/common/djangoapps/pipeline_mako/templates/mako/js.html
@@ -1,8 +1 @@
-
+
diff --git a/common/djangoapps/track/backends/mongodb.py b/common/djangoapps/track/backends/mongodb.py
index 07687eae9899..1969f2f344ee 100644
--- a/common/djangoapps/track/backends/mongodb.py
+++ b/common/djangoapps/track/backends/mongodb.py
@@ -63,14 +63,14 @@ def __init__(self, **kwargs):
self.connection = MongoClient(
host=host,
port=port,
+ username=user,
+ password=password,
+ authSource=auth_source,
**extra
)
database = self.connection[db_name]
- if user or password:
- database.authenticate(user, password, source=auth_source)
-
self.collection = database[collection_name]
self._create_indexes()
@@ -84,13 +84,13 @@ def _create_indexes(self):
# TODO: The creation of indexes can be moved to a Django
# management command or equivalent. There is also an option to
# run the indexing on the background, without locking.
- self.collection.ensure_index([('time', pymongo.DESCENDING)], background=True)
- self.collection.ensure_index('event_type', background=True)
+ self.collection.create_index([('time', pymongo.DESCENDING)], background=True)
+ self.collection.create_index('event_type', background=True)
def send(self, event):
"""Insert the event in to the Mongo collection"""
try:
- self.collection.insert(event, manipulate=False)
+ self.collection.insert_one(event)
except (PyMongoError, BSONError):
# The event will be lost in case of a connection error or any error
# that occurs when trying to insert the event into Mongo.
diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py
index 00a123e1d108..74b52b11961d 100644
--- a/common/lib/xmodule/xmodule/modulestore/__init__.py
+++ b/common/lib/xmodule/xmodule/modulestore/__init__.py
@@ -899,7 +899,7 @@ def _value_matches(self, target, criteria):
"""
if isinstance(target, list):
return any(self._value_matches(ele, criteria) for ele in target)
- elif isinstance(criteria, re._pattern_type): # pylint: disable=protected-access
+ elif isinstance(criteria, re.Pattern):
return criteria.search(target) is not None
elif callable(criteria):
return criteria(target)
diff --git a/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py b/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py
index f177ac01c817..22d36fed3f35 100644
--- a/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py
+++ b/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py
@@ -11,7 +11,6 @@
import ddt
import pytest
-from bson.code import Code
from path import Path as path
from xmodule.assetstore import AssetMetadata
@@ -172,29 +171,25 @@ def test_asset_sizes(self, source_ms, num_assets):
asset_collection = source_ms.asset_collection()
# Ensure the asset collection exists.
- if asset_collection.name in asset_collection.database.collection_names():
-
- # Map gets the size of each structure.
- mapper = Code("""
- function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) }
- """)
-
- # Reduce finds the largest structure size and returns only it.
- reducer = Code("""
- function(key, values) {
- var max_size = 0;
- for (var i=0; i < values.length; i++) {
- if (values[i] > max_size) {
- max_size = values[i];
- }
+ if asset_collection.name in asset_collection.database.list_collection_names():
+ # Get the size of each structure, and finds the largest structure size
+ pipeline = [
+ {
+ "$project": {
+ "size": {"$bsonSize": "$$ROOT"}
+ }
+ },
+ {
+ "$group": {
+ "_id": None,
+ "max_size": {"$max": "$size"}
}
- return max_size;
}
- """)
+ ]
- results = asset_collection.map_reduce(mapper, reducer, "size_results")
+ results = list(asset_collection.aggregate(pipeline))
result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format(
- self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()]
+ self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, results
)
with open("bson_sizes.txt", "a") as f:
f.write(result_str)
diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
index 413880e6ea87..ed42c3251a08 100644
--- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
+++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
@@ -538,7 +538,8 @@ def delete_course_index(self, course_key):
key_attr: getattr(course_key, key_attr)
for key_attr in ('org', 'course', 'run')
}
- return self.course_index.remove(query)
+ result = self.course_index.delete_many(query)
+ return result.deleted_count
def get_definition(self, key, course_context=None):
"""
@@ -614,9 +615,9 @@ def _drop_database(self, database=True, collections=True, connections=True):
self.structures.drop()
self.definitions.drop()
else:
- self.course_index.remove({})
- self.structures.remove({})
- self.definitions.remove({})
+ self.course_index.delete_many({})
+ self.structures.delete_many({})
+ self.definitions.delete_many({})
if connections:
connection.close()
diff --git a/common/lib/xmodule/xmodule/mongo_utils.py b/common/lib/xmodule/xmodule/mongo_utils.py
index 76f476bd43bf..35a8108e3ebd 100644
--- a/common/lib/xmodule/xmodule/mongo_utils.py
+++ b/common/lib/xmodule/xmodule/mongo_utils.py
@@ -32,18 +32,6 @@ def connect_to_mongodb(
handles AutoReconnect errors by retrying read operations, since these exceptions
typically indicate a temporary step-down condition for MongoDB.
"""
- # The MongoReplicaSetClient class is deprecated in Mongo 3.x, in favor of using
- # the MongoClient class for all connections. Update/simplify this code when using
- # PyMongo 3.x.
- if kwargs.get('replicaSet'):
- # Enable reading from secondary nodes in the MongoDB replicaset by using the
- # MongoReplicaSetClient class.
- # The 'replicaSet' parameter in kwargs is required for secondary reads.
- # The read_preference should be set to a proper value, like SECONDARY_PREFERRED.
- mongo_client_class = pymongo.MongoReplicaSetClient
- else:
- # No 'replicaSet' in kwargs - so no secondary reads.
- mongo_client_class = pymongo.MongoClient
# If the MongoDB server uses a separate authentication database that should be specified here
auth_source = kwargs.get('authsource', '') or None
@@ -67,11 +55,14 @@ def connect_to_mongodb(
kwargs['read_preference'] = read_preference
mongo_conn = pymongo.database.Database(
- mongo_client_class(
+ pymongo.MongoClient(
host=host,
port=port,
tz_aware=tz_aware,
document_class=dict,
+ username=user,
+ password=password,
+ authSource=auth_source,
**kwargs
),
db
@@ -82,9 +73,6 @@ def connect_to_mongodb(
mongo_conn,
wait_time=retry_wait_time
)
- # If credentials were provided, authenticate the user.
- if user is not None and password is not None:
- mongo_conn.authenticate(user, password, source=auth_source)
return mongo_conn
diff --git a/requirements/edx/base.in b/requirements/edx/base.in
index 9cc3da794c2d..f34196016581 100644
--- a/requirements/edx/base.in
+++ b/requirements/edx/base.in
@@ -65,6 +65,7 @@ django-user-tasks
django-waffle==0.18.0
django-webpack-loader # Used to wire webpack bundles into the django asset pipeline
djangorestframework==3.9.4
+dnspython
edx-ace
edx-analytics-data-api-client
edx-api-doc-tools
@@ -88,7 +89,6 @@ edx-submissions
edx-user-state-client
edx-when
edxval
-event-tracking
fs==2.0.18
fs-s3fs==0.1.8
geoip2 # Python API for the GeoIP web services and databases
diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
index 775e61c4ed32..288b3a061bf3 100644
--- a/requirements/edx/base.txt
+++ b/requirements/edx/base.txt
@@ -90,6 +90,7 @@ djangorestframework-xml==2.0.0 # via edx-enterprise
djangorestframework==3.9.4 # via -r requirements/edx/base.in, django-config-models, django-user-tasks, drf-jwt, drf-yasg, edx-api-doc-tools, edx-completion, edx-drf-extensions, edx-enterprise, edx-organizations, edx-proctoring, edx-submissions, ora2, rest-condition, super-csv
docopt==0.6.2 # via xmodule
docutils==0.16 # via botocore
+dnspython==1.16.0 # via -r requirements/edx/base.in
drf-jwt==1.14.0 # via -c requirements/edx/../constraints.txt, edx-drf-extensions
drf-yasg==1.17.0 # via -c requirements/edx/../constraints.txt, edx-api-doc-tools
edx-ace==0.1.15 # via -r requirements/edx/base.in
@@ -121,7 +122,7 @@ edx-when==1.2.3 # via -r requirements/edx/base.in, edx-proctoring
edxval==1.3.4 # via -r requirements/edx/base.in
elasticsearch==1.9.0 # via edx-search
enum34==1.1.10 # via edxval
-event-tracking==0.3.2 # via -r requirements/edx/base.in, edx-proctoring, edx-search
+git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2 # via -r requirements/edx/github.in
fs-s3fs==0.1.8 # via -r requirements/edx/base.in, django-pyfs
fs==2.0.18 # via -r requirements/edx/base.in, django-pyfs, fs-s3fs, xblock
future==0.18.2 # via django-ses, edx-celeryutils, edx-enterprise, pycontracts, pyjwkest
@@ -157,7 +158,7 @@ markey==0.8 # via django-babel-underscore
markupsafe==1.1.1 # via -r requirements/edx/paver.txt, chem, jinja2, mako, xblock
maxminddb==1.5.4 # via geoip2
mock==3.0.5 # via -c requirements/edx/../constraints.txt, -r requirements/edx/paver.txt, xblock-drag-and-drop-v2, xblock-poll
-git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/github.in
+git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/github.in
mongoengine==0.10.0 # via -r requirements/edx/base.in
more-itertools==8.3.0 # via -r requirements/edx/paver.txt, zipp
mpmath==1.1.0 # via sympy
@@ -190,7 +191,7 @@ pycryptodomex==3.9.7 # via -r requirements/edx/base.in, edx-proctoring, pyj
pygments==2.6.1 # via -r requirements/edx/base.in
pyjwkest==1.4.2 # via -r requirements/edx/base.in, edx-drf-extensions
pyjwt==1.5.2 # via -r requirements/edx/base.in, drf-jwt, edx-rest-api-client, social-auth-core
-pymongo==3.9.0 # via -r requirements/edx/base.in, -r requirements/edx/paver.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
+pymongo==4.2.0 # via -r requirements/edx/base.in, -r requirements/edx/paver.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
pynliner==0.8.0 # via -r requirements/edx/base.in
pyparsing==2.4.7 # via chem, openedx-calc, packaging, pycontracts
pysrt==1.1.2 # via -r requirements/edx/base.in, edxval
diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt
index 616984a8598e..dc9c71f384d1 100644
--- a/requirements/edx/development.txt
+++ b/requirements/edx/development.txt
@@ -135,7 +135,7 @@ edx-when==1.2.3 # via -r requirements/edx/testing.txt, edx-proctoring
edxval==1.3.4 # via -r requirements/edx/testing.txt
elasticsearch==1.9.0 # via -r requirements/edx/testing.txt, edx-search
enum34==1.1.10 # via -r requirements/edx/testing.txt, edxval
-event-tracking==0.3.2 # via -r requirements/edx/testing.txt, edx-proctoring, edx-search
+git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2 # via -r requirements/edx/testing.txt, edx-proctoring, edx-search
execnet==1.7.1 # via -r requirements/edx/testing.txt, pytest-xdist
factory-boy==2.8.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt
faker==4.1.0 # via -r requirements/edx/testing.txt, factory-boy
@@ -190,7 +190,7 @@ maxminddb==1.5.4 # via -r requirements/edx/testing.txt, geoip2
mccabe==0.6.1 # via -r requirements/edx/testing.txt, flake8, pylint
mistune==0.8.4 # via m2r
mock==3.0.5 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.txt, xblock-drag-and-drop-v2, xblock-poll
-git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/testing.txt
+git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/testing.txt
mongoengine==0.10.0 # via -r requirements/edx/testing.txt
more-itertools==8.3.0 # via -r requirements/edx/testing.txt, pytest, zipp
mpmath==1.1.0 # via -r requirements/edx/testing.txt, sympy
diff --git a/requirements/edx/github.in b/requirements/edx/github.in
index 9a8b2966e725..408892c416ed 100644
--- a/requirements/edx/github.in
+++ b/requirements/edx/github.in
@@ -56,7 +56,8 @@ git+https://github.com/edx/openedx-chem.git@ff4e3a03d3c7610e47a9af08eb648d8aabe2
# Third-party:
-e git+https://github.com/edx/django-wiki.git@0.0.27#egg=django-wiki
-git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2
+git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2
+git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2
-e git+https://github.com/dementrock/pystache_custom.git@776973740bdaad83a3b029f96e415a7d1e8bec2f#egg=pystache_custom-dev
-e git+https://github.com/jazkarta/edx-jsme.git@690dbf75441fa91c7c4899df0b83d77f7deb5458#egg=edx-jsme
diff --git a/requirements/edx/paver.in b/requirements/edx/paver.in
index 1a4266a2bee7..c25f0317da60 100644
--- a/requirements/edx/paver.in
+++ b/requirements/edx/paver.in
@@ -18,7 +18,7 @@ mock # Stub out code with mock objects and make a
path # Easier manipulation of filesystem paths
paver # Build, distribution and deployment scripting tool
psutil==1.2.1 # Library for retrieving information on running processes and system utilization
-pymongo==3.9.0 # via edx-opaque-keys
+pymongo==4.2.0 # via edx-opaque-keys
python-memcached # Python interface to the memcached memory cache daemon
requests # Simple interface for making HTTP requests
stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins
diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt
index ef360465d765..abc896a67cdb 100644
--- a/requirements/edx/testing.txt
+++ b/requirements/edx/testing.txt
@@ -131,7 +131,7 @@ edx-when==1.2.3 # via -r requirements/edx/base.txt, edx-proctoring
edxval==1.3.4 # via -r requirements/edx/base.txt
elasticsearch==1.9.0 # via -r requirements/edx/base.txt, edx-search
enum34==1.1.10 # via -r requirements/edx/base.txt, edxval
-event-tracking==0.3.2 # via -r requirements/edx/base.txt, edx-proctoring, edx-search
+git+https://github.com/appsembler/event-tracking.git@7d173191931c512ce6139c0abfc127bdf5279198#egg=event-tracking==0.3.2 # via -r requirements/edx/base.txt, edx-proctoring, edx-search
execnet==1.7.1 # via pytest-xdist
factory-boy==2.8.1 # via -c requirements/edx/../constraints.txt, -r requirements/edx/testing.in
faker==4.1.0 # via factory-boy
@@ -182,7 +182,7 @@ markupsafe==1.1.1 # via -r requirements/edx/base.txt, -r requirements/ed
maxminddb==1.5.4 # via -r requirements/edx/base.txt, geoip2
mccabe==0.6.1 # via flake8, pylint
mock==3.0.5 # via -c requirements/edx/../constraints.txt, -r requirements/edx/base.txt, xblock-drag-and-drop-v2, xblock-poll
-git+https://github.com/edx/MongoDBProxy.git@d92bafe9888d2940f647a7b2b2383b29c752f35a#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/base.txt
+git+https://github.com/appsembler/MongoDBProxy.git@02665ac118a79edae186c5bb1ad8c4f6d98778f4#egg=MongoDBProxy==0.1.0+edx.2 # via -r requirements/edx/base.txt
mongoengine==0.10.0 # via -r requirements/edx/base.txt
more-itertools==8.3.0 # via -r requirements/edx/base.txt, -r requirements/edx/coverage.txt, pytest, zipp
mpmath==1.1.0 # via -r requirements/edx/base.txt, sympy
@@ -227,7 +227,7 @@ pylint-celery==0.3 # via edx-lint
pylint-django==2.0.11 # via edx-lint
pylint-plugin-utils==0.6 # via pylint-celery, pylint-django
pylint==2.4.2 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils
-pymongo==3.9.0 # via -r requirements/edx/base.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
+pymongo==4.2.0 # via -r requirements/edx/base.txt, edx-opaque-keys, event-tracking, mongodbproxy, mongoengine
pynliner==0.8.0 # via -r requirements/edx/base.txt
pyparsing==2.4.7 # via -r requirements/edx/base.txt, chem, openedx-calc, packaging, pycontracts
pyquery==1.4.1 # via -r requirements/edx/testing.in
diff --git a/tox.ini b/tox.ini
index b76bb0d699b5..10724fae2538 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,7 +15,7 @@ skipsdist=True
toxworkdir={env:TOX_WORKDIR:{homedir}/edxapp_toxenv}
[testenv]
-basepython=python3.5
+basepython=python3.7
# TODO: Remove tox-pip-version once we upgrade to Koa+, or whenever we have addressed pip 20.3 strict issues.
pip_version=pip==20.2.4
# This ensures "-e ." is installed, so that a link back to the top-level