Merge pull request #764 from UCL-INGI/pymongo4

[pymongo] migrate to pymongo 4

inginious/frontend/app.py

@@ -147,7 +147,7 @@ def get_app(config):
         database.user_tasks.create_index([("courseid", pymongo.ASCENDING), ("taskid", pymongo.ASCENDING)])
         database.user_tasks.create_index([("courseid", pymongo.ASCENDING)])
         database.user_tasks.create_index([("username", pymongo.ASCENDING)])
-        database.db_version.insert({"db_version": DB_VERSION})
+        database.db_version.insert_one({"db_version": DB_VERSION})
     elif db_version.get("db_version", 0) != DB_VERSION:
         raise Exception("Please update the database before running INGInious")
 

inginious/frontend/flask/mongo_sessions.py

@@ -53,7 +53,7 @@ def __init__(self, client, db, collection, use_signer=False,
                  permanent=True):
         self.client = client
         self.store = client[db][collection]
-        self.store.ensure_index('expiration')
+        self.store.create_index('expiration')  # ensure index
         self.use_signer = use_signer
         self.permanent = permanent
 
@@ -106,7 +106,7 @@ def open_session(self, app, request):
         document = self.store.find_one({'_id': store_id})
         if document and document.get('expiration') <= datetime.utcnow():
             # Delete expired session
-            self.store.remove({'_id': store_id})
+            self.store.delete_one({'_id': store_id})
             document = None
         if document is not None:
             try:
@@ -123,7 +123,7 @@ def save_session(self, app, session, response):
         store_id = session.sid
         if not session:
             if session.modified:
-                self.store.remove({'_id': store_id})
+                self.store.delete_one({'_id': store_id})
                 response.delete_cookie(app.session_cookie_name,
                                        domain=domain, path=path)
             return
@@ -133,10 +133,9 @@ def save_session(self, app, session, response):
         expires = self.get_expiration_time(app, session)
         cookieless = session.cookieless
         val = self.serializer.dumps(dict(session))
-        self.store.update({'_id': store_id},
-                          {'data': val,
-                           'expiration': expires,
-                           'cookieless': cookieless}, True)
+        self.store.update_one({'_id': store_id},
+                              {"$set": {'data': val, 'expiration': expires, 'cookieless': cookieless}},
+                              upsert=True)
         if self.use_signer:
             session_id = self._get_signer(app).sign(want_bytes(session.sid))
         else:

inginious/frontend/installer.py

@@ -553,12 +553,12 @@ def configure_authentication(self, database):
         email = self._ask_with_default("Enter the email address of the superadmin", "superadmin@inginious.org")
         password = self._ask_with_default("Enter the password of the superadmin", "superadmin")
 
-        database.users.insert({"username": username,
-                               "realname": realname,
-                               "email": email,
-                               "password": hashlib.sha512(password.encode("utf-8")).hexdigest(),
-                               "bindings": {},
-                               "language": "en"})
+        database.users.insert_one({"username": username,
+                                   "realname": realname,
+                                   "email": email,
+                                   "password": hashlib.sha512(password.encode("utf-8")).hexdigest(),
+                                   "bindings": {},
+                                   "language": "en"})
 
         options["superadmins"].append(username)
 

inginious/frontend/pages/course_admin/danger_zone.py

@@ -31,10 +31,10 @@ def wipe_course(self, courseid):
                     gridfs.delete(submission[key])
 
         self.database.courses.update_one({"_id": courseid}, {"$set": {"students": []}})
-        self.database.audiences.remove({"courseid": courseid})
-        self.database.groups.remove({"courseid": courseid})
-        self.database.user_tasks.remove({"courseid": courseid})
-        self.database.submissions.remove({"courseid": courseid})
+        self.database.audiences.delete_many({"courseid": courseid})
+        self.database.groups.delete_many({"courseid": courseid})
+        self.database.user_tasks.delete_many({"courseid": courseid})
+        self.database.submissions.delete_many({"courseid": courseid})
 
         self._logger.info("Course %s wiped.", courseid)
 
@@ -94,15 +94,15 @@ def restore_course(self, courseid, backup):
 
             audiences = bson.json_util.loads(zipf.read("audiences.json").decode("utf-8"))
             if len(audiences) > 0:
-                self.database.audiences.insert(audiences)
+                self.database.audiences.insert_many(audiences)
 
             groups = bson.json_util.loads(zipf.read("groups.json").decode("utf-8"))
             if len(groups) > 0:
-                self.database.groups.insert(groups)
+                self.database.groups.insert_many(groups)
 
             user_tasks = bson.json_util.loads(zipf.read("user_tasks.json").decode("utf-8"))
             if len(user_tasks) > 0:
-                self.database.user_tasks.insert(user_tasks)
+                self.database.user_tasks.insert_many(user_tasks)
 
             submissions = bson.json_util.loads(zipf.read("submissions.json").decode("utf-8"))
             for submission in submissions:
@@ -111,7 +111,7 @@ def restore_course(self, courseid, backup):
                         submission[key] = self.submission_manager.get_gridfs().put(zipf.read(key + "/" + str(submission[key]) + ".data"))
 
             if len(submissions) > 0:
-                self.database.submissions.insert(submissions)
+                self.database.submissions.insert_many(submissions)
 
         self._logger.info("Course %s restored from backup directory.", courseid)
 

inginious/frontend/pages/course_admin/student_list.py

@@ -180,9 +180,9 @@ def post_student_list(self, course, data):
     def post_audiences(self, course, data, active_tab, msg, error):
         try:
             if 'audience' in data:
-                self.database.audiences.insert({"courseid": course.get_id(), "students": [],
-                                                 "tutors": [],
-                                                 "description": data['audience']})
+                self.database.audiences.insert_one({"courseid": course.get_id(), "students": [],
+                                                    "tutors": [],
+                                                    "description": data['audience']})
                 msg["audiences"] = _("New audience created.")
                 active_tab = "tab_audiences"
 

inginious/frontend/pages/course_admin/submissions.py

@@ -201,7 +201,7 @@ def get_selected_submissions(self, course,
                                                                     keep_only_crashes=keep_only_crashes)
 
         submissions = self.database.submissions.find(filter)
-        submissions_count = submissions.count()
+        submissions_count = self.database.submissions.count_documents(filter)
 
         if sort_by[0] not in ["submitted_on", "username", "grade", "taskid"]:
             sort_by[0] = "submitted_on"

inginious/frontend/pages/course_admin/task_edit.py

@@ -90,8 +90,8 @@ def wipe_task(self, courseid, taskid):
                 if key in submission and type(submission[key]) == bson.objectid.ObjectId:
                     self.submission_manager.get_gridfs().delete(submission[key])
 
-        self.database.user_tasks.remove({"courseid": courseid, "taskid": taskid})
-        self.database.submissions.remove({"courseid": courseid, "taskid": taskid})
+        self.database.user_tasks.delete_many({"courseid": courseid, "taskid": taskid})
+        self.database.submissions.delete_many({"courseid": courseid, "taskid": taskid})
 
         self._logger.info("Task %s/%s wiped.", courseid, taskid)
 

inginious/frontend/pages/course_admin/task_list.py

@@ -80,8 +80,8 @@ def wipe_task(self, courseid, taskid):
                 if key in submission and type(submission[key]) == bson.objectid.ObjectId:
                     self.submission_manager.get_gridfs().delete(submission[key])
 
-        self.database.user_tasks.remove({"courseid": courseid, "taskid": taskid})
-        self.database.submissions.remove({"courseid": courseid, "taskid": taskid})
+        self.database.user_tasks.delete_many({"courseid": courseid, "taskid": taskid})
+        self.database.submissions.delete_many({"courseid": courseid, "taskid": taskid})
 
         logging.getLogger("inginious.webapp.task_edit").info("Task %s/%s wiped.", courseid, taskid)
 

inginious/frontend/pages/preferences/delete.py

@@ -28,8 +28,8 @@ def delete_account(self, data):
             error = True
             msg = _("The specified email is incorrect.")
         else:
-            self.database.submissions.remove({"username": username})
-            self.database.user_tasks.remove({"username": username})
+            self.database.submissions.delete_many({"username": username})
+            self.database.user_tasks.delete_many({"username": username})
 
             all_courses = self.course_factory.get_all_courses()
 

inginious/frontend/pages/register.py

@@ -106,15 +106,15 @@ def register_user(self, data):
             else:
                 passwd_hash = hashlib.sha512(data["passwd"].encode("utf-8")).hexdigest()
                 activate_hash = hashlib.sha512(str(random.getrandbits(256)).encode("utf-8")).hexdigest()
-                self.database.users.insert({"username": data["username"],
-                                            "realname": data["realname"],
-                                            "email": data["email"],
-                                            "password": passwd_hash,
-                                            "activate": activate_hash,
-                                            "bindings": {},
-                                            "language": self.user_manager._session.get("language", "en"),
-                                            "tos_accepted": True
-                                            })
+                self.database.users.insert_one({"username": data["username"],
+                                                "realname": data["realname"],
+                                                "email": data["email"],
+                                                "password": passwd_hash,
+                                                "activate": activate_hash,
+                                                "bindings": {},
+                                                "language": self.user_manager._session.get("language", "en"),
+                                                "tos_accepted": True
+                                                })
                 try:
                     subject = _("Welcome on INGInious")
                     body = _("""Welcome on INGInious !
@@ -129,7 +129,7 @@ def register_user(self, data):
                     msg = _("You are succesfully registered. An email has been sent to you for activation.")
                 except Exception as ex:
                     # Remove newly inserted user (do not add after to prevent email sending in case of failure)
-                    self.database.users.remove({"username": data["username"]})
+                    self.database.users.delete_one({"username": data["username"]})
                     error = True
                     msg = _("Something went wrong while sending you activation email. Please contact the administrator.")
                     self._logger.error("Couldn't send email : {}".format(str(ex)))

inginious/frontend/submission_manager.py

@@ -190,10 +190,10 @@ def replay_job(self, task, submission, copy=False, debug=False):
             submission["input"] = self._gridfs.put(bson.BSON.encode(inputdata))
             submission["tests"] = {}  # Be sure tags are reinitialized
             submission["user_ip"] = flask.request.remote_addr
-            submissionid = self._database.submissions.insert(submission)
+            submissionid = self._database.submissions.insert_one(submission).inserted_id
 
         # Clean the submission document in db
-        self._database.submissions.update(
+        self._database.submissions.update_one(
             {"_id": submission["_id"]},
             {"$set": {"status": "waiting", "response_type": task.get_response_type()},
              "$unset": {"result": "", "grade": "", "text": "", "tests": "", "problems": "", "archive": "", "state": "",
@@ -207,7 +207,7 @@ def replay_job(self, task, submission, copy=False, debug=False):
                                      "Frontend - {}".format(submission["username"]), debug, ssh_callback)
 
 
-        self._database.submissions.update(
+        self._database.submissions.update_one(
             {"_id": submission["_id"], "status": "waiting"},
             {"$set": {"jobid": jobid,"last_replay": datetime.now()}}
         )
@@ -299,7 +299,7 @@ def add_job(self, task, inputdata, debug=False):
 
         self._before_submission_insertion(task, inputdata, debug, obj)
         obj["input"] = self._gridfs.put(bson.BSON.encode(inputdata))
-        submissionid = self._database.submissions.insert(obj)
+        submissionid = self._database.submissions.insert_one(obj).inserted_id
         to_remove = self._after_submission_insertion(task, inputdata, debug, obj, submissionid)
 
         ssh_callback = lambda host, port, user, password: self._handle_ssh_callback(submissionid, host, port, user, password)
@@ -310,7 +310,7 @@ def add_job(self, task, inputdata, debug=False):
                                                               custom, state, archive, stdout, stderr, True)),
                                      "Frontend - {}".format(username), debug, ssh_callback)
 
-        self._database.submissions.update(
+        self._database.submissions.update_one(
             {"_id": submissionid, "status": "waiting"},
             {"$set": {"jobid": jobid}}
         )
@@ -710,7 +710,6 @@ def update_pending_jobs(database):
     """ Updates pending jobs status in the database """
 
     # Updates the submissions that are waiting with the status error, as the server restarted
-    database.submissions.update({'status': 'waiting'},
+    database.submissions.update_many({'status': 'waiting'},
                                 {"$unset": {'jobid': ""},
-                                 "$set": {'status': 'error', 'grade': 0.0, 'text': 'Internal error. Server restarted'}},
-                                multi=True)
+                                 "$set": {'status': 'error', 'grade': 0.0, 'text': 'Internal error. Server restarted'}})

inginious/frontend/user_manager.py

@@ -421,11 +421,11 @@ def bind_user(self, auth_id, user):
                 return False
             else:
                 # New user, create an account using email address
-                self._database.users.insert({"username": "",
-                                             "realname": realname,
-                                             "email": email,
-                                             "bindings": {auth_id: [username, additional]},
-                                             "language": self._session.get("language", "en")})
+                self._database.users.insert_one({"username": "",
+                                                 "realname": realname,
+                                                 "email": email,
+                                                 "bindings": {auth_id: [username, additional]},
+                                                 "language": self._session.get("language", "en")})
                 self.connect_user("", realname, email, self._session.get("language", "en"), False)
 
         return True
@@ -550,11 +550,11 @@ def get_task_caches(self, usernames, courseid, taskid):
 
     def user_saw_task(self, username, courseid, taskid):
         """ Set in the database that the user has viewed this task """
-        self._database.user_tasks.update({"username": username, "courseid": courseid, "taskid": taskid},
-                                         {"$setOnInsert": {"username": username, "courseid": courseid, "taskid": taskid,
-                                                           "tried": 0, "succeeded": False, "grade": 0.0,
-                                                           "submissionid": None, "state": ""}},
-                                         upsert=True)
+        self._database.user_tasks.update_one({"username": username, "courseid": courseid, "taskid": taskid},
+                                             {"$setOnInsert": {"username": username, "courseid": courseid, "taskid": taskid,
+                                                               "tried": 0, "succeeded": False, "grade": 0.0,
+                                                               "submissionid": None, "state": ""}},
+                                             upsert=True)
 
     def update_user_stats(self, username, task, submission, result_str, grade, state, newsub):
         """ Update stats with a new submission """

utils/database_updater/inginious-database-update

@@ -111,10 +111,11 @@ if __name__ == "__main__":
 
         db_version = 15
 
-    database.db_version.update_one({}, {"$set": {"db_version": db_version}}, upsert=True)
-
     if db_version < 16:
         print("Updating database to db_version 16")
         database.submissions.create_index([("status", pymongo.ASCENDING)])
+        db_version = 16
+
+    database.db_version.update_one({}, {"$set": {"db_version": db_version}}, upsert=True)
 
     print("Database up to date")