From 9abfa5dbe67289ad2170caa49f25c9bf11d35615 Mon Sep 17 00:00:00 2001 From: Nicolas Chang <25302138+NicolasCwy@users.noreply.github.com> Date: Wed, 20 Mar 2024 22:05:50 +0800 Subject: [PATCH 01/14] Add classes to migrate test json data --- .../ConvertDatastoreJsonToSqlJson.java | 251 +++++++++++++ .../DataStoreToSqlConverter.java | 345 ++++++++++++++++++ .../testdataconversion/UUIDGenerator.java | 25 ++ 3 files changed, 621 insertions(+) create mode 100644 src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java create mode 100644 src/client/java/teammates/client/scripts/testdataconversion/DataStoreToSqlConverter.java create mode 100644 src/client/java/teammates/client/scripts/testdataconversion/UUIDGenerator.java diff --git a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java new file mode 100644 index 00000000000..6ac765e992f --- /dev/null +++ b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java @@ -0,0 +1,251 @@ +package teammates.client.scripts.testdataconversion; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +import teammates.common.datatransfer.DataBundle; +import teammates.common.datatransfer.SqlDataBundle; +import teammates.common.exception.InvalidParametersException; +import teammates.common.util.JsonUtils; +import teammates.storage.sqlentity.Account; +import teammates.storage.sqlentity.AccountRequest; +import teammates.storage.sqlentity.Course; +import teammates.storage.sqlentity.DeadlineExtension; +import teammates.storage.sqlentity.FeedbackQuestion; +import teammates.storage.sqlentity.FeedbackResponse; +import teammates.storage.sqlentity.FeedbackResponseComment; +import teammates.storage.sqlentity.FeedbackSession; +import teammates.storage.sqlentity.Instructor; +import teammates.storage.sqlentity.Notification; +import teammates.storage.sqlentity.ReadNotification; +import teammates.storage.sqlentity.Section; +import teammates.storage.sqlentity.Student; +import teammates.storage.sqlentity.Team; +import teammates.test.FileHelper; + +public class ConvertDatastoreJsonToSqlJson { + // Get all entities in JSON file + // Convert entities into SQL entities + // Create corresponding links between entities + // Collate entities and write back to json + + DataStoreToSqlConverter entityConverter; + DataBundle dataStoreBundle; + + protected ConvertDatastoreJsonToSqlJson() throws IOException { + this.entityConverter = new DataStoreToSqlConverter(); + File file = new File("./src/client/java/teammates/client/scripts/typicalDataBundle.json"); + this.dataStoreBundle = loadDataBundle(file.getCanonicalPath()); + } + + private void migrate() throws IOException, InvalidParametersException{ + String outputFileName = "output.json"; + File outputFile = new File("./src/client/java/teammates/client/scripts/" + outputFileName); + createSqlJson(outputFile); + } + + private String removeWhitespace(String string) { + return string.replaceAll("\\s", ""); + } + + private DataBundle loadDataBundle(String pathToJsonFile) throws IOException { + String jsonString = FileHelper.readFile(pathToJsonFile); + return JsonUtils.fromJson(jsonString, DataBundle.class); + } + + private void saveFile(String filePath, String content) throws IOException { + FileHelper.saveFile(filePath, content); + System.out.println(filePath + " created!"); + } + + private void createSqlJson(File outputFile) throws IOException, InvalidParametersException { + SqlDataBundle sqlDataBundle = new SqlDataBundle(); + + migrateIndepedentEntities(sqlDataBundle); + + // TODO how to remove course information except ID + // Feedback ses (course) + // section (course) + // team (section) + migrateDependentEntities(sqlDataBundle); + + + String sqlJsonString = JsonUtils.toJson(sqlDataBundle); + saveFile(outputFile.getCanonicalPath(), sqlJsonString + System.lineSeparator()); + } + + private void migrateIndepedentEntities(SqlDataBundle sqlDataBundle) { + assert sqlDataBundle != null; + + dataStoreBundle.accounts.forEach((k, datastoreAccount) -> { + Account sqlAccount = entityConverter.convert(datastoreAccount); + sqlDataBundle.accounts.put(k, sqlAccount); + }); + + dataStoreBundle.courses.forEach((k, datastoreCourse) -> { + Course sqlCourse = entityConverter.convert(datastoreCourse); + sqlDataBundle.courses.put(k, sqlCourse); + }); + + dataStoreBundle.accountRequests.forEach((k, accountRequest) -> { + AccountRequest sqlAccountRequest = entityConverter.convert(accountRequest); + sqlDataBundle.accountRequests.put(k, sqlAccountRequest); + }); + + dataStoreBundle.notifications.forEach((k, notification) -> { + Notification sqlNotification = entityConverter.convert(notification); + sqlDataBundle.notifications.put(k, sqlNotification); + }); + } + + private void migrateDependentEntities(SqlDataBundle sqlDataBundle) { + // ['feedback_response_comment', + // 'feedback_response', 'readNotification', , + // 'feednackxxxresponse', 'feedbackxxxquestion', 'feedback_question', + // 'deadline_extension', + + // 'instructor', 'student', + // 'team', 'section', 'feedback_session ] + + dataStoreBundle.feedbackSessions.forEach((k, feedbackSession) -> { + FeedbackSession sqlFeedbackSession = entityConverter.convert(feedbackSession); + sqlDataBundle.feedbackSessions.put(k, sqlFeedbackSession); + }); + + dataStoreBundle.students.forEach((k, student) -> { + // Assumes that section name is unique in JSON file + String jsonKey = removeWhitespace(student.getSection()); + + if (!sqlDataBundle.sections.containsKey(jsonKey)) { + Section sqlSection = entityConverter.createSection(student); + sqlDataBundle.sections.put(jsonKey, sqlSection); + } + }); + + dataStoreBundle.students.forEach((k, student) -> { + // Assumes that team name is unique in JSON file + String jsonKey = removeWhitespace(student.getTeam()); + + if (!sqlDataBundle.teams.containsKey(jsonKey)) { + Team sqlTeam = entityConverter.createTeam(student); + sqlDataBundle.teams.put(jsonKey, sqlTeam); + } + }); + + dataStoreBundle.instructors.forEach((k, instructor) -> { + // Sets instructor key as "courseid-instructorName" + String jsonKey = removeWhitespace(instructor.getCourseId() + "-" + instructor.getName()); + + if (!sqlDataBundle.instructors.containsKey(jsonKey)) { + Instructor sqlInstructor = entityConverter.convert(instructor); + sqlDataBundle.instructors.put(jsonKey, sqlInstructor); + } + }); + + dataStoreBundle.students.forEach((k, student) -> { + // Sets instructor key as "courseid-instructorName" + String jsonKey = removeWhitespace(student.getCourse() + "-" + student.getName()); + + if (!sqlDataBundle.students.containsKey(jsonKey)) { + Student sqlStudent = entityConverter.convert(student); + sqlDataBundle.students.put(jsonKey, sqlStudent); + } + }); + + dataStoreBundle.deadlineExtensions.forEach((k, deadlineExtension) -> { + DeadlineExtension sqlDeadline = entityConverter.convert(deadlineExtension); + sqlDataBundle.deadlineExtensions.put(k, sqlDeadline); + }); + + dataStoreBundle.feedbackQuestions.forEach((k, feedbackQuestion) -> { + FeedbackQuestion sqlFeedbackQuestion = entityConverter.convert(feedbackQuestion); + sqlDataBundle.feedbackQuestions.put(k, sqlFeedbackQuestion); + }); + + dataStoreBundle.accounts.forEach((k, account) -> { + List sqlReadNotifications = entityConverter.createReadNotifications(account); + sqlReadNotifications.forEach((notif) -> { + String jsonKey = removeWhitespace(notif.getNotification().getTitle() + "-" + account.getName()); + sqlDataBundle.readNotifications.put(jsonKey, notif); + }); + }); + + dataStoreBundle.feedbackResponses.forEach((k, feedbackResponse) -> { + FeedbackResponse sqlFeedbackResponse = entityConverter.convert(feedbackResponse); + sqlDataBundle.feedbackResponses.put(k, sqlFeedbackResponse); + }); + + dataStoreBundle.feedbackResponseComments.forEach((k, feedbackReponseComment) -> { + FeedbackResponseComment sqlFeedbackResponseComment = entityConverter.convert(feedbackReponseComment); + sqlDataBundle.feedbackResponseComments.put(k, sqlFeedbackResponseComment); + }); + + // dataStoreBundle.accounts.forEach((k, account) -> { + // List sqlReadNotifications = entityConverter.createReadNotifications(account); + // }); + + // dataStoreBundle.feedbackSessions.forEach((k, feedbackSession) -> { + // FeedbackSession sqlFeedbackSession = entityConverter.convert(feedbackSession); + // sqlDataBundle.feedbackSessions.put(k, sqlFeedbackSession); + // }); + } + + + + public static void main(String[] args) throws IOException, InvalidParametersException { + // Topo sort, last element created first + // ['feedback_response_comment', + // 'feedback_response', 'readNotification', , + // 'feednackxxxresponse', 'feedbackxxxquestion', 'feedback_question', + // 'deadline_extension', 'instructor', 'student', + // user', 'team', 'section', 'feedback_session ] + + + // Independent entities + // acc req + // usage stats + // course + // account + // notification + + // Dependencies + // deadline extensions -> user + // deadline extension -> feedback-session + // feedback question -> feedback-session + // feedback response comments -> feedback responses + // feedback response comments -> sections responses + // feedback response -> sections + // feedback response -> feedback_questions + // feedback session -> courses + // instructors -> users + // sections -> courses + // students -> users + // teams -> sections + // users -> courses + // users -> teams + // users -> accounts + // readnotification -> account + // readNotification -> notification + + // Entities + // Course + // feedback session + // User + // Section + // Team + // Student + // Instructor + // Deadlineextension + // Feedback question / Feedbackxxxquestion + // Account + // ReadNotification + // Notification + // Feedbackresponse / Feednack xxx response + // feedbackResponseComment + // Account requests + // Usage statistics + ConvertDatastoreJsonToSqlJson script = new ConvertDatastoreJsonToSqlJson(); + script.migrate(); + } +} diff --git a/src/client/java/teammates/client/scripts/testdataconversion/DataStoreToSqlConverter.java b/src/client/java/teammates/client/scripts/testdataconversion/DataStoreToSqlConverter.java new file mode 100644 index 00000000000..b8cef98785b --- /dev/null +++ b/src/client/java/teammates/client/scripts/testdataconversion/DataStoreToSqlConverter.java @@ -0,0 +1,345 @@ +package teammates.client.scripts.testdataconversion; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + + +import teammates.common.datatransfer.InstructorPermissionRole; +import teammates.common.datatransfer.attributes.AccountAttributes; +import teammates.common.datatransfer.attributes.AccountRequestAttributes; +import teammates.common.datatransfer.attributes.CourseAttributes; +import teammates.common.datatransfer.attributes.DeadlineExtensionAttributes; +import teammates.common.datatransfer.attributes.FeedbackQuestionAttributes; +import teammates.common.datatransfer.attributes.FeedbackResponseAttributes; +import teammates.common.datatransfer.attributes.FeedbackResponseCommentAttributes; +import teammates.common.datatransfer.attributes.FeedbackSessionAttributes; +import teammates.common.datatransfer.attributes.InstructorAttributes; +import teammates.common.datatransfer.attributes.NotificationAttributes; +import teammates.common.datatransfer.attributes.StudentAttributes; +import teammates.storage.sqlentity.Account; +import teammates.storage.sqlentity.AccountRequest; +import teammates.storage.sqlentity.Course; +import teammates.storage.sqlentity.DeadlineExtension; +import teammates.storage.sqlentity.FeedbackQuestion; +import teammates.storage.sqlentity.FeedbackResponse; +import teammates.storage.sqlentity.FeedbackResponseComment; +import teammates.storage.sqlentity.FeedbackSession; +import teammates.storage.sqlentity.Instructor; +import teammates.storage.sqlentity.Notification; +import teammates.storage.sqlentity.ReadNotification; +import teammates.storage.sqlentity.Section; +import teammates.storage.sqlentity.Student; +import teammates.storage.sqlentity.Team; + +public class DataStoreToSqlConverter { + private String UUIDPrefix = "00000000-0000-4000-8000-"; + private int initialAccountNumber = 1; + private int initialAccountRequestNumber = 101; + private int initialSectionNumber = 201; + private int initialTeamNumber = 301; + private int initialDeadlineExtensionNumber = 401; + private int initialInstructorNumber = 501; + private int initialStudentNumber = 601; + private int intitialFeedbackSessionNumber = 701; + private int initialFeedbackQuestionNumber = 801; + private int intialFeedbackResponseNumber = 901; + private int initialNotificationNumber = 1101; + private int initialReadNotificationNumber = 1201; + + private UUIDGenerator accountUUIDGenerator = new UUIDGenerator(initialAccountNumber, UUIDPrefix); + private UUIDGenerator accounRequestUUIDGenerator = new UUIDGenerator(initialAccountRequestNumber, UUIDPrefix); + private UUIDGenerator sectionUUIDGenerator = new UUIDGenerator(initialSectionNumber, UUIDPrefix); + private UUIDGenerator teamUUIDGenerator = new UUIDGenerator(initialTeamNumber, UUIDPrefix); + private UUIDGenerator deadlineExtensionUUIDGenerator = new UUIDGenerator(initialDeadlineExtensionNumber, UUIDPrefix); + private UUIDGenerator instructorUUIDGenerator = new UUIDGenerator(initialInstructorNumber, UUIDPrefix); + private UUIDGenerator studentUUIDGenerator = new UUIDGenerator(initialStudentNumber, UUIDPrefix); + private UUIDGenerator feedbackSessionUUIDGenerator = new UUIDGenerator(intitialFeedbackSessionNumber, UUIDPrefix); + private UUIDGenerator feedbackQuestionUUIDGenerator = new UUIDGenerator(initialFeedbackQuestionNumber, UUIDPrefix); + private UUIDGenerator feedbackResponseUUIDGenerator = new UUIDGenerator(intialFeedbackResponseNumber, UUIDPrefix); + private UUIDGenerator notificationUUIDGenerator = new UUIDGenerator(initialNotificationNumber, UUIDPrefix); + private UUIDGenerator readNotificationUUIDGenerator = new UUIDGenerator(initialReadNotificationNumber, UUIDPrefix); + + private long initialFeedbackResponseCommentId = 0; + private long getNextFeedbackResponseCommentId() { + long nextId = initialFeedbackResponseCommentId; + initialFeedbackResponseCommentId += 1; + return nextId; + } + + + // Maps google id to account + Map accounts = new HashMap<>(); + // Maps old id to courses + Map courses = new HashMap<>(); + // Map course%section to section + Map sections = new HashMap<>(); + + // Map course%feedbackSession to feedbackSection + Map feedbackSessions = new HashMap<>(); + + // Maps notification id to notification + Map notifications = new HashMap<>(); + + // Maps question id to question + Map feedbackQuestions = new HashMap<>(); + + protected DataStoreToSqlConverter() { + } + + private String generateSectionKey(StudentAttributes student) { + return String.format("%s-%s", student.getCourse(), student.getSection()); + } + + private String generateSectionKey(String courseId, String sectionName) { + return String.format("%s-%s", courseId, sectionName); + } + + private String generatefeedbackSessionKey(FeedbackSessionAttributes feedbackSession) { + return String.format("%s-%s", feedbackSession.getCourseId(), feedbackSession.getFeedbackSessionName()); + } + + private String generatefeedbackSessionKey(String courseId, String feedbackSessionName) { + return String.format("%s-%s", courseId, feedbackSessionName); + } + + // private String generatefeedbackQuestionKey(FeedbackResponse feedbackQuestion) { + // return String.format("%s-%s-%s", feedbackQuestion.getCourseId(), + // feedbackQuestion.getFeedbackSessionName(), feedbackQuestion.getQuestionNumber()); + // } + + + protected Account convert(AccountAttributes accAttr) { + Account sqlAccount = new Account(accAttr.getGoogleId(), + accAttr.getName(), + accAttr.getEmail()); + + UUID uuid = accountUUIDGenerator.generateUUID(); + sqlAccount.setId(uuid); + + accounts.put(accAttr.getGoogleId(), sqlAccount); + return sqlAccount; + } + + protected AccountRequest convert(AccountRequestAttributes accReqAttr) { + AccountRequest sqlAccountRequest = new AccountRequest(accReqAttr.getEmail(), + accReqAttr.getName(), + accReqAttr.getInstitute()); + + sqlAccountRequest.setCreatedAt(accReqAttr.getCreatedAt()); + sqlAccountRequest.setRegisteredAt(accReqAttr.getRegisteredAt()); + sqlAccountRequest.setRegistrationKey(accReqAttr.getRegistrationKey()); + + UUID uuid = accounRequestUUIDGenerator.generateUUID(); + sqlAccountRequest.setId(uuid); + return sqlAccountRequest; + } + + protected Course convert(CourseAttributes courseAttr) { + Course sqlCourse = new Course(courseAttr.getId(), + courseAttr.getName(), + courseAttr.getTimeZone(), + courseAttr.getInstitute()); + + sqlCourse.setDeletedAt(courseAttr.getDeletedAt()); + sqlCourse.setCreatedAt(courseAttr.getCreatedAt()); + + courses.put(courseAttr.getId(), sqlCourse); + return sqlCourse; + } + + protected Notification convert(NotificationAttributes notifAttr) { + Notification sqlNotification = new Notification(notifAttr.getStartTime(), + notifAttr.getEndTime(), + notifAttr.getStyle(), + notifAttr.getTargetUser(), + notifAttr.getTitle(), + notifAttr.getMessage()); + + sqlNotification.setCreatedAt(notifAttr.getCreatedAt()); + if (notifAttr.isShown()) { + sqlNotification.setShown(); + } + + UUID uuid = notificationUUIDGenerator.generateUUID(); + sqlNotification.setId(uuid); + + notifications.put(notifAttr.getNotificationId(), sqlNotification); + return sqlNotification; + } + + protected FeedbackSession convert(FeedbackSessionAttributes fsAttr) { + Duration gracePeriod = Duration.ofMinutes(fsAttr.getGracePeriodMinutes()); + Course sqlCourse = courses.get(fsAttr.getCourseId()); + FeedbackSession sqlFs = new FeedbackSession( + fsAttr.getFeedbackSessionName(), + sqlCourse, + fsAttr.getCreatorEmail(), + fsAttr.getInstructions(), + fsAttr.getStartTime(), + fsAttr.getEndTime(), + fsAttr.getSessionVisibleFromTime(), + fsAttr.getResultsVisibleFromTime(), + gracePeriod, + fsAttr.isOpeningEmailEnabled(), + fsAttr.isClosingEmailEnabled(), + fsAttr.isPublishedEmailEnabled()); + + sqlFs.setCreatedAt(fsAttr.getCreatedTime()); + sqlFs.setDeletedAt(fsAttr.getDeletedTime()); + sqlFs.setId(feedbackSessionUUIDGenerator.generateUUID()); + + feedbackSessions.put(generatefeedbackSessionKey(fsAttr) ,sqlFs); + return sqlFs; + } + + protected Instructor convert(InstructorAttributes instructor) { + Course sqlCourse = courses.get(instructor.getCourseId()); + Account sqlAccount = accounts.get(instructor.getGoogleId()); + + InstructorPermissionRole role = InstructorPermissionRole.getEnum(instructor.getRole()); + + Instructor sqlInstructor = new Instructor(sqlCourse, + instructor.getName(), + instructor.getEmail(), + instructor.isDisplayedToStudents(), + instructor.getDisplayedName(), + role, + instructor.getPrivileges()); + sqlInstructor.setId(instructorUUIDGenerator.generateUUID()); + sqlInstructor.setAccount(sqlAccount); + + return sqlInstructor; + } + + protected Student convert(StudentAttributes student) { + Course sqlCourse = courses.get(student.getCourse()); + Account sqlAccount = accounts.get(student.getGoogleId()); + + + Student sqlStudent = new Student(sqlCourse, + student.getName(), + student.getEmail(), + student.getComments()); + sqlStudent.setId(studentUUIDGenerator.generateUUID()); + sqlStudent.setAccount(sqlAccount); + + return sqlStudent; + } + + protected DeadlineExtension convert(DeadlineExtensionAttributes deadlineExtension) { + FeedbackSession sqlFeedbackSession = feedbackSessions.get( + generatefeedbackSessionKey(deadlineExtension.getCourseId(), deadlineExtension.getFeedbackSessionName())); + + // User is not included since DataBundleLogic.java does not read users from this attribute + DeadlineExtension sqlDE = new DeadlineExtension(null, + sqlFeedbackSession, + deadlineExtension.getEndTime()); + sqlDE.setClosingSoonEmailSent(deadlineExtension.getSentClosingEmail()); + sqlDE.setCreatedAt(deadlineExtension.getCreatedAt()); + sqlDE.setId(deadlineExtensionUUIDGenerator.generateUUID()); + + return sqlDE; + } + + public FeedbackQuestion convert(FeedbackQuestionAttributes feedbackQuestion) { + FeedbackSession sqlFeedbackSession = feedbackSessions.get( + generatefeedbackSessionKey(feedbackQuestion.getCourseId(), feedbackQuestion.getFeedbackSessionName())); + + FeedbackQuestion sqlFq = FeedbackQuestion.makeQuestion(sqlFeedbackSession, + feedbackQuestion.getQuestionNumber(), + feedbackQuestion.getQuestionDescription(), + feedbackQuestion.getGiverType(), + feedbackQuestion.getRecipientType(), + feedbackQuestion.getNumberOfEntitiesToGiveFeedbackTo(), + feedbackQuestion.getShowResponsesTo(), + feedbackQuestion.getShowGiverNameTo(), + feedbackQuestion.getShowRecipientNameTo(), + feedbackQuestion.getQuestionDetails()); + + sqlFq.setCreatedAt(feedbackQuestion.getCreatedAt()); + sqlFq.setId(feedbackQuestionUUIDGenerator.generateUUID()); + + return sqlFq; + } + + public FeedbackResponse convert(FeedbackResponseAttributes feedbackResponse) { + FeedbackQuestion sqlFeedbackQuestion = feedbackQuestions.get(feedbackResponse.getFeedbackQuestionId()); + Section sqlGiverSection = sections.get(generateSectionKey(feedbackResponse.getCourseId(), + feedbackResponse.getGiverSection())); + Section sqlReceiverSection = sections.get(generateSectionKey(feedbackResponse.getCourseId(), + feedbackResponse.getRecipientSection())); + + FeedbackResponse sqlFeedbackResponse = FeedbackResponse.makeResponse( + sqlFeedbackQuestion, + feedbackResponse.getGiver(), + sqlGiverSection, + feedbackResponse.getRecipient(), + sqlReceiverSection, + feedbackResponse.getResponseDetails()); + + sqlFeedbackResponse.setId(feedbackResponseUUIDGenerator.generateUUID()); + sqlFeedbackResponse.setCreatedAt(feedbackResponse.getCreatedAt()); + + return sqlFeedbackResponse; + } + + public FeedbackResponseComment convert(FeedbackResponseCommentAttributes feedbackReponseComment) { + Section sqlGiverSection = sections.get(generateSectionKey(feedbackReponseComment.getCourseId(), + feedbackReponseComment.getGiverSection())); + Section sqlReceiverSection = sections.get(generateSectionKey(feedbackReponseComment.getCourseId(), + feedbackReponseComment.getReceiverSection())); + + FeedbackResponseComment sqlFrc = new FeedbackResponseComment(null, + feedbackReponseComment.getCommentGiver(), + feedbackReponseComment.getCommentGiverType(), + sqlGiverSection, + sqlReceiverSection, + feedbackReponseComment.getCommentText(), + feedbackReponseComment.isVisibilityFollowingFeedbackQuestion(), + feedbackReponseComment.isCommentFromFeedbackParticipant(), + feedbackReponseComment.getShowCommentTo(), + feedbackReponseComment.getShowGiverNameTo(), + feedbackReponseComment.getLastEditorEmail()); + + sqlFrc.setId(getNextFeedbackResponseCommentId()); + sqlFrc.setCreatedAt(feedbackReponseComment.getCreatedAt()); + return sqlFrc; + } + + protected Section createSection(StudentAttributes student) { + Course sqlCourse = courses.get(student.getCourse()); + Section sqlSection = new Section(sqlCourse, student.getSection()); + sqlSection.setId(sectionUUIDGenerator.generateUUID()); + + sections.put(generateSectionKey(student), sqlSection); + + return sqlSection; + } + + + protected Team createTeam(StudentAttributes student) { + Section sqlSection = sections.get(generateSectionKey(student)); + Team sqlTeam = new Team(sqlSection, student.getTeam()); + sqlTeam.setId(teamUUIDGenerator.generateUUID()); + + return sqlTeam; + } + + protected List createReadNotifications(AccountAttributes account) { + List sqlReadNotifications = new ArrayList<>(); + Account sqlAccount = accounts.get(account.getGoogleId()); + + account.getReadNotifications().forEach((notifId, endTime) -> { + Notification sqlNotification = notifications.get(notifId); + ReadNotification sqlReadNotification = new ReadNotification(sqlAccount, sqlNotification); + sqlReadNotification.setId(readNotificationUUIDGenerator.generateUUID()); + sqlReadNotifications.add(sqlReadNotification); + }); + + return sqlReadNotifications; + } +} diff --git a/src/client/java/teammates/client/scripts/testdataconversion/UUIDGenerator.java b/src/client/java/teammates/client/scripts/testdataconversion/UUIDGenerator.java new file mode 100644 index 00000000000..38344e1705a --- /dev/null +++ b/src/client/java/teammates/client/scripts/testdataconversion/UUIDGenerator.java @@ -0,0 +1,25 @@ +package teammates.client.scripts.testdataconversion; + +import java.util.UUID; + +public class UUIDGenerator { + int currId; + String UUIDPrefix; + + protected UUIDGenerator(int startId, String UUIDPrefix) { + this.currId = startId; + this.UUIDPrefix = UUIDPrefix; + } + + + private String leftPad(int digits, String string, Character paddingChar) { + return String.format("%10s", string).replace(' ', paddingChar); + } + + protected UUID generateUUID() { + String trailingUUID = leftPad(12, Integer.toString(this.currId), '0'); + UUID uuid = UUID.fromString(UUIDPrefix + trailingUUID); + this.currId += 1; + return uuid; + } +} From 291693c505291add31dafe32294076e12cc31d9a Mon Sep 17 00:00:00 2001 From: Nicolas Chang <25302138+NicolasCwy@users.noreply.github.com> Date: Wed, 20 Mar 2024 22:06:14 +0800 Subject: [PATCH 02/14] Add toposort script --- .../scripts/testdataconversion/toposort.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 src/client/java/teammates/client/scripts/testdataconversion/toposort.py diff --git a/src/client/java/teammates/client/scripts/testdataconversion/toposort.py b/src/client/java/teammates/client/scripts/testdataconversion/toposort.py new file mode 100644 index 00000000000..a2a4b7ffdf5 --- /dev/null +++ b/src/client/java/teammates/client/scripts/testdataconversion/toposort.py @@ -0,0 +1,114 @@ +#Python program to print topological sorting of a DAG +from collections import defaultdict + +#Class to represent a graph +class Graph: + def __init__(self,vertices): + self.graph = defaultdict(list) #dictionary containing adjacency List + self.V = vertices #No. of vertices + + # function to add an edge to graph + def addEdge(self,u,v): + self.graph[u].append(v) + + # A recursive function used by topologicalSort + def topologicalSortUtil(self,v,visited,stack): + + # Mark the current node as visited. + visited[v] = True + + # Recur for all the vertices adjacent to this vertex + for i in self.graph[v]: + if visited[i] == False: + self.topologicalSortUtil(i,visited,stack) + + # Push current vertex to stack which stores result + stack.insert(0,v) + + # The function to do Topological Sort. It uses recursive + # topologicalSortUtil() + def topologicalSort(self): + # Mark all the vertices as not visited + visited = [False]*self.V + stack =[] + + # Call the recursive helper function to store Topological + # Sort starting from all vertices one by one + for i in range(self.V): + if visited[i] == False: + self.topologicalSortUtil(i,visited,stack) + + # Print contents of stack + return stack.copy() +g = Graph(18) +course = 0 +feedback_session = 1 +user = 2 +section = 3 +team = 4 +student = 5 +instructor = 6 +deadline_extension = 7 +feedback_question = 8 +feedbackxxxquestion = 9 +feednackxxxresponse = 10 +account = 11 +readNotification = 12 +notification = 13 +feedback_response = 14 +feedback_response_comment = 15 +acc_req = 16 +usage_stats = 17 + +lookup = { + 0: "course", + 1: "feedback_session", + 2: "user", + 3: "section", + 4: "team", + 5:"student", + 6: "instructor", + 7: "deadline_extension", + 8: "feedback_question", + 9: "feedbackxxxquestion", + 10: "feednackxxxresponse", + 11: "account", + 12: "readNotification", + 13: "notification", + 14: "feedback_response", + 15: "feedback_response_comment", + 16: "acc_req", + 17: "usage_stats" +} + +dependencies = [ + (deadline_extension, user), + (deadline_extension, feedback_session), + (feedback_question, feedback_session), + (feedback_response_comment, feedback_response), + (feedback_response_comment, section), + (feedback_response, section), + (feedback_response, feedback_question), + (feedback_session, course), + (instructor, user), + (section, course), + (student, user), + (team, section), + (user, course), + (user, team), + (user, account), + (readNotification, account), + (readNotification, notification), +] + +for start, end in dependencies: + g.addEdge(start, end) +# g.addEdge("Course", 2) +# g.addEdge(5, 0) +# g.addEdge(4, 0) +# g.addEdge(4, 1) +# g.addEdge(2, 3) +# g.addEdge(1, 2) + +print ("Following is a Topological Sort of the given graph") +print(list(map(lambda x: lookup[x], g.topologicalSort()))) \ No newline at end of file From 72472654f16a8c797602cf7e5b1c6ff76837b2f1 Mon Sep 17 00:00:00 2001 From: nicolascwy <25302138+NicolasCwy@users.noreply.github.com> Date: Thu, 21 Mar 2024 20:05:32 +0800 Subject: [PATCH 03/14] Add function to remove foreign key data --- .../ConvertDatastoreJsonToSqlJson.java | 117 ++++++------------ .../java/teammates/common/util/JsonUtils.java | 7 ++ 2 files changed, 46 insertions(+), 78 deletions(-) diff --git a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java index 6ac765e992f..769e840a1ad 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java +++ b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.util.List; +import com.google.gson.JsonObject; + import teammates.common.datatransfer.DataBundle; import teammates.common.datatransfer.SqlDataBundle; import teammates.common.exception.InvalidParametersException; @@ -25,11 +27,6 @@ import teammates.test.FileHelper; public class ConvertDatastoreJsonToSqlJson { - // Get all entities in JSON file - // Convert entities into SQL entities - // Create corresponding links between entities - // Collate entities and write back to json - DataStoreToSqlConverter entityConverter; DataBundle dataStoreBundle; @@ -59,22 +56,46 @@ private void saveFile(String filePath, String content) throws IOException { System.out.println(filePath + " created!"); } + String[] entitiesAsForeignKeys = new String[]{"course", "feedbackSession", "section", "account", "giverSection", "recipientSection", "notification"}; + + private void removeForeignKeyData(JsonObject obj) { + for (String entityName : entitiesAsForeignKeys) { + if (obj.get(entityName) != null) { + JsonObject entity = obj.get(entityName).getAsJsonObject(); + for (String field : entity.deepCopy().keySet()) { + if (field != "id") { + entity.remove(field); + } + } + } + }; + } + private void createSqlJson(File outputFile) throws IOException, InvalidParametersException { SqlDataBundle sqlDataBundle = new SqlDataBundle(); migrateIndepedentEntities(sqlDataBundle); - - // TODO how to remove course information except ID - // Feedback ses (course) - // section (course) - // team (section) migrateDependentEntities(sqlDataBundle); + // Iterates through all entities in JSON file and removes foreign entitity data except its ID + JsonObject sqlJsonString = JsonUtils.toJsonObject(sqlDataBundle); + for (String entityCollectionName: sqlJsonString.keySet()) { + JsonObject entityCollection = sqlJsonString.get(entityCollectionName).getAsJsonObject(); + for (String entityName : entityCollection.getAsJsonObject().keySet()) { + JsonObject entity = entityCollection.get(entityName).getAsJsonObject(); + removeForeignKeyData(entity); + } + } - String sqlJsonString = JsonUtils.toJson(sqlDataBundle); - saveFile(outputFile.getCanonicalPath(), sqlJsonString + System.lineSeparator()); + String jsonString = JsonUtils.toJson(sqlJsonString); + saveFile(outputFile.getCanonicalPath(), jsonString + System.lineSeparator()); } + /** + * Migrate entities with no foreign key reference account requests, usage statistics + * courses, accouns, notifications + * @param sqlDataBundle + */ private void migrateIndepedentEntities(SqlDataBundle sqlDataBundle) { assert sqlDataBundle != null; @@ -99,14 +120,13 @@ private void migrateIndepedentEntities(SqlDataBundle sqlDataBundle) { }); } + /** + * Migrate entities which have dependence on each other or on the independent entities. + * feedback sessions, sections, teams, users, students, instructors, deadline extensions, feedback questions, + * read notifications, feedback responses and feedback response comments + * @param sqlDataBundle + */ private void migrateDependentEntities(SqlDataBundle sqlDataBundle) { - // ['feedback_response_comment', - // 'feedback_response', 'readNotification', , - // 'feednackxxxresponse', 'feedbackxxxquestion', 'feedback_question', - // 'deadline_extension', - - // 'instructor', 'student', - // 'team', 'section', 'feedback_session ] dataStoreBundle.feedbackSessions.forEach((k, feedbackSession) -> { FeedbackSession sqlFeedbackSession = entityConverter.convert(feedbackSession); @@ -181,70 +201,11 @@ private void migrateDependentEntities(SqlDataBundle sqlDataBundle) { sqlDataBundle.feedbackResponseComments.put(k, sqlFeedbackResponseComment); }); - // dataStoreBundle.accounts.forEach((k, account) -> { - // List sqlReadNotifications = entityConverter.createReadNotifications(account); - // }); - - // dataStoreBundle.feedbackSessions.forEach((k, feedbackSession) -> { - // FeedbackSession sqlFeedbackSession = entityConverter.convert(feedbackSession); - // sqlDataBundle.feedbackSessions.put(k, sqlFeedbackSession); - // }); } public static void main(String[] args) throws IOException, InvalidParametersException { - // Topo sort, last element created first - // ['feedback_response_comment', - // 'feedback_response', 'readNotification', , - // 'feednackxxxresponse', 'feedbackxxxquestion', 'feedback_question', - // 'deadline_extension', 'instructor', 'student', - // user', 'team', 'section', 'feedback_session ] - - - // Independent entities - // acc req - // usage stats - // course - // account - // notification - - // Dependencies - // deadline extensions -> user - // deadline extension -> feedback-session - // feedback question -> feedback-session - // feedback response comments -> feedback responses - // feedback response comments -> sections responses - // feedback response -> sections - // feedback response -> feedback_questions - // feedback session -> courses - // instructors -> users - // sections -> courses - // students -> users - // teams -> sections - // users -> courses - // users -> teams - // users -> accounts - // readnotification -> account - // readNotification -> notification - - // Entities - // Course - // feedback session - // User - // Section - // Team - // Student - // Instructor - // Deadlineextension - // Feedback question / Feedbackxxxquestion - // Account - // ReadNotification - // Notification - // Feedbackresponse / Feednack xxx response - // feedbackResponseComment - // Account requests - // Usage statistics ConvertDatastoreJsonToSqlJson script = new ConvertDatastoreJsonToSqlJson(); script.migrate(); } diff --git a/src/main/java/teammates/common/util/JsonUtils.java b/src/main/java/teammates/common/util/JsonUtils.java index d921ef06324..f507441fe3b 100644 --- a/src/main/java/teammates/common/util/JsonUtils.java +++ b/src/main/java/teammates/common/util/JsonUtils.java @@ -82,6 +82,13 @@ private static Gson getGsonInstance(boolean prettyPrint) { return builder.create(); } + /** + * This creates a Gson object that can be reformatted to modify JSON output + */ + public static JsonObject toJsonObject(Object src) { + return (JsonObject) getGsonInstance(true).toJsonTree(src); + } + /** * Serializes and pretty-prints the specified object into its equivalent JSON string. * From b19e22c34824b6c5ec5db493f555525e4b92ad51 Mon Sep 17 00:00:00 2001 From: nicolascwy <25302138+NicolasCwy@users.noreply.github.com> Date: Thu, 21 Mar 2024 20:05:37 +0800 Subject: [PATCH 04/14] Cleanup --- .../scripts/testdataconversion/toposort.py | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/src/client/java/teammates/client/scripts/testdataconversion/toposort.py b/src/client/java/teammates/client/scripts/testdataconversion/toposort.py index a2a4b7ffdf5..3002225a509 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/toposort.py +++ b/src/client/java/teammates/client/scripts/testdataconversion/toposort.py @@ -1,43 +1,43 @@ #Python program to print topological sorting of a DAG from collections import defaultdict - + #Class to represent a graph class Graph: def __init__(self,vertices): self.graph = defaultdict(list) #dictionary containing adjacency List self.V = vertices #No. of vertices - + # function to add an edge to graph def addEdge(self,u,v): self.graph[u].append(v) - + # A recursive function used by topologicalSort def topologicalSortUtil(self,v,visited,stack): - + # Mark the current node as visited. visited[v] = True - + # Recur for all the vertices adjacent to this vertex for i in self.graph[v]: if visited[i] == False: self.topologicalSortUtil(i,visited,stack) - + # Push current vertex to stack which stores result stack.insert(0,v) - + # The function to do Topological Sort. It uses recursive # topologicalSortUtil() def topologicalSort(self): # Mark all the vertices as not visited visited = [False]*self.V stack =[] - + # Call the recursive helper function to store Topological # Sort starting from all vertices one by one for i in range(self.V): if visited[i] == False: self.topologicalSortUtil(i,visited,stack) - + # Print contents of stack return stack.copy() g = Graph(18) @@ -81,6 +81,7 @@ def topologicalSort(self): 17: "usage_stats" } +# (x, y) where x is dependent on y, y has to be created to be used dependencies = [ (deadline_extension, user), (deadline_extension, feedback_session), @@ -103,12 +104,6 @@ def topologicalSort(self): for start, end in dependencies: g.addEdge(start, end) -# g.addEdge("Course", 2) -# g.addEdge(5, 0) -# g.addEdge(4, 0) -# g.addEdge(4, 1) -# g.addEdge(2, 3) -# g.addEdge(1, 2) - + print ("Following is a Topological Sort of the given graph") print(list(map(lambda x: lookup[x], g.topologicalSort()))) \ No newline at end of file From 5925dc13715f0ba78dd058fdfb742218c0b84764 Mon Sep 17 00:00:00 2001 From: Nicolas Chang <25302138+NicolasCwy@users.noreply.github.com> Date: Thu, 21 Mar 2024 20:36:58 +0800 Subject: [PATCH 05/14] WIP --- .../ConvertDatastoreJsonToSqlJson.java | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java index 769e840a1ad..99eb523cd00 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java +++ b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java @@ -27,21 +27,24 @@ import teammates.test.FileHelper; public class ConvertDatastoreJsonToSqlJson { - DataStoreToSqlConverter entityConverter; - DataBundle dataStoreBundle; + private DataStoreToSqlConverter entityConverter; + private DataBundle dataStoreBundle; - protected ConvertDatastoreJsonToSqlJson() throws IOException { + private String[] entitiesReferencedForeignKeys = new String[]{"course", + "feedbackSession", + "section", + "account", + "giverSection", + "recipientSection", + "notification"}; + + + protected ConvertDatastoreJsonToSqlJson(String inputFilePath) throws IOException { this.entityConverter = new DataStoreToSqlConverter(); - File file = new File("./src/client/java/teammates/client/scripts/typicalDataBundle.json"); + File file = new File(inputFilePath); this.dataStoreBundle = loadDataBundle(file.getCanonicalPath()); } - private void migrate() throws IOException, InvalidParametersException{ - String outputFileName = "output.json"; - File outputFile = new File("./src/client/java/teammates/client/scripts/" + outputFileName); - createSqlJson(outputFile); - } - private String removeWhitespace(String string) { return string.replaceAll("\\s", ""); } @@ -56,10 +59,12 @@ private void saveFile(String filePath, String content) throws IOException { System.out.println(filePath + " created!"); } - String[] entitiesAsForeignKeys = new String[]{"course", "feedbackSession", "section", "account", "giverSection", "recipientSection", "notification"}; + /** + * Amends foreign key references to only have ID field + */ private void removeForeignKeyData(JsonObject obj) { - for (String entityName : entitiesAsForeignKeys) { + for (String entityName : entitiesReferencedForeignKeys) { if (obj.get(entityName) != null) { JsonObject entity = obj.get(entityName).getAsJsonObject(); for (String field : entity.deepCopy().keySet()) { @@ -71,6 +76,9 @@ private void removeForeignKeyData(JsonObject obj) { }; } + /** + * Read datstore json file and creates a SQL equivalent + */ private void createSqlJson(File outputFile) throws IOException, InvalidParametersException { SqlDataBundle sqlDataBundle = new SqlDataBundle(); @@ -206,7 +214,13 @@ private void migrateDependentEntities(SqlDataBundle sqlDataBundle) { public static void main(String[] args) throws IOException, InvalidParametersException { - ConvertDatastoreJsonToSqlJson script = new ConvertDatastoreJsonToSqlJson(); - script.migrate(); + if (args.length > 0) { + ConvertDatastoreJsonToSqlJson script = new ConvertDatastoreJsonToSqlJson(args[0]); + String outputFileName = "output.json"; + File outputFile = new File("./src/client/java/teammates/client/scripts/" + outputFileName); + script.createSqlJson(outputFile); + } else { + throw new InvalidParametersException("Required the path of the script to convert"); + } } } From cbb6e52398e9a8b12a7259faee3dc06e2de8eada Mon Sep 17 00:00:00 2001 From: Nicolas Chang <25302138+NicolasCwy@users.noreply.github.com> Date: Thu, 21 Mar 2024 21:29:13 +0800 Subject: [PATCH 06/14] Simplify keys for students and instructors --- .../ConvertDatastoreJsonToSqlJson.java | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java index 99eb523cd00..3dbb7f78441 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java +++ b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java @@ -157,28 +157,18 @@ private void migrateDependentEntities(SqlDataBundle sqlDataBundle) { if (!sqlDataBundle.teams.containsKey(jsonKey)) { Team sqlTeam = entityConverter.createTeam(student); - sqlDataBundle.teams.put(jsonKey, sqlTeam); + sqlDataBundle.teams.put(k, sqlTeam); } }); - dataStoreBundle.instructors.forEach((k, instructor) -> { - // Sets instructor key as "courseid-instructorName" - String jsonKey = removeWhitespace(instructor.getCourseId() + "-" + instructor.getName()); - - if (!sqlDataBundle.instructors.containsKey(jsonKey)) { - Instructor sqlInstructor = entityConverter.convert(instructor); - sqlDataBundle.instructors.put(jsonKey, sqlInstructor); - } + dataStoreBundle.instructors.forEach((k, instructor) -> { + Instructor sqlInstructor = entityConverter.convert(instructor); + sqlDataBundle.instructors.put(k, sqlInstructor); }); dataStoreBundle.students.forEach((k, student) -> { - // Sets instructor key as "courseid-instructorName" - String jsonKey = removeWhitespace(student.getCourse() + "-" + student.getName()); - - if (!sqlDataBundle.students.containsKey(jsonKey)) { - Student sqlStudent = entityConverter.convert(student); - sqlDataBundle.students.put(jsonKey, sqlStudent); - } + Student sqlStudent = entityConverter.convert(student); + sqlDataBundle.students.put(k, sqlStudent); }); dataStoreBundle.deadlineExtensions.forEach((k, deadlineExtension) -> { From 090511b56333a3c524b704a0a0d91006ae10d38e Mon Sep 17 00:00:00 2001 From: Nicolas Chang <25302138+NicolasCwy@users.noreply.github.com> Date: Thu, 21 Mar 2024 22:20:12 +0800 Subject: [PATCH 07/14] Fix lint issues --- .../ConvertDatastoreJsonToSqlJson.java | 58 ++- .../DataStoreToSqlConverter.java | 339 ++++++++++-------- .../testdataconversion/UUIDGenerator.java | 25 -- .../testdataconversion/UuidGenerator.java | 31 ++ .../testdataconversion/package-info.java | 4 + .../java/teammates/common/util/JsonUtils.java | 2 +- 6 files changed, 257 insertions(+), 202 deletions(-) delete mode 100644 src/client/java/teammates/client/scripts/testdataconversion/UUIDGenerator.java create mode 100644 src/client/java/teammates/client/scripts/testdataconversion/UuidGenerator.java create mode 100644 src/client/java/teammates/client/scripts/testdataconversion/package-info.java diff --git a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java index 3dbb7f78441..07b9c628d71 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java +++ b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java @@ -26,18 +26,22 @@ import teammates.storage.sqlentity.Team; import teammates.test.FileHelper; +/** + * Class to create JSON test data in SQL format from a noSQL JSON file. + */ public class ConvertDatastoreJsonToSqlJson { private DataStoreToSqlConverter entityConverter; private DataBundle dataStoreBundle; + private SqlDataBundle sqlDataBundle; - private String[] entitiesReferencedForeignKeys = new String[]{"course", - "feedbackSession", - "section", - "account", - "giverSection", - "recipientSection", - "notification"}; - + private String[] entitiesReferencedForeignKeys = new String[] { + "course", + "feedbackSession", + "section", + "account", + "giverSection", + "recipientSection", + "notification"}; protected ConvertDatastoreJsonToSqlJson(String inputFilePath) throws IOException { this.entityConverter = new DataStoreToSqlConverter(); @@ -59,35 +63,34 @@ private void saveFile(String filePath, String content) throws IOException { System.out.println(filePath + " created!"); } - /** - * Amends foreign key references to only have ID field + * Amends foreign key references to only have ID field. */ private void removeForeignKeyData(JsonObject obj) { for (String entityName : entitiesReferencedForeignKeys) { if (obj.get(entityName) != null) { JsonObject entity = obj.get(entityName).getAsJsonObject(); for (String field : entity.deepCopy().keySet()) { - if (field != "id") { + if (!"id".equals(field)) { entity.remove(field); } } } - }; + } } /** - * Read datstore json file and creates a SQL equivalent + * Read datstore json file and creates a SQL equivalent. */ private void createSqlJson(File outputFile) throws IOException, InvalidParametersException { - SqlDataBundle sqlDataBundle = new SqlDataBundle(); + sqlDataBundle = new SqlDataBundle(); + + migrateIndepedentEntities(); + migrateDependentEntities(); - migrateIndepedentEntities(sqlDataBundle); - migrateDependentEntities(sqlDataBundle); - // Iterates through all entities in JSON file and removes foreign entitity data except its ID JsonObject sqlJsonString = JsonUtils.toJsonObject(sqlDataBundle); - for (String entityCollectionName: sqlJsonString.keySet()) { + for (String entityCollectionName : sqlJsonString.keySet()) { JsonObject entityCollection = sqlJsonString.get(entityCollectionName).getAsJsonObject(); for (String entityName : entityCollection.getAsJsonObject().keySet()) { JsonObject entity = entityCollection.get(entityName).getAsJsonObject(); @@ -100,11 +103,10 @@ private void createSqlJson(File outputFile) throws IOException, InvalidParameter } /** - * Migrate entities with no foreign key reference account requests, usage statistics - * courses, accouns, notifications - * @param sqlDataBundle + * Migrate entities with no foreign key reference. + * Entities are account requests, usage statistics, courses, accouns, notifications */ - private void migrateIndepedentEntities(SqlDataBundle sqlDataBundle) { + private void migrateIndepedentEntities() { assert sqlDataBundle != null; dataStoreBundle.accounts.forEach((k, datastoreAccount) -> { @@ -131,10 +133,9 @@ private void migrateIndepedentEntities(SqlDataBundle sqlDataBundle) { /** * Migrate entities which have dependence on each other or on the independent entities. * feedback sessions, sections, teams, users, students, instructors, deadline extensions, feedback questions, - * read notifications, feedback responses and feedback response comments - * @param sqlDataBundle + * read notifications, feedback responses and feedback response comments. */ - private void migrateDependentEntities(SqlDataBundle sqlDataBundle) { + private void migrateDependentEntities() { dataStoreBundle.feedbackSessions.forEach((k, feedbackSession) -> { FeedbackSession sqlFeedbackSession = entityConverter.convert(feedbackSession); @@ -161,7 +162,7 @@ private void migrateDependentEntities(SqlDataBundle sqlDataBundle) { } }); - dataStoreBundle.instructors.forEach((k, instructor) -> { + dataStoreBundle.instructors.forEach((k, instructor) -> { Instructor sqlInstructor = entityConverter.convert(instructor); sqlDataBundle.instructors.put(k, sqlInstructor); }); @@ -183,7 +184,7 @@ private void migrateDependentEntities(SqlDataBundle sqlDataBundle) { dataStoreBundle.accounts.forEach((k, account) -> { List sqlReadNotifications = entityConverter.createReadNotifications(account); - sqlReadNotifications.forEach((notif) -> { + sqlReadNotifications.forEach(notif -> { String jsonKey = removeWhitespace(notif.getNotification().getTitle() + "-" + account.getName()); sqlDataBundle.readNotifications.put(jsonKey, notif); }); @@ -198,11 +199,8 @@ private void migrateDependentEntities(SqlDataBundle sqlDataBundle) { FeedbackResponseComment sqlFeedbackResponseComment = entityConverter.convert(feedbackReponseComment); sqlDataBundle.feedbackResponseComments.put(k, sqlFeedbackResponseComment); }); - } - - public static void main(String[] args) throws IOException, InvalidParametersException { if (args.length > 0) { ConvertDatastoreJsonToSqlJson script = new ConvertDatastoreJsonToSqlJson(args[0]); diff --git a/src/client/java/teammates/client/scripts/testdataconversion/DataStoreToSqlConverter.java b/src/client/java/teammates/client/scripts/testdataconversion/DataStoreToSqlConverter.java index b8cef98785b..69acaad7975 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/DataStoreToSqlConverter.java +++ b/src/client/java/teammates/client/scripts/testdataconversion/DataStoreToSqlConverter.java @@ -7,7 +7,6 @@ import java.util.Map; import java.util.UUID; - import teammates.common.datatransfer.InstructorPermissionRole; import teammates.common.datatransfer.attributes.AccountAttributes; import teammates.common.datatransfer.attributes.AccountRequestAttributes; @@ -35,8 +34,11 @@ import teammates.storage.sqlentity.Student; import teammates.storage.sqlentity.Team; +/** + * Helper class to convert entities from its noSQL to SQL format. + */ public class DataStoreToSqlConverter { - private String UUIDPrefix = "00000000-0000-4000-8000-"; + private String uuidPrefix = "00000000-0000-4000-8000-"; private int initialAccountNumber = 1; private int initialAccountRequestNumber = 101; private int initialSectionNumber = 201; @@ -50,44 +52,42 @@ public class DataStoreToSqlConverter { private int initialNotificationNumber = 1101; private int initialReadNotificationNumber = 1201; - private UUIDGenerator accountUUIDGenerator = new UUIDGenerator(initialAccountNumber, UUIDPrefix); - private UUIDGenerator accounRequestUUIDGenerator = new UUIDGenerator(initialAccountRequestNumber, UUIDPrefix); - private UUIDGenerator sectionUUIDGenerator = new UUIDGenerator(initialSectionNumber, UUIDPrefix); - private UUIDGenerator teamUUIDGenerator = new UUIDGenerator(initialTeamNumber, UUIDPrefix); - private UUIDGenerator deadlineExtensionUUIDGenerator = new UUIDGenerator(initialDeadlineExtensionNumber, UUIDPrefix); - private UUIDGenerator instructorUUIDGenerator = new UUIDGenerator(initialInstructorNumber, UUIDPrefix); - private UUIDGenerator studentUUIDGenerator = new UUIDGenerator(initialStudentNumber, UUIDPrefix); - private UUIDGenerator feedbackSessionUUIDGenerator = new UUIDGenerator(intitialFeedbackSessionNumber, UUIDPrefix); - private UUIDGenerator feedbackQuestionUUIDGenerator = new UUIDGenerator(initialFeedbackQuestionNumber, UUIDPrefix); - private UUIDGenerator feedbackResponseUUIDGenerator = new UUIDGenerator(intialFeedbackResponseNumber, UUIDPrefix); - private UUIDGenerator notificationUUIDGenerator = new UUIDGenerator(initialNotificationNumber, UUIDPrefix); - private UUIDGenerator readNotificationUUIDGenerator = new UUIDGenerator(initialReadNotificationNumber, UUIDPrefix); - - private long initialFeedbackResponseCommentId = 0; - private long getNextFeedbackResponseCommentId() { - long nextId = initialFeedbackResponseCommentId; - initialFeedbackResponseCommentId += 1; - return nextId; - } - + private UuidGenerator accountUuidGenerator = new UuidGenerator(initialAccountNumber, uuidPrefix); + private UuidGenerator accounRequestUuidGenerator = new UuidGenerator(initialAccountRequestNumber, uuidPrefix); + private UuidGenerator sectionUuidGenerator = new UuidGenerator(initialSectionNumber, uuidPrefix); + private UuidGenerator teamUuidGenerator = new UuidGenerator(initialTeamNumber, uuidPrefix); + private UuidGenerator deadlineExtensionUuidGenerator = new UuidGenerator(initialDeadlineExtensionNumber, uuidPrefix); + private UuidGenerator instructorUuidGenerator = new UuidGenerator(initialInstructorNumber, uuidPrefix); + private UuidGenerator studentUuidGenerator = new UuidGenerator(initialStudentNumber, uuidPrefix); + private UuidGenerator feedbackSessionUuidGenerator = new UuidGenerator(intitialFeedbackSessionNumber, uuidPrefix); + private UuidGenerator feedbackQuestionUuidGenerator = new UuidGenerator(initialFeedbackQuestionNumber, uuidPrefix); + private UuidGenerator feedbackResponseUuidGenerator = new UuidGenerator(intialFeedbackResponseNumber, uuidPrefix); + private UuidGenerator notificationUuidGenerator = new UuidGenerator(initialNotificationNumber, uuidPrefix); + private UuidGenerator readNotificationUuidGenerator = new UuidGenerator(initialReadNotificationNumber, uuidPrefix); + + private long initialFeedbackResponseCommentId; // Maps google id to account - Map accounts = new HashMap<>(); + private Map accounts = new HashMap<>(); // Maps old id to courses - Map courses = new HashMap<>(); - // Map course%section to section - Map sections = new HashMap<>(); + private Map courses = new HashMap<>(); + + // Maps question id to question + private Map feedbackQuestions = new HashMap<>(); // Map course%feedbackSession to feedbackSection - Map feedbackSessions = new HashMap<>(); + private Map feedbackSessions = new HashMap<>(); // Maps notification id to notification - Map notifications = new HashMap<>(); + private Map notifications = new HashMap<>(); - // Maps question id to question - Map feedbackQuestions = new HashMap<>(); + // Map course%section to section + private Map sections = new HashMap<>(); - protected DataStoreToSqlConverter() { + private long getNextFeedbackResponseCommentId() { + long nextId = initialFeedbackResponseCommentId; + initialFeedbackResponseCommentId += 1; + return nextId; } private String generateSectionKey(StudentAttributes student) { @@ -106,96 +106,114 @@ private String generatefeedbackSessionKey(String courseId, String feedbackSessio return String.format("%s-%s", courseId, feedbackSessionName); } - // private String generatefeedbackQuestionKey(FeedbackResponse feedbackQuestion) { - // return String.format("%s-%s-%s", feedbackQuestion.getCourseId(), - // feedbackQuestion.getFeedbackSessionName(), feedbackQuestion.getQuestionNumber()); - // } - - + /** + * Converts Account from its noSQL to SQL entity. + */ protected Account convert(AccountAttributes accAttr) { Account sqlAccount = new Account(accAttr.getGoogleId(), - accAttr.getName(), - accAttr.getEmail()); + accAttr.getName(), + accAttr.getEmail()); + + UUID uuid = accountUuidGenerator.generateUuid(); + sqlAccount.setId(uuid); - UUID uuid = accountUUIDGenerator.generateUUID(); - sqlAccount.setId(uuid); + accounts.put(accAttr.getGoogleId(), sqlAccount); - accounts.put(accAttr.getGoogleId(), sqlAccount); return sqlAccount; } - + + /** + * Converts Account Request from its noSQL to SQL entity. + */ protected AccountRequest convert(AccountRequestAttributes accReqAttr) { AccountRequest sqlAccountRequest = new AccountRequest(accReqAttr.getEmail(), - accReqAttr.getName(), - accReqAttr.getInstitute()); + accReqAttr.getName(), + accReqAttr.getInstitute()); + + sqlAccountRequest.setCreatedAt(accReqAttr.getCreatedAt()); + sqlAccountRequest.setRegisteredAt(accReqAttr.getRegisteredAt()); + sqlAccountRequest.setRegistrationKey(accReqAttr.getRegistrationKey()); - sqlAccountRequest.setCreatedAt(accReqAttr.getCreatedAt()); - sqlAccountRequest.setRegisteredAt(accReqAttr.getRegisteredAt()); - sqlAccountRequest.setRegistrationKey(accReqAttr.getRegistrationKey()); + UUID uuid = accounRequestUuidGenerator.generateUuid(); + sqlAccountRequest.setId(uuid); - UUID uuid = accounRequestUUIDGenerator.generateUUID(); - sqlAccountRequest.setId(uuid); - return sqlAccountRequest; + return sqlAccountRequest; } + /** + * Converts Course from its noSQL to SQL entity. + */ protected Course convert(CourseAttributes courseAttr) { Course sqlCourse = new Course(courseAttr.getId(), - courseAttr.getName(), - courseAttr.getTimeZone(), - courseAttr.getInstitute()); + courseAttr.getName(), + courseAttr.getTimeZone(), + courseAttr.getInstitute()); - sqlCourse.setDeletedAt(courseAttr.getDeletedAt()); - sqlCourse.setCreatedAt(courseAttr.getCreatedAt()); + sqlCourse.setDeletedAt(courseAttr.getDeletedAt()); + sqlCourse.setCreatedAt(courseAttr.getCreatedAt()); courses.put(courseAttr.getId(), sqlCourse); + return sqlCourse; } + /** + * Converts Notification from its noSQL to SQL entity. + */ protected Notification convert(NotificationAttributes notifAttr) { Notification sqlNotification = new Notification(notifAttr.getStartTime(), - notifAttr.getEndTime(), - notifAttr.getStyle(), - notifAttr.getTargetUser(), - notifAttr.getTitle(), - notifAttr.getMessage()); + notifAttr.getEndTime(), + notifAttr.getStyle(), + notifAttr.getTargetUser(), + notifAttr.getTitle(), + notifAttr.getMessage()); + + sqlNotification.setCreatedAt(notifAttr.getCreatedAt()); + + if (notifAttr.isShown()) { + sqlNotification.setShown(); + } - sqlNotification.setCreatedAt(notifAttr.getCreatedAt()); - if (notifAttr.isShown()) { - sqlNotification.setShown(); - } + UUID uuid = notificationUuidGenerator.generateUuid(); + sqlNotification.setId(uuid); - UUID uuid = notificationUUIDGenerator.generateUUID(); - sqlNotification.setId(uuid); + notifications.put(notifAttr.getNotificationId(), sqlNotification); - notifications.put(notifAttr.getNotificationId(), sqlNotification); return sqlNotification; } + /** + * Converts Feedback Session from its noSQL to SQL entity. + */ protected FeedbackSession convert(FeedbackSessionAttributes fsAttr) { Duration gracePeriod = Duration.ofMinutes(fsAttr.getGracePeriodMinutes()); Course sqlCourse = courses.get(fsAttr.getCourseId()); FeedbackSession sqlFs = new FeedbackSession( - fsAttr.getFeedbackSessionName(), - sqlCourse, - fsAttr.getCreatorEmail(), - fsAttr.getInstructions(), - fsAttr.getStartTime(), - fsAttr.getEndTime(), - fsAttr.getSessionVisibleFromTime(), - fsAttr.getResultsVisibleFromTime(), - gracePeriod, - fsAttr.isOpeningEmailEnabled(), - fsAttr.isClosingEmailEnabled(), - fsAttr.isPublishedEmailEnabled()); - - sqlFs.setCreatedAt(fsAttr.getCreatedTime()); - sqlFs.setDeletedAt(fsAttr.getDeletedTime()); - sqlFs.setId(feedbackSessionUUIDGenerator.generateUUID()); - - feedbackSessions.put(generatefeedbackSessionKey(fsAttr) ,sqlFs); + fsAttr.getFeedbackSessionName(), + sqlCourse, + fsAttr.getCreatorEmail(), + fsAttr.getInstructions(), + fsAttr.getStartTime(), + fsAttr.getEndTime(), + fsAttr.getSessionVisibleFromTime(), + fsAttr.getResultsVisibleFromTime(), + gracePeriod, + fsAttr.isOpeningEmailEnabled(), + fsAttr.isClosingEmailEnabled(), + fsAttr.isPublishedEmailEnabled()); + + sqlFs.setCreatedAt(fsAttr.getCreatedTime()); + sqlFs.setDeletedAt(fsAttr.getDeletedTime()); + sqlFs.setId(feedbackSessionUuidGenerator.generateUuid()); + + feedbackSessions.put(generatefeedbackSessionKey(fsAttr), sqlFs); + return sqlFs; } + /** + * Converts Instructor from its noSQL to SQL entity. + */ protected Instructor convert(InstructorAttributes instructor) { Course sqlCourse = courses.get(instructor.getCourseId()); Account sqlAccount = accounts.get(instructor.getGoogleId()); @@ -203,132 +221,161 @@ protected Instructor convert(InstructorAttributes instructor) { InstructorPermissionRole role = InstructorPermissionRole.getEnum(instructor.getRole()); Instructor sqlInstructor = new Instructor(sqlCourse, - instructor.getName(), - instructor.getEmail(), - instructor.isDisplayedToStudents(), - instructor.getDisplayedName(), - role, - instructor.getPrivileges()); - sqlInstructor.setId(instructorUUIDGenerator.generateUUID()); + instructor.getName(), + instructor.getEmail(), + instructor.isDisplayedToStudents(), + instructor.getDisplayedName(), + role, + instructor.getPrivileges()); + sqlInstructor.setId(instructorUuidGenerator.generateUuid()); sqlInstructor.setAccount(sqlAccount); return sqlInstructor; } + /** + * Converts Student from its noSQL to SQL entity. + */ protected Student convert(StudentAttributes student) { Course sqlCourse = courses.get(student.getCourse()); Account sqlAccount = accounts.get(student.getGoogleId()); - Student sqlStudent = new Student(sqlCourse, - student.getName(), - student.getEmail(), - student.getComments()); - sqlStudent.setId(studentUUIDGenerator.generateUUID()); + student.getName(), + student.getEmail(), + student.getComments()); + + sqlStudent.setId(studentUuidGenerator.generateUuid()); sqlStudent.setAccount(sqlAccount); return sqlStudent; } + /** + * Converts Deadline Extension from its noSQL to SQL entity. + */ protected DeadlineExtension convert(DeadlineExtensionAttributes deadlineExtension) { FeedbackSession sqlFeedbackSession = feedbackSessions.get( - generatefeedbackSessionKey(deadlineExtension.getCourseId(), deadlineExtension.getFeedbackSessionName())); + generatefeedbackSessionKey(deadlineExtension.getCourseId(), deadlineExtension.getFeedbackSessionName())); // User is not included since DataBundleLogic.java does not read users from this attribute DeadlineExtension sqlDE = new DeadlineExtension(null, - sqlFeedbackSession, - deadlineExtension.getEndTime()); + sqlFeedbackSession, + deadlineExtension.getEndTime()); + sqlDE.setClosingSoonEmailSent(deadlineExtension.getSentClosingEmail()); sqlDE.setCreatedAt(deadlineExtension.getCreatedAt()); - sqlDE.setId(deadlineExtensionUUIDGenerator.generateUUID()); + sqlDE.setId(deadlineExtensionUuidGenerator.generateUuid()); return sqlDE; } - public FeedbackQuestion convert(FeedbackQuestionAttributes feedbackQuestion) { + /** + * Converts Feedback Question from its noSQL to SQL entity. + */ + protected FeedbackQuestion convert(FeedbackQuestionAttributes feedbackQuestion) { FeedbackSession sqlFeedbackSession = feedbackSessions.get( - generatefeedbackSessionKey(feedbackQuestion.getCourseId(), feedbackQuestion.getFeedbackSessionName())); + generatefeedbackSessionKey(feedbackQuestion.getCourseId(), feedbackQuestion.getFeedbackSessionName())); FeedbackQuestion sqlFq = FeedbackQuestion.makeQuestion(sqlFeedbackSession, - feedbackQuestion.getQuestionNumber(), - feedbackQuestion.getQuestionDescription(), - feedbackQuestion.getGiverType(), - feedbackQuestion.getRecipientType(), - feedbackQuestion.getNumberOfEntitiesToGiveFeedbackTo(), - feedbackQuestion.getShowResponsesTo(), - feedbackQuestion.getShowGiverNameTo(), - feedbackQuestion.getShowRecipientNameTo(), - feedbackQuestion.getQuestionDetails()); + feedbackQuestion.getQuestionNumber(), + feedbackQuestion.getQuestionDescription(), + feedbackQuestion.getGiverType(), + feedbackQuestion.getRecipientType(), + feedbackQuestion.getNumberOfEntitiesToGiveFeedbackTo(), + feedbackQuestion.getShowResponsesTo(), + feedbackQuestion.getShowGiverNameTo(), + feedbackQuestion.getShowRecipientNameTo(), + feedbackQuestion.getQuestionDetails()); sqlFq.setCreatedAt(feedbackQuestion.getCreatedAt()); - sqlFq.setId(feedbackQuestionUUIDGenerator.generateUUID()); + sqlFq.setId(feedbackQuestionUuidGenerator.generateUuid()); return sqlFq; } - public FeedbackResponse convert(FeedbackResponseAttributes feedbackResponse) { + /** + * Converts Feedback Response from its noSQL to SQL entity. + */ + protected FeedbackResponse convert(FeedbackResponseAttributes feedbackResponse) { FeedbackQuestion sqlFeedbackQuestion = feedbackQuestions.get(feedbackResponse.getFeedbackQuestionId()); + Section sqlGiverSection = sections.get(generateSectionKey(feedbackResponse.getCourseId(), - feedbackResponse.getGiverSection())); + feedbackResponse.getGiverSection())); + Section sqlReceiverSection = sections.get(generateSectionKey(feedbackResponse.getCourseId(), - feedbackResponse.getRecipientSection())); + feedbackResponse.getRecipientSection())); FeedbackResponse sqlFeedbackResponse = FeedbackResponse.makeResponse( - sqlFeedbackQuestion, - feedbackResponse.getGiver(), - sqlGiverSection, - feedbackResponse.getRecipient(), - sqlReceiverSection, - feedbackResponse.getResponseDetails()); - - sqlFeedbackResponse.setId(feedbackResponseUUIDGenerator.generateUUID()); + sqlFeedbackQuestion, + feedbackResponse.getGiver(), + sqlGiverSection, + feedbackResponse.getRecipient(), + sqlReceiverSection, + feedbackResponse.getResponseDetails()); + + sqlFeedbackResponse.setId(feedbackResponseUuidGenerator.generateUuid()); sqlFeedbackResponse.setCreatedAt(feedbackResponse.getCreatedAt()); - + return sqlFeedbackResponse; } - public FeedbackResponseComment convert(FeedbackResponseCommentAttributes feedbackReponseComment) { + /** + * Converts Feedback Response Comment from its noSQL to SQL entity. + */ + protected FeedbackResponseComment convert(FeedbackResponseCommentAttributes feedbackReponseComment) { Section sqlGiverSection = sections.get(generateSectionKey(feedbackReponseComment.getCourseId(), - feedbackReponseComment.getGiverSection())); + feedbackReponseComment.getGiverSection())); + Section sqlReceiverSection = sections.get(generateSectionKey(feedbackReponseComment.getCourseId(), - feedbackReponseComment.getReceiverSection())); + feedbackReponseComment.getReceiverSection())); FeedbackResponseComment sqlFrc = new FeedbackResponseComment(null, - feedbackReponseComment.getCommentGiver(), - feedbackReponseComment.getCommentGiverType(), - sqlGiverSection, - sqlReceiverSection, - feedbackReponseComment.getCommentText(), - feedbackReponseComment.isVisibilityFollowingFeedbackQuestion(), - feedbackReponseComment.isCommentFromFeedbackParticipant(), - feedbackReponseComment.getShowCommentTo(), - feedbackReponseComment.getShowGiverNameTo(), - feedbackReponseComment.getLastEditorEmail()); - + feedbackReponseComment.getCommentGiver(), + feedbackReponseComment.getCommentGiverType(), + sqlGiverSection, + sqlReceiverSection, + feedbackReponseComment.getCommentText(), + feedbackReponseComment.isVisibilityFollowingFeedbackQuestion(), + feedbackReponseComment.isCommentFromFeedbackParticipant(), + feedbackReponseComment.getShowCommentTo(), + feedbackReponseComment.getShowGiverNameTo(), + feedbackReponseComment.getLastEditorEmail()); + sqlFrc.setId(getNextFeedbackResponseCommentId()); sqlFrc.setCreatedAt(feedbackReponseComment.getCreatedAt()); + return sqlFrc; } + /** + * Creates SQL Section from noSQL Student attribute. + */ protected Section createSection(StudentAttributes student) { Course sqlCourse = courses.get(student.getCourse()); Section sqlSection = new Section(sqlCourse, student.getSection()); - sqlSection.setId(sectionUUIDGenerator.generateUUID()); + + sqlSection.setId(sectionUuidGenerator.generateUuid()); sections.put(generateSectionKey(student), sqlSection); - + return sqlSection; } - + /** + * Creates SQL Team from noSQL Student attribute. + */ protected Team createTeam(StudentAttributes student) { Section sqlSection = sections.get(generateSectionKey(student)); Team sqlTeam = new Team(sqlSection, student.getTeam()); - sqlTeam.setId(teamUUIDGenerator.generateUUID()); - + sqlTeam.setId(teamUuidGenerator.generateUuid()); + return sqlTeam; } + /** + * Creates SQL Read Notifications from Account attributes. + */ protected List createReadNotifications(AccountAttributes account) { List sqlReadNotifications = new ArrayList<>(); Account sqlAccount = accounts.get(account.getGoogleId()); @@ -336,7 +383,7 @@ protected List createReadNotifications(AccountAttributes accou account.getReadNotifications().forEach((notifId, endTime) -> { Notification sqlNotification = notifications.get(notifId); ReadNotification sqlReadNotification = new ReadNotification(sqlAccount, sqlNotification); - sqlReadNotification.setId(readNotificationUUIDGenerator.generateUUID()); + sqlReadNotification.setId(readNotificationUuidGenerator.generateUuid()); sqlReadNotifications.add(sqlReadNotification); }); diff --git a/src/client/java/teammates/client/scripts/testdataconversion/UUIDGenerator.java b/src/client/java/teammates/client/scripts/testdataconversion/UUIDGenerator.java deleted file mode 100644 index 38344e1705a..00000000000 --- a/src/client/java/teammates/client/scripts/testdataconversion/UUIDGenerator.java +++ /dev/null @@ -1,25 +0,0 @@ -package teammates.client.scripts.testdataconversion; - -import java.util.UUID; - -public class UUIDGenerator { - int currId; - String UUIDPrefix; - - protected UUIDGenerator(int startId, String UUIDPrefix) { - this.currId = startId; - this.UUIDPrefix = UUIDPrefix; - } - - - private String leftPad(int digits, String string, Character paddingChar) { - return String.format("%10s", string).replace(' ', paddingChar); - } - - protected UUID generateUUID() { - String trailingUUID = leftPad(12, Integer.toString(this.currId), '0'); - UUID uuid = UUID.fromString(UUIDPrefix + trailingUUID); - this.currId += 1; - return uuid; - } -} diff --git a/src/client/java/teammates/client/scripts/testdataconversion/UuidGenerator.java b/src/client/java/teammates/client/scripts/testdataconversion/UuidGenerator.java new file mode 100644 index 00000000000..feb83abd66b --- /dev/null +++ b/src/client/java/teammates/client/scripts/testdataconversion/UuidGenerator.java @@ -0,0 +1,31 @@ +package teammates.client.scripts.testdataconversion; + +import java.util.UUID; + +/** + * Generator that counts up to generate an ID for an entity. + */ +public class UuidGenerator { + int currId; + String uuidPrefix; + + protected UuidGenerator(int startId, String uuidPrefix) { + this.currId = startId; + this.uuidPrefix = uuidPrefix; + } + + private String leftPad(int digits, String string, Character paddingChar) { + return String.format("%" + digits + "s", string).replace(' ', paddingChar); + } + + /** + * Generates an ID for the test entity. + * This does not guarantee uniqueness between entities and is merely a counter + */ + protected UUID generateUuid() { + String trailingUuid = leftPad(12, Integer.toString(this.currId), '0'); + UUID uuid = UUID.fromString(uuidPrefix + trailingUuid); + this.currId += 1; + return uuid; + } +} diff --git a/src/client/java/teammates/client/scripts/testdataconversion/package-info.java b/src/client/java/teammates/client/scripts/testdataconversion/package-info.java new file mode 100644 index 00000000000..181de6513fe --- /dev/null +++ b/src/client/java/teammates/client/scripts/testdataconversion/package-info.java @@ -0,0 +1,4 @@ +/** + * Contains classes to migrate noSQL test data. + */ +package teammates.client.scripts.testdataconversion; diff --git a/src/main/java/teammates/common/util/JsonUtils.java b/src/main/java/teammates/common/util/JsonUtils.java index f507441fe3b..7f121801e05 100644 --- a/src/main/java/teammates/common/util/JsonUtils.java +++ b/src/main/java/teammates/common/util/JsonUtils.java @@ -83,7 +83,7 @@ private static Gson getGsonInstance(boolean prettyPrint) { } /** - * This creates a Gson object that can be reformatted to modify JSON output + * This creates a Gson object that can be reformatted to modify JSON output. */ public static JsonObject toJsonObject(Object src) { return (JsonObject) getGsonInstance(true).toJsonTree(src); From 8fbc05c6c5d29644a6ec0295f008585581f07da6 Mon Sep 17 00:00:00 2001 From: nicolascwy <25302138+NicolasCwy@users.noreply.github.com> Date: Thu, 21 Mar 2024 23:07:00 +0800 Subject: [PATCH 08/14] Output SQL JSON in same folder as JSON --- .../ConvertDatastoreJsonToSqlJson.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java index 07b9c628d71..9cecfe24b88 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java +++ b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java @@ -43,10 +43,10 @@ public class ConvertDatastoreJsonToSqlJson { "recipientSection", "notification"}; - protected ConvertDatastoreJsonToSqlJson(String inputFilePath) throws IOException { + protected ConvertDatastoreJsonToSqlJson(File inputFile) throws IOException { this.entityConverter = new DataStoreToSqlConverter(); - File file = new File(inputFilePath); - this.dataStoreBundle = loadDataBundle(file.getCanonicalPath()); + + this.dataStoreBundle = loadDataBundle(inputFile.getCanonicalPath()); } private String removeWhitespace(String string) { @@ -203,9 +203,10 @@ private void migrateDependentEntities() { public static void main(String[] args) throws IOException, InvalidParametersException { if (args.length > 0) { - ConvertDatastoreJsonToSqlJson script = new ConvertDatastoreJsonToSqlJson(args[0]); + File inputFile = new File(args[0]); + ConvertDatastoreJsonToSqlJson script = new ConvertDatastoreJsonToSqlJson(inputFile); String outputFileName = "output.json"; - File outputFile = new File("./src/client/java/teammates/client/scripts/" + outputFileName); + File outputFile = new File(inputFile.getParent() + "/" + outputFileName); script.createSqlJson(outputFile); } else { throw new InvalidParametersException("Required the path of the script to convert"); From 0a6265e764eb47cbeeffc2d2bb9406c0c027511e Mon Sep 17 00:00:00 2001 From: Nicolas Chang <25302138+NicolasCwy@users.noreply.github.com> Date: Sat, 23 Mar 2024 07:57:37 +0800 Subject: [PATCH 09/14] Change output file name --- .../ConvertDatastoreJsonToSqlJson.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java index 9cecfe24b88..03935021e41 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java +++ b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java @@ -4,6 +4,8 @@ import java.io.IOException; import java.util.List; +import org.apache.commons.io.FilenameUtils; + import com.google.gson.JsonObject; import teammates.common.datatransfer.DataBundle; @@ -28,6 +30,8 @@ /** * Class to create JSON test data in SQL format from a noSQL JSON file. + * File can be run using the gradle execScript task and accepts a single argument which is the JSON path + * ./gradlew execScript -PuserScript="testdataconversion/ConvertDatastoreJsonToSqlJson" --args="JSON_FILE_PATH_HERE" */ public class ConvertDatastoreJsonToSqlJson { private DataStoreToSqlConverter entityConverter; @@ -132,8 +136,11 @@ private void migrateIndepedentEntities() { /** * Migrate entities which have dependence on each other or on the independent entities. - * feedback sessions, sections, teams, users, students, instructors, deadline extensions, feedback questions, - * read notifications, feedback responses and feedback response comments. + * The order which the entities were migrated was generated using a topological sort + * of its foreign key dependencies. + * Dependent entities: feedback sessions, sections, teams, users, students, instructors, + * deadline extensions, feedback questions, read notifications, + * feedback responses and feedback response comments. */ private void migrateDependentEntities() { @@ -204,9 +211,14 @@ private void migrateDependentEntities() { public static void main(String[] args) throws IOException, InvalidParametersException { if (args.length > 0) { File inputFile = new File(args[0]); + String fileExtension = FilenameUtils.getExtension(inputFile.getName()); + if (!fileExtension.equals("json")) { + throw new InvalidParametersException("The file provided is not a JSON file"); + } + ConvertDatastoreJsonToSqlJson script = new ConvertDatastoreJsonToSqlJson(inputFile); - String outputFileName = "output.json"; - File outputFile = new File(inputFile.getParent() + "/" + outputFileName); + String outputFileName = FilenameUtils.getBaseName(inputFile.getName()) + "Sql.json"; + File outputFile = new File(inputFile.getParent(), outputFileName); script.createSqlJson(outputFile); } else { throw new InvalidParametersException("Required the path of the script to convert"); From d23d0d0cbe14d73ac8a145e6b21243a93d8f2cc2 Mon Sep 17 00:00:00 2001 From: Nicolas Chang <25302138+NicolasCwy@users.noreply.github.com> Date: Sat, 23 Mar 2024 08:01:42 +0800 Subject: [PATCH 10/14] Fix bug: wrong jsonkey used --- .../testdataconversion/ConvertDatastoreJsonToSqlJson.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java index 03935021e41..1ee55af87e4 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java +++ b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java @@ -165,7 +165,7 @@ private void migrateDependentEntities() { if (!sqlDataBundle.teams.containsKey(jsonKey)) { Team sqlTeam = entityConverter.createTeam(student); - sqlDataBundle.teams.put(k, sqlTeam); + sqlDataBundle.teams.put(jsonKey, sqlTeam); } }); From 5d8a2f3042b6fc5b7c747fc84adbd67b51082b7e Mon Sep 17 00:00:00 2001 From: Nicolas Chang <25302138+NicolasCwy@users.noreply.github.com> Date: Sat, 23 Mar 2024 08:06:38 +0800 Subject: [PATCH 11/14] Fix lint error --- .../testdataconversion/ConvertDatastoreJsonToSqlJson.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java index 1ee55af87e4..db10704b7ce 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java +++ b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java @@ -212,10 +212,10 @@ public static void main(String[] args) throws IOException, InvalidParametersExce if (args.length > 0) { File inputFile = new File(args[0]); String fileExtension = FilenameUtils.getExtension(inputFile.getName()); - if (!fileExtension.equals("json")) { + if (!"json".equals(fileExtension)) { throw new InvalidParametersException("The file provided is not a JSON file"); } - + ConvertDatastoreJsonToSqlJson script = new ConvertDatastoreJsonToSqlJson(inputFile); String outputFileName = FilenameUtils.getBaseName(inputFile.getName()) + "Sql.json"; File outputFile = new File(inputFile.getParent(), outputFileName); From aa5d8255cb759c83b916178544a1e0e6c469201e Mon Sep 17 00:00:00 2001 From: Nicolas Chang <25302138+NicolasCwy@users.noreply.github.com> Date: Sat, 23 Mar 2024 08:23:50 +0800 Subject: [PATCH 12/14] Make section and team name unique --- .../testdataconversion/ConvertDatastoreJsonToSqlJson.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java index db10704b7ce..6d0a7844137 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java +++ b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java @@ -150,8 +150,8 @@ private void migrateDependentEntities() { }); dataStoreBundle.students.forEach((k, student) -> { - // Assumes that section name is unique in JSON file - String jsonKey = removeWhitespace(student.getSection()); + String jsonKey = removeWhitespace(String.format("%s-%s", + student.getCourse(), student.getSection())); if (!sqlDataBundle.sections.containsKey(jsonKey)) { Section sqlSection = entityConverter.createSection(student); @@ -160,8 +160,8 @@ private void migrateDependentEntities() { }); dataStoreBundle.students.forEach((k, student) -> { - // Assumes that team name is unique in JSON file - String jsonKey = removeWhitespace(student.getTeam()); + String jsonKey = removeWhitespace(String.format("%s-%s-%s", + student.getCourse(), student.getSection(), student.getTeam())); if (!sqlDataBundle.teams.containsKey(jsonKey)) { Team sqlTeam = entityConverter.createTeam(student); From 6c0ff703b2889d7e5477fd53c76ff1daf986ebc6 Mon Sep 17 00:00:00 2001 From: Nicolas Chang <25302138+NicolasCwy@users.noreply.github.com> Date: Sat, 23 Mar 2024 08:39:02 +0800 Subject: [PATCH 13/14] Set read notification key to be unique --- .../testdataconversion/ConvertDatastoreJsonToSqlJson.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java index 6d0a7844137..e705008232c 100644 --- a/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java +++ b/src/client/java/teammates/client/scripts/testdataconversion/ConvertDatastoreJsonToSqlJson.java @@ -192,7 +192,8 @@ private void migrateDependentEntities() { dataStoreBundle.accounts.forEach((k, account) -> { List sqlReadNotifications = entityConverter.createReadNotifications(account); sqlReadNotifications.forEach(notif -> { - String jsonKey = removeWhitespace(notif.getNotification().getTitle() + "-" + account.getName()); + String jsonKey = removeWhitespace(String.format("%s-%s", + notif.getNotification().getTitle(), account.getEmail())); sqlDataBundle.readNotifications.put(jsonKey, notif); }); }); From 12cdf02d4a0e926ee590e406666b0e72324434d1 Mon Sep 17 00:00:00 2001 From: Nicolas Chang <25302138+NicolasCwy@users.noreply.github.com> Date: Mon, 25 Mar 2024 16:54:29 +0800 Subject: [PATCH 14/14] Delete python file --- .../scripts/testdataconversion/toposort.py | 109 ------------------ 1 file changed, 109 deletions(-) delete mode 100644 src/client/java/teammates/client/scripts/testdataconversion/toposort.py diff --git a/src/client/java/teammates/client/scripts/testdataconversion/toposort.py b/src/client/java/teammates/client/scripts/testdataconversion/toposort.py deleted file mode 100644 index 3002225a509..00000000000 --- a/src/client/java/teammates/client/scripts/testdataconversion/toposort.py +++ /dev/null @@ -1,109 +0,0 @@ -#Python program to print topological sorting of a DAG -from collections import defaultdict - -#Class to represent a graph -class Graph: - def __init__(self,vertices): - self.graph = defaultdict(list) #dictionary containing adjacency List - self.V = vertices #No. of vertices - - # function to add an edge to graph - def addEdge(self,u,v): - self.graph[u].append(v) - - # A recursive function used by topologicalSort - def topologicalSortUtil(self,v,visited,stack): - - # Mark the current node as visited. - visited[v] = True - - # Recur for all the vertices adjacent to this vertex - for i in self.graph[v]: - if visited[i] == False: - self.topologicalSortUtil(i,visited,stack) - - # Push current vertex to stack which stores result - stack.insert(0,v) - - # The function to do Topological Sort. It uses recursive - # topologicalSortUtil() - def topologicalSort(self): - # Mark all the vertices as not visited - visited = [False]*self.V - stack =[] - - # Call the recursive helper function to store Topological - # Sort starting from all vertices one by one - for i in range(self.V): - if visited[i] == False: - self.topologicalSortUtil(i,visited,stack) - - # Print contents of stack - return stack.copy() -g = Graph(18) -course = 0 -feedback_session = 1 -user = 2 -section = 3 -team = 4 -student = 5 -instructor = 6 -deadline_extension = 7 -feedback_question = 8 -feedbackxxxquestion = 9 -feednackxxxresponse = 10 -account = 11 -readNotification = 12 -notification = 13 -feedback_response = 14 -feedback_response_comment = 15 -acc_req = 16 -usage_stats = 17 - -lookup = { - 0: "course", - 1: "feedback_session", - 2: "user", - 3: "section", - 4: "team", - 5:"student", - 6: "instructor", - 7: "deadline_extension", - 8: "feedback_question", - 9: "feedbackxxxquestion", - 10: "feednackxxxresponse", - 11: "account", - 12: "readNotification", - 13: "notification", - 14: "feedback_response", - 15: "feedback_response_comment", - 16: "acc_req", - 17: "usage_stats" -} - -# (x, y) where x is dependent on y, y has to be created to be used -dependencies = [ - (deadline_extension, user), - (deadline_extension, feedback_session), - (feedback_question, feedback_session), - (feedback_response_comment, feedback_response), - (feedback_response_comment, section), - (feedback_response, section), - (feedback_response, feedback_question), - (feedback_session, course), - (instructor, user), - (section, course), - (student, user), - (team, section), - (user, course), - (user, team), - (user, account), - (readNotification, account), - (readNotification, notification), -] - -for start, end in dependencies: - g.addEdge(start, end) - -print ("Following is a Topological Sort of the given graph") -print(list(map(lambda x: lookup[x], g.topologicalSort()))) \ No newline at end of file