Skip to content

Commit

Permalink
Merge branch 'develop' into refactor/rodrigo-promise-await
Browse files Browse the repository at this point in the history
  • Loading branch information
@kodiakhq
kodiakhq[bot] committed Mar 20, 2023
2 parents fa42790 + 6d17b2b commit f54a487
Show file tree
Hide file tree
Showing 12 changed files with 263 additions and 349 deletions.
288 changes: 146 additions & 142 deletions apps/meteor/app/importer-csv/server/importer.js
View file
Expand Up @@ -55,165 +55,169 @@ export class CsvImporter extends Base {
};

zip.forEach((entry) => {
this.logger.debug(`Entry: ${entry.entryName}`);

// Ignore anything that has `__MACOSX` in it's name, as sadly these things seem to mess everything up
if (entry.entryName.indexOf('__MACOSX') > -1) {
this.logger.debug(`Ignoring the file: ${entry.entryName}`);
return increaseProgressCount();
}

// Directories are ignored, since they are "virtual" in a zip file
if (entry.isDirectory) {
this.logger.debug(`Ignoring the directory entry: ${entry.entryName}`);
return increaseProgressCount();
}
Promise.await(
(async () => {
this.logger.debug(`Entry: ${entry.entryName}`);

// Ignore anything that has `__MACOSX` in it's name, as sadly these things seem to mess everything up
if (entry.entryName.indexOf('__MACOSX') > -1) {
this.logger.debug(`Ignoring the file: ${entry.entryName}`);
return increaseProgressCount();
}

// Parse the channels
if (entry.entryName.toLowerCase() === 'channels.csv') {
super.updateProgress(ProgressStep.PREPARING_CHANNELS);
const parsedChannels = this.csvParser(entry.getData().toString());
channelsCount = parsedChannels.length;

for (const c of parsedChannels) {
const name = c[0].trim();
const id = getRoomId(name);
const creator = c[1].trim();
const isPrivate = c[2].trim().toLowerCase() === 'private';
const members = c[3]
.trim()
.split(';')
.map((m) => m.trim())
.filter((m) => m);

this.converter.addChannel({
importIds: [id],
u: {
_id: creator,
},
name,
users: members,
t: isPrivate ? 'p' : 'c',
});
}
// Directories are ignored, since they are "virtual" in a zip file
if (entry.isDirectory) {
this.logger.debug(`Ignoring the directory entry: ${entry.entryName}`);
return increaseProgressCount();
}

super.updateRecord({ 'count.channels': channelsCount });
return increaseProgressCount();
}
// Parse the channels
if (entry.entryName.toLowerCase() === 'channels.csv') {
super.updateProgress(ProgressStep.PREPARING_CHANNELS);
const parsedChannels = this.csvParser(entry.getData().toString());
channelsCount = parsedChannels.length;

for await (const c of parsedChannels) {
const name = c[0].trim();
const id = getRoomId(name);
const creator = c[1].trim();
const isPrivate = c[2].trim().toLowerCase() === 'private';
const members = c[3]
.trim()
.split(';')
.map((m) => m.trim())
.filter((m) => m);

await this.converter.addChannel({
importIds: [id],
u: {
_id: creator,
},
name,
users: members,
t: isPrivate ? 'p' : 'c',
});
}

// Parse the users
if (entry.entryName.toLowerCase() === 'users.csv') {
super.updateProgress(ProgressStep.PREPARING_USERS);
const parsedUsers = this.csvParser(entry.getData().toString());
usersCount = parsedUsers.length;

for (const u of parsedUsers) {
const username = u[0].trim();
availableUsernames.add(username);

const email = u[1].trim();
const name = u[2].trim();

this.converter.addUser({
importIds: [username],
emails: [email],
username,
name,
});
}
super.updateRecord({ 'count.channels': channelsCount });
return increaseProgressCount();
}

super.updateRecord({ 'count.users': usersCount });
return increaseProgressCount();
}
// Parse the users
if (entry.entryName.toLowerCase() === 'users.csv') {
super.updateProgress(ProgressStep.PREPARING_USERS);
const parsedUsers = this.csvParser(entry.getData().toString());
usersCount = parsedUsers.length;

// Parse the messages
if (entry.entryName.indexOf('/') > -1) {
if (this.progress.step !== ProgressStep.PREPARING_MESSAGES) {
super.updateProgress(ProgressStep.PREPARING_MESSAGES);
}
for (const u of parsedUsers) {
const username = u[0].trim();
availableUsernames.add(username);

const item = entry.entryName.split('/'); // random/messages.csv
const folderName = item[0]; // random
const email = u[1].trim();
const name = u[2].trim();

let msgs = [];
this.converter.addUser({
importIds: [username],
emails: [email],
username,
name,
});
}

try {
msgs = this.csvParser(entry.getData().toString());
} catch (e) {
this.logger.warn(`The file ${entry.entryName} contains invalid syntax`, e);
return increaseProgressCount();
}
super.updateRecord({ 'count.users': usersCount });
return increaseProgressCount();
}

let data;
const msgGroupData = item[1].split('.')[0]; // messages
let isDirect = false;

if (folderName.toLowerCase() === 'directmessages') {
isDirect = true;
data = msgs.map((m) => ({
username: m[0],
ts: m[2],
text: m[3],
otherUsername: m[1],
isDirect: true,
}));
} else {
data = msgs.map((m) => ({ username: m[0], ts: m[1], text: m[2] }));
}
// Parse the messages
if (entry.entryName.indexOf('/') > -1) {
if (this.progress.step !== ProgressStep.PREPARING_MESSAGES) {
super.updateProgress(ProgressStep.PREPARING_MESSAGES);
}

messagesCount += data.length;
const channelName = `${folderName}/${msgGroupData}`;
const item = entry.entryName.split('/'); // random/messages.csv
const folderName = item[0]; // random

super.updateRecord({ messagesstatus: channelName });
let msgs = [];

if (isDirect) {
for (const msg of data) {
const sourceId = [msg.username, msg.otherUsername].sort().join('/');
try {
msgs = this.csvParser(entry.getData().toString());
} catch (e) {
this.logger.warn(`The file ${entry.entryName} contains invalid syntax`, e);
return increaseProgressCount();
}

if (!dmRooms.has(sourceId)) {
this.converter.addChannel({
importIds: [sourceId],
users: [msg.username, msg.otherUsername],
t: 'd',
});
let data;
const msgGroupData = item[1].split('.')[0]; // messages
let isDirect = false;

if (folderName.toLowerCase() === 'directmessages') {
isDirect = true;
data = msgs.map((m) => ({
username: m[0],
ts: m[2],
text: m[3],
otherUsername: m[1],
isDirect: true,
}));
} else {
data = msgs.map((m) => ({ username: m[0], ts: m[1], text: m[2] }));
}

dmRooms.set(sourceId, true);
messagesCount += data.length;
const channelName = `${folderName}/${msgGroupData}`;

super.updateRecord({ messagesstatus: channelName });

if (isDirect) {
for await (const msg of data) {
const sourceId = [msg.username, msg.otherUsername].sort().join('/');

if (!dmRooms.has(sourceId)) {
await this.converter.addChannel({
importIds: [sourceId],
users: [msg.username, msg.otherUsername],
t: 'd',
});

dmRooms.set(sourceId, true);
}

const newMessage = {
rid: sourceId,
u: {
_id: msg.username,
},
ts: new Date(parseInt(msg.ts)),
msg: msg.text,
};

usedUsernames.add(msg.username);
usedUsernames.add(msg.otherUsername);
await this.converter.addMessage(newMessage);
}
} else {
const rid = getRoomId(folderName);

for await (const msg of data) {
const newMessage = {
rid,
u: {
_id: msg.username,
},
ts: new Date(parseInt(msg.ts)),
msg: msg.text,
};

usedUsernames.add(msg.username);
await this.converter.addMessage(newMessage);
}
}

const newMessage = {
rid: sourceId,
u: {
_id: msg.username,
},
ts: new Date(parseInt(msg.ts)),
msg: msg.text,
};

usedUsernames.add(msg.username);
usedUsernames.add(msg.otherUsername);
this.converter.addMessage(newMessage);
}
} else {
const rid = getRoomId(folderName);

for (const msg of data) {
const newMessage = {
rid,
u: {
_id: msg.username,
},
ts: new Date(parseInt(msg.ts)),
msg: msg.text,
};

usedUsernames.add(msg.username);
this.converter.addMessage(newMessage);
super.updateRecord({ 'count.messages': messagesCount, 'messagesstatus': null });
return increaseProgressCount();
}
}

super.updateRecord({ 'count.messages': messagesCount, 'messagesstatus': null });
return increaseProgressCount();
}
})(),
);

increaseProgressCount();
});
Expand Down
20 changes: 10 additions & 10 deletions apps/meteor/app/importer-hipchat-enterprise/server/importer.js
View file
Expand Up @@ -62,8 +62,8 @@ export class HipChatEnterpriseImporter extends Base {
super.updateProgress(ProgressStep.PREPARING_CHANNELS);
let count = 0;

for (const r of file) {
this.converter.addChannel({
for await (const r of file) {
await this.converter.addChannel({
u: {
_id: r.Room.owner,
},
Expand All @@ -88,7 +88,7 @@ export class HipChatEnterpriseImporter extends Base {
let count = 0;
const dmRooms = [];

for (const m of file) {
for await (const m of file) {
if (!m.PrivateUserMessage) {
continue;
}
Expand All @@ -104,7 +104,7 @@ export class HipChatEnterpriseImporter extends Base {
const users = [senderId, receiverId].sort();

if (!dmRooms[receiverId]) {
dmRooms[receiverId] = this.converter.findDMForImportedUsers(senderId, receiverId);
dmRooms[receiverId] = await this.converter.findDMForImportedUsers(senderId, receiverId);

if (!dmRooms[receiverId]) {
const room = {
Expand All @@ -113,15 +113,15 @@ export class HipChatEnterpriseImporter extends Base {
t: 'd',
ts: new Date(m.PrivateUserMessage.timestamp.split(' ')[0]),
};
this.converter.addChannel(room);
await this.converter.addChannel(room);
dmRooms[receiverId] = room;
}
}

const rid = dmRooms[receiverId].importIds[0];
const newMessage = this.convertImportedMessage(m.PrivateUserMessage, rid, 'private');
count++;
this.converter.addMessage(newMessage);
await this.converter.addMessage(newMessage);
}

return count;
Expand Down Expand Up @@ -199,23 +199,23 @@ export class HipChatEnterpriseImporter extends Base {

await this.loadTurndownService();

for (const m of file) {
for await (const m of file) {
if (m.UserMessage) {
const newMessage = this.convertImportedMessage(m.UserMessage, rid, 'user');
this.converter.addMessage(newMessage);
await this.converter.addMessage(newMessage);
count++;
} else if (m.NotificationMessage) {
const newMessage = this.convertImportedMessage(m.NotificationMessage, rid, 'notif');
newMessage.u._id = 'rocket.cat';
newMessage.alias = m.NotificationMessage.sender;

this.converter.addMessage(newMessage);
await this.converter.addMessage(newMessage);
count++;
} else if (m.TopicRoomMessage) {
const newMessage = this.convertImportedMessage(m.TopicRoomMessage, rid, 'topic');
newMessage.t = 'room_changed_topic';

this.converter.addMessage(newMessage);
await this.converter.addMessage(newMessage);
count++;
} else if (m.ArchiveRoomMessage) {
this.logger.warn('Archived Room Notification was ignored.');
Expand Down

0 comments on commit f54a487

Please sign in to comment.