Skip to content

Commit

Permalink
perf: io-parallelization (#7458)
Browse files Browse the repository at this point in the history
  • Loading branch information
@CurryYangxx
CurryYangxx authored May 30, 2024
1 parent 2898c2f commit 41911f0
Show file tree
Hide file tree
Showing 4 changed files with 127 additions and 130 deletions.
65 changes: 37 additions & 28 deletions packages/insomnia/src/sync/vcs/migrate-projects-into-organization.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,16 @@ import { Project, RemoteProject } from '../../models/project';
// which is now the remoteId for tracking the projects within an org

export const shouldMigrateProjectUnderOrganization = async () => {
const localProjectCount = await database.count<Project>(models.project.type, {
remoteId: null,
parentId: null,
_id: { $ne: models.project.SCRATCHPAD_PROJECT_ID },
});

const legacyRemoteProjectCount = await database.count<RemoteProject>(models.project.type, {
remoteId: { $ne: null },
parentId: null,
});
const [localProjectCount, legacyRemoteProjectCount] = await Promise.all([
database.count(models.project.type, {
remoteId: null,
parentId: null,
}),
database.count(models.project.type, {
remoteId: { $ne: null },
parentId: null,
}),
]);

return localProjectCount > 0 || legacyRemoteProjectCount > 0;
};
Expand All @@ -37,29 +37,38 @@ export const migrateProjectsIntoOrganization = async ({
personalOrganization: Organization;
}) => {
// Legacy remote projects without organizations
const legacyRemoteProjects = await database.find<RemoteProject>(models.project.type, {
remoteId: { $ne: null },
parentId: null,
});
// Local projects without organizations except scratchpad
const [legacyRemoteProjects, localProjects] = await Promise.all([
database.find<RemoteProject>(models.project.type, {
remoteId: { $ne: null },
parentId: null,
}),
database.find<Project>(models.project.type, {
remoteId: null,
parentId: null,
_id: { $ne: models.project.SCRATCHPAD_PROJECT_ID },
}),
]);

const updatePromises = [];
// Legacy remoteId should be orgId and legacy _id should be remoteId
for (const remoteProject of legacyRemoteProjects) {
await models.project.update(remoteProject, {
parentId: remoteProject.remoteId,
remoteId: remoteProject._id,
});
updatePromises.push(
models.project.update(remoteProject, {
parentId: remoteProject.remoteId,
remoteId: remoteProject._id,
})
);
}

// Local projects without organizations except scratchpad
const localProjects = await database.find<Project>(models.project.type, {
remoteId: null,
parentId: null,
_id: { $ne: models.project.SCRATCHPAD_PROJECT_ID },
});

// Assign all local projects to personal organization
for (const localProject of localProjects) {
await models.project.update(localProject, {
parentId: personalOrganization.id,
});
updatePromises.push(
models.project.update(localProject, {
parentId: personalOrganization.id,
})
);
}

await Promise.all(updatePromises);
};
1 change: 1 addition & 0 deletions packages/insomnia/src/sync/vcs/vcs.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,7 @@ export class VCS {
return this._queryCreateProject(rootDocumentId, name, teamId, teamProjectId, teamKeys.memberKeys);
}

// TODO: may be we can create another push function for initial push, so that we can reduce some api calls
async push({ teamId, teamProjectId }: { teamId: string; teamProjectId: string }) {
await this._getOrCreateRemoteBackendProject({ teamId, teamProjectId });
const branch = await this._getCurrentBranch();
Expand Down
80 changes: 28 additions & 52 deletions packages/insomnia/src/ui/routes/organization.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,41 +127,46 @@ function sortOrganizations(accountId: string, organizations: Organization[]): Or
];
}

export const indexLoader: LoaderFunction = async () => {
const { id: sessionId, accountId } = await userSession.getOrCreate();
if (sessionId) {
try {
const organizationsResult = await insomniaFetch<OrganizationsResponse | void>({
async function syncOrganization(sessionId: string, accountId: string) {
try {
const [organizationsResult, user, currentPlan] = await Promise.all([
insomniaFetch<OrganizationsResponse | void>({
method: 'GET',
path: '/v1/organizations',
sessionId,
});
const user = await insomniaFetch<UserProfileResponse | void>({
}),
insomniaFetch<UserProfileResponse | void>({
method: 'GET',
path: '/v1/user/profile',
sessionId,
});

const currentPlan = await insomniaFetch<CurrentPlan | void>({
}),
insomniaFetch<CurrentPlan | void>({
method: 'GET',
path: '/v1/billing/current-plan',
sessionId,
});
}),
]);

invariant(organizationsResult && organizationsResult.organizations, 'Failed to load organizations');
invariant(user && user.id, 'Failed to load user');
invariant(currentPlan && currentPlan.planId, 'Failed to load current plan');
invariant(organizationsResult && organizationsResult.organizations, 'Failed to load organizations');
invariant(user && user.id, 'Failed to load user');
invariant(currentPlan && currentPlan.planId, 'Failed to load current plan');

const { organizations } = organizationsResult;
const { organizations } = organizationsResult;

invariant(accountId, 'Account ID is not defined');
invariant(accountId, 'Account ID is not defined');

localStorage.setItem(`${accountId}:organizations`, JSON.stringify(sortOrganizations(accountId, organizations)));
localStorage.setItem(`${accountId}:user`, JSON.stringify(user));
localStorage.setItem(`${accountId}:currentPlan`, JSON.stringify(currentPlan));
} catch (error) {
console.log('Failed to load Organizations', error);
}
localStorage.setItem(`${accountId}:organizations`, JSON.stringify(sortOrganizations(accountId, organizations)));
localStorage.setItem(`${accountId}:user`, JSON.stringify(user));
localStorage.setItem(`${accountId}:currentPlan`, JSON.stringify(currentPlan));
} catch (error) {
console.log('Failed to load Organizations', error);
}
}

export const indexLoader: LoaderFunction = async () => {
const { id: sessionId, accountId } = await userSession.getOrCreate();
if (sessionId) {
await syncOrganization(sessionId, accountId);

const organizations = JSON.parse(localStorage.getItem(`${accountId}:organizations`) || '[]') as Organization[];
invariant(organizations, 'Failed to fetch organizations.');
Expand Down Expand Up @@ -214,36 +219,7 @@ export const syncOrganizationsAction: ActionFunction = async () => {
const { id: sessionId, accountId } = await userSession.getOrCreate();

if (sessionId) {
try {

const organizationsResult = await insomniaFetch<OrganizationsResponse | void>({
method: 'GET',
path: '/v1/organizations',
sessionId,
});

const user = await insomniaFetch<UserProfileResponse | void>({
method: 'GET',
path: '/v1/user/profile',
sessionId,
});

const currentPlan = await insomniaFetch<CurrentPlan | void>({
method: 'GET',
path: '/v1/billing/current-plan',
sessionId,
});

invariant(organizationsResult, 'Failed to load organizations');
invariant(user, 'Failed to load user');
invariant(currentPlan, 'Failed to load current plan');
invariant(accountId, 'Account ID is not defined');
localStorage.setItem(`${accountId}:organizations`, JSON.stringify(sortOrganizations(accountId, organizationsResult.organizations)));
localStorage.setItem(`${accountId}:user`, JSON.stringify(user));
localStorage.setItem(`${accountId}:currentPlan`, JSON.stringify(currentPlan));
} catch (error) {
console.log('Failed to load Organizations', error);
}
await syncOrganization(sessionId, accountId);
}

return null;
Expand Down
111 changes: 61 additions & 50 deletions packages/insomnia/src/ui/routes/project.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -320,21 +320,23 @@ async function getAllLocalFiles({
projectId: string;
}) {
const projectWorkspaces = await models.workspace.findByParentId(projectId);
const workspaceMetas = await database.find<WorkspaceMeta>(models.workspaceMeta.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
});
const apiSpecs = await database.find<ApiSpec>(models.apiSpec.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
});
const mockServers = await database.find<MockServer>(models.mockServer.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
});
const [workspaceMetas, apiSpecs, mockServers] = await Promise.all([
database.find<WorkspaceMeta>(models.workspaceMeta.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
}),
database.find<ApiSpec>(models.apiSpec.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
}),
database.find<MockServer>(models.mockServer.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
}),
]);

const files: InsomniaFile[] = projectWorkspaces.map(workspace => {
const apiSpec = apiSpecs.find(spec => spec.parentId === workspace._id);
Expand Down Expand Up @@ -421,9 +423,11 @@ async function getAllRemoteFiles({
invariant(remoteId, 'Project is not a remote project');
const vcs = VCSInstance();

const allPulledBackendProjectsForRemoteId = (await vcs.localBackendProjects()).filter(p => p.id === remoteId);
const [allPulledBackendProjectsForRemoteId, allFetchedRemoteBackendProjectsForRemoteId] = await Promise.all([
vcs.localBackendProjects().then(projects => projects.filter(p => p.id === remoteId)),
// Remote backend projects are fetched from the backend since they are not stored locally
const allFetchedRemoteBackendProjectsForRemoteId = await vcs.remoteBackendProjects({ teamId: organizationId, teamProjectId: remoteId });
vcs.remoteBackendProjects({ teamId: organizationId, teamProjectId: remoteId }),
]);

// Get all workspaces that are connected to backend projects and under the current project
const workspacesWithBackendProjects = await database.find<Workspace>(models.workspace.type, {
Expand Down Expand Up @@ -496,6 +500,37 @@ export const projectIdLoader: LoaderFunction = async ({ params }): Promise<Proje
};
};

interface LearningFeature {
active: boolean;
title: string;
message: string;
cta: string;
url: string;
}
const getLearningFeature = async (fallbackLearningFeature: LearningFeature) => {
let learningFeature = fallbackLearningFeature;
const lastFetchedString = window.localStorage.getItem('learning-feature-last-fetch');
const lastFetched = lastFetchedString ? parseInt(lastFetchedString, 10) : 0;
const oneDay = 86400000;
const hasOneDayPassedSinceLastFetch = (Date.now() - lastFetched) > oneDay;
const wasDismissed = window.localStorage.getItem('learning-feature-dismissed');
const wasNotDismissedAndOneDayHasPassed = !wasDismissed && hasOneDayPassedSinceLastFetch;
if (wasNotDismissedAndOneDayHasPassed) {
try {
learningFeature = await insomniaFetch<LearningFeature>({
method: 'GET',
path: '/insomnia-production-public-assets/inapp-learning.json',
origin: 'https://storage.googleapis.com',
sessionId: '',
});
window.localStorage.setItem('learning-feature-last-fetch', Date.now().toString());
} catch (err) {
console.log('Could not fetch learning feature data.');
}
}
return learningFeature;
};

export const loader: LoaderFunction = async ({
params,
}): Promise<ProjectLoaderData> => {
Expand Down Expand Up @@ -533,43 +568,19 @@ export const loader: LoaderFunction = async ({
const project = await models.project.getById(projectId);
invariant(project, `Project was not found ${projectId}`);

const localFiles = await getAllLocalFiles({ projectId });
const remoteFiles = await getAllRemoteFiles({ projectId, organizationId });
const files = [...localFiles, ...remoteFiles];
const [localFiles, remoteFiles, organizationProjects = [], learningFeature] = await Promise.all([
getAllLocalFiles({ projectId }),
getAllRemoteFiles({ projectId, organizationId }),
database.find<Project>(models.project.type, {
parentId: organizationId,
}),
getLearningFeature(fallbackLearningFeature),
]);

const organizationProjects = await database.find<Project>(models.project.type, {
parentId: organizationId,
}) || [];
const files = [...localFiles, ...remoteFiles];

const projects = sortProjects(organizationProjects);

let learningFeature = fallbackLearningFeature;
const lastFetchedString = window.localStorage.getItem('learning-feature-last-fetch');
const lastFetched = lastFetchedString ? parseInt(lastFetchedString, 10) : 0;
const oneDay = 86400000;
const hasOneDayPassedSinceLastFetch = (Date.now() - lastFetched) > oneDay;
const wasDismissed = window.localStorage.getItem('learning-feature-dismissed');
const wasNotDismissedAndOneDayHasPassed = !wasDismissed && hasOneDayPassedSinceLastFetch;
if (wasNotDismissedAndOneDayHasPassed) {
try {
learningFeature = await insomniaFetch<{
active: boolean;
title: string;
message: string;
cta: string;
url: string;
}>({
method: 'GET',
path: '/insomnia-production-public-assets/inapp-learning.json',
origin: 'https://storage.googleapis.com',
sessionId: '',
});
window.localStorage.setItem('learning-feature-last-fetch', Date.now().toString());
} catch (err) {
console.log('Could not fetch learning feature data.');
}
}

return {
files,
learningFeature,
Expand Down

0 comments on commit 41911f0

Please sign in to comment.