Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

perf: io-parallelization [INS-3911] #7458

Merged
merged 3 commits into from
May 30, 2024
Merged

perf: io-parallelization [INS-3911] #7458

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2070f38
perf: io-parallelization
CurryYangxx May 28, 2024
d39a4e9
Merge branch 'develop' into perf/poor-network-improvement
CurryYangxx May 29, 2024
b2cb267
Merge branch 'develop' into perf/poor-network-improvement
CurryYangxx May 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 37 additions & 28 deletions packages/insomnia/src/sync/vcs/migrate-projects-into-organization.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,16 @@ import { Project, RemoteProject } from '../../models/project';
// which is now the remoteId for tracking the projects within an org

export const shouldMigrateProjectUnderOrganization = async () => {
const localProjectCount = await database.count<Project>(models.project.type, {
remoteId: null,
parentId: null,
_id: { $ne: models.project.SCRATCHPAD_PROJECT_ID },
});

const legacyRemoteProjectCount = await database.count<RemoteProject>(models.project.type, {
remoteId: { $ne: null },
parentId: null,
});
const [localProjectCount, legacyRemoteProjectCount] = await Promise.all([
database.count(models.project.type, {
remoteId: null,
parentId: null,
}),
database.count(models.project.type, {
remoteId: { $ne: null },
parentId: null,
}),
]);

return localProjectCount > 0 || legacyRemoteProjectCount > 0;
};
Expand All @@ -37,29 +37,38 @@ export const migrateProjectsIntoOrganization = async ({
personalOrganization: Organization;
}) => {
// Legacy remote projects without organizations
const legacyRemoteProjects = await database.find<RemoteProject>(models.project.type, {
remoteId: { $ne: null },
parentId: null,
});
// Local projects without organizations except scratchpad
const [legacyRemoteProjects, localProjects] = await Promise.all([
database.find<RemoteProject>(models.project.type, {
remoteId: { $ne: null },
parentId: null,
}),
database.find<Project>(models.project.type, {
remoteId: null,
parentId: null,
_id: { $ne: models.project.SCRATCHPAD_PROJECT_ID },
}),
]);

const updatePromises = [];
// Legacy remoteId should be orgId and legacy _id should be remoteId
for (const remoteProject of legacyRemoteProjects) {
await models.project.update(remoteProject, {
parentId: remoteProject.remoteId,
remoteId: remoteProject._id,
});
updatePromises.push(
models.project.update(remoteProject, {
parentId: remoteProject.remoteId,
remoteId: remoteProject._id,
})
);
}

// Local projects without organizations except scratchpad
const localProjects = await database.find<Project>(models.project.type, {
remoteId: null,
parentId: null,
_id: { $ne: models.project.SCRATCHPAD_PROJECT_ID },
});

// Assign all local projects to personal organization
for (const localProject of localProjects) {
await models.project.update(localProject, {
parentId: personalOrganization.id,
});
updatePromises.push(
models.project.update(localProject, {
parentId: personalOrganization.id,
})
);
}

await Promise.all(updatePromises);
};
1 change: 1 addition & 0 deletions packages/insomnia/src/sync/vcs/vcs.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,7 @@ export class VCS {
return this._queryCreateProject(rootDocumentId, name, teamId, teamProjectId, teamKeys.memberKeys);
}

// TODO: may be we can create another push function for initial push, so that we can reduce some api calls
async push({ teamId, teamProjectId }: { teamId: string; teamProjectId: string }) {
await this._getOrCreateRemoteBackendProject({ teamId, teamProjectId });
const branch = await this._getCurrentBranch();
Expand Down
80 changes: 28 additions & 52 deletions packages/insomnia/src/ui/routes/organization.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,41 +127,46 @@ function sortOrganizations(accountId: string, organizations: Organization[]): Or
];
}

export const indexLoader: LoaderFunction = async () => {
const { id: sessionId, accountId } = await userSession.getOrCreate();
if (sessionId) {
try {
const organizationsResult = await insomniaFetch<OrganizationsResponse | void>({
async function syncOrganization(sessionId: string, accountId: string) {
try {
const [organizationsResult, user, currentPlan] = await Promise.all([
insomniaFetch<OrganizationsResponse | void>({
method: 'GET',
path: '/v1/organizations',
sessionId,
});
const user = await insomniaFetch<UserProfileResponse | void>({
}),
insomniaFetch<UserProfileResponse | void>({
method: 'GET',
path: '/v1/user/profile',
sessionId,
});

const currentPlan = await insomniaFetch<CurrentPlan | void>({
}),
insomniaFetch<CurrentPlan | void>({
method: 'GET',
path: '/v1/billing/current-plan',
sessionId,
});
}),
]);

invariant(organizationsResult && organizationsResult.organizations, 'Failed to load organizations');
invariant(user && user.id, 'Failed to load user');
invariant(currentPlan && currentPlan.planId, 'Failed to load current plan');
invariant(organizationsResult && organizationsResult.organizations, 'Failed to load organizations');
invariant(user && user.id, 'Failed to load user');
invariant(currentPlan && currentPlan.planId, 'Failed to load current plan');

const { organizations } = organizationsResult;
const { organizations } = organizationsResult;

invariant(accountId, 'Account ID is not defined');
invariant(accountId, 'Account ID is not defined');

localStorage.setItem(`${accountId}:organizations`, JSON.stringify(sortOrganizations(accountId, organizations)));
localStorage.setItem(`${accountId}:user`, JSON.stringify(user));
localStorage.setItem(`${accountId}:currentPlan`, JSON.stringify(currentPlan));
} catch (error) {
console.log('Failed to load Organizations', error);
}
localStorage.setItem(`${accountId}:organizations`, JSON.stringify(sortOrganizations(accountId, organizations)));
localStorage.setItem(`${accountId}:user`, JSON.stringify(user));
localStorage.setItem(`${accountId}:currentPlan`, JSON.stringify(currentPlan));
} catch (error) {
console.log('Failed to load Organizations', error);
}
}

export const indexLoader: LoaderFunction = async () => {
const { id: sessionId, accountId } = await userSession.getOrCreate();
if (sessionId) {
await syncOrganization(sessionId, accountId);

const organizations = JSON.parse(localStorage.getItem(`${accountId}:organizations`) || '[]') as Organization[];
invariant(organizations, 'Failed to fetch organizations.');
Expand Down Expand Up @@ -214,36 +219,7 @@ export const syncOrganizationsAction: ActionFunction = async () => {
const { id: sessionId, accountId } = await userSession.getOrCreate();

if (sessionId) {
try {

const organizationsResult = await insomniaFetch<OrganizationsResponse | void>({
method: 'GET',
path: '/v1/organizations',
sessionId,
});

const user = await insomniaFetch<UserProfileResponse | void>({
method: 'GET',
path: '/v1/user/profile',
sessionId,
});

const currentPlan = await insomniaFetch<CurrentPlan | void>({
method: 'GET',
path: '/v1/billing/current-plan',
sessionId,
});

invariant(organizationsResult, 'Failed to load organizations');
invariant(user, 'Failed to load user');
invariant(currentPlan, 'Failed to load current plan');
invariant(accountId, 'Account ID is not defined');
localStorage.setItem(`${accountId}:organizations`, JSON.stringify(sortOrganizations(accountId, organizationsResult.organizations)));
localStorage.setItem(`${accountId}:user`, JSON.stringify(user));
localStorage.setItem(`${accountId}:currentPlan`, JSON.stringify(currentPlan));
} catch (error) {
console.log('Failed to load Organizations', error);
}
await syncOrganization(sessionId, accountId);
}

return null;
Expand Down
111 changes: 61 additions & 50 deletions packages/insomnia/src/ui/routes/project.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -320,21 +320,23 @@ async function getAllLocalFiles({
projectId: string;
}) {
const projectWorkspaces = await models.workspace.findByParentId(projectId);
const workspaceMetas = await database.find<WorkspaceMeta>(models.workspaceMeta.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
});
const apiSpecs = await database.find<ApiSpec>(models.apiSpec.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
});
const mockServers = await database.find<MockServer>(models.mockServer.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
});
const [workspaceMetas, apiSpecs, mockServers] = await Promise.all([
database.find<WorkspaceMeta>(models.workspaceMeta.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
}),
database.find<ApiSpec>(models.apiSpec.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
}),
database.find<MockServer>(models.mockServer.type, {
parentId: {
$in: projectWorkspaces.map(w => w._id),
},
}),
]);

const files: InsomniaFile[] = projectWorkspaces.map(workspace => {
const apiSpec = apiSpecs.find(spec => spec.parentId === workspace._id);
Expand Down Expand Up @@ -421,9 +423,11 @@ async function getAllRemoteFiles({
invariant(remoteId, 'Project is not a remote project');
const vcs = VCSInstance();

const allPulledBackendProjectsForRemoteId = (await vcs.localBackendProjects()).filter(p => p.id === remoteId);
const [allPulledBackendProjectsForRemoteId, allFetchedRemoteBackendProjectsForRemoteId] = await Promise.all([
vcs.localBackendProjects().then(projects => projects.filter(p => p.id === remoteId)),
// Remote backend projects are fetched from the backend since they are not stored locally
const allFetchedRemoteBackendProjectsForRemoteId = await vcs.remoteBackendProjects({ teamId: organizationId, teamProjectId: remoteId });
vcs.remoteBackendProjects({ teamId: organizationId, teamProjectId: remoteId }),
]);

// Get all workspaces that are connected to backend projects and under the current project
const workspacesWithBackendProjects = await database.find<Workspace>(models.workspace.type, {
Expand Down Expand Up @@ -496,6 +500,37 @@ export const projectIdLoader: LoaderFunction = async ({ params }): Promise<Proje
};
};

interface LearningFeature {
active: boolean;
title: string;
message: string;
cta: string;
url: string;
}
const getLearningFeature = async (fallbackLearningFeature: LearningFeature) => {
let learningFeature = fallbackLearningFeature;
const lastFetchedString = window.localStorage.getItem('learning-feature-last-fetch');
const lastFetched = lastFetchedString ? parseInt(lastFetchedString, 10) : 0;
const oneDay = 86400000;
const hasOneDayPassedSinceLastFetch = (Date.now() - lastFetched) > oneDay;
const wasDismissed = window.localStorage.getItem('learning-feature-dismissed');
const wasNotDismissedAndOneDayHasPassed = !wasDismissed && hasOneDayPassedSinceLastFetch;
if (wasNotDismissedAndOneDayHasPassed) {
try {
learningFeature = await insomniaFetch<LearningFeature>({
method: 'GET',
path: '/insomnia-production-public-assets/inapp-learning.json',
origin: 'https://storage.googleapis.com',
sessionId: '',
});
window.localStorage.setItem('learning-feature-last-fetch', Date.now().toString());
} catch (err) {
console.log('Could not fetch learning feature data.');
}
}
return learningFeature;
};

export const loader: LoaderFunction = async ({
params,
}): Promise<ProjectLoaderData> => {
Expand Down Expand Up @@ -533,43 +568,19 @@ export const loader: LoaderFunction = async ({
const project = await models.project.getById(projectId);
invariant(project, `Project was not found ${projectId}`);

const localFiles = await getAllLocalFiles({ projectId });
const remoteFiles = await getAllRemoteFiles({ projectId, organizationId });
const files = [...localFiles, ...remoteFiles];
const [localFiles, remoteFiles, organizationProjects = [], learningFeature] = await Promise.all([
getAllLocalFiles({ projectId }),
getAllRemoteFiles({ projectId, organizationId }),
database.find<Project>(models.project.type, {
parentId: organizationId,
}),
getLearningFeature(fallbackLearningFeature),
]);

const organizationProjects = await database.find<Project>(models.project.type, {
parentId: organizationId,
}) || [];
const files = [...localFiles, ...remoteFiles];

const projects = sortProjects(organizationProjects);

let learningFeature = fallbackLearningFeature;
const lastFetchedString = window.localStorage.getItem('learning-feature-last-fetch');
const lastFetched = lastFetchedString ? parseInt(lastFetchedString, 10) : 0;
const oneDay = 86400000;
const hasOneDayPassedSinceLastFetch = (Date.now() - lastFetched) > oneDay;
const wasDismissed = window.localStorage.getItem('learning-feature-dismissed');
const wasNotDismissedAndOneDayHasPassed = !wasDismissed && hasOneDayPassedSinceLastFetch;
if (wasNotDismissedAndOneDayHasPassed) {
try {
learningFeature = await insomniaFetch<{
active: boolean;
title: string;
message: string;
cta: string;
url: string;
}>({
method: 'GET',
path: '/insomnia-production-public-assets/inapp-learning.json',
origin: 'https://storage.googleapis.com',
sessionId: '',
});
window.localStorage.setItem('learning-feature-last-fetch', Date.now().toString());
} catch (err) {
console.log('Could not fetch learning feature data.');
}
}

return {
files,
learningFeature,
Expand Down
Loading