From 21bdbe9f841b53f0e8fd23ba1d8fc1b53acc18ed Mon Sep 17 00:00:00 2001 From: MoizAdnan Date: Thu, 5 Oct 2023 17:48:31 +0500 Subject: [PATCH] Updated project hooks --- .../src/common/services/ProjectService.ts | 2 +- .../schemas/projects/project-build.schema.ts | 3 +- .../project-github-push.class.ts | 2 +- .../project-permission.test.ts | 2 +- .../src/projects/project/project-helper.ts | 20 +- .../src/projects/project/project.class.ts | 357 +---------------- .../src/projects/project/project.hooks.ts | 374 +++++++++++++++++- .../src/projects/scene/scene.class.ts | 8 +- .../server-core/src/route/route/route.test.ts | 2 +- scripts/install-projects.js | 49 ++- scripts/push-project.ts | 2 +- scripts/update-project.ts | 2 +- 12 files changed, 417 insertions(+), 406 deletions(-) diff --git a/packages/client-core/src/common/services/ProjectService.ts b/packages/client-core/src/common/services/ProjectService.ts index f121a738827..415228e7707 100644 --- a/packages/client-core/src/common/services/ProjectService.ts +++ b/packages/client-core/src/common/services/ProjectService.ts @@ -127,7 +127,7 @@ export const ProjectService = { // restricted to admin scope uploadProject: async (data: ProjectBuildUpdateItemType) => { - const result = await API.instance.client.service(projectPath).update({ + const result = await API.instance.client.service(projectPath).update('', { sourceURL: data.sourceURL, destinationURL: data.destinationURL, name: data.name, diff --git a/packages/engine/src/schemas/projects/project-build.schema.ts b/packages/engine/src/schemas/projects/project-build.schema.ts index d0cf08d0c88..e0940dd4466 100644 --- a/packages/engine/src/schemas/projects/project-build.schema.ts +++ b/packages/engine/src/schemas/projects/project-build.schema.ts @@ -33,7 +33,8 @@ export const ProjectBuildUpdateItemSchema = Type.Object( sourceURL: Type.String(), destinationURL: Type.String(), name: Type.String(), - reset: Type.Boolean(), + needsRebuild: Type.Optional(Type.Boolean()), + reset: Type.Optional(Type.Boolean()), commitSHA: Type.String(), sourceBranch: Type.String(), updateType: StringEnum(projectUpdateTypes), diff --git a/packages/server-core/src/projects/project-github-push/project-github-push.class.ts b/packages/server-core/src/projects/project-github-push/project-github-push.class.ts index 388875cde53..e7f8f41a261 100644 --- a/packages/server-core/src/projects/project-github-push/project-github-push.class.ts +++ b/packages/server-core/src/projects/project-github-push/project-github-push.class.ts @@ -32,7 +32,7 @@ export class ProjectGithubPushService implements ServiceInterface { } async patch(id: NullableId, data: any, params?: RootParams): Promise { - const project = await this.app.service(projectPath)._get(id!) + const project = await this.app.service(projectPath).get(id!) return pushProjectToGithub(this.app, project, params!.user!) } } diff --git a/packages/server-core/src/projects/project-permission/project-permission.test.ts b/packages/server-core/src/projects/project-permission/project-permission.test.ts index c296b2692f2..e9cd7bb7594 100644 --- a/packages/server-core/src/projects/project-permission/project-permission.test.ts +++ b/packages/server-core/src/projects/project-permission/project-permission.test.ts @@ -48,7 +48,7 @@ const cleanup = async (app: Application) => { const project1Dir = path.resolve(appRootPath.path, `packages/projects/projects/${newProjectName1}/`) deleteFolderRecursive(project1Dir) try { - await app.service(projectPath)._remove(null, { query: { name: newProjectName1 } }) + await app.service(projectPath).remove(null, { query: { name: newProjectName1 } }) } catch (e) { // } diff --git a/packages/server-core/src/projects/project/project-helper.ts b/packages/server-core/src/projects/project/project-helper.ts index 8d6beb27d1d..80756e2b987 100644 --- a/packages/server-core/src/projects/project/project-helper.ts +++ b/packages/server-core/src/projects/project/project-helper.ts @@ -130,7 +130,7 @@ export const updateBuilder = async ( } if (data.updateProjects) { - await Promise.all(data.projectsToUpdate.map((project) => app.service(projectPath).update(project, null, params))) + await Promise.all(data.projectsToUpdate.map((project) => app.service(projectPath).update('', project, params))) } const helmSettingsResult = await app.service(helmSettingPath).find() @@ -1190,7 +1190,7 @@ export async function getDirectoryArchiveJobBody( } export const createOrUpdateProjectUpdateJob = async (app: Application, projectName: string): Promise => { - const projectData = (await app.service(projectPath)._find({ + const projectData = (await app.service(projectPath).find({ query: { name: projectName, $limit: 1 @@ -1246,7 +1246,7 @@ export const removeProjectUpdateJob = async (app: Application, projectName: stri export const checkProjectAutoUpdate = async (app: Application, projectName: string): Promise => { let commitSHA - const projectData = (await app.service(projectPath)._find({ + const projectData = (await app.service(projectPath).find({ query: { name: projectName, $limit: 1 @@ -1275,6 +1275,7 @@ export const checkProjectAutoUpdate = async (app: Application, projectName: stri } if (commitSHA) await app.service(projectPath).update( + '', { sourceURL: project.sourceRepo!, destinationURL: project.repositoryPath, @@ -1285,7 +1286,6 @@ export const checkProjectAutoUpdate = async (app: Application, projectName: stri updateType: project.updateType, updateSchedule: project.updateSchedule! }, - null, { user: user } ) } @@ -1413,7 +1413,7 @@ export const updateProject = async ( deleteFolderRecursive(projectDirectory) } - const projectResult = (await app.service(projectPath)._find({ + const projectResult = (await app.service(projectPath).find({ query: { name: projectName } @@ -1463,7 +1463,7 @@ export const updateProject = async ( const projectConfig = getProjectConfig(projectName) ?? {} // when we have successfully re-installed the project, remove the database entry if it already exists - const existingProjectResult = (await app.service(projectPath)._find({ + const existingProjectResult = (await app.service(projectPath).find({ query: { name: { $like: projectName @@ -1480,7 +1480,7 @@ export const updateProject = async ( const returned = !existingProject ? // Add to DB - await app.service(projectPath)._create( + await app.service(projectPath).create( { id: v4(), name: projectName, @@ -1498,7 +1498,7 @@ export const updateProject = async ( }, params || {} ) - : await app.service(projectPath)._patch(existingProject.id, { + : await app.service(projectPath).patch(existingProject.id, { commitSHA, commitDate: toDateTimeSql(commitDate), sourceRepo: data.sourceURL, @@ -1518,7 +1518,7 @@ export const updateProject = async ( } if (returned.name !== projectName) - await app.service(projectPath)._patch(existingProject!.id, { + await app.service(projectPath).patch(existingProject!.id, { name: projectName }) @@ -1529,7 +1529,7 @@ export const updateProject = async ( await git.raw(['lfs', 'fetch', '--all']) await git.push('destination', branchName, ['-f', '--tags']) const { commitSHA, commitDate } = await getCommitSHADate(projectName) - await app.service(projectPath)._patch(returned.id, { + await app.service(projectPath).patch(returned.id, { commitSHA, commitDate: toDateTimeSql(commitDate) }) diff --git a/packages/server-core/src/projects/project/project.class.ts b/packages/server-core/src/projects/project/project.class.ts index f18350f1ba1..3bf70203579 100644 --- a/packages/server-core/src/projects/project/project.class.ts +++ b/packages/server-core/src/projects/project/project.class.ts @@ -23,61 +23,33 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ -import { BadRequest, Forbidden } from '@feathersjs/errors' -import { Id, NullableId, Paginated, Params } from '@feathersjs/feathers' +import { Params } from '@feathersjs/feathers' import appRootPath from 'app-root-path' import fs from 'fs' import path from 'path' -import { GITHUB_URL_REGEX } from '@etherealengine/common/src/constants/GitHubConstants' import { DefaultUpdateSchedule } from '@etherealengine/common/src/interfaces/ProjectPackageJsonType' -import { routePath } from '@etherealengine/engine/src/schemas/route/route.schema' -import { locationPath } from '@etherealengine/engine/src/schemas/social/location.schema' -import { AvatarType, avatarPath } from '@etherealengine/engine/src/schemas/user/avatar.schema' -import { - GithubRepoAccessType, - githubRepoAccessPath -} from '@etherealengine/engine/src/schemas/user/github-repo-access.schema' -import templateProjectJson from '@etherealengine/projects/template-project/package.json' -import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' -import { projectPermissionPath } from '@etherealengine/engine/src/schemas/projects/project-permission.schema' +import { ProjectBuildUpdateItemType } from '@etherealengine/engine/src/schemas/projects/project-build.schema' import { ProjectData, ProjectPatch, ProjectQuery, - ProjectType, - projectPath + ProjectType } from '@etherealengine/engine/src/schemas/projects/project.schema' -import { - IdentityProviderType, - identityProviderPath -} from '@etherealengine/engine/src/schemas/user/identity-provider.schema' import { UserType } from '@etherealengine/engine/src/schemas/user/user.schema' -import { KnexAdapter, KnexAdapterOptions } from '@feathersjs/knex' -import { Knex } from 'knex' +import { KnexAdapterOptions, KnexService } from '@feathersjs/knex' import { v4 } from 'uuid' import { Application } from '../../../declarations' import logger from '../../ServerLogger' import { RootParams } from '../../api/root-params' -import config from '../../appconfig' -import { cleanString } from '../../util/cleanString' import { getDateTimeSql, toDateTimeSql } from '../../util/datetime-sql' -import { copyFolderRecursiveSync } from '../../util/fsHelperFunctions' -import { useGit } from '../../util/gitHelperFunctions' -import { checkAppOrgStatus, checkUserOrgWriteStatus, checkUserRepoWriteStatus } from './github-helper' import { - createExecutorJob, deleteProjectFilesInStorageProvider, getCommitSHADate, - getEnginePackageJson, getGitProjectData, getProjectConfig, - getProjectPackageJson, - getProjectUpdateJobBody, onProjectEvent, - removeProjectUpdateJob, - updateProject, uploadLocalProjectToProvider } from './project-helper' @@ -96,9 +68,9 @@ export interface ProjectParams extends RootParams, ProjectUpdatePa export type ProjectParamsClient = Omit -export class ProjectService extends KnexAdapter< +export class ProjectService extends KnexService< ProjectType, - ProjectData, + ProjectData | ProjectBuildUpdateItemType, ProjectParams, ProjectPatch > { @@ -111,323 +83,6 @@ export class ProjectService this._callOnLoad()) } - async create(data: ProjectData, params?: ProjectParams) { - const projectName = cleanString(data.name!) - const projectLocalDirectory = path.resolve(projectsRootFolder, projectName) - - const projectExists = (await this._find({ query: { name: projectName, $limit: 1 } })) as Paginated - - if (projectExists.total > 0) throw new Error(`[Projects]: Project with name ${projectName} already exists`) - - if ((!config.db.forceRefresh && projectName === 'default-project') || projectName === 'template-project') - throw new Error(`[Projects]: Project name ${projectName} not allowed`) - - copyFolderRecursiveSync(templateFolderDirectory, projectsRootFolder) - fs.renameSync(path.resolve(projectsRootFolder, 'template-project'), projectLocalDirectory) - - fs.mkdirSync(path.resolve(projectLocalDirectory, '.git'), { recursive: true }) - - const git = useGit(path.resolve(projectLocalDirectory, '.git')) - try { - await git.init(true) - } catch (e) { - logger.warn(e) - } - - const packageData = Object.assign({}, templateProjectJson) as any - packageData.name = projectName - packageData.etherealEngine.version = getEnginePackageJson().version - fs.writeFileSync(path.resolve(projectLocalDirectory, 'package.json'), JSON.stringify(packageData, null, 2)) - - await uploadLocalProjectToProvider(this.app, projectName, false) - - return super._create( - { - id: v4(), - name: projectName, - needsRebuild: true, - createdAt: await getDateTimeSql(), - updatedAt: await getDateTimeSql() - }, - params - ) - } - - /** - * 1. Clones the repo to the local FS - * 2. If in production mode, uploads it to the storage provider - * 3. Creates a database entry - * @param data - * @param placeholder This is where data normally goes, but we've put data as the first parameter - * @param params - * @returns - */ - // @ts-ignore - async update( - data: { - sourceURL: string - destinationURL: string - name: string - needsRebuild?: boolean - reset?: boolean - commitSHA?: string - sourceBranch: string - updateType: ProjectType['updateType'] - updateSchedule: string - }, - placeholder?: null, - params?: ProjectParams - ) { - if (!config.kubernetes.enabled || params?.isJob) return updateProject(this.app, data, params) - else { - const urlParts = data.sourceURL.split('/') - let projectName = data.name || urlParts.pop() - if (!projectName) throw new Error('Git repo must be plain URL') - projectName = projectName.toLowerCase() - if (projectName.substring(projectName.length - 4) === '.git') projectName = projectName.slice(0, -4) - if (projectName.substring(projectName.length - 1) === '/') projectName = projectName.slice(0, -1) - const jobBody = await getProjectUpdateJobBody(data, this.app, params!.user!.id) - const jobLabelSelector = `etherealengine/projectField=${data.name},etherealengine/release=${process.env.RELEASE_NAME},etherealengine/autoUpdate=false` - const jobFinishedPromise = createExecutorJob(this.app, jobBody, jobLabelSelector, 1000) - try { - await jobFinishedPromise - const result = (await super._find({ - query: { - name: { - $like: projectName - } - } - })) as Paginated - let returned = {} as ProjectType - if (result.total > 0) returned = result.data[0] - else throw new BadRequest('Project did not exist after update') - returned.needsRebuild = typeof data.needsRebuild === 'boolean' ? data.needsRebuild : true - return returned - } catch (err) { - console.log('Error: project did not exist after completing update', projectName, err) - throw err - } - } - } - - async get(id: Id, params?: ProjectParams) { - return super._get(id, params) - } - - async patch(id: NullableId, data: ProjectPatch, params?: ProjectParams) { - if (data.repositoryPath) { - const repoPath = data.repositoryPath - const user = params!.user! - - const githubIdentityProvider = (await this.app.service(identityProviderPath).find({ - query: { - userId: user.id, - type: 'github', - $limit: 1 - } - })) as Paginated - - const githubPathRegexExec = GITHUB_URL_REGEX.exec(repoPath) - if (!githubPathRegexExec) throw new BadRequest('Invalid Github URL') - if (githubIdentityProvider.data.length === 0) - throw new Error('Must be logged in with GitHub to link a project to a GitHub repo') - const split = githubPathRegexExec[2].split('/') - const org = split[0] - const repo = split[1].replace('.git', '') - const appOrgAccess = await checkAppOrgStatus(org, githubIdentityProvider.data[0].oauthToken) - if (!appOrgAccess) - throw new Forbidden( - `The organization ${org} needs to install the GitHub OAuth app ${config.authentication.oauth.github.key} in order to push code to its repositories` - ) - const repoWriteStatus = await checkUserRepoWriteStatus(org, repo, githubIdentityProvider.data[0].oauthToken) - if (repoWriteStatus !== 200) { - if (repoWriteStatus === 404) { - const orgWriteStatus = await checkUserOrgWriteStatus(org, githubIdentityProvider.data[0].oauthToken) - if (orgWriteStatus !== 200) throw new Forbidden('You do not have write access to that organization') - } else { - throw new Forbidden('You do not have write access to that repo') - } - } - } - return super._patch(id, data, params) - } - - async remove(id: Id, params?: ProjectParams) { - if (!id) return - const { name } = await super._get(id, params) - - const projectConfig = getProjectConfig(name) - - // run project uninstall script - if (projectConfig?.onEvent) { - await onProjectEvent(this.app, name, projectConfig.onEvent, 'onUninstall') - } - - if (fs.existsSync(path.resolve(projectsRootFolder, name))) { - fs.rmSync(path.resolve(projectsRootFolder, name), { recursive: true }) - } - - logger.info(`[Projects]: removing project id "${id}", name: "${name}".`) - await deleteProjectFilesInStorageProvider(name) - - await this.app.service(locationPath).remove(null, { - query: { - sceneId: { - $like: `${name}/%` - } - } - }) - - await this.app.service(routePath).remove(null, { - query: { - project: name - } - }) - - const avatarItems = (await this.app.service(avatarPath).find({ - query: { - project: name - }, - paginate: false - })) as AvatarType[] - - await Promise.all( - avatarItems.map(async (avatar) => { - await this.app.service(avatarPath).remove(avatar.id) - }) - ) - - const staticResourceItems = (await this.app.service(staticResourcePath).find({ - query: { - project: name - }, - paginate: false - })) as StaticResourceType[] - staticResourceItems.length && - staticResourceItems.forEach(async (staticResource) => { - await this.app.service(staticResourcePath).remove(staticResource.id) - }) - - await removeProjectUpdateJob(this.app, name) - - return super._remove(id, params) - } - - async find(params?: ProjectParams) { - let projectPushIds: string[] = [] - if (params?.query?.allowed) { - // See if the user has a GitHub identity-provider, and if they do, also determine which GitHub repos they personally - // can push to. - - const githubIdentityProvider = (await this.app.service(identityProviderPath).find({ - query: { - userId: params.user!.id, - type: 'github', - $limit: 1 - } - })) as Paginated - - // Get all of the projects that this user has permissions for, then calculate push status by whether the user - // can push to it. This will make sure no one tries to push to a repo that they do not have write access to. - const knexClient: Knex = this.app.get('knexClient') - const projectPermissions = await knexClient - .from(projectPermissionPath) - .join(projectPath, `${projectPath}.id`, `${projectPermissionPath}.projectId`) - .where(`${projectPermissionPath}.userId`, params.user!.id) - .select() - .options({ nestTables: true }) - - const allowedProjects = await projectPermissions.map((permission) => permission.project) - const repoAccess = - githubIdentityProvider.data.length > 0 - ? ((await this.app.service(githubRepoAccessPath).find({ - query: { - identityProviderId: githubIdentityProvider.data[0].id - }, - paginate: false - })) as any as GithubRepoAccessType[]) - : [] - const pushRepoPaths = repoAccess.filter((repo) => repo.hasWriteAccess).map((item) => item.repo.toLowerCase()) - let allowedProjectGithubRepos = allowedProjects.filter((project) => project.repositoryPath != null) - allowedProjectGithubRepos = await Promise.all( - allowedProjectGithubRepos.map(async (project) => { - const regexExec = GITHUB_URL_REGEX.exec(project.repositoryPath) - if (!regexExec) return { repositoryPath: '', name: '' } - const split = regexExec[2].split('/') - project.repositoryPath = `https://github.com/${split[0]}/${split[1]}` - return project - }) - ) - const pushableAllowedProjects = allowedProjectGithubRepos.filter( - (project) => pushRepoPaths.indexOf(project.repositoryPath.toLowerCase().replace(/.git$/, '')) > -1 - ) - projectPushIds = projectPushIds.concat(pushableAllowedProjects.map((project) => project.id)) - - if (githubIdentityProvider) { - const repositoryPaths: string[] = [] - repoAccess.forEach((item) => { - if (item.hasWriteAccess) { - const url = item.repo.toLowerCase() - repositoryPaths.push(url) - repositoryPaths.push(`${url}.git`) - const regexExec = GITHUB_URL_REGEX.exec(url) - if (regexExec) { - const split = regexExec[2].split('/') - repositoryPaths.push(`git@github.com:${split[0]}/${split[1]}`) - repositoryPaths.push(`git@github.com:${split[0]}/${split[1]}.git`) - } - } - }) - - const matchingAllowedRepos = (await super._find({ - query: { repositoryPath: { $in: repositoryPaths } }, - paginate: false - })) as ProjectType[] - - projectPushIds = projectPushIds.concat(matchingAllowedRepos.map((repo) => repo.id)) - } - - if (!params.user!.scopes?.find((scope) => scope.type === 'admin:admin')) - params.query.id = { $in: [...new Set(allowedProjects.map((project) => project.id))] } - } - - let paramsWithoutExtras = { - ...params, - // Explicitly cloned sort object because otherwise it was affecting default params object as well. - query: params?.query ? JSON.parse(JSON.stringify(params?.query)) : {} - } - - paramsWithoutExtras = { - ...paramsWithoutExtras, - query: { - ...paramsWithoutExtras.query, - $limit: params?.query?.$limit || 1000, - $sort: params?.query?.$sort || { name: 1 } - } - } - - if (paramsWithoutExtras?.query?.allowed) delete paramsWithoutExtras.query.allowed - - const result = (await super._find(paramsWithoutExtras)) as Paginated | ProjectType[] - - const data: ProjectType[] = result['data'] ? result['data'] : result - for (const item of data) { - try { - const packageJson = getProjectPackageJson(item.name) - const config = getProjectConfig(item.name) - item.thumbnail = config.thumbnail! - item.version = packageJson.version - item.engineVersion = packageJson.etherealEngine?.version - item.description = packageJson.description - item.hasWriteAccess = projectPushIds.indexOf(item.id) > -1 - } catch (err) { - // - } - } - - return result - } - async _callOnLoad() { const projects = (await super._find({ query: { $select: ['name'] }, diff --git a/packages/server-core/src/projects/project/project.hooks.ts b/packages/server-core/src/projects/project/project.hooks.ts index b7becba4690..fc69c472de3 100644 --- a/packages/server-core/src/projects/project/project.hooks.ts +++ b/packages/server-core/src/projects/project/project.hooks.ts @@ -27,16 +27,56 @@ import { iff, isProvider } from 'feathers-hooks-common' import { projectPermissionPath } from '@etherealengine/engine/src/schemas/projects/project-permission.schema' import { + ProjectType, projectDataValidator, projectPatchValidator, + projectPath, projectQueryValidator } from '@etherealengine/engine/src/schemas/projects/project.schema' +import fs from 'fs' +import path from 'path' import authenticate from '../../hooks/authenticate' import projectPermissionAuthenticate from '../../hooks/project-permission-authenticate' import verifyScope from '../../hooks/verify-scope' import { projectPermissionDataResolver } from '../project-permission/project-permission.resolvers' -import { HookContext } from '@feathersjs/feathers' +import { GITHUB_URL_REGEX } from '@etherealengine/common/src/constants/GitHubConstants' +import { StaticResourceType, staticResourcePath } from '@etherealengine/engine/src/schemas/media/static-resource.schema' +import { routePath } from '@etherealengine/engine/src/schemas/route/route.schema' +import { locationPath } from '@etherealengine/engine/src/schemas/social/location.schema' +import { AvatarType, avatarPath } from '@etherealengine/engine/src/schemas/user/avatar.schema' +import { + GithubRepoAccessType, + githubRepoAccessPath +} from '@etherealengine/engine/src/schemas/user/github-repo-access.schema' +import { + IdentityProviderType, + identityProviderPath +} from '@etherealengine/engine/src/schemas/user/identity-provider.schema' +import templateProjectJson from '@etherealengine/projects/template-project/package.json' +import { BadRequest, Forbidden } from '@feathersjs/errors' +import { HookContext, Paginated } from '@feathersjs/feathers' +import appRootPath from 'app-root-path' +import { Knex } from 'knex' +import logger from '../../ServerLogger' +import config from '../../appconfig' +import enableClientPagination from '../../hooks/enable-client-pagination' +import { cleanString } from '../../util/cleanString' +import { copyFolderRecursiveSync } from '../../util/fsHelperFunctions' +import { useGit } from '../../util/gitHelperFunctions' +import { checkAppOrgStatus, checkUserOrgWriteStatus, checkUserRepoWriteStatus } from './github-helper' +import { + createExecutorJob, + deleteProjectFilesInStorageProvider, + getEnginePackageJson, + getProjectConfig, + getProjectPackageJson, + getProjectUpdateJobBody, + onProjectEvent, + removeProjectUpdateJob, + updateProject, + uploadLocalProjectToProvider +} from './project-helper' import { projectDataResolver, projectExternalResolver, @@ -45,6 +85,219 @@ import { projectResolver } from './project.resolvers' +const templateFolderDirectory = path.join(appRootPath.path, `packages/projects/template-project/`) + +const projectsRootFolder = path.join(appRootPath.path, 'packages/projects/projects/') + +const ensurePushStatus = async (context: HookContext) => { + context.projectPushIds = [] + if (context.params?.query?.allowed) { + // See if the user has a GitHub identity-provider, and if they do, also determine which GitHub repos they personally + // can push to. + + const githubIdentityProvider = (await context.app.service(identityProviderPath).find({ + query: { + userId: context.params.user!.id, + type: 'github', + $limit: 1 + } + })) as Paginated + + // Get all of the projects that this user has permissions for, then calculate push status by whether the user + // can push to it. This will make sure no one tries to push to a repo that they do not have write access to. + const knexClient: Knex = context.app.get('knexClient') + const projectPermissions = await knexClient + .from(projectPermissionPath) + .join(projectPath, `${projectPath}.id`, `${projectPermissionPath}.projectId`) + .where(`${projectPermissionPath}.userId`, context.params.user!.id) + .select() + .options({ nestTables: true }) + + const allowedProjects = await projectPermissions.map((permission) => permission.project) + const repoAccess = + githubIdentityProvider.data.length > 0 + ? ((await context.app.service(githubRepoAccessPath).find({ + query: { + identityProviderId: githubIdentityProvider.data[0].id + }, + paginate: false + })) as GithubRepoAccessType[]) + : [] + const pushRepoPaths = repoAccess.filter((repo) => repo.hasWriteAccess).map((item) => item.repo.toLowerCase()) + let allowedProjectGithubRepos = allowedProjects.filter((project) => project.repositoryPath != null) + allowedProjectGithubRepos = await Promise.all( + allowedProjectGithubRepos.map(async (project) => { + const regexExec = GITHUB_URL_REGEX.exec(project.repositoryPath) + if (!regexExec) return { repositoryPath: '', name: '' } + const split = regexExec[2].split('/') + project.repositoryPath = `https://github.com/${split[0]}/${split[1]}` + return project + }) + ) + const pushableAllowedProjects = allowedProjectGithubRepos.filter( + (project) => pushRepoPaths.indexOf(project.repositoryPath.toLowerCase().replace(/.git$/, '')) > -1 + ) + context.projectPushIds = context.projectPushIds.concat(pushableAllowedProjects.map((project) => project.id)) + + if (githubIdentityProvider) { + const repositoryPaths: string[] = [] + repoAccess.forEach((item) => { + if (item.hasWriteAccess) { + const url = item.repo.toLowerCase() + repositoryPaths.push(url) + repositoryPaths.push(`${url}.git`) + const regexExec = GITHUB_URL_REGEX.exec(url) + if (regexExec) { + const split = regexExec[2].split('/') + repositoryPaths.push(`git@github.com:${split[0]}/${split[1]}`) + repositoryPaths.push(`git@github.com:${split[0]}/${split[1]}.git`) + } + } + }) + + const matchingAllowedRepos = (await context.app.service(projectPath).find({ + query: { repositoryPath: { $in: repositoryPaths } }, + paginate: false + })) as ProjectType[] + + context.projectPushIds = context.projectPushIds.concat(matchingAllowedRepos.map((repo) => repo.id)) + } + + if (!context.params.user!.scopes?.find((scope) => scope.type === 'admin:admin')) + context.params.query.id = { $in: [...new Set(allowedProjects.map((project) => project.id))] } + } +} + +const addLimitToParams = async (context: HookContext) => { + context.params.query = { + ...context.params.query, + $limit: context.params?.query?.$limit || 1000, + $sort: context.params?.query?.$sort || { name: 1 } + } + if (context.params?.query?.allowed) delete context.params.query.allowed +} + +const addDataToProjectResult = async (context: HookContext) => { + const data: ProjectType[] = context.result['data'] ? context.result['data'] : context.result + for (const item of data) { + try { + const packageJson = getProjectPackageJson(item.name) + const config = getProjectConfig(item.name) + item.thumbnail = config.thumbnail! + item.version = packageJson.version + item.engineVersion = packageJson.etherealEngine?.version + item.description = packageJson.description + item.hasWriteAccess = context.projectPushIds.indexOf(item.id) > -1 + } catch (err) { + // + } + } + + context.result = + context.params.paginate === false + ? data + : { + data: data, + total: data.length, + limit: context.params.query.$limit || 1000, + skip: context.params.query.$skip || 0 + } +} + +const createAndUploadProject = async (context: HookContext) => { + const projectName = cleanString(context.data.name!) + const projectLocalDirectory = path.resolve(projectsRootFolder, projectName) + + const projectExists = (await context.app + .service(projectPath) + .find({ query: { name: projectName, $limit: 1 } })) as Paginated + + if (projectExists.total > 0) throw new Error(`[Projects]: Project with name ${projectName} already exists`) + + if ((!config.db.forceRefresh && projectName === 'default-project') || projectName === 'template-project') + throw new Error(`[Projects]: Project name ${projectName} not allowed`) + + copyFolderRecursiveSync(templateFolderDirectory, projectsRootFolder) + fs.renameSync(path.resolve(projectsRootFolder, 'template-project'), projectLocalDirectory) + + fs.mkdirSync(path.resolve(projectLocalDirectory, '.git'), { recursive: true }) + + const git = useGit(path.resolve(projectLocalDirectory, '.git')) + try { + await git.init(true) + } catch (e) { + logger.warn(e) + } + + const packageData = Object.assign({}, templateProjectJson) as any + packageData.name = projectName + packageData.etherealEngine.version = getEnginePackageJson().version + fs.writeFileSync(path.resolve(projectLocalDirectory, 'package.json'), JSON.stringify(packageData, null, 2)) + + await uploadLocalProjectToProvider(context.service, projectName, false) + + context.data = { name: projectName, needsRebuild: true } +} + +const linkGithubToProject = async (context: HookContext) => { + if (context.data.repositoryPath) { + const repoPath = context.data.repositoryPath + const user = context.params!.user! + + const githubIdentityProvider = (await context.app.service(identityProviderPath).find({ + query: { + userId: user.id, + type: 'github', + $limit: 1 + } + })) as Paginated + + const githubPathRegexExec = GITHUB_URL_REGEX.exec(repoPath) + if (!githubPathRegexExec) throw new BadRequest('Invalid Github URL') + if (githubIdentityProvider.data.length === 0) + throw new Error('Must be logged in with GitHub to link a project to a GitHub repo') + const split = githubPathRegexExec[2].split('/') + const org = split[0] + const repo = split[1].replace('.git', '') + const appOrgAccess = await checkAppOrgStatus(org, githubIdentityProvider.data[0].oauthToken) + if (!appOrgAccess) + throw new Forbidden( + `The organization ${org} needs to install the GitHub OAuth app ${config.authentication.oauth.github.key} in order to push code to its repositories` + ) + const repoWriteStatus = await checkUserRepoWriteStatus(org, repo, githubIdentityProvider.data[0].oauthToken) + if (repoWriteStatus !== 200) { + if (repoWriteStatus === 404) { + const orgWriteStatus = await checkUserOrgWriteStatus(org, githubIdentityProvider.data[0].oauthToken) + if (orgWriteStatus !== 200) throw new Forbidden('You do not have write access to that organization') + } else { + throw new Forbidden('You do not have write access to that repo') + } + } + } +} + +const getProjectName = async (context: HookContext) => { + if (!context.id) throw new BadRequest('You need to pass project id') + context.name = ((await context.app.service(projectPath).get(context.id, context.params)) as ProjectType).name +} + +const runProjectUninstallScript = async (context: HookContext) => { + const projectConfig = getProjectConfig(context.name) + + if (projectConfig?.onEvent) { + await onProjectEvent(context.service, context.name, projectConfig.onEvent, 'onUninstall') + } +} + +const removeProjectFiles = async (context: HookContext) => { + if (fs.existsSync(path.resolve(projectsRootFolder, context.name))) { + fs.rmSync(path.resolve(projectsRootFolder, context.name), { recursive: true }) + } + + logger.info(`[Projects]: removing project id "${context.id}", name: "${name}".`) + await deleteProjectFilesInStorageProvider(context.name) +} + const createProjectPermission = async (context: HookContext) => { if (context.params?.user?.id) { const projectPermissionData = await projectPermissionDataResolver.resolve( @@ -60,6 +313,99 @@ const createProjectPermission = async (context: HookContext) => { return context } +const removeLocationFromProject = async (context: HookContext) => { + await context.app.service(locationPath).remove(null, { + query: { + sceneId: { + $like: `${name}/%` + } + } + }) +} + +const removeRouteFromProject = async (context: HookContext) => { + await context.app.service(routePath).remove(null, { + query: { + project: name + } + }) +} + +const removeAvatarsFromProject = async (context: HookContext) => { + const avatarItems = (await context.app.service(avatarPath).find({ + query: { + project: name + }, + paginate: false + })) as AvatarType[] + + await Promise.all( + avatarItems.map(async (avatar) => { + await context.app.service(avatarPath).remove(avatar.id) + }) + ) +} + +const removeStaticResourcesFromProject = async (context: HookContext) => { + const staticResourceItems = (await context.app.service(staticResourcePath).find({ + query: { + project: name + }, + paginate: false + })) as StaticResourceType[] + staticResourceItems.length && + staticResourceItems.forEach(async (staticResource) => { + await context.app.service(staticResourcePath).remove(staticResource.id) + }) +} + +const removeProjectUpdate = async (context: HookContext) => { + await removeProjectUpdateJob(context.service, context.name) +} + +/** + * 1. Clones the repo to the local FS + * 2. If in production mode, uploads it to the storage provider + * 3. Creates a database entry + * @param data + * @param placeholder This is where data normally goes, but we've put data as the first parameter + * @param params + * @returns + */ +const updateProjectJob = async (context: HookContext) => { + if (!config.kubernetes.enabled || context.params?.isJob) + context.result = updateProject(context.service, context.data, context.params) + else { + const urlParts = context.data.sourceURL.split('/') + let projectName = context.data.name || urlParts.pop() + if (!projectName) throw new Error('Git repo must be plain URL') + projectName = projectName.toLowerCase() + if (projectName.substring(projectName.length - 4) === '.git') projectName = projectName.slice(0, -4) + if (projectName.substring(projectName.length - 1) === '/') projectName = projectName.slice(0, -1) + const jobBody = await getProjectUpdateJobBody(context.data, context.service, context.params!.user!.id) + const jobLabelSelector = `etherealengine/projectField=${context.data.name},etherealengine/release=${process.env.RELEASE_NAME},etherealengine/autoUpdate=false` + const jobFinishedPromise = createExecutorJob(context.service, jobBody, jobLabelSelector, 1000) + try { + await jobFinishedPromise + const result = (await context.app.service(projectPath).find({ + query: { + name: { + $like: projectName + } + } + })) as Paginated + let returned = {} as ProjectType + if (result.total > 0) returned = result.data[0] + else throw new BadRequest('Project did not exist after update') + returned.needsRebuild = typeof context.data.needsRebuild === 'boolean' ? context.data.needsRebuild : true + context.result = returned + } catch (err) { + console.log('Error: project did not exist after completing update', projectName, err) + throw err + } + } +} + export default { around: { all: [schemaHooks.resolveExternal(projectExternalResolver), schemaHooks.resolveResult(projectResolver)] @@ -71,30 +417,44 @@ export default { () => schemaHooks.validateQuery(projectQueryValidator), schemaHooks.resolveQuery(projectQueryResolver) ], - find: [], + find: [enableClientPagination(), ensurePushStatus, addLimitToParams], get: [], create: [ iff(isProvider('external'), verifyScope('editor', 'write')), () => schemaHooks.validateData(projectDataValidator), - schemaHooks.resolveData(projectDataResolver) + schemaHooks.resolveData(projectDataResolver), + createAndUploadProject ], update: [ iff(isProvider('external'), verifyScope('editor', 'write')), projectPermissionAuthenticate(false), - () => schemaHooks.validateData(projectPatchValidator) + () => schemaHooks.validateData(projectPatchValidator), + updateProjectJob ], patch: [ iff(isProvider('external'), verifyScope('editor', 'write')), projectPermissionAuthenticate(false), () => schemaHooks.validateData(projectPatchValidator), - schemaHooks.resolveData(projectPatchResolver) + schemaHooks.resolveData(projectPatchResolver), + linkGithubToProject ], - remove: [iff(isProvider('external'), verifyScope('editor', 'write')), projectPermissionAuthenticate(false)] + remove: [ + iff(isProvider('external'), verifyScope('editor', 'write')), + projectPermissionAuthenticate(false), + getProjectName, + runProjectUninstallScript, + removeProjectFiles, + removeLocationFromProject, + removeRouteFromProject, + removeAvatarsFromProject, + removeStaticResourcesFromProject, + removeProjectUpdate + ] }, after: { all: [], - find: [], + find: [addDataToProjectResult], get: [], create: [createProjectPermission], update: [], diff --git a/packages/server-core/src/projects/scene/scene.class.ts b/packages/server-core/src/projects/scene/scene.class.ts index 88376d711b3..2d9d9fb7d9b 100644 --- a/packages/server-core/src/projects/scene/scene.class.ts +++ b/packages/server-core/src/projects/scene/scene.class.ts @@ -106,7 +106,7 @@ export class SceneService const sceneName = params?.query?.name?.toString() const project = (await this.app .service(projectPath) - ._find({ ...params, query: { name: projectName!, $limit: 1 } })) as Paginated + .find({ ...params, query: { name: projectName!, $limit: 1 } })) as Paginated if (project.data.length === 0) throw new Error(`No project named ${projectName!} exists`) const sceneData = await getSceneData(projectName!, sceneName!, metadataOnly, params!.provider == null) @@ -123,7 +123,7 @@ export class SceneService const projectResult = (await this.app .service(projectPath) - ._find({ ...params, query: { name: project, $limit: 1 } })) as Paginated + .find({ ...params, query: { name: project, $limit: 1 } })) as Paginated if (projectResult.data.length === 0) throw new Error(`No project named ${project} exists`) const projectRoutePath = `projects/${project}/` @@ -179,7 +179,7 @@ export class SceneService const projectResult = (await this.app .service(projectPath) - ._find({ ...params, query: { name: project, $limit: 1 } })) as Paginated + .find({ ...params, query: { name: project, $limit: 1 } })) as Paginated if (projectResult.data.length === 0) throw new Error(`No project named ${project} exists`) const projectRoutePath = `projects/${project}/` @@ -305,7 +305,7 @@ export class SceneService const project = (await this.app .service(projectPath) - .find({ ...params, query: { name: projectName }, paginate: false })) as ProjectType[] + .find({ ...params, query: { name: projectName }, paginate: false })) as any as ProjectType[] if (project.length === 0) throw new Error(`No project named ${projectName} exists`) for (const ext of sceneAssetFiles) { diff --git a/packages/server-core/src/route/route/route.test.ts b/packages/server-core/src/route/route/route.test.ts index 1d0a81e350a..480a80d2ae3 100644 --- a/packages/server-core/src/route/route/route.test.ts +++ b/packages/server-core/src/route/route/route.test.ts @@ -44,7 +44,7 @@ const cleanup = async (app: Application, projectName: string) => { const projectDir = path.resolve(appRootPath.path, `packages/projects/projects/${projectName}/`) deleteFolderRecursive(projectDir) try { - await app.service(projectPath)._remove(null, { query: { name: projectName } }) + await app.service(projectPath).remove(null, { query: { name: projectName } }) } catch (e) { // } diff --git a/scripts/install-projects.js b/scripts/install-projects.js index eb5fbb53720..dda6e7aa154 100755 --- a/scripts/install-projects.js +++ b/scripts/install-projects.js @@ -1,4 +1,3 @@ - /* CPAL-1.0 License @@ -24,32 +23,29 @@ All portions of the code written by the Ethereal Engine team are Copyright © 20 Ethereal Engine. All Rights Reserved. */ - -import { download } from "@etherealengine/server-core/src/projects/project/downloadProjects"; -import { createDefaultStorageProvider } from "@etherealengine/server-core/src/media/storageprovider/storageprovider"; -import dotenv from 'dotenv'; -import path from "path"; -import fs from "fs"; -import appRootPath from 'app-root-path' +import { projectPath } from '@etherealengine/engine/src/schemas/projects/project.schema' import logger from '@etherealengine/server-core/src/ServerLogger' -import { createFeathersKoaApp } from '@etherealengine/server-core/src/createApp' import { ServerMode } from '@etherealengine/server-core/src/ServerState' +import { createFeathersKoaApp } from '@etherealengine/server-core/src/createApp' +import { createDefaultStorageProvider } from '@etherealengine/server-core/src/media/storageprovider/storageprovider' +import { download } from '@etherealengine/server-core/src/projects/project/downloadProjects' import { getProjectConfig, onProjectEvent } from '@etherealengine/server-core/src/projects/project/project-helper' -import { projectPath, ProjectType } from "@etherealengine/engine/src/schemas/projects/project.schema"; +import appRootPath from 'app-root-path' +import dotenv from 'dotenv' +import fs from 'fs' +import path from 'path' -dotenv.config(); +dotenv.config() const db = { - username: process.env.MYSQL_USER ?? 'server', - password: process.env.MYSQL_PASSWORD ?? 'password', - database: process.env.MYSQL_DATABASE ?? 'etherealengine', - host: process.env.MYSQL_HOST ?? '127.0.0.1', - port: process.env.MYSQL_PORT ?? 3306, - dialect: 'mysql' -}; - -db.url = process.env.MYSQL_URL ?? - `mysql://${db.username}:${db.password}@${db.host}:${db.port}/${db.database}`; + username: process.env.MYSQL_USER ?? 'server', + password: process.env.MYSQL_PASSWORD ?? 'password', + database: process.env.MYSQL_DATABASE ?? 'etherealengine', + host: process.env.MYSQL_HOST ?? '127.0.0.1', + port: process.env.MYSQL_PORT ?? 3306, + dialect: 'mysql' +} +db.url = process.env.MYSQL_URL ?? `mysql://${db.username}:${db.password}@${db.host}:${db.port}/${db.database}` async function installAllProjects() { try { @@ -60,19 +56,18 @@ async function installAllProjects() { if (!fs.existsSync(localProjectDirectory)) fs.mkdirSync(localProjectDirectory, { recursive: true }) logger.info('running installAllProjects') - const projects = await app.service(projectPath)._find({paginate: false}) + const projects = await app.service(projectPath).find({ paginate: false }) logger.info('found projects %o', projects) await Promise.all(projects.map((project) => download(project.name))) - await app.service(projectPath).update({ sourceURL: 'default-project' }, null, { isInternal: true, isJob: true }) + await app.service(projectPath).update('', { sourceURL: 'default-project' }, { isInternal: true, isJob: true }) const projectConfig = getProjectConfig('default-project') ?? {} if (projectConfig.onEvent) await onProjectEvent(app, 'default-project', projectConfig.onEvent, 'onUpdate') process.exit(0) } catch (e) { logger.fatal(e) - console.error(e) - process.exit(1) + console.error(e) + process.exit(1) } - } -installAllProjects(); +installAllProjects() diff --git a/scripts/push-project.ts b/scripts/push-project.ts index af4f539707b..c08bd8b1b58 100644 --- a/scripts/push-project.ts +++ b/scripts/push-project.ts @@ -66,7 +66,7 @@ cli.main(async () => { await app.setup() const { userId, projectId, reset, commitSHA, storageProviderName } = options const user = await app.service(userPath).get(userId) - const project = await app.service(projectPath)._get(projectId) + const project = await app.service(projectPath).get(projectId) await pushProjectToGithub(app, project, user, reset, commitSHA, storageProviderName || undefined, true) cli.exit(0) } catch (err) { diff --git a/scripts/update-project.ts b/scripts/update-project.ts index 423cbd3a224..caa377e6fa3 100644 --- a/scripts/update-project.ts +++ b/scripts/update-project.ts @@ -72,7 +72,7 @@ cli.main(async () => { data.reset = data.reset === 'true' data.needsRebuild = data.needsRebuild === true const user = await app.service(userPath).get(userId) - await app.service(projectPath).update(data, null, { user: user, isJob: true }) + await app.service(projectPath).update('', data, { user: user, isJob: true }) cli.exit(0) } catch (err) { console.log(err)