Skip to content

Commit

Permalink
Fixed some bugs in cache flow (#7993)
Browse files Browse the repository at this point in the history
Running npm install from packages/projects was not correct. To just install
projects' dependencies, we're now using a second root package.json that for
workspaces only has the packages that the builder comes preinstalled with
and packages/projects/projects/*. This ensures that all projects'
dependencies are installed, all necessary packages for running the builder
stay installed, and packages that are not necessary for root builder tasks
are not installed unnecessarily.

Dockerfile-root runs npm install; this had been commented out for testing
purposes and mistakenly not uncommented.
  • Loading branch information
barankyle committed May 18, 2023
1 parent 50a30a5 commit 2eb0326
Show file tree
Hide file tree
Showing 5 changed files with 180 additions and 32 deletions.
2 changes: 1 addition & 1 deletion dockerfiles/root/Dockerfile-root
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@ COPY packages/xrui/package.json ./packages/xrui/
COPY project-package-jsons ./

ARG NODE_ENV
#RUN npm install --loglevel notice --legacy-peer-deps
RUN npm install --loglevel notice --legacy-peer-deps
174 changes: 174 additions & 0 deletions package-root-build.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
{
"name": "etherealengine",
"description": "Engine, Server & Devops powering Ethereal Engine. Please support at https://opencollective.com/etherealengine",
"version": "0.0.0",
"homepage": "etherealengine.org",
"private": true,
"bin": {
"etherealengine": "npx/cli.js"
},
"workspaces": [
"packages/common",
"packages/engine",
"packages/hyperflux",
"packages/matchmaking",
"packages/projects",
"packages/server",
"packages/server-core",
"packages/xrui",
"packages/projects/projects/*"
],
"keywords": [
"three",
"three.js",
"ecs",
"webgl",
"mmo",
"game engine",
"webrtc",
"productivity",
"xr",
"vr"
],
"author": {
"name": "Ethereal Engine",
"email": "info@etherealengine.org"
},
"entrypoint": {},
"contributors": [],
"bugs": {},
"directories": {
"lib": "packages/server-core/src/user",
"config": "config/"
},
"engines": {
"node": ">= 18.12.0"
},
"npmClient": "npm",
"scripts": {
"build-client": "cd packages/client && npm run build",
"check": "npm run lint && npm run check-errors && npm run test && npm run test:projects && npm run build-client",
"check-db-exists": "cross-env ts-node --swc scripts/check-db-exists.ts",
"check-db-exists-only": "cross-env ts-node --swc scripts/check-db-exists-only.ts",
"check-errors": "lerna run check-errors",
"clean-node-modules": "npx rimraf node_modules && npx rimraf package-lock.json && npx lerna exec npx rimraf node_modules && npx lerna exec npx rimraf package-lock.json",
"cleanup-tests": "node tests/cleanup-tests.js",
"clear-projects-rebuild": "cross-env ts-node --swc scripts/clear-projects-rebuild",
"depcheck": "lerna exec --no-bail --stream -- depcheck",
"dev": "npm run dev-docker && concurrently -n agones,server,client,taskserver npm:dev-agones-silent npm:dev-server npm:dev-client npm:dev-taskserver",
"dev-agones": "cd scripts && ./start-agones.sh",
"dev-agones-silent": "npm run dev-agones &> /dev/null",
"dev-client": "cd packages/client && npm run dev",
"dev-taskserver": "cd packages/taskserver && npm run dev",
"dev-docker": "cd scripts && docker-compose up -d",
"dev-tabs": "cd scripts && ./dev-tabs.sh",
"fetch-projects": "lerna exec 'git fetch -p && git rebase' --parallel --ignore @etherealengine/*",
"dev-nginx": "",
"dev-reinit": "npm run dev-docker && cd packages/server && npm run dev-reinit-db",
"dev-server": "cd packages/server && npm run dev",
"dev-windows": "npm run dev-docker && concurrently -n agones,server,client npm:dev-agones-silent npm:dev-server npm:dev-client",
"diff": "lerna diff",
"edit": "node scripts/open-editor.js",
"format": "prettier --write \"packages/**/*.{ts,tsx}\"",
"format-scss": "stylelint \"packages/**/*.scss\" --fix",
"format-staged": "lint-staged",
"init-db-production": "cross-env APP_ENV=production FORCE_DB_REFRESH=true EXIT_ON_DB_INIT=true ts-node --swc packages/server/src/index.ts",
"install-projects": "ts-node --swc scripts/install-projects.js",
"lint": "prettier --check \"packages/**/*.{ts,tsx}\"",
"local": "npm run dev-docker && cross-env VITE_LOCAL_BUILD=true LOCAL=true concurrently -n agones,server,worldserver,mediaserver,client,files npm:dev-agones-silent \"cd packages/server && npm run start\" \"cd packages/instanceserver && npm run start\" \"cd packages/instanceserver && npm run start-channel\" \"cd packages/client && npm run local\" \"cd packages/server && npm run serve-local-files\"",
"make-user-admin": "ts-node --swc scripts/make-user-admin.js",
"migrate": "cd packages/server-core && npm run migrate",
"postinstall": "patch-package",
"precommit": "no-master-commits -b master",
"prepare-database": "cross-env APP_ENV=production PREPARE_DATABASE=true EXIT_ON_DB_INIT=true ts-node --swc packages/server/src/index.ts",
"publish": "lerna publish from-package --yes --registry https://registry.npmjs.org",
"publish-npm": "lerna publish from-package --yes --no-verify-access --ignore-scripts --registry https://registry.npmjs.org",
"publish-github": "lerna publish from-package --yes --no-verify-access --ignore-scripts --registry https://npm.pkg.github.com",
"list-client-s3-files-to-delete": "cross-env APP_ENV=production ts-node --swc scripts/get-deletable-client-files.ts",
"delete-old-s3-files": "cross-env APP_ENV=production ts-node --swc scripts/delete-old-s3-files.ts",
"push-client-dist-to-s3": "cross-env APP_ENV=production ts-node --swc scripts/push-client-to-s3.ts",
"copy-client-to-localstorage": "cross-env APP_ENV=production ts-node --swc scripts/copy-client-to-localstorage.ts",
"resave-all-scenes": "ts-node --swc scripts/resave-all-scenes.ts",
"start": "npm run dev",
"test": "cross-env TEST=true lerna run test",
"test-e2e": "ts-node --swc scripts/run_e2e_tests.ts",
"test:projects": "cross-env TEST=true lerna run test:projects",
"test:ci": "cpy --no-overwrite --rename=.env.local '.env.local.default' . && cross-env CI=true npm run test",
"validate": "npm run lint && lerna run validate",
"version-increment": "lerna version --conventional-commits --yes",
"version-increment-no-tag": "lerna version --conventional-commits --yes --no-git-tag-version",
"bump-project-versions": "cross-env ts-node --swc scripts/bump-project-versions.js",
"create-build-status": "cross-env ts-node --swc scripts/create-build-status.ts",
"record-build-error": "cross-env ts-node --swc scripts/record-build-error.ts",
"record-build-success": "cross-env ts-node --swc scripts/record-build-success",
"updateProject": "cross-env ts-node --swc scripts/update-project.ts",
"updateCronjobImage": "cross-env ts-node --swc scripts/update-cronjob-image.ts"
},
"types": "lib/",
"pre-commit": [
"format-staged"
],
"lint-staged": {
"*.{ts,tsx}": [
"prettier --write"
],
"*.scss": [
"stylelint --fix"
]
},
"devDependencies": {
"@ianvs/prettier-plugin-sort-imports": "3.7.1",
"@originjs/vite-plugin-commonjs": "^1.0.3",
"@swc/core": "^1.3.41",
"@types/app-root-path": "1.2.4",
"@types/cli": "0.11.21",
"@types/mocha": "10.0.1",
"@types/primus": "^7.3.6",
"@typescript-eslint/eslint-plugin": "5.56.0",
"@typescript-eslint/parser": "5.56.0",
"concurrently": "7.6.0",
"depcheck": "1.4.3",
"eslint": "8.36.0",
"jsdom": "^21.1.1",
"kill-port": "2.0.1",
"lint-staged": "13.2.0",
"mocha": "10.2.0",
"no-master-commits": "1.1.1",
"package-json-type": "1.0.3",
"pre-commit": "1.2.2",
"prettier": "2.8.6",
"react-dnd": "16.0.1",
"react-router-dom": "6.9.0",
"rimraf": "4.4.0",
"stylelint": "^15.3.0",
"stylelint-config-standard-scss": "^7.0.1",
"stylelint-scss": "^4.5.0",
"supertest": "6.3.3"
},
"dependencies": {
"@feathersjs/commons": "5.0.1",
"@feathersjs/errors": "5.0.1",
"@feathersjs/feathers": "5.0.1",
"@feathersjs/schema": "5.0.1",
"@feathersjs/typebox": "5.0.1",
"app-root-path": "3.1.0",
"cli": "1.0.1",
"daisyui": "^2.51.4",
"dotenv": "16.0.3",
"fix-esm": "^1.0.1",
"lerna": "6.5.1",
"patch-package": "^6.5.1",
"pino": "^8.11.0",
"pino-pretty": "^10.0.0",
"postcss": "^8.4.21",
"qrcode.react": "^3.1.0",
"react-json-view": "^1.21.3",
"tailwindcss": "^3.2.7",
"three": "0.150.1",
"three.quarks": "^0.10.2",
"ts-node": "10.9.1",
"typescript": "5.0.2",
"vite-plugin-node": "^3.0.2",
"ws": "8.13.0"
}
}
2 changes: 1 addition & 1 deletion scripts/build_and_publish_package.sh
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ else
fi

BUILD_END_TIME=`date +"%d-%m-%yT%H-%M-%S"`
echo "Ending ${PACKAGE} build at ${BUILD_END_TIME}"
echo "Ending ${PACKAGE} build at ${BUILD_END_TIME}, start time was ${BUILD_START_TIME}"
# cache links to use once ECR supports cache manifests
# --cache-to type=registry,ref=$ECR_URL/$REPO_NAME-$PACKAGE:latest_$STAGE_cache,mode=max \
# --cache-from type=registry,ref=$ECR_URL/$REPO_NAME-$PACKAGE:latest_$STAGE_cache
26 changes: 0 additions & 26 deletions scripts/publish_ecr.sh

This file was deleted.

8 changes: 4 additions & 4 deletions scripts/run-builder.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,11 @@ npm run create-build-status
BUILDER_RUN=$(tail -1 builder-run.txt)
npm run install-projects >project-install-build-logs.txt 2>project-install-build-error.txt || npm run record-build-error -- --service=project-install
test -s project-install-build-error.txt && npm run record-build-error -- --service=project-install
mv package.json package.jsonmoved
mv package-root-build.json package.json
npm install
mv package.json package-root-build.json
mv package.jsonmoved package.json
npm run prepare-database >prepare-database-build-logs.txt 2>prepare-database-build-error.txt || npm run record-build-error -- --service=prepare-database
test -s prepare-database-build-error.txt && npm run record-build-error -- --service=prepare-database
cd packages/client && npm run buildenv >buildenv-build-logs.txt 2>buildenv-build-error.txt || npm run record-build-error -- --service=buildenv
Expand All @@ -44,13 +48,9 @@ mkdir -p ./project-package-jsons/projects/default-project
cp packages/projects/default-project/package.json ./project-package-jsons/projects/default-project
find packages/projects/projects/ -name package.json -exec bash -c 'mkdir -p ./project-package-jsons/$(dirname $1) && cp $1 ./project-package-jsons/$(dirname $1)' - '{}' \;

ROOT_START_TIME=`date +"%d-%m-%yT%H-%M-%S"`
bash ./scripts/build_and_publish_package.sh $RELEASE_NAME $DOCKER_LABEL root $START_TIME $AWS_REGION $NODE_ENV $PRIVATE_ECR >root-build-logs.txt 2>root-build-error.txt
ROOT_END_TIME=`date +"%d-%m-%yT%H-%M-%S"`
npm run record-build-error -- --service=root --isDocker=true

echo "Started root build at $ROOT_START_TIME, ended at $ROOT_END_TIME"

npm install -g cli aws-sdk

if [ "$SERVE_CLIENT_FROM_STORAGE_PROVIDER" = "true" ] && [ "$STORAGE_PROVIDER" = "aws" ] ; then npm run list-client-s3-files-to-delete ; fi
Expand Down

0 comments on commit 2eb0326

Please sign in to comment.