From 11f71049011fbd5cbe1ad60c063a1fc4ba94d969 Mon Sep 17 00:00:00 2001 From: tharvik Date: Tue, 27 Feb 2024 17:44:24 +0100 Subject: [PATCH] github/actions: rolling deploy Closes: #666 --- .github/workflows/deploy-server.yml | 47 -------------- .github/workflows/github-pages.yml | 50 -------------- .github/workflows/lint-test-build.yml | 5 +- .github/workflows/publish-deploy.yml | 94 +++++++++++++++++++++++++++ .github/workflows/publish.yml | 92 -------------------------- README.md | 10 +-- docs/CONTRIBUTING.md | 2 +- docs/TASK.md | 8 +-- webapp/.env.production | 2 +- 9 files changed, 109 insertions(+), 201 deletions(-) delete mode 100644 .github/workflows/deploy-server.yml delete mode 100644 .github/workflows/github-pages.yml create mode 100644 .github/workflows/publish-deploy.yml delete mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/deploy-server.yml b/.github/workflows/deploy-server.yml deleted file mode 100644 index 2e25cdd5c..000000000 --- a/.github/workflows/deploy-server.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: deploy-server -on: - workflow_run: - workflows: ["lint-test-build"] - branches: [production] - types: [completed] - -env: - server_url: https://deai-313515.ew.r.appspot.com/ - -jobs: - deploy: - name: deploy - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - with: - ref: production - - uses: actions/setup-node@v4 - with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: | - discojs/package-lock.json - server/package-lock.json - - - run: npm ci - working-directory: discojs - - run: npm run build - working-directory: discojs/discojs-node - - - name: authenticate-to-google-cloud - uses: "google-github-actions/auth@v0" - with: - credentials_json: "${{ secrets.GCP_CREDENTIALS }}" - - - name: deploy-to-google-cloud - uses: google-github-actions/deploy-appengine@v0.8.0 - with: - working_directory: . - deliverables: app.yaml - version: prod - project_id: ${{ secrets.GCP_PROJECT }} - - - name: test - run: curl "${{ env.server_url }}" diff --git a/.github/workflows/github-pages.yml b/.github/workflows/github-pages.yml deleted file mode 100644 index 7e1932272..000000000 --- a/.github/workflows/github-pages.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: github-pages -on: - workflow_run: - workflows: ["lint-test-build"] - branches: [production] - types: [completed] - -jobs: - gh-pages-build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version-file: .nvmrc - cache: npm - cache-dependency-path: | - discojs/package-lock.json - web-client/package-lock.json - - - run: npm ci - working-directory: discojs - - run: npm run build - working-directory: discojs/discojs-web - - run: npm ci && npm run build - working-directory: web-client - - name: Upload artifact - uses: actions/upload-pages-artifact@v1 - with: - path: "./web-client/dist" - - gh-pages-deploy: - needs: gh-pages-build - - # Grant GITHUB_TOKEN the permissions required to make a Pages deployment - permissions: - pages: write # to deploy to Pages - id-token: write # to verify the deployment originates from an appropriate source - - # Deploy to the github-pages environment - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - # Specify runner + deployment step - runs-on: ubuntu-latest - steps: - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v1 diff --git a/.github/workflows/lint-test-build.yml b/.github/workflows/lint-test-build.yml index e72b1e421..479df3a6a 100644 --- a/.github/workflows/lint-test-build.yml +++ b/.github/workflows/lint-test-build.yml @@ -1,9 +1,12 @@ name: lint-test-build on: push: - branches: production + branches: [develop] pull_request: +permissions: + contents: read + jobs: download-datasets: runs-on: ubuntu-latest diff --git a/.github/workflows/publish-deploy.yml b/.github/workflows/publish-deploy.yml new file mode 100644 index 000000000..49338637f --- /dev/null +++ b/.github/workflows/publish-deploy.yml @@ -0,0 +1,94 @@ +name: publish-deploy +on: + push: + branches: [develop] + +jobs: + publish-npm: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + registry-url: "https://registry.npmjs.org" + scope: "@epfml" + - run: npm ci + - run: npm --workspace=discojs{,-node,-web} version prerelease --preid=p`date +%Y%m%d%H%M%S` + - run: npm --workspace=discojs{,-node,-web} run build + - run: npm --workspace=discojs{,-node,-web} publish --access public + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + + build-webapp: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - run: npm --workspace={discojs,discojs-web,webapp} run build + - uses: actions/upload-pages-artifact@v3 + with: + path: webapp/dist + deploy-pages: + needs: build-webapp + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 + + publish-github-container: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + outputs: + digest: ${{ steps.build.outputs.digest }} + steps: + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: docker/metadata-action@v5 + id: meta + with: + images: ghcr.io/epfml/disco + - uses: docker/build-push-action@v5 + id: build + with: + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + deploy-server: + needs: [publish-github-container] + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@v4 + - uses: google-github-actions/auth@v2 + with: + project_id: disco-367212 + workload_identity_provider: projects/905419434797/locations/global/workloadIdentityPools/github/providers/disco + - uses: google-github-actions/deploy-cloudrun@v2 + with: + service: disco + region: europe-west6 + image: europe-west6-docker.pkg.dev/disco-367212/github/epfml/disco@${{ needs.publish-github-container.outputs.digest }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index a31cacf7b..000000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: publish - -on: - workflow_dispatch: - inputs: - version: - required: true - type: string - -jobs: - publish-and-tag: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version-file: .nvmrc - registry-url: https://registry.npmjs.org - cache: npm - cache-dependency-path: | - discojs/package-lock.json - server/package-lock.json - - - name: Build @epfml/discojs and @epfml/discojs-node - run: | - npm ci - npm run build - working-directory: discojs - - - name: Build @epfml/disco-server - run: | - npm ci - npm link ../discojs/discojs-node - npm run build - working-directory: server - - - name: Bump the @epfml/discojs version - run: npm version ${{ inputs.version }} - working-directory: discojs/discojs-web - - - name: Bump the @epfml/discojs-node version - run: npm version ${{ inputs.version }} - working-directory: discojs/discojs-node - - - name: Bump the @epfml/disco-server version - run: npm version ${{ inputs.version }} - working-directory: server - - - name: Bump @epfml/disco-server dependency to @epfml/discojs-node - run: npx ts-node publish.ts - working-directory: server - - - name: Create version branch - run: | - git config --global user.name 'GitHub Action Bot' - git config --global user.email 'bot@users.noreply.github.com' - git switch -c branch-v${{ inputs.version }} - git commit -am "publish v${{ inputs.version }}" - git push -u origin branch-v${{ inputs.version }} - - - name: Write dependencies for @epfml/discojs and @epfml/discojs-node - run: npx ts-node publish.ts - working-directory: discojs - - - name: Publish @epfml/discojs - run: npm publish --access public - working-directory: discojs/discojs-web - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - - name: Publish @epfml/discojs-node - run: npm publish --access public - working-directory: discojs/discojs-node - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - - name: Publish @epfml/disco-server - run: npm publish --access public - working-directory: server - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - - - name: Create version commit and tag - run: | - npm i @epfml/discojs-node - git switch branch-v${{ inputs.version }} - git add package-lock.json - git commit -m "v${{ inputs.version }}" - git tag -a "v${{ inputs.version }}" -m "disco: v${{ inputs.version }}" - git push - git push origin v${{ inputs.version }} - working-directory: server diff --git a/README.md b/README.md index 60a35b319..a95b960e5 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,9 @@ DISCO leverages federated :star2: and decentralized :sparkles: learning to allow several data owners to collaboratively build machine learning models without sharing any original data. -The latest version is always running on the following link, [directly in your browser](https://epfml.github.io/disco/), for web and mobile: +The latest version is always running on the following link, [directly in your browser](https://discolab.ai/), for web and mobile: - :man_dancing: [https://epfml.github.io/disco/](https://epfml.github.io/disco/) :man_dancing: + :man_dancing: [https://discolab.ai/](https://discolab.ai/) :man_dancing: ___ :magic_wand: DEVELOPERS: Have a look at our [developer guide](DEV.md) @@ -24,7 +24,7 @@ ___ - a central server : **federated** learning ( :star2: ) - directly between users : **decentralized** learning ( :sparkles: ) i.e. no central coordination - Model updates are then securely aggregated into a trained model -- See more [HERE](https://epfml.github.io/disco/#/information) +- See more [HERE](https://discolab.ai/#/information) ___ :question: **DISCO TECHNOLOGY** @@ -50,9 +50,9 @@ ___ :checkered_flag: **HOW TO USE DISCO** -- Start by exploring our example *DISCOllaboratives* in the [`Tasks` page](https://epfml.github.io/disco/#/list). +- Start by exploring our example *DISCOllaboratives* in the [`Tasks` page](https://discolab.ai/#/list). - The example models are based on popular datasets such as [Titanic](https://www.kaggle.com/c/titanic), [MNIST](https://www.kaggle.com/c/digit-recognizer) or [CIFAR-10](https://www.kaggle.com/pankrzysiu/cifar10-python) -- It is also possible to create your own task without coding on the [custom training page](https://epfml.github.io/disco/#/create): +- It is also possible to create your own task without coding on the [custom training page](https://discolab.ai/#/create): - Upload the initial model - You can choose from several existing dataloaders - Choose between federated and decentralized for your DISCO training scheme ... connect your data and... done! :bar_chart: diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 6d414ae03..6e1ab4721 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -23,7 +23,7 @@ with both to a certain extent. If not, the following references might be useful: DISCO is a complex project composed of the Disco.js library (`discojs`, `discojs-node` and `discojs-node`), a front-end (`webapp`), a `server` and a `cli` (e.g., for benchmarking). Depending on what your goal is, you might only use a subset of them, e.g. you won't need an in-depth understanding of the webapp and Vue.js to add a new decentralized learning feature. Instead, you will probably rely on the CLI. -1. If you are going to work, contribute and improve the project, I first recommend you get a good understand of what DISCO does: play around with the [website](https://epfml.github.io/disco/#/), train a model from the pre-defined tasks, or even create your own custom task. Feedback is always appreciated, feel free to let us know on slack/in the github issues/in person if you noticed any issues or thought of an improvement. +1. If you are going to work, contribute and improve the project, I first recommend you get a good understand of what DISCO does: play around with the [website](https://discolab.ai/#/), train a model from the pre-defined tasks, or even create your own custom task. Feedback is always appreciated, feel free to let us know on slack/in the github issues/in person if you noticed any issues or thought of an improvement. 2. Then, get a high-level understanding of the different parts of the projects in the [developer guide](../DEV.md), even if you're planning on working on a subset of the project. If you want to know more about a specific part of the project, refer to the table of contents at the end of the DEV guide. diff --git a/docs/TASK.md b/docs/TASK.md index 068fcea3e..b3206fb2a 100644 --- a/docs/TASK.md +++ b/docs/TASK.md @@ -3,7 +3,7 @@ DISCO currently allows learning of arbitrary machine learning tasks, where tasks can be defined in three possible ways: 1. **Predefined tasks**: As examples, DISCO already hosts several pre-defined popular tasks such as [Titanic](../discojs/src/tasks/titanic.ts), [CIFAR-10](../discojs/src/tasks/cifar10.ts), and [MNIST](../discojs/src/tasks/mnist.ts) among others. -2. **Task creation UI**: new tasks can be defined via the [**task creation form**](https://epfml.github.io/disco/#/create) +2. **Task creation UI**: new tasks can be defined via the [**task creation form**](https://discolab.ai/#/create) 3. **Implementing custom tasks**: tasks too specific for the UI form need to be implemented in the repository directly. In any case, one user needs to upload the initial model that is going to be trained collaboratively. @@ -38,13 +38,13 @@ tensorflowjs_converter --input_format=tf_saved_model my_tensorflow_saved_model / ## 1. Predefined tasks -Predefined tasks are example use cases available in the [DISCO website](https://epfml.github.io/disco/#/list) where users can upload their respective data and train collaboratively. For predefined tasks, the initial model to train is already defined and doesn't need to be uploaded. +Predefined tasks are example use cases available in the [DISCO website](https://discolab.ai/#/list) where users can upload their respective data and train collaboratively. For predefined tasks, the initial model to train is already defined and doesn't need to be uploaded. ## 2. Task creation UI -The [task creation form](https://epfml.github.io/disco/#/create) lets users create a custom task DISCO without programming. In this case, users can choose between the data modalities and preprocessing that are already supported (such as tabular, images, text etc) and upload an initial model. +The [task creation form](https://discolab.ai/#/create) lets users create a custom task DISCO without programming. In this case, users can choose between the data modalities and preprocessing that are already supported (such as tabular, images, text etc) and upload an initial model. -1. On the [DISCO website](https://epfml.github.io/disco/#), click on `Get Started` and then `Create`. +1. On the [DISCO website](https://discolab.ai/), click on `Get Started` and then `Create`. 2. Fill in all the relevant information for the task 3. Upload model files: 1) a TF.js architecture file in JSON format (cf. the _Uploading ML models_ section) as well as a weight file (`.bin` format), which is necessary in this case. This is the initial weights provided to new users joining your task (pre-trained or random initialization). diff --git a/webapp/.env.production b/webapp/.env.production index 63f32a129..f6db2652b 100644 --- a/webapp/.env.production +++ b/webapp/.env.production @@ -1 +1 @@ -VITE_SERVER_URL=https://deai-313515.ew.r.appspot.com +VITE_SERVER_URL=https://disco-zbkj3i466a-oa.a.run.app