Integrate DVC into repo (#861)

* Integrate DVC into repo, move esrgan-slim models to Google Drive and S3

* Update Github Actions to pull models from DVC

* Trigger test run

* Fix github workflows

* Move all model files into DVC

* Reduce timeout time

* Add gdrive credentials

* Make verbose

* Bump

* Retrigger

* Update dvc to use service account

* Attempt to add two separate gdrive accounts

* Bump timeout to 5 minutes

.dvc/.gitignore

@@ -0,0 +1,2 @@
+/config.local
+/cache

.dvc/config

@@ -0,0 +1,11 @@
+[core]
+    remote = gdrive
+    autostage = true
+['remote "gdrive"']
+    url = gdrive://1tGm1wnv7pAhbSuy4u9Ci61xA8WD3DXXr
+    gdrive_use_service_account = false
+['remote "s3"']
+    url = s3://upscalerjs/models
+['remote "gdrive-service-account"']
+    url = gdrive://1tGm1wnv7pAhbSuy4u9Ci61xA8WD3DXXr
+    gdrive_use_service_account = true

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

.github/workflows/tests.yml

@@ -218,9 +218,18 @@ jobs:
           username: ${{ secrets.BROWSERSTACK_USERNAME }}
           access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
 
+      - name: 'Setup DVC'
+        uses: iterative/setup-dvc@v1
+
       - name: 'Checkout repository'
         uses: actions/checkout@v2
 
+      - name: 'Checkout Models'
+        run : dvc pull -v -r gdrive-service-account
+        timeout-minutes: 5
+        env:
+          GDRIVE_CREDENTIALS_DATA : ${{ secrets.GDRIVE_CREDENTIALS_DATA }}          
+
       - uses: pnpm/action-setup@v2.0.1
         with:
           version: 7
@@ -262,9 +271,18 @@ jobs:
           username: ${{ secrets.BROWSERSTACK_USERNAME }}
           access-key: ${{ secrets.BROWSERSTACK_ACCESS_KEY }}
 
+      - name: 'Setup DVC'
+        uses: iterative/setup-dvc@v1
+
       - name: 'Checkout repository'
         uses: actions/checkout@v2
 
+      - name: 'Checkout Models'
+        run : dvc pull -v -r gdrive-service-account
+        timeout-minutes: 5
+        env:
+          GDRIVE_CREDENTIALS_DATA : ${{ secrets.GDRIVE_CREDENTIALS_DATA }}          
+
       - uses: pnpm/action-setup@v2.0.1
         with:
           version: 7
@@ -299,9 +317,19 @@ jobs:
       matrix:
         node-version: [16]
     steps:
+      - name: 'Setup DVC'
+        uses: iterative/setup-dvc@v1
+
       - name: 'Checkout repository'
         uses: actions/checkout@v2
 
+      - name: 'Checkout Models'
+        run : dvc pull -v -r gdrive-service-account
+        timeout-minutes: 5
+        env:
+          GDRIVE_CREDENTIALS_DATA : ${{ secrets.GDRIVE_CREDENTIALS_DATA }}          
+
+
       - uses: pnpm/action-setup@v2.0.1
         with:
           version: 7
@@ -334,9 +362,18 @@ jobs:
       matrix:
         node-version: [16]
     steps:
+      - name: 'Setup DVC'
+        uses: iterative/setup-dvc@v1
+
       - name: 'Checkout repository'
         uses: actions/checkout@v2
 
+      - name: 'Checkout Models'
+        run : dvc pull -v -r gdrive-service-account
+        timeout-minutes: 5
+        env:
+          GDRIVE_CREDENTIALS_DATA : ${{ secrets.GDRIVE_CREDENTIALS_DATA }}          
+
       - uses: pnpm/action-setup@v2.0.1
         with:
           version: 7
@@ -369,9 +406,18 @@ jobs:
       matrix:
         node-version: [16]
     steps:
+      - name: 'Setup DVC'
+        uses: iterative/setup-dvc@v1
+
       - name: 'Checkout repository'
         uses: actions/checkout@v2
 
+      - name: 'Checkout Models'
+        run : dvc pull -v -r gdrive-service-account
+        timeout-minutes: 5
+        env:
+          GDRIVE_CREDENTIALS_DATA : ${{ secrets.GDRIVE_CREDENTIALS_DATA }}          
+
       - uses: pnpm/action-setup@v2.0.1
         with:
           version: 7

.gitignore

@@ -30,3 +30,4 @@ test-output
 tmp
 .nyc_output
 output.json
+service-account.json

README.md

@@ -7,37 +7,32 @@
 <a href="https://deepsource.io/gh/thekevinscott/UpscalerJS/?ref=repository-badge"><img alt="DeepSource" src="https://deepsource.io/gh/thekevinscott/UpscalerJS.svg/?label=active+issues&show_trend=true" /></a>
 
 
-UpscalerJS is a tool for increasing image resolution in Javascript via a [Neural Network](https://github.com/thekevinscott/upscalerjs-models) up to 4x.
+UpscalerJS is a tool for enhancing images in Javascript using Neural Networks. UpscalerJS features models enabling the following:
 
-![Demo](assets/demo.gif)
+- Super Resolution
+- Deblurring
+- Denoising
+- Deraining
+- Dehazing
+- Low Light Enhancement
+- Retouching
+- Inpainting (soon!)
 
-[A live demo is here](https://upscaler.ai).
+![Demo](docs/assets/assets/demo.png)
 
-## Announcement
+[A live demo is here](https://upscalerjs.com/demo).
 
+## Announcement
 
 [A beta version of `1.0.0`](http://npmjs.com/package/upscaler) was released on 7/15/22. [Learn more here](https://twitter.com/upscalerjs/status/1547662175950344192) about what's included.
 
 If you are migrating from `<=0.12`, and you are using a custom model, [you will need to modify how you load models](https://upscalerjs.com/#/?id=instantiation). If you are using the default model (e.g., not providing a `model` argument) then no changes are needed.
 
 
-## Examples
-
-You can [view runnable code examples](https://github.com/thekevinscott/UpscalerJS/tree/master/examples) on CodeSandbox.
-
-To run an example locally, navigate to the relevant folder and run:
-
-```
-pnpm start
-```
-
-If you are developing UpscalerJS locally and wish to run an example during development, use the following command:
+## Guides
 
-```
-pnpm example {folder-name-of-example}
-```
+You can [view runnable code examples](https://upscalerjs.com/documentation/guides/). You can also find the [guides here on Github](https://github.com/thekevinscott/UpscalerJS/tree/main/examples).
 
-This will automatically start a watcher in the example folder, as well as set the local UpscalerJS package to build in the background.
 
 ## Quick Start
 
@@ -53,6 +48,10 @@ upscaler.upscale('/path/to/image').then(upscaledImage => {
 
 [View the docs here.](https://upscalerjs.com)
 
+## Developing
+
+If cloning the repo, [ensure that `dvc` is installed](http://dvc.orgv). `dvc` is used to manage the model files and has git pre-hooks enabled.
+
 ## License
 
 [MIT License](https://oss.ninja/mit/developit/) © [Kevin Scott](https://thekevinscott.com)

models/default-model/.gitignore

@@ -1,3 +1,4 @@
 dist
 node_modules
 *.generated.ts
+/models

models/default-model/models.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: b33f1f6aa2be21da44cbbd212427c505.dir
+  size: 900636
+  nfiles: 2
+  path: models

models/esrgan-legacy/.gitignore

@@ -1,3 +1,4 @@
 dist
 node_modules
 *.generated.ts
+/models

models/esrgan-legacy/models.dvc

@@ -0,0 +1,5 @@
+outs:
+- md5: ead96e667ff201a0bb4acbf170c7769e.dir
+  size: 15495994
+  nfiles: 11
+  path: models