feat(backend): store files in binary as compressed (#263)

* feat(backend): store files in binary as compressed

No need to compress them during runtime, so smaller and faster binary

* refactor(build): test best compression

* refactor(backend): test brotli compression instead

* fix: only compress files once

Do it in the Makefile

* ci(build): add missing compression feature

* ci(build): remove docker cache

* ci(build): go back to gzip compression

Brotli is only supported on https and localhost connections

.github/workflows/push-build.yml

@@ -9,13 +9,6 @@ jobs:
       - uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Install Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-          components: rustfmt, clippy
       - name: Lint (rustfmt)
         working-directory: src/backend
         run: cargo fmt -- --check
@@ -47,20 +40,20 @@ jobs:
       - name: Build frontend
         working-directory: src/frontend
         run: yarn build
-      - name: Copy "dist" folder
-        run: cp -r src/frontend/dist/. src/backend/dist/
+      - name: Compress frontend files
+        run: make compress
       - name: Build x86_64
         working-directory: src/backend
-        run: cross build --release --target x86_64-unknown-linux-musl
+        run: cross build --release --target x86_64-unknown-linux-musl --features compression
       - name: Build armv6l
         working-directory: src/backend
-        run: cross build --release --target arm-unknown-linux-musleabihf
+        run: cross build --release --target arm-unknown-linux-musleabihf --features compression
       - name: Build armv7l
         working-directory: src/backend
-        run: cross build --release --target armv7-unknown-linux-musleabihf
+        run: cross build --release --target armv7-unknown-linux-musleabihf --features compression
       - name: Build aarch64
         working-directory: src/backend
-        run: cross build --release --target aarch64-unknown-linux-musl
+        run: cross build --release --target aarch64-unknown-linux-musl --features compression
       - uses: actions/upload-artifact@v2
         with:
           name: dietpi-dashboard-x86_64

Makefile

@@ -1,18 +1,30 @@
-default: yarn publiccopy fmt
+build: yarn rustbuild
 
+rustbuild: distcopy fmt
+ifdef TARGET
+	cd src/backend; cargo build --target $(TARGET)
+	mv src/backend/target/$(TARGET)/debug/dietpi-dashboard .
+else
 	cd src/backend; cargo build
+	mv src/backend/target/debug/dietpi-dashboard .
+endif
 
-	$(MAKE) publicdelete
-
-	mv src/backend/target/debug/dietpi-dashboard ./dietpi-dashboard
-
-rust: publiccopy fmt 
+	rm -r src/backend/dist
 
-	cd src/backend; cargo build
+release: yarn compress
+ifdef TARGET
+	cd src/backend; cargo build --target $(TARGET) --release --features compression
+	mv src/backend/target/$(TARGET)/release/dietpi-dashboard .
+else
+	cd src/backend; cargo build --release --features compression
+	mv src/backend/target/release/dietpi-dashboard .
+endif
 
-	$(MAKE) publicdelete
+	rm -r src/backend/dist
 
-	mv src/backend/target/debug/dietpi-dashboard ./dietpi-dashboard
+# There may be a better, more 'make'y way of doing this, but find works for now
+compress: distcopy
+	find src/backend/dist ! -name '*.png' -type f -exec gzip -9 {} \; -exec mv {}.gz {} \;
 
 yarn:
 	cd src/frontend; yarn build
@@ -23,12 +35,6 @@ else
 	rm -f src/backend/target/debug/deps/dietpi_dashboard-*
 endif
 
-publiccopy:
-	cp -r src/frontend/dist src/backend
-
-publicdelete:
-	rm -r src/backend/dist
-
 fmt:
 	cd src/backend; cargo fmt
 ifdef TARGET
@@ -37,10 +43,5 @@ else
 	cd src/backend; cargo clippy
 endif
 
-rustdev: publiccopy fmt
-	cd src/backend; cargo build --target $(TARGET)
-	mv src/backend/target/$(TARGET)/debug/dietpi-dashboard ./dietpi-dashboard
-
-	$(MAKE) publicdelete
-
-dev: yarn rustdev
+distcopy:
+	cp -r src/frontend/dist src/backend

src/backend/Cargo.toml

@@ -28,7 +28,8 @@ anyhow = "1.0.57"
 
 [features]
 default = ["frontend"]
-frontend = ["include_dir", "warp/compression"]
+frontend = ["include_dir"]
+compression = ["frontend"]
 
 [profile.dev]
 debug = 1

src/backend/src/main.rs

@@ -42,6 +42,8 @@ fn main() -> anyhow::Result<()> {
             headers.insert("X-Permitted-Cross-Domain_Policies", header::HeaderValue::from_static("none"));
             headers.insert(header::REFERRER_POLICY, header::HeaderValue::from_static("no-referrer"));
             headers.insert("Content-Security-Policy", header::HeaderValue::from_static("default-src 'self'; font-src 'self'; img-src 'self' blob:; script-src 'self'; style-src 'unsafe-inline' 'self'; connect-src * ws:;"));
+            #[cfg(feature = "compression")]
+            headers.insert(header::CONTENT_ENCODING, header::HeaderValue::from_static("gzip"));
             }
 
             #[cfg(feature = "frontend")]
@@ -58,15 +60,16 @@ fn main() -> anyhow::Result<()> {
                 .and(warp::path::param())
                 .map(|path: String| {
                     let ext = path.rsplit('.').next().unwrap_or("plain");
-                    warp::reply::with_header(
+                    #[allow(unused_mut)] // Mute warning, variable is mut because it's used with the compression feature
+                    let mut reply = warp::reply::with_header(
                         match DIR.get_file(format!("assets/{}", path)) {
                             Some(file) => file.contents(),
                             None => {
                                 log::info!("Couldn't get asset {}", path);
                                 &[]
                             }
                         },
-                        "content-type",
+                        header::CONTENT_TYPE,
                         if ext == "js" {
                             "text/javascript".to_string()
                         } else if ext == "svg" {
@@ -76,7 +79,14 @@ fn main() -> anyhow::Result<()> {
                         } else {
                             format!("text/{}", ext)
                         },
-                    )
+                    ).into_response();
+
+                    #[cfg(feature = "compression")]
+                    if ext != "png" {
+                        reply.headers_mut().insert(header::CONTENT_ENCODING, header::HeaderValue::from_static("gzip"));
+                    };
+
+                    reply
                 });
 
             let login_route = warp::path("login")
@@ -137,16 +147,14 @@ fn main() -> anyhow::Result<()> {
 
             #[cfg(feature = "frontend")]
             let main_route = warp::any().map(|| {
-                let file_bytes = handle_error!(DIR.get_file("index.html").context("Couldn't get main HTML file"), return warp::reply::with_status("Couldn't get main HTML file", warp::http::StatusCode::INTERNAL_SERVER_ERROR).into_response());
-                let file = handle_error!(file_bytes.contents_utf8().context("Invalid main HTML file"), return warp::reply::with_status("Invalid main HTML file", warp::http::StatusCode::INTERNAL_SERVER_ERROR).into_response());
+                let file = handle_error!(DIR.get_file("index.html").context("Couldn't get main HTML file"), return warp::reply::with_status("Couldn't get main HTML file", warp::http::StatusCode::INTERNAL_SERVER_ERROR).into_response()).contents();
                 warp::reply::html(file).into_response()
             }).with(warp::reply::with::headers(headers));
 
             #[cfg(feature = "frontend")]
             let page_routes = favicon_route
                 .or(assets_route)
-                .or(main_route)
-                .with(warp::compression::gzip());
+                .or(main_route);
 
             let socket_routes = terminal_route.or(file_route).or(socket_route);