Headless windows & Data pipeline (#119)

* Implement headless rendering

* Simplify window traits

* Simplify and fix headless rendering

* Rename HttpClient

* Ignore wasm-pack output

* Refactor stages and context

* Rename runnable to eventloop and fix variable names

* Fix tests

* Add pipeline draft

* Start implementing pipeline steps

* Use pipeline in headless rendering

* Improve the style significantly. The colors are taken from the default OSM map style

* Start working on a specific headless API

* Refactor thread state and pipeline

* Refactor thread state and pipeline

* Fix web

* Remove mem::replace usage

* Improve pipeline types

* Simplify pipeline API

* Add comment

* Fix tests

* Remove dynamic dispatch for schedule

* Add Run Test config

* Revive old legacy webworker example

* Fix resize detection

* Cleanup dependencies and simplify bounds

* Conditionally enable create_png

* Add some comments and rename

* Run more tests in CI

* Fix tests for various platforms

* Install missing dependency

.cargo/config.toml

@@ -7,4 +7,5 @@ rustflags = [
     # Enables the possibility to import memory into wasm.
     # Without --shared-memory it is not possible to use shared WebAssembly.Memory.
     "-C", "link-args=--shared-memory --import-memory",
-]
+]
+runner = 'wasm-bindgen-test-runner'

.github/actions/android/action.yml

@@ -12,6 +12,7 @@ runs:
     - name: Build
       shell: bash
       run: just build-android
+      # TODO: Additional clippy checks for different targets
     - name: Check x86_64
       shell: bash
       run: |
@@ -24,3 +25,8 @@ runs:
         env "AR_aarch64-linux-android=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-ar" \
         env "CC_aarch64-linux-android=$ANDROID_NDK_ROOT/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android30-clang" \
         just check maplibre-android aarch64-linux-android
+    # FIXME: Requires cross-compilation
+    #- name: Test
+    #  shell: bash
+    #  # TODO: Additional test runs for different targets
+    #  run: just test maplibre-android aarch64-linux-android

.github/actions/apple/action.yml

@@ -22,10 +22,14 @@ runs:
     - name: Check x86_64 darwin
       shell: bash
       run: just check apple x86_64-apple-darwin
-    - name: Check aarch64 darwin
+    - name: Check x86_64 darwin
+      shell: bash
+      # TODO: Additional clippy checks for different targets (iOS)
+      run: just check apple x86_64-apple-darwin
+    - name: Test x86_64 darwin
       shell: bash
-      run: just check apple aarch64-apple-darwin
-    # TODO: Additional clippy checks for iOS
+      # TODO: Additional test runs for different targets (Different targets might require emulation)
+      run: just test apple x86_64-apple-darwin
     - name: Build Example
       shell: bash
       run: cd apple/xcode && xcodebuild -scheme "example (iOS)" -arch arm64 -sdk iphoneos build CODE_SIGNING_ALLOWED=NO

.github/actions/demo/linux/action.yml

@@ -15,12 +15,12 @@ runs:
     - name: Build
       shell: bash
       run: cargo build -p maplibre-demo
-    - name: Test
-      shell: bash
-      run: just test maplibre-demo x86_64-unknown-linux-gnu
     - name: Check
       shell: bash
       run: just check maplibre-demo x86_64-unknown-linux-gnu
+    - name: Test x86_64 linux
+      shell: bash
+      run: just test maplibre-demo x86_64-unknown-linux-gnu
     - uses: actions/upload-artifact@v2
       with:
         name: maplibre-rs

.github/actions/demo/macos/action.yml

@@ -19,6 +19,12 @@ runs:
     - name: Build
       shell: bash
       run: cd apple/xcode && xcodebuild -scheme "example (macOS)" build CODE_SIGNING_ALLOWED=NO MACOSX_DEPLOYMENT_TARGET=10.9 -derivedDataPath build
+    - name: Check x86_64 darwin
+      shell: bash
+      run: just check maplibre-demo x86_64-apple-darwin
+    - name: Test x86_64 darwin
+      shell: bash
+      run: just test maplibre-demo x86_64-apple-darwin
     - uses: actions/upload-artifact@v3
       with:
         name: maplibre-x86_64-apple-darwin-demo

.github/actions/demo/windows/action.yml

@@ -22,7 +22,10 @@ runs:
     - name: Build
       shell: bash
       run: cargo build -p maplibre-demo --release --target x86_64-pc-windows-msvc
-    - name: Test
+    - name: Check x86_64 windows
+      shell: bash
+      run: just check maplibre-demo x86_64-pc-windows-msvc
+    - name: Test x86_64 windows
       shell: bash
       run: just test maplibre-demo x86_64-pc-windows-msvc
     - uses: actions/upload-artifact@v3

.github/actions/tests/action.yml

@@ -0,0 +1,18 @@
+name: tests
+description: Run tests
+
+runs:
+  using: "composite"
+  steps:
+    - uses: extractions/setup-just@v1
+    - name: Install toolchain
+      shell: bash
+      run: just default-toolchain
+    - uses: Swatinem/rust-cache@v1
+    - name: Install Dependencies
+      shell: bash
+      run: sudo apt-get install -y libwayland-dev libxkbcommon-dev # Required for winit
+    - name: Test
+      shell: bash
+      # TODO: Additional test runs for different targets
+      run: just test maplibre x86_64-unknown-linux-gnu

.github/actions/webgl/action.yml

@@ -18,3 +18,8 @@ runs:
     - name: Check
       shell: bash
       run: just check web wasm32-unknown-unknown
+    - name: Test
+      shell: bash
+      run: |
+        cargo install wasm-bindgen-cli --version "0.2.80"
+        just web-test "web-webgl"

.github/actions/webgpu/action.yml

@@ -17,4 +17,9 @@ runs:
       run: just web-demo build
     - name: Check
       shell: bash
-      run: just check web wasm32-unknown-unknown
+      run: just check web wasm32-unknown-unknown
+    - name: Test
+      shell: bash
+      run: |
+        cargo install wasm-bindgen-cli --version "0.2.80"
+        just web-test ""

.github/workflows/on_main_push.yml

@@ -17,6 +17,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/benchmarks
+  run-tests:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/tests
   build-android:
     runs-on: ubuntu-20.04
     steps:
@@ -57,7 +62,7 @@ jobs:
           source: docs/book/.
           destination: docs
           key: ${{ secrets.SSH_KEY_MAXAMMANN_ORG }}
-  build-ios:
+  build-apple:
     runs-on: macos-11
     steps:
       - uses: actions/checkout@v2

.github/workflows/on_pull_request.yml

@@ -17,6 +17,11 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/benchmarks
+  run-tests:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ./.github/actions/tests
   build-android:
     runs-on: ubuntu-20.04
     steps:
@@ -37,7 +42,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - uses: ./.github/actions/docs
-  build-ios:
+  build-apple:
     runs-on: macos-11
     steps:
       - uses: actions/checkout@v2

justfile

@@ -45,12 +45,18 @@ web-lib TARGET: nightly-toolchain (web-install "lib")
 web-demo TARGET: (web-install "demo")
   cd web/demo && npm run {{TARGET}}
 
+web-test FEATURES: nightly-toolchain
+  export RUSTUP_TOOLCHAIN=$NIGHTLY_TOOLCHAIN && cargo test -p web --features "{{FEATURES}}" --target wasm32-unknown-unknown -Z build-std=std,panic_abort
+
 #profile-bench:
 # cargo flamegraph --bench render -- --bench
 
-build-android: print-android-env
+build-android: nightly-toolchain print-android-env
   export RUSTUP_TOOLCHAIN=$NIGHTLY_TOOLCHAIN && cd android/gradle && ./gradlew assembleDebug
 
+test-android TARGET: nightly-toolchain print-android-env
+  export RUSTUP_TOOLCHAIN=$NIGHTLY_TOOLCHAIN && cargo test -p maplibre-android --target {{TARGET}} -Z build-std=std,panic_abort
+
 # language=bash
 print-android-env:
   #!/usr/bin/env bash

maplibre-demo/Cargo.toml

@@ -14,7 +14,7 @@ trace = ["maplibre/trace", "tracing-subscriber", "tracing-tracy", "tracy-client"
 
 [dependencies]
 env_logger = "0.9"
-maplibre = { path = "../maplibre", version = "0.0.2"  }
+maplibre = { path = "../maplibre", version = "0.0.2", features = ["headless"]  }
 maplibre-winit = { path = "../maplibre-winit", version = "0.0.1"  }
 
 tracing = { version = "0.1" }

maplibre-demo/src/main.rs

@@ -1,8 +1,24 @@
+use maplibre::benchmarking::tessellation::{IndexDataType, OverAlignedVertexBuffer};
+use maplibre::coords::{WorldTileCoords, ZoomLevel};
+use maplibre::error::Error;
+use maplibre::io::pipeline::Processable;
+use maplibre::io::pipeline::{PipelineContext, PipelineProcessor};
+use maplibre::io::scheduler::ScheduleMethod;
+use maplibre::io::source_client::{HttpClient, HttpSourceClient};
+use maplibre::io::tile_pipelines::build_vector_tile_pipeline;
+use maplibre::io::tile_repository::StoredLayer;
+use maplibre::io::{RawLayer, TileRequest, TileRequestID};
+use maplibre::map_schedule::{EventuallyMapContext, InteractiveMapSchedule};
 use maplibre::platform::http_client::ReqwestHttpClient;
 use maplibre::platform::run_multithreaded;
 use maplibre::platform::schedule_method::TokioScheduleMethod;
+use maplibre::render::settings::{RendererSettings, TextureFormat};
+use maplibre::render::ShaderVertex;
+use maplibre::window::{EventLoop, MapWindow, MapWindowConfig, WindowSize};
 use maplibre::MapBuilder;
 use maplibre_winit::winit::{WinitEventLoop, WinitMapWindow, WinitMapWindowConfig, WinitWindow};
+use std::any::Any;
+use std::collections::HashSet;
 
 #[cfg(feature = "trace")]
 fn enable_tracing() {
@@ -13,6 +29,27 @@ fn enable_tracing() {
 
     tracing::subscriber::set_global_default(subscriber).expect("setting default subscriber failed");
 }
+pub struct HeadlessMapWindowConfig {
+    size: WindowSize,
+}
+
+impl MapWindowConfig for HeadlessMapWindowConfig {
+    type MapWindow = HeadlessMapWindow;
+
+    fn create(&self) -> Self::MapWindow {
+        Self::MapWindow { size: self.size }
+    }
+}
+
+pub struct HeadlessMapWindow {
+    size: WindowSize,
+}
+
+impl MapWindow for HeadlessMapWindow {
+    fn size(&self) -> WindowSize {
+        self.size
+    }
+}
 
 fn run_in_window() {
     run_multithreaded(async {
@@ -27,11 +64,99 @@ fn run_in_window() {
     })
 }
 
+#[derive(Default)]
+struct HeadlessPipelineProcessor {
+    layers: Vec<StoredLayer>,
+}
+
+impl PipelineProcessor for HeadlessPipelineProcessor {
+    fn layer_tesselation_finished(
+        &mut self,
+        coords: &WorldTileCoords,
+        buffer: OverAlignedVertexBuffer<ShaderVertex, IndexDataType>,
+        feature_indices: Vec<u32>,
+        layer_data: RawLayer,
+    ) {
+        self.layers.push(StoredLayer::TessellatedLayer {
+            coords: *coords,
+            buffer,
+            feature_indices,
+            layer_data,
+        })
+    }
+}
+
+fn run_headless() {
+    run_multithreaded(async {
+        let mut map = MapBuilder::new()
+            .with_map_window_config(HeadlessMapWindowConfig {
+                size: WindowSize::new(1000, 1000).unwrap(),
+            })
+            .with_http_client(ReqwestHttpClient::new(None))
+            .with_schedule_method(TokioScheduleMethod::new())
+            .with_renderer_settings(RendererSettings {
+                texture_format: TextureFormat::Rgba8UnormSrgb,
+                ..RendererSettings::default()
+            })
+            .build()
+            .initialize_headless()
+            .await;
+
+        let http_source_client: HttpSourceClient<ReqwestHttpClient> =
+            HttpSourceClient::new(ReqwestHttpClient::new(None));
+
+        let coords = WorldTileCoords::from((0, 0, ZoomLevel::default()));
+        let request_id = 0;
+
+        let data = http_source_client
+            .fetch(&coords)
+            .await
+            .unwrap()
+            .into_boxed_slice();
+
+        let processor = HeadlessPipelineProcessor::default();
+        let mut pipeline_context = PipelineContext::new(processor);
+        let pipeline = build_vector_tile_pipeline();
+        pipeline.process(
+            (
+                TileRequest {
+                    coords,
+                    layers: HashSet::from(["boundary".to_owned(), "water".to_owned()]),
+                },
+                request_id,
+                data,
+            ),
+            &mut pipeline_context,
+        );
+
+        let mut processor = pipeline_context
+            .take_processor::<HeadlessPipelineProcessor>()
+            .unwrap();
+
+        while let Some(v) = processor.layers.pop() {
+            map.map_schedule_mut()
+                .map_context
+                .tile_repository
+                .put_tessellated_layer(v);
+        }
+
+        match map.map_schedule_mut().update_and_redraw() {
+            Ok(_) => {}
+            Err(Error::Render(e)) => {
+                eprintln!("{}", e);
+                if e.should_exit() {}
+            }
+            e => eprintln!("{:?}", e),
+        };
+    })
+}
+
 fn main() {
     env_logger::init_from_env(env_logger::Env::default().default_filter_or("info"));
 
     #[cfg(feature = "trace")]
     enable_tracing();
 
-    run_in_window()
+    //run_headless();
+    run_in_window();
 }