From fa8118888776ace454ca1fe6373d5a2a4a150a68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Casper=20K=C3=BCthe?=
<43839798+Casper64@users.noreply.github.com>
Date: Sat, 30 Dec 2023 21:10:10 +0100
Subject: [PATCH] x.vweb.assets: reimplement assets module for x.vweb (#20280)
---
vlib/x/vweb/assets/README.md | 164 ++++++++++++++++
vlib/x/vweb/assets/assets.v | 311 +++++++++++++++++++++++++++++++
vlib/x/vweb/assets/assets_test.v | 190 +++++++++++++++++++
3 files changed, 665 insertions(+)
create mode 100644 vlib/x/vweb/assets/README.md
create mode 100644 vlib/x/vweb/assets/assets.v
create mode 100644 vlib/x/vweb/assets/assets_test.v
diff --git a/vlib/x/vweb/assets/README.md b/vlib/x/vweb/assets/README.md
new file mode 100644
index 00000000000000..33e431c10b9516
--- /dev/null
+++ b/vlib/x/vweb/assets/README.md
@@ -0,0 +1,164 @@
+# Assets
+
+The asset manager for vweb. You can use this asset manager to minify CSS and Javscript files,
+combine them into a single file and to make sure the asset you're using exists.
+
+## Usage
+
+Add `AssetManager` to your App struct to use the asset manager.
+
+**Example:**
+```v
+module main
+
+import x.vweb
+import x.vweb.assets
+
+pub struct Context {
+ vweb.Context
+}
+
+pub struct App {
+pub mut:
+ am assets.AssetManager
+}
+
+fn main() {
+ mut app := &App{}
+ vweb.run[App, Context](mut app, 8080)
+}
+```
+
+### Including assets
+
+If you want to include an asset in your templates you can use the `include` method.
+First pass the type of asset (css or js), then specify the "include name" of an asset.
+
+**Example:**
+```html
+@{app.am.include(.css, 'main.css')}
+```
+Will generate
+```html
+
+```
+
+### Adding assets
+
+To add an asset use the `add` method. You must specify the path of the asset and what its
+include name will be: the name that you will use in templates.
+
+**Example:**
+```v ignore
+// add a css file at the path "css/main.css" and set its include name to "main.css"
+app.am.add(.css, 'css/main.css', 'main.css')
+```
+
+### Minify and Combine assets
+
+If you want to minify each asset you must set the `minify` field and specify the cache
+folder. Each assest you add is minifed and outputted in `cache_dir`.
+
+**Example:**
+```v ignore
+pub struct App {
+pub mut:
+ am assets.AssetManager = assets.AssetManager{
+ cache_dir: 'dist'
+ minify: true
+ }
+}
+```
+
+To combine the all currently added assets into a single file you must call the `combine` method
+and specify which asset type you want to combine.
+
+**Example:**
+```v ignore
+// `combine` returns the path of the minified file
+minified_file := app.am.combine(.css)!
+```
+
+### Handle folders
+
+You can use the asset manger in combination with vweb's `StaticHandler` to serve
+assets in a folder as static assets.
+
+**Example:**
+```v ignore
+pub struct App {
+ vweb.StaticHandler
+pub mut:
+ am assets.AssetManager
+}
+```
+Let's say we have the following folder structure:
+```
+assets/
+├── css/
+│ └── main.css
+└── js/
+ └── main.js
+```
+
+We can tell the asset manager to add all assets in the `static` folder
+
+**Example:**
+```v ignore
+fn main() {
+ mut app := &App{}
+ // add all assets in the "assets" folder
+ app.am.handle_assets('assets')!
+ // serve all files in the "assets" folder as static files
+ app.handle_static('assets', false)!
+ // start the app
+ vweb.run[App, Context](mut app, 8080)
+}
+```
+
+The include name of each minified asset will be set to its relative path,
+so if you want to include `main.css` in your template you would write
+`@{app.am.include('css/main.css')}`
+
+#### Minify
+
+If you add an asset folder and want to minify those assets you can call the
+`cleanup_cache` method to remove old files from the cache folder
+that are no longer needed.
+
+**Example:**
+```v ignore
+pub struct App {
+ vweb.StaticHandler
+pub mut:
+ am assets.AssetManager = assets.AssetManager{
+ cache_dir: 'dist'
+ minify: true
+ }
+}
+
+fn main() {
+ mut app := &App{}
+ // add all assets in the "assets" folder
+ app.am.handle_assets('assets')!
+ // remove all old cached files from the cache folder
+ app.am.cleanup_cache()!
+ // serve all files in the "assets" folder as static files
+ app.handle_static('assets', false)!
+ // start the app
+ vweb.run[App, Context](mut app, 8080)
+}
+```
+
+#### Prefix the include name
+
+You can add a custom prefix to the include name of assets when adding a folder.
+
+**Example:**
+```v ignore
+// add all assets in the "assets" folder
+app.am.handle_assets_at('assets', 'static')!
+```
+
+Now if you want to include `main.css` you would write
+``@{app.am.include('static/css/main.css')}`
diff --git a/vlib/x/vweb/assets/assets.v b/vlib/x/vweb/assets/assets.v
new file mode 100644
index 00000000000000..e5d8da4ba51fab
--- /dev/null
+++ b/vlib/x/vweb/assets/assets.v
@@ -0,0 +1,311 @@
+module assets
+
+import crypto.md5
+import os
+import strings
+import time
+import x.vweb
+
+pub enum AssetType {
+ css
+ js
+ all
+}
+
+pub struct Asset {
+pub:
+ kind AssetType
+ file_path string
+ last_modified time.Time
+ include_name string
+}
+
+pub struct AssetManager {
+mut:
+ css []Asset
+ js []Asset
+ cached_file_names []string
+pub mut:
+ // when true assets will be minified
+ minify bool
+ // the directory to store the cached/combined files
+ cache_dir string
+ // how a combined file should be named. For example for css the extension '.css'
+ // will be added to the end of `combined_file_name`
+ combined_file_name string = 'combined'
+}
+
+fn (mut am AssetManager) add_asset_directory(directory_path string, traversed_path string) ! {
+ files := os.ls(directory_path)!
+ if files.len > 0 {
+ for file in files {
+ full_path := os.join_path(directory_path, file)
+ relative_path := os.join_path(traversed_path, file)
+
+ if os.is_dir(full_path) {
+ am.add_asset_directory(full_path, relative_path)!
+ } else {
+ ext := os.file_ext(full_path)
+ match ext {
+ '.css' { am.add(.css, full_path, relative_path)! }
+ '.js' { am.add(.js, full_path, relative_path)! }
+ // ignore non css/js files
+ else {}
+ }
+ }
+ }
+ }
+}
+
+// handle_assets recursively walks `directory_path` and adds any assets to the asset manager
+pub fn (mut am AssetManager) handle_assets(directory_path string) ! {
+ return am.add_asset_directory(directory_path, '')
+}
+
+// handle_assets_at recursively walks `directory_path` and adds any assets to the asset manager.
+// The include name of assets are prefixed with `prepend`
+pub fn (mut am AssetManager) handle_assets_at(directory_path string, prepend string) ! {
+ // remove trailing '/'
+ return am.add_asset_directory(directory_path, prepend.trim_right('/'))
+}
+
+// get all assets of type `asset_type`
+pub fn (am AssetManager) get_assets(asset_type AssetType) []Asset {
+ return match asset_type {
+ .css {
+ am.css
+ }
+ .js {
+ am.js
+ }
+ .all {
+ mut assets := []Asset{}
+ assets << am.css
+ assets << am.js
+ assets
+ }
+ }
+}
+
+// add an asset to the asset manager
+pub fn (mut am AssetManager) add(asset_type AssetType, file_path string, include_name string) ! {
+ if asset_type == .all {
+ return error('cannot minify asset of type "all"')
+ }
+ if !os.exists(file_path) {
+ return error('cnanot add asset: file "${file_path}" does not exist')
+ }
+
+ last_modified_unix := os.file_last_mod_unix(file_path)
+
+ mut real_path := file_path
+
+ if am.minify {
+ // minify and cache file if it was modified
+ output_path, is_cached := am.minify_and_cache(asset_type, real_path, last_modified_unix,
+ include_name)!
+
+ if is_cached == false && am.exists(asset_type, include_name) {
+ // file was not modified between the last call to `add`
+ // and the file was already in the asset manager, so we don't need to
+ // add it again
+ return
+ }
+
+ real_path = output_path
+ }
+
+ asset := Asset{
+ kind: asset_type
+ file_path: real_path
+ last_modified: time.unix(last_modified_unix)
+ include_name: include_name
+ }
+
+ match asset_type {
+ .css { am.css << asset }
+ .js { am.js << asset }
+ else {}
+ }
+}
+
+fn (mut am AssetManager) minify_and_cache(asset_type AssetType, file_path string, last_modified i64, include_name string) !(string, bool) {
+ if asset_type == .all {
+ return error('cannot minify asset of type "all"')
+ }
+
+ if am.cache_dir == '' {
+ return error('cannot minify asset: cache directory is not valid')
+ } else if !os.exists(am.cache_dir) {
+ os.mkdir_all(am.cache_dir)!
+ }
+
+ cache_key := am.get_cache_key(file_path, last_modified)
+ output_file := '${cache_key}.${asset_type}'
+ output_path := os.join_path(am.cache_dir, output_file)
+
+ if os.exists(output_path) {
+ // the output path already exists, this means that the file has
+ // been minifed and cached before and hasn't changed in the meantime
+ am.cached_file_names << output_file
+ return output_path, false
+ } else {
+ // check if the file has been minified before, but is modified.
+ // if that's the case we remove the old cached file
+ cached_files := os.ls(am.cache_dir)!
+ hash := cache_key.all_before('-')
+ for file in cached_files {
+ if file.starts_with(hash) {
+ os.rm(os.join_path(am.cache_dir, file))!
+ }
+ }
+ }
+
+ txt := os.read_file(file_path)!
+ minified := match asset_type {
+ .css { minify_css(txt) }
+ .js { minify_js(txt) }
+ else { '' }
+ }
+ os.write_file(output_path, minified)!
+
+ am.cached_file_names << output_file
+ return output_path, true
+}
+
+fn (mut am AssetManager) get_cache_key(file_path string, last_modified i64) string {
+ abs_path := if os.is_abs_path(file_path) { file_path } else { os.resource_abs_path(file_path) }
+ hash := md5.sum(abs_path.bytes())
+ return '${hash.hex()}-${last_modified}'
+}
+
+// cleanup_cache removes all files in the cache directory that aren't cached at the time
+// this function is called
+pub fn (mut am AssetManager) cleanup_cache() ! {
+ if am.cache_dir == '' {
+ return error('[vweb.assets]: cache directory is not valid')
+ }
+ cached_files := os.ls(am.cache_dir)!
+
+ // loop over all the files in the cache directory. If a file isn't cached, remove it
+ for file in cached_files {
+ ext := os.file_ext(file)
+ if ext !in ['.css', '.js'] || file in am.cached_file_names {
+ continue
+ } else if !file.starts_with(am.combined_file_name) {
+ os.rm(os.join_path(am.cache_dir, file))!
+ }
+ }
+}
+
+// check if an asset is already added to the asset manager
+pub fn (am AssetManager) exists(asset_type AssetType, include_name string) bool {
+ assets := am.get_assets(asset_type)
+
+ return assets.any(it.include_name == include_name)
+}
+
+// include css/js files in your vweb app from templates
+// Example:
+// ```html
+// @{app.am.include(.css, 'main.css')}
+// ```
+pub fn (am AssetManager) include(asset_type AssetType, include_name string) vweb.RawHtml {
+ assets := am.get_assets(asset_type)
+ for asset in assets {
+ if asset.include_name == include_name {
+ // always add link/src from root of web server ('/css/main.css'),
+ // but leave absolute paths intact
+ mut real_path := asset.file_path
+ if real_path[0] != `/` && !os.is_abs_path(real_path) {
+ real_path = '/${asset.file_path}'
+ }
+
+ return match asset_type {
+ .css {
+ ''
+ }
+ .js {
+ ''
+ }
+ else {
+ eprintln('[vweb.assets] can only include css or js assets')
+ ''
+ }
+ }
+ }
+ }
+ eprintln('[vweb.assets] no asset with include name "${include_name}" exists!')
+ return ''
+}
+
+// combine assets of type `asset_type` into a single file and return the outputted file path.
+// If you call `combine` with asset type `all` the function will return an empty string,
+// the minified files will be available at `combined_file_name`.`asset_type`
+pub fn (mut am AssetManager) combine(asset_type AssetType) !string {
+ if asset_type == .all {
+ am.combine(.css)!
+ am.combine(.js)!
+ return ''
+ }
+ if am.cache_dir == '' {
+ return error('cannot combine assets: cache directory is not valid')
+ } else if !os.exists(am.cache_dir) {
+ os.mkdir_all(am.cache_dir)!
+ }
+
+ assets := am.get_assets(asset_type)
+ combined_file_path := os.join_path(am.cache_dir, '${am.combined_file_name}.${asset_type}')
+ mut f := os.create(combined_file_path)!
+
+ for asset in assets {
+ bytes := os.read_bytes(asset.file_path)!
+ f.write(bytes)!
+ f.write_string('\n')!
+ }
+
+ f.close()
+
+ return combined_file_path
+}
+
+// TODO: implement proper minification
+@[manualfree]
+pub fn minify_css(css string) string {
+ mut lines := css.split('\n')
+ // estimate arbitrary number of characters for a line of css
+ mut sb := strings.new_builder(lines.len * 20)
+ defer {
+ unsafe { sb.free() }
+ }
+
+ for line in lines {
+ trimmed := line.trim_space()
+ if trimmed.len > 0 {
+ sb.write_string(trimmed)
+ }
+ }
+
+ return sb.str()
+}
+
+// TODO: implement proper minification
+@[manualfree]
+pub fn minify_js(js string) string {
+ mut lines := js.split('\n')
+ // estimate arbitrary number of characters for a line of js
+ mut sb := strings.new_builder(lines.len * 40)
+ defer {
+ unsafe { sb.free() }
+ }
+
+ for line in lines {
+ trimmed := line.trim_space()
+ if trimmed.len > 0 {
+ sb.write_string(trimmed)
+ sb.write_u8(` `)
+ }
+ }
+
+ return sb.str()
+}
diff --git a/vlib/x/vweb/assets/assets_test.v b/vlib/x/vweb/assets/assets_test.v
new file mode 100644
index 00000000000000..b8d2d5b3bc89b3
--- /dev/null
+++ b/vlib/x/vweb/assets/assets_test.v
@@ -0,0 +1,190 @@
+import x.vweb.assets
+import os
+
+const base_cache_dir = os.join_path(os.vtmp_dir(), 'xvweb_assets_test_cache')
+
+fn testsuite_begin() {
+ os.mkdir_all(base_cache_dir) or {}
+}
+
+fn testsuite_end() {
+ os.rmdir_all(base_cache_dir) or {}
+}
+
+// clean_cache_dir used before and after tests that write to a cache directory.
+// Because of parallel compilation and therefore test running,
+// unique cache dirs are needed per test function.
+fn clean_cache_dir(dir string) {
+ os.rmdir_all(dir) or {}
+}
+
+fn cache_dir(test_name string) string {
+ return os.join_path(base_cache_dir, test_name)
+}
+
+fn get_test_file_path(file string) string {
+ path := os.join_path(base_cache_dir, file)
+ os.rm(path) or {}
+ os.write_file(path, get_test_file_contents(file)) or { panic(err) }
+ return path
+}
+
+fn get_test_file_contents(file string) string {
+ contents := match file {
+ 'test1.js' { '{"one": 1}\n' }
+ 'test2.js' { '{"two": 2}\n' }
+ 'test1.css' { '.one {\n\tcolor: #336699;\n}\n' }
+ 'test2.css' { '.two {\n\tcolor: #996633;\n}\n' }
+ else { 'wibble\n' }
+ }
+ return contents
+}
+
+fn test_add() {
+ mut am := assets.AssetManager{}
+
+ mut errored := false
+ am.add(.css, 'test.css', 'test.css') or { errored = true }
+ assert errored == true, 'am.add should error'
+
+ errored = false
+ am.add(.css, get_test_file_path('test1.css'), 'included.css') or {
+ eprintln(err)
+ errored = true
+ }
+ assert errored == false, 'am.add should not error'
+
+ css_assets := am.get_assets(.css)
+ assert css_assets.len == 1
+ assert css_assets[0].file_path == get_test_file_path('test1.css')
+ assert css_assets[0].include_name == 'included.css'
+}
+
+fn test_add_minify_missing_cache_dir() {
+ mut am := assets.AssetManager{
+ minify: true
+ }
+ mut errored := false
+ am.add(.js, get_test_file_path('test1.css'), 'included.js') or {
+ assert err.msg() == 'cannot minify asset: cache directory is not valid'
+ errored = true
+ }
+
+ assert errored == true, 'am.add should return an error'
+}
+
+fn test_add_minified() {
+ mut am := assets.AssetManager{
+ minify: true
+ cache_dir: cache_dir('test_add_minified')
+ }
+ clean_cache_dir(am.cache_dir)
+
+ am.add(.js, get_test_file_path('test1.js'), 'included.js')!
+
+ js_assets := am.get_assets(.js)
+ assert js_assets.len == 1
+ assert js_assets[0].file_path.starts_with(am.cache_dir) == true
+}
+
+fn test_combine() {
+ mut am := assets.AssetManager{
+ cache_dir: cache_dir('test_combine')
+ }
+ clean_cache_dir(am.cache_dir)
+
+ am.add(.css, get_test_file_path('test1.css'), 'test1.css')!
+ am.add(.css, get_test_file_path('test2.css'), 'test2.css')!
+
+ combined_path := am.combine(.css)!
+ combined := os.read_file(combined_path)!
+
+ expected := get_test_file_contents('test1.css') + '\n' + get_test_file_contents('test2.css') +
+ '\n'
+ assert combined == expected
+}
+
+fn test_combine_minified() {
+ // minify test is simple for now, because assets are not properly minified yet
+ mut am := assets.AssetManager{
+ cache_dir: cache_dir('test_combine_minified')
+ minify: true
+ }
+ clean_cache_dir(am.cache_dir)
+
+ am.add(.css, get_test_file_path('test1.css'), 'test1.css')!
+ am.add(.css, get_test_file_path('test2.css'), 'test2.css')!
+
+ combined_path := am.combine(.css)!
+ combined := os.read_file(combined_path)!
+
+ // minified version should be 2 lines + one extra newline
+ assert combined.split('\n').len == 3
+}
+
+fn test_minify_cache_last_modified() {
+ mut am := assets.AssetManager{
+ minify: true
+ cache_dir: cache_dir('test_cache_last_modified')
+ }
+ clean_cache_dir(am.cache_dir)
+
+ // first we write the file and add it
+ am.add(.js, get_test_file_path('test1.js'), 'included.js')!
+ mut js_assets := am.get_assets(.js)
+ assert js_assets.len == 1
+ old_cached_path := js_assets[0].file_path
+
+ // then we only add the file, the file is not modified so the "last modified is the same".
+ // we expect that the asset manager doesn't cache a minified file if it hasn't been changed
+ // the last time it was added
+ am.add(.js, os.join_path(base_cache_dir, 'test1.js'), 'included.js')!
+
+ js_assets = am.get_assets(.js)
+ // check if the file isn't added twice
+ assert js_assets.len == 1
+ // if the file path was not modified, vweb.assets didn't overwrite the file
+ assert js_assets[0].file_path == old_cached_path
+}
+
+fn test_cleanup_cache() {
+ mut am := assets.AssetManager{
+ minify: true
+ cache_dir: cache_dir('test_cleanup_cache')
+ }
+ clean_cache_dir(am.cache_dir)
+ // manually make the cache dir
+ os.mkdir_all(am.cache_dir) or {}
+
+ // write a file to the cache dir isn't added to the asset manager to represent
+ // a previously cached file
+ path1 := os.join_path(am.cache_dir, 'test1.css')
+ os.write_file(path1, 'h1 { color: red; }')!
+ assert os.exists(path1) == true
+
+ // add a file to the asset manager and write it
+ am.add(.css, get_test_file_path('test2.css'), 'test2.css')!
+ css_assets := am.get_assets(.css)
+ // get the cached path
+ assert css_assets.len == 1
+ path2 := css_assets[0].file_path
+ assert os.exists(path2) == true
+
+ am.cleanup_cache()!
+
+ // the first asset wasn't added to the asset manager, so it should not exist
+ assert os.exists(path1) == false
+ assert os.exists(path2) == true
+}
+
+fn test_include() {
+ mut am := assets.AssetManager{}
+
+ css_path := get_test_file_path('test1.css')
+ js_path := get_test_file_path('test1.js')
+ am.add(.css, css_path, 'other.css')!
+ am.add(.js, js_path, 'js/test.js')!
+
+ assert am.include(.css, 'other.css') == ''
+ assert am.include(.js, 'js/test.js') == ''
+}