Skip to content

Commit

Permalink
feat: zstd compression
Browse files Browse the repository at this point in the history
disabled by default due to lack of server support
close #156
  • Loading branch information
curbengh committed Jun 6, 2024
1 parent e99e110 commit 7f2033e
Show file tree
Hide file tree
Showing 5 changed files with 330 additions and 2 deletions.
29 changes: 28 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
[![NPM Dependencies](https://img.shields.io/librariesio/release/npm/hexo-yam)](https://libraries.io/npm/hexo-yam)
[![Known Vulnerabilities](https://img.shields.io/snyk/vulnerabilities/npm/hexo-yam?logo=snyk)](https://snyk.io/test/npm/hexo-yam)

Yet Another Minifier for Hexo. Minify and compress HTML, JS, CSS, SVG, XML and JSON. [Other files](https://github.com/curbengh/hexo-yam/blob/ba77db0094a7c07ea9f70f010bfc15541d4105ca/index.js#L64) are also compressed. Support gzip and [brotli](https://en.wikipedia.org/wiki/Brotli) [compressions](https://en.wikipedia.org/wiki/HTTP_compression).
Yet Another Minifier for Hexo. Minify and compress HTML, JS, CSS, SVG, XML and JSON. [Other files](https://github.com/curbengh/hexo-yam/blob/ba77db0094a7c07ea9f70f010bfc15541d4105ca/index.js#L64) are also compressed. Support gzip, brotli and zstd [compressions](https://en.wikipedia.org/wiki/HTTP_compression).


## Table of contents
Expand All @@ -19,6 +19,7 @@ Yet Another Minifier for Hexo. Minify and compress HTML, JS, CSS, SVG, XML and J
- [SVG](#svg)
- [Gzip](#gzip)
- [Brotli](#brotli)
- [Zstd](#zstd)
- [XML](#xml)
- [JSON](#json)
- [Globbing](#globbing)
Expand Down Expand Up @@ -190,6 +191,32 @@ minify:
- **globOptions** - See [globbing](#globbing) section.
- **level** - Compression level. Range `1-11`. Defaults to `11`, or the value of [`zlib.constants.BROTLI_MAX_QUALITY`](https://nodejs.org/docs/latest-v12.x/api/zlib.html#zlib_brotli_constants)

## Zstd

``` yaml
minify:
zstd:
enable: false
include:
- '*.html'
- '*.css'
- '*.js'
- '*.txt'
- '*.ttf'
- '*.atom'
- '*.stl'
- '*.xml'
- '*.svg'
- '*.eot'
- '*.json'
```
- **enable** - Enable the plugin. Defaults to `false`.
- **priority** - Plugin's priority. Defaults to `10`.
- **verbose** - Verbose output. Defaults to `false`.
- **include** - Include files. Support [wildcard](http://www.globtester.com/) pattern(s) in a string or array.
- **globOptions** - See [globbing](#globbing) section.
- **level** - Compression level. Range `1-22`. Defaults to `3`, or the value of [`DEFAULT_LEVEL`](https://github.com/mongodb-js/zstd/blob/a3a08c61c9045411c8275e248498dbc583457fb5/src/lib.rs#L9)

## XML

Remove whitespaces in xml.
Expand Down
9 changes: 9 additions & 0 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,14 @@ hexo.config.minify.brotli = Object.assign({
globOptions: { basename: true }
}, hexo.config.minify.brotli)

hexo.config.minify.zstd = Object.assign({
enable: false,
priority: 10,
verbose: false,
include: ['*.html', '*.css', '*.js', '*.txt', '*.ttf', '*.atom', '*.stl', '*.xml', '*.svg', '*.eot', '*.json'],
globOptions: { basename: true }
}, hexo.config.minify.zstd)

hexo.config.minify.xml = Object.assign({
enable: false,
priority: 10,
Expand All @@ -93,6 +101,7 @@ if (hexo.config.minify.enable === true) {
hexo.extend.filter.register('after_generate', filter.minifySvg, hexo.config.minify.svg.priority)
hexo.extend.filter.register('after_generate', filter.gzipFn, hexo.config.minify.gzip.priority)
hexo.extend.filter.register('after_generate', filter.brotliFn, hexo.config.minify.brotli.priority)
hexo.extend.filter.register('after_generate', filter.zstdFn, hexo.config.minify.zstd.priority)
hexo.extend.filter.register('after_generate', filter.minifyXml, hexo.config.minify.xml.priority)
hexo.extend.filter.register('after_generate', filter.minifyJson, hexo.config.minify.json.priority)
}
35 changes: 35 additions & 0 deletions lib/filter.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ const gzip = promisify(zlib.gzip)
const br = promisify(zlib.brotliCompress)
const { minify: compressXml } = require('minify-xml')
const micromatch = require('micromatch')
const { compress: zstd } = require('@mongodb-js/zstd')

const isMatch = (path = '', patterns = [], options = {}) => {
if (path && patterns) {
Expand Down Expand Up @@ -230,6 +231,39 @@ function brotliFn () {
}))
}

function zstdFn () {
const hexo = this
const options = hexo.config.minify.zstd
if (options.enable === false) return

const { route } = hexo
const routeList = route.list()
const { globOptions, include, verbose } = options
let { level } = options
if (typeof level !== 'number') level = undefined

return Promise.all((match(routeList, include, globOptions)).map((path) => {
return new Promise((resolve, reject) => {
const assetPath = route.get(path)
let assetTxt = ''
assetPath.on('data', (chunk) => (assetTxt += chunk))
assetPath.on('end', async () => {
if (assetTxt.length) {
try {
const input = Buffer.from(assetTxt, 'utf-8')
const result = await zstd(input, level)
if (verbose) logFn.call(this, assetTxt, result, path, 'zstd')
resolve(route.set(path + '.zst', result))
} catch (err) {
reject(new Error(`Path: ${path}\n${err}`))
}
}
resolve()
})
})
}))
}

function minifyXml () {
const hexo = this
const options = hexo.config.minify.xml
Expand Down Expand Up @@ -297,6 +331,7 @@ module.exports = {
minifySvg,
gzipFn,
brotliFn,
zstdFn,
minifyXml,
minifyJson
}
3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@
"micromatch": "^4.0.2",
"minify-xml": "^3.2.0",
"svgo": "^3.0.0",
"terser": "^5.3.0"
"terser": "^5.3.0",
"@mongodb-js/zstd": "^1.2.0"
},
"devDependencies": {
"hexo": "^7.1.0",
Expand Down
256 changes: 256 additions & 0 deletions test/zstd.test.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,256 @@
/* eslint-env jest */
'use strict'

const Hexo = require('hexo')
const { compress: zstd, decompress: unzstd } = require('@mongodb-js/zstd')

describe('zstd', () => {
const hexo = new Hexo(__dirname)
const z = require('../lib/filter').zstdFn.bind(hexo)
const path = 'foo.txt'
const input = 'Lorem ipsum dolor sit amet consectetur adipiscing elit fusce'
const inputBuf = Buffer.from(input, 'utf8')

beforeEach(() => {
hexo.config.minify = {
zstd: {
enable: true,
verbose: false,
include: ['*.html', '*.css', '*.js', '*.txt', '*.ttf', '*.atom', '*.stl', '*.xml', '*.svg', '*.eot', '*.json'],
globOptions: { basename: true }
}
}
hexo.route.set(path, input)
})

afterEach(() => {
const routeList = hexo.route.list()
routeList.forEach((path) => hexo.route.remove(path))
})

test('default', async () => {
await z()

const output = hexo.route.get(path.concat('.zst'))
const buf = []
output.on('data', (chunk) => (buf.push(chunk)))
output.on('end', async () => {
const result = Buffer.concat(buf)
const expected = await zstd(inputBuf)
const resultUnzst = await unzstd(result)
const expectedUnzst = await unzstd(expected)

expect(result.equals(expected)).toBe(true)
expect(resultUnzst.toString()).toBe(input)
expect(expectedUnzst.toString()).toBe(input)
})
})

test('disable', async () => {
hexo.config.minify.zstd.enable = false
const result = await z()

expect(result).toBeUndefined()
})

test('empty file', async () => {
hexo.route.set(path, '')

const routeList = hexo.route.list()
expect(routeList).not.toContain(path.concat('.zst'))

const result = await z()
expect(result).toBeDefined()
expect(result[0]).toBeUndefined()
})

test('option', async () => {
const level = 1
hexo.config.minify.zstd.level = level
await z()

const output = hexo.route.get(path.concat('.zst'))
const buf = []
output.on('data', (chunk) => (buf.push(chunk)))
output.on('end', async () => {
const result = Buffer.concat(buf)
const expected = await zstd(inputBuf, level)

expect(result.equals(expected)).toBe(true)
})
})

test('option - verbose', async () => {
hexo.config.minify.zstd.verbose = true
hexo.log.log = jest.fn()
await z()

expect(hexo.log.log.mock.calls[0][0]).toContain(`zstd: ${path}`)
})

test('option - level is string', async () => {
const level = 'foo'
hexo.config.minify.zstd.level = level
await z()

const output = hexo.route.get(path.concat('.zst'))
const buf = []
output.on('data', (chunk) => (buf.push(chunk)))
output.on('end', async () => {
const result = Buffer.concat(buf)
const expected = await zstd(inputBuf, undefined)

expect(result.equals(expected)).toBe(true)
})
})

test('include - exclude non-text file by default', async () => {
const path = 'foo.jpg'
hexo.route.set(path, input)
await z()

const result = hexo.route.get(path.concat('.zst'))

expect(result).toBeUndefined()
})

test('include - basename', async () => {
hexo.config.minify.zstd.include = 'bar.txt'
const path = 'foo/bar.txt'
hexo.route.set(path, input)
await z()

const result = hexo.route.get(path.concat('.zst'))

expect(result).toBeDefined()
})

test('include - slash in pattern', async () => {
hexo.config.minify.zstd.include = '**/lectus/**/*.txt'
const path = 'eleifend/lectus/nullam/dapibus/netus.txt'
hexo.route.set(path, input)
await z()

const result = hexo.route.get(path.concat('.zst'))

expect(result).toBeDefined()
})

test('include - basename + slash + basename enabled', async () => {
hexo.route.remove(path)

const paths = [
'lorem/ipsum/dolor.html',
'gravida/sociis/erat/ante.css',
'aptent/elementum.js',
'felis/blandit/cursus.svg'
]
hexo.config.minify.zstd.include = [
'*.html',
'**/sociis/**/*.css'
]

paths.forEach((inpath) => {
hexo.route.set(inpath, input)
})
await z()

const routeList = hexo.route.list()
const expected = [
'lorem/ipsum/dolor.html.zst',
'gravida/sociis/erat/ante.css.zst'
]
const notExpected = [
'aptent/elementum.js.zst',
'felis/blandit/cursus.svg.zst'
]

expect(routeList).toEqual(expect.arrayContaining(expected))
expect(routeList).toEqual(expect.not.arrayContaining(notExpected))
})

test('include - basename + slash + basename disabled', async () => {
hexo.route.remove(path)

const paths = [
'lorem/ipsum/dolor.html',
'gravida/sociis/erat/ante.css',
'aptent/elementum.js',
'felis/blandit/cursus.svg'
]
hexo.config.minify.zstd.include = [
'*.html',
'**/sociis/**/*.css'
]
hexo.config.minify.zstd.globOptions = {
basename: false
}

paths.forEach((inpath) => {
hexo.route.set(inpath, input)
})
await z()

const routeList = hexo.route.list()
const expected = [
'gravida/sociis/erat/ante.css.zst'
]
const notExpected = [
'lorem/ipsum/dolor.html.zst',
'aptent/elementum.js.zst',
'felis/blandit/cursus.svg.zst'
]

expect(routeList).toEqual(expect.arrayContaining(expected))
expect(routeList).toEqual(expect.not.arrayContaining(notExpected))
})

test('include - reverse pattern + basename disabled', async () => {
hexo.route.remove(path)

const paths = [
'lorem/ipsum/dolor.html',
'gravida/sociis/erat/ante.css',
'aptent/elementum.js',
'felis/blandit/cursus.svg'
]
hexo.config.minify.zstd.include = [
'!dolor.html'
]
hexo.config.minify.zstd.globOptions = {
basename: false
}

paths.forEach((inpath) => {
hexo.route.set(inpath, input)
})
await z()

const routeList = hexo.route.list()
const expected = paths.map((path) => path.concat('.zst'))

expect(routeList).toEqual(expect.arrayContaining(expected))
})

test('blns', async () => {
const blns = require('./fixtures/blns.json')

for (const nStr of blns) {
hexo.route.remove(path)

hexo.route.set(path, nStr)

await z()

const output = hexo.route.get(path.concat('.zst'))
const buf = []
output.on('data', (chunk) => (buf.push(chunk)))
output.on('end', async () => {
const result = Buffer.concat(buf)
const resultUnzst = await unzstd(result)

expect(resultUnzst.toString()).toBe(nStr)
})
}
})
})

0 comments on commit 7f2033e

Please sign in to comment.