Skip to content

Commit

Permalink
queue preview generation and cache results on disk
Browse files Browse the repository at this point in the history
  • Loading branch information
@axe312ger
axe312ger committed Feb 25, 2018
1 parent f21aae0 commit ae8fc3b
Show file tree
Hide file tree
Showing 2 changed files with 91 additions and 62 deletions.
2 changes: 2 additions & 0 deletions packages/gatsby-transformer-sqip/package.json
View file
Expand Up @@ -9,6 +9,8 @@
"dependencies": {
"babel-runtime": "^6.26.0",
"fs-extra": "^4.0.2",
"p-queue": "^2.3.0",
"progress": "^2.0.0",
"sqip": "^0.2.2"
},
"devDependencies": {
Expand Down
151 changes: 89 additions & 62 deletions packages/gatsby-transformer-sqip/src/extend-node-type.js
View file
@@ -1,16 +1,32 @@
const crypto = require(`crypto`)
const { createWriteStream } = require(`fs`)
const { extname, join, resolve } = require(`path`)

const {
GraphQLString,
GraphQLInt,
} = require(`graphql`)
const fs = require(`fs-extra`)
const { GraphQLObjectType, GraphQLString, GraphQLInt } = require(`graphql`)
const PQueue = require(`p-queue`)
const ProgressBar = require(`progress`)
const sqip = require(`sqip`)

module.exports = async (args) => {
const SUPPORTED_NODES = [`ImageSharp`, `ContentfulAsset`]
const queue = new PQueue({ concurrency: 1 })
let bar

module.exports = async args => {
const { type: { name } } = args

if (!SUPPORTED_NODES.includes(name)) {
return {}
}

// @todo progress bar does not show up some times
bar = new ProgressBar(
`Generating sqip's [:bar] :current/:total :percent :eta left`,
{
total: 0,
width: process.stdout.columns || 40,
}
)

if (name === `ImageSharp`) {
return sqipSharp(args)
}
Expand All @@ -22,27 +38,21 @@ module.exports = async (args) => {
return {}
}

async function sqipSharp({ type, cache, getNodeAndSavePathDependency }) {
if (type.name !== `ImageSharp`) {
return {}
}

async function sqipSharp({ type, cache, getNodeAndSavePathDependency, store }) {
return {
sqip: {
type: GraphQLString,
args: {
blur: {
type: GraphQLInt,
defaultValue: 1,
},
numberOfPrimitives: {
type: GraphQLInt,
defaultValue: 10,
},
mode: {
type: GraphQLInt,
defaultValue: 0,
type: new GraphQLObjectType({
name: `Sqip`,
fields: {
svg: { type: GraphQLString },
dataURI: { type: GraphQLString },
dataURIbase64: { type: GraphQLString },
},
}),
args: {
blur: { type: GraphQLInt, defaultValue: 1 },
numberOfPrimitives: { type: GraphQLInt, defaultValue: 10 },
mode: { type: GraphQLInt, defaultValue: 0 },
},
async resolve(image, fieldArgs, context) {
const { blur, numberOfPrimitives, mode } = fieldArgs
Expand All @@ -56,21 +66,20 @@ async function sqipSharp({ type, cache, getNodeAndSavePathDependency }) {
numberOfPrimitives,
blur,
mode,
store,
})
},
},
}
}

async function sqipContentful({ type, store, cache }) {
if (type.name !== `ContentfulAsset`) {
return {}
}

const fs = require(`fs-extra`)
const { createWriteStream } = require(`fs`)
const axios = require(`axios`)

const { schemes: { ImageResizingBehavior, ImageCropFocusType } } = require(`gatsby-source-contentful`)
const {
schemes: { ImageResizingBehavior, ImageCropFocusType },
} = require(`gatsby-source-contentful`)

const cacheDir = join(
store.getState().program.directory,
Expand Down Expand Up @@ -117,11 +126,7 @@ async function sqipContentful({ type, store, cache }) {
},
},
async resolve(asset, fieldArgs, context) {
const {
id,
file: { url, fileName, details, contentType },
node_locale: locale,
} = asset
const { id, file: { url, fileName, details, contentType } } = asset
const {
blur,
numberOfPrimitives,
Expand All @@ -137,19 +142,14 @@ async function sqipContentful({ type, store, cache }) {
return null
}

console.log(`Processing: ${id}-${locale}`)

// Downloading small version of the image with same aspect ratio
const assetWidth = width || details.image.width
const assetHeight = height || details.image.height
const aspectRatio = assetHeight / assetWidth
const previewWidth = 256
const previewHeight = Math.floor(previewWidth * aspectRatio)

const params = [
`w=${previewWidth}`,
`h=${previewHeight}`,
]
const params = [`w=${previewWidth}`, `h=${previewHeight}`]
if (resizingBehavior) {
params.push(`fit=${resizingBehavior}`)
}
Expand All @@ -175,12 +175,10 @@ async function sqipContentful({ type, store, cache }) {

const alreadyExists = await fs.pathExists(absolutePath)

console.log(`Calculated path: ${absolutePath}`)

if (!alreadyExists) {
const previewUrl = `http:${url}?${params.join(`&`)}`

console.log(`Downloading: ${previewUrl}`)
bar.interrupt(`Downloading: ${previewUrl}`)

const response = await axios({
method: `get`,
Expand All @@ -191,9 +189,7 @@ async function sqipContentful({ type, store, cache }) {
await new Promise((resolve, reject) => {
const file = createWriteStream(absolutePath)
response.data.pipe(file)
file.on(`finish`, () => {
resolve()
})
file.on(`finish`, resolve)
file.on(`error`, reject)
})
}
Expand All @@ -204,14 +200,17 @@ async function sqipContentful({ type, store, cache }) {
numberOfPrimitives,
blur,
mode,
store,
})
},
},
}
}

async function generateSqip(options) {
const { cache, absolutePath, numberOfPrimitives, blur, mode } = options
const { cache, absolutePath, numberOfPrimitives, blur, mode, store } = options

const dataDir = join(store.getState().program.directory, `.cache-sqip`)

// @todo add check if file actually exists

Expand All @@ -228,22 +227,50 @@ async function generateSqip(options) {
.digest(`hex`)

const cacheKey = `sqip-${optionsHash}`

let svgThumbnail = await cache.get(cacheKey)

if (!svgThumbnail) {
console.log(`Calculating low quality svg thumbnail: ${absolutePath}`)
const result = sqip(sqipOptions)
// @todo make blur setting in sqip via PR
svgThumbnail = result.final_svg.replace(new RegExp(`<feGaussianBlur stdDeviation="[0-9]+"\\s*/>`), `<feGaussianBlur stdDeviation="${sqipOptions.blur}" />`)

await cache.set(cacheKey, svgThumbnail)
console.log(`done calculating primitive ${absolutePath}`)
const cachePath = resolve(dataDir, `sqip-${optionsHash}.json`)
let primitiveData = await cache.get(cacheKey)

await fs.ensureDir(dataDir)

bar.total++

if (!primitiveData) {
if (await fs.exists(cachePath)) {
const cacheData = await fs.readFile(cachePath)
primitiveData = JSON.parse(cacheData)
} else {
const result = await queue.add(
async () =>
new Promise((resolve, reject) => {
try {
resolve(sqip(sqipOptions))
} catch (error) {
reject(error)
}
})
)

// @todo make blur setting in sqip via PR
result.final_svg = result.final_svg.replace(
new RegExp(`<feGaussianBlur stdDeviation="[0-9]+"\\s*/>`),
`<feGaussianBlur stdDeviation="${sqipOptions.blur}" />`
)

primitiveData = {
svg: result.final_svg,
dataURI: encodeOptimizedSVGDataUri(result.final_svg),
dataURIbase64: `data:image/svg+xml;base64,${result.svg_base64encoded}`,
}

const json = JSON.stringify(primitiveData, null, 2)
await fs.writeFile(cachePath, json)
}

bar.tick()
await cache.set(cacheKey, primitiveData)
}

const dataURI = encodeOptimizedSVGDataUri(svgThumbnail)

return dataURI
return primitiveData
}

// https://codepen.io/tigt/post/optimizing-svgs-in-data-uris
Expand Down

0 comments on commit ae8fc3b

Please sign in to comment.