adds first two tickers btc-usd, btc-eur 🚀

.github/workflows/deploy.yml

@@ -1,8 +1,12 @@
 name: Deploy to GitHub Pages
 on:
-  push:
-    branches: [main] # todo: this should somehow be triggered only if test passes, which is another workflow though... this could just be a continuation of a previous job!
   workflow_dispatch:
+  schedule:
+    - cron: '0 0 * * *'
+  # workflow_run:
+  #   workflows: [Test]
+  #   types: [completed]
+  #   branches: [main]
 permissions:
   contents: read
   pages: write
@@ -15,11 +19,26 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      # todo: right about here at this point there should be some build steps to update caches, and build www api structure
+      - uses: actions/setup-node@v3
+        with:
+          check-latest: true
+          node-version-file: .node-version
+      - run: npm ci
+      - uses: actions/cache/restore@v3
+        id: cache
+        with:
+          path: data
+          key: ${{ runner.os }}-finance-data
+      - run: npm run sync-coinbase
+      - uses: actions/cache/save@v3
+        with:
+          path: data
+          key: ${{ steps.cache.outputs.cache-primary-key }}
+      - run: npm run build-coinbase
       - uses: actions/configure-pages@v3
       - uses: actions/upload-pages-artifact@v2
         with:
-          path: 'www' # fixme: rename to dist or something else
+          path: 'www'
   deploy:
     needs: build
     environment:

.gitignore

@@ -2,3 +2,6 @@
 .DS_Store
 *todo.md
 node_modules
+www/api/*/
+!www/api/*.*
+www/api/.DS_Store

bin/build-coinbase.mjs

@@ -0,0 +1,39 @@
+#!/usr/bin/env node --experimental-modules
+
+import { createReadStream, createWriteStream } from 'node:fs'
+import { mkdir } from 'node:fs/promises'
+import { readCacheBy } from '../lib/cache.mjs'
+import { fromJsonl as jsonlToCsv } from '../lib/stream/csv.mjs'
+import { fromJsonl as jsonlToJson } from '../lib/stream/json.mjs'
+import { fromJsonl as jsonlToXml } from '../lib/stream/xml.mjs'
+
+for (const file of await readCacheBy(name => name.startsWith('coinbase,'))) {
+  console.log('► bin/build-coinbase loading:%s', file)
+  const [directory, exchange , id, interval, format] = file.split(/[/,.]/)
+  await mkdir(`www/api/${id}`, { recursive: true })
+  console.log('↳ exchange:', directory)
+  console.log('↳ exchange:', exchange)
+  console.log('↳ id:', id)
+  console.log('↳ interval:', interval)
+  console.log('↳ source:', format)
+  // jsonl
+  console.time(`◄ bin/build-coinbase created: ${file} ➡️ www/api/${id}/${interval}.jsonl elapsed`)
+  createReadStream(file)
+    .pipe(createWriteStream(`www/api/${id}/${interval}.jsonl`))
+    .on('close', () => console.timeEnd(`◄ bin/build-coinbase created: ${file} ➡️ www/api/${id}/${interval}.jsonl elapsed`))
+  // csv
+  console.time(`◄ bin/build-coinbase created: ${file} ➡️ www/api/${id}/${interval}.csv elapsed`)
+  jsonlToCsv(createReadStream(file))
+    .pipe(createWriteStream(`www/api/${id}/${interval}.csv`))
+    .on('close', () => console.timeEnd(`◄ bin/build-coinbase created: ${file} ➡️ www/api/${id}/${interval}.csv elapsed`))
+  // json
+  console.time(`◄ bin/build-coinbase created: ${file} ➡️ www/api/${id}/${interval}.json elapsed`)
+  jsonlToJson(createReadStream(file))
+    .pipe(createWriteStream(`www/api/${id}/${interval}.json`))
+    .on('close', () => console.timeEnd(`◄ bin/build-coinbase created: ${file} ➡️ www/api/${id}/${interval}.json elapsed`))
+  // xml
+  console.time(`◄ bin/build-coinbase created: ${file} ➡️ www/api/${id}/${interval}.xml elapsed`)
+  jsonlToXml(createReadStream(file))
+    .pipe(createWriteStream(`www/api/${id}/${interval}.xml`))
+    .on('close', () => console.timeEnd(`◄ bin/build-coinbase created: ${file} ➡️ www/api/${id}/${interval}.xml elapsed`))
+}

bin/sync-coinbase.mjs

@@ -0,0 +1,39 @@
+#!/usr/bin/env node --experimental-modules
+
+// import path from 'node:path'
+// import fs from 'node:fs/promises'
+// import readline from 'node:readline'
+
+import { createReadStream, createWriteStream } from 'node:fs'
+import { EOL } from 'node:os'
+
+import { readCacheBy, readLastCachedJsonLineOf } from '../lib/cache.mjs'
+import { fetchCandlesSince, coinbaseIntervalFor, coinbaseIdFor } from '../lib/coinbase.mjs'
+import { dateReviver } from '../lib/json.mjs'
+import { daysBetween, INTERVALS } from '../lib/date.mjs'
+
+// move this
+export function intervalFor(file) {
+  const [,,interval] = file.replace('.', ',').split(',')
+  return interval
+}
+
+for (const filePath of await readCacheBy(name => name.startsWith('coinbase,'))) {
+  console.log('[bin/sync-coinbase] Loading:%s', filePath)
+  const [lastCachedCandleDate] = await readLastCachedJsonLineOf(createReadStream(filePath), dateReviver)
+  console.log('↳ lastCachedCandleDate:', lastCachedCandleDate)
+  console.log('↳ interval:', intervalFor(filePath))
+  console.log('↳ coinbaseId:', coinbaseIdFor(filePath))
+  console.log('↳ coinbaseInterval:', coinbaseIntervalFor(filePath))
+
+  const nextUncachedCandleDate = new Date(lastCachedCandleDate.getTime() + INTERVALS.get(intervalFor(filePath)))
+  const numberOfCandlesToSync = daysBetween(nextUncachedCandleDate, new Date(new Date().setTime(new Date().getTime() - INTERVALS.get(intervalFor(filePath)))))
+  console.log('↳ numberOfCandlesToSync:', numberOfCandlesToSync)
+  if (numberOfCandlesToSync === 0) continue
+  const stream = createWriteStream(filePath, { flags: 'a' })
+  for await (const [date, open, high, low, close, volume] of fetchCandlesSince(nextUncachedCandleDate, coinbaseIdFor(filePath), coinbaseIntervalFor(filePath))) {
+    stream.write(JSON.stringify([date, open, high, low, close, volume]) + EOL)
+    break
+  }
+  stream.close()
+}

data/coinbase,btc-eur,1d.jsonl

@@ -0,0 +1 @@
+["2015-04-23T00:00:00.000Z",300,300,200,220,0.03,{"source":"coinbase-v3"}]

data/coinbase,btc-usd,1d.jsonl

@@ -0,0 +1 @@
+["2015-07-20T00:00:00.000Z",277.98,280,277.37,280,782.88341959,{"source":"coinbase-v3"}]

lib/cache.mjs

@@ -0,0 +1,51 @@
+#!/usr/bin/env node --experimental-modules
+
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import readline from 'node:readline'
+import { env } from 'node:process'
+
+export async function readCacheBy(filterFn = () => true, { npm_package_config_data } = env, { readdir } = fs, { join } = path) {
+  return (
+    await readdir(npm_package_config_data)
+  )
+    .filter(filterFn)
+    .map(file => join(npm_package_config_data, file))
+}
+
+export async function readLastCachedLineOf(input, {createInterface} = readline) {
+  let lastLine
+  for await (lastLine of createInterface({ input, terminal: false })) {}
+  return lastLine
+}
+
+export async function readLastCachedJsonLineOf(input, jsonDateReviver, {createInterface} = readline) {
+  return JSON.parse(await readLastCachedLineOf(input), jsonDateReviver)
+}
+
+// export async function createFileRecursively (filePath, data = '') {
+//   try {
+//     const dir = dirname(filePath)
+//     await access(dir)
+//   } catch (error) {
+//     await createDirectoryRecursively(dir)
+//   }
+//   await writeFile(filePath, data, 'utf8')
+// }
+
+// export async function createDirectoryRecursively (directoryPath) {
+//   const parentDir = dirname(directoryPath)
+//   try {
+//     await access(parentDir)
+//   } catch (error) {
+//     await createDirectoryRecursively(parentDir)
+//   }
+//   await mkdir(directoryPath)
+// }
+
+// Example usage:
+// const filePath = './myFolder/mySubFolder/myFile.txt';
+// const fileData = 'Hello, World!';
+// createFileRecursively(filePath, fileData)
+//   .then(() => console.log('File created successfully'))
+//   .catch(error => console.error(`Error: ${error.message}`));

lib/coinbase.mjs

@@ -0,0 +1,135 @@
+import { createHmac } from 'node:crypto'
+import { URL } from 'node:url'
+import { env } from 'node:process'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import readline from 'node:readline'
+
+export function iso8601DateReviver (_, value, iso8601Regex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/) {
+  return typeof value === 'string' && iso8601Regex.test(value)
+    ? new Date(value)
+    : value
+}
+
+export function withJsonBody (data) {
+  return {
+    method: 'POST',
+    body: JSON.stringify(data),
+    headers: {
+      'Content-Type': 'application/json'
+    }
+  }
+}
+
+// declare function fetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response>;
+
+export function fetchCoinbase (input,init = {method: 'GET',headers: {'Content-Type': 'application/json'}},{env: {npm_config_coinbase_api_key: key,npm_config_coinbase_api_secret: secret,npm_config_coinbase_api_base: base}} = process) {
+  const url = new URL(input, base) // base is set right at https://api.coinbase.com/api/v3/ for easy calls ie. coinbase.fetch('brokerage/products')
+
+  init.headers['CB-ACCESS-TIMESTAMP'] = Math.floor(1e-3 * Date.now())
+  init.headers['CB-ACCESS-KEY'] = key
+  init.headers['CB-ACCESS-SIGN'] = createHmac('sha256', secret).update(init.headers['CB-ACCESS-TIMESTAMP'] + init.method.toUpperCase() + url.pathname + (init.body || String.prototype)).digest('hex')
+
+  console.time(`[lib/coinbase] fetch: ${url} (json) duration`)
+
+  return fetch(url, init)
+    .then(response => {
+      console.timeEnd(`[lib/coinbase] fetch: ${url} (json) duration`)
+      return response.text()
+    })
+    .then(text => {
+      // console.log(`[lib/coinbase] fetch: ${url} (json) response: %s`, text)
+      return JSON.parse(text, iso8601DateReviver)
+    })
+}
+
+export async function * fetchCandlesSince (start, id, size = 'ONE_DAY') {
+  const granularity = new Map([
+    ['UNKNOWN_GRANULARITY', null],
+    ['ONE_MINUTE', 60000],
+    ['FIVE_MINUTE', 300000],
+    ['FIFTEEN_MINUTE', 900000],
+    ['THIRTY_MINUTE', 1800000],
+    ['ONE_HOUR', 3600000],
+    ['TWO_HOUR', 7200000],
+    ['SIX_HOUR', 21600000],
+    ['ONE_DAY', 86400000]
+  ])
+  const yesterday = new Date(new Date().setTime(new Date().getTime() - granularity.get(size)))
+  do {
+    const end = new Date(
+      Math.min(
+        start.getTime() + granularity.get(size) * 299, // readme: magic number 299 simply works; while 300, which is the actual limit, does skip candles sometimes!
+        yesterday // warning: today's candle is continuously changing!
+      )
+    )
+    // console.log('[lib/coinbase] generateCandles: %s » %s', start.toJSON(), end.toJSON())
+    const { candles = [], error = null } = await fetchCoinbase(`brokerage/products/${id.toUpperCase()}/candles?start=${stringifyCoinbaseDate(start)}&end=${stringifyCoinbaseDate(end)}&granularity=${size}`)
+
+    if (error) {
+      console.error(error)
+      break
+    }
+    candles.reverse()
+    yield * candles.map(parseCoinbaseCandle)
+    start = new Date(end.getTime() + granularity.get(size)) // can this assignment be moved in the while condition?
+  } while (start < yesterday)
+}
+
+// export const fetchCoinbase = (uri, data = '', headers = { 'Content-Type': 'application/json' }, { env: { npm_config_coinbase_api_key: key, npm_config_coinbase_api_secret: secret } } = process) => {
+//   const url = new URL(uri, 'https://api.coinbase.com/api/v3/')
+//   const timestamp = Math.floor(1e-3 * Date.now())
+//   const method = data ? 'POST' : 'GET'
+//   const sign = createHmac('sha256', secret)
+//     .update(timestamp + method + url.pathname + data)
+//     .digest('hex')
+
+//   return fetch(url, { method, data, headers: { 'CB-ACCESS-TIMESTAMP': timestamp, 'CB-ACCESS-KEY': key, 'CB-ACCESS-SIGN': sign, ...headers } })
+//     .then((response) => response.json())
+// }
+
+export function coinbaseIntervalFor(file) {
+  const [,,size] = file.replace('.', ',').split(',')
+  return new Map([['1', 'ONE_MINUTE'], ['5', 'FIVE_MINUTE'], ['15', 'FIFTEEN_MINUTE'], ['30', 'THIRTY_MINUTE'], ['1h', 'ONE_HOUR'], ['2h', 'TWO_HOUR'], ['6h', 'SIX_HOUR'], ['1d', 'ONE_DAY']]).get(size)
+}
+
+export function coinbaseIdFor(file) {
+  const [,id] = file.replace('.', ',').split(',')
+  return id.toUpperCase()
+}
+
+
+/*
+These are pure functions so they can be tested in isolation.
+Reference: https://docs.cloud.coinbase.com/advanced-trade-api/reference/retailbrokerageapi_getcandles
+Todo: Might move them to lib/coinbase.mjs later!
+*/
+
+export function toUnixTimestamp (date, fromMilliseconds = 1e-3) { // todo: rename to generic utils.toUnixTimestamp
+  return Math.floor(fromMilliseconds * date.getTime())
+}
+
+export function fromUnixTimestamp (unixTimestamp, toMilliseconds = 1e3) { // todo: rename to generic utils.fromUnixTimestamp
+  return new Date(toMilliseconds * unixTimestamp)
+}
+
+/*
+
+It makes sense however to keep the following two functions here until refactored away... 🚀
+
+*/
+
+export function stringifyCoinbaseDate (date) {
+  return toUnixTimestamp(date)
+}
+
+export function parseCoinbaseCandle ({ start, low, high, open, close, volume }) { // this is indeed a mapper for the coinbaseResponse
+  return  [
+    fromUnixTimestamp(start),
+    parseFloat(open),
+    parseFloat(high),
+    parseFloat(low),
+    parseFloat(close),
+    parseFloat(volume),
+  ]
+}