From 5e21f8e3ef86408e9a7ee1218bbbdd758e4ce642 Mon Sep 17 00:00:00 2001 From: Josh Reed Date: Fri, 23 Dec 2022 11:14:36 -0800 Subject: [PATCH 1/6] Implemented selective querying from cache and cache invalidation on mutations to a given table. Co-authored-by: Jonathan Fangon Co-authored-by: Liam Johnson Co-authored-by: Derek Okuno Co-authored-by: Liam Jeon Co-authored-by: Josh Reed --- ObsidianWrapper/ObsidianWrapper.jsx | 36 ++--- src/Obsidian.ts | 85 ++++++---- src/invalidateCacheCheck.ts | 61 +++++-- src/mapSelections.js | 13 +- src/normalize.ts | 135 +++++++++------- src/quickCache.js | 112 ++++++++----- src/restructure.ts | 238 +++++++++++++++------------- src/transformResponse.ts | 141 ++++++++++------ 8 files changed, 501 insertions(+), 320 deletions(-) diff --git a/ObsidianWrapper/ObsidianWrapper.jsx b/ObsidianWrapper/ObsidianWrapper.jsx index 57933d7..77b0ed8 100644 --- a/ObsidianWrapper/ObsidianWrapper.jsx +++ b/ObsidianWrapper/ObsidianWrapper.jsx @@ -1,6 +1,6 @@ -import React from "https://dev.jspm.io/react"; -import BrowserCache from "../src/Browser/CacheClassBrowser.js"; -import { insertTypenames } from "../src/Browser/insertTypenames.js"; +import React from 'https://dev.jspm.io/react'; +import BrowserCache from '../src/Browser/CacheClassBrowser.js'; +import { insertTypenames } from '../src/Browser/insertTypenames.js'; const cacheContext = React.createContext(); @@ -8,23 +8,23 @@ function ObsidianWrapper(props) { const [cache, setCache] = React.useState(new BrowserCache()); // You have to put your Google Chrome Obsidian developer tool extension id to connect Obsidian Wrapper with dev tool - const chromeExtensionId = "dkbfipkapkljpdbhdihnlnbieffhjdmh"; - window.localStorage.setItem("cache", JSON.stringify(cache)); + const chromeExtensionId = 'apcpdmmbhhephobnmnllbklplpaoiemo'; + window.localStorage.setItem('cache', JSON.stringify(cache)); async function query(query, options = {}) { // dev tool messages const startTime = Date.now(); chrome.runtime.sendMessage(chromeExtensionId, { query: query }); chrome.runtime.sendMessage(chromeExtensionId, { - cache: window.localStorage.getItem("cache"), + cache: window.localStorage.getItem('cache'), }); console.log( "Here's the message content: ", - window.localStorage.getItem("cache") + window.localStorage.getItem('cache') ); // set the options object default properties if not provided const { - endpoint = "/graphql", + endpoint = '/graphql', cacheRead = true, cacheWrite = true, pollInterval = null, @@ -50,7 +50,7 @@ function ObsidianWrapper(props) { // when the developer decides to only utilize whole query for cache if (wholeQuery) resObj = await cache.readWholeQuery(query); else resObj = await cache.read(query); - console.log("query function resObj: ", resObj); + console.log('query function resObj: ', resObj); // check if query is stored in cache if (resObj) { // returning cached response as a promise @@ -74,10 +74,10 @@ function ObsidianWrapper(props) { try { // send fetch request with query const resJSON = await fetch(endpoint, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", - Accept: "application/json", + 'Content-Type': 'application/json', + Accept: 'application/json', }, body: JSON.stringify({ query }), }); @@ -117,7 +117,7 @@ function ObsidianWrapper(props) { const startTime = Date.now(); mutation = insertTypenames(mutation); const { - endpoint = "/graphql", + endpoint = '/graphql', cacheWrite = true, toDelete = false, update = null, @@ -153,7 +153,7 @@ function ObsidianWrapper(props) { } // always write/over-write to cache (add/update) // GQL call to make changes and synchronize database - console.log("WriteThrough - true ", responseObj); + console.log('WriteThrough - true ', responseObj); const addOrUpdateMutationResponseTime = Date.now() - startTime; chrome.runtime.sendMessage(chromeExtensionId, { addOrUpdateMutationResponseTime: addOrUpdateMutationResponseTime, @@ -165,10 +165,10 @@ function ObsidianWrapper(props) { // use cache.write instead of cache.writeThrough const responseObj = await fetch(endpoint, { - method: "POST", + method: 'POST', headers: { - "Content-Type": "application/json", - Accept: "application/json", + 'Content-Type': 'application/json', + Accept: 'application/json', }, body: JSON.stringify({ query: mutation }), }).then((resp) => resp.json()); @@ -184,7 +184,7 @@ function ObsidianWrapper(props) { } // third behaviour just for normal update (no-delete, no update function) cache.write(mutation, responseObj); - console.log("WriteThrough - false ", responseObj); + console.log('WriteThrough - false ', responseObj); return responseObj; } } catch (e) { diff --git a/src/Obsidian.ts b/src/Obsidian.ts index ab2cce9..bafff40 100644 --- a/src/Obsidian.ts +++ b/src/Obsidian.ts @@ -2,15 +2,17 @@ import { graphql } from 'https://cdn.pika.dev/graphql@15.0.0'; import { renderPlaygroundPage } from 'https://deno.land/x/oak_graphql@0.6.2/graphql-playground-html/render-playground-html.ts'; import { makeExecutableSchema } from 'https://deno.land/x/oak_graphql@0.6.2/graphql-tools/schema/makeExecutableSchema.ts'; import { Cache } from './quickCache.js'; +import LFUCache from './Browser/lfuBrowserCache.js'; import queryDepthLimiter from './DoSSecurity.ts'; import { restructure } from './restructure.ts'; -import { rebuildFromQuery } from './rebuild.js' -import { normalizeObject } from './normalize.ts' -import { transformResponse, detransformResponse } from './transformResponse.ts' -import { isMutation, invalidateCache } from './invalidateCacheCheck.ts' +import { rebuildFromQuery } from './rebuild.js'; +import { normalizeObject } from './normalize.ts'; +import { transformResponse, detransformResponse } from './transformResponse.ts'; +import { isMutation, invalidateCache } from './invalidateCacheCheck.ts'; +import { mapSelectionSet } from './mapSelections.js'; interface Constructable { - new(...args: any): T & OakRouter; + new (...args: any): T & OakRouter; } interface OakRouter { @@ -34,6 +36,7 @@ export interface ObsidianRouterOptions { useQueryCache?: boolean; // trivial parameter useRebuildCache?: boolean; customIdentifier?: Array; + mutationTableMap?: Record; // Deno recommended type name } export interface ResolversProps { @@ -46,9 +49,9 @@ export interface ResolversProps { export let redisPortExport: number = 6379; /** - * - * @param param0 - * @returns + * + * @param param0 + * @returns */ export async function ObsidianRouter({ Router, @@ -64,7 +67,8 @@ export async function ObsidianRouter({ maxQueryDepth = 0, useQueryCache = true, useRebuildCache = true, - customIdentifier = ["id", "__typename"], + customIdentifier = ['id', '__typename'], + mutationTableMap = {}, }: ObsidianRouterOptions): Promise { redisPortExport = redisPort; const router = new Router(); @@ -72,11 +76,13 @@ export async function ObsidianRouter({ // const cache = new LFUCache(50); // If using LFU Browser Caching, uncomment line const cache = new Cache(); // If using Redis caching, uncomment line cache.cacheClear(); - if (policy || maxmemory) { // set redis configurations + if (policy || maxmemory) { + // set redis configurations cache.configSet('maxmemory-policy', policy); cache.configSet('maxmemory', maxmemory); } + //post await router.post(path, async (ctx: any) => { const t0 = performance.now(); const { response, request } = ctx; @@ -84,12 +90,23 @@ export async function ObsidianRouter({ try { const contextResult = context ? await context(ctx) : undefined; let body = await request.body().value; + + // Gets requested data point from query and saves into an array + const selectionsArray = mapSelectionSet(body.query); + if (maxQueryDepth) queryDepthLimiter(body.query, maxQueryDepth); // If a securty limit is set for maxQueryDepth, invoke queryDepthLimiter, which throws error if query depth exceeds maximum - body = { query: restructure(body) }; // Restructre gets rid of variables and fragments from the query - let cacheQueryValue = await cache.read(body.query) - // Is query in cache? + let restructuredBody = { query: restructure(body) }; // Restructre gets rid of variables and fragments from the query + + // Parses query string into query key and checks cach for that key + let cacheQueryValue = await cache.read(body.query); + + // Is query in cache? if (useCache && useQueryCache && cacheQueryValue) { - let detransformedCacheQueryValue = await detransformResponse(body.query, cacheQueryValue) + let detransformedCacheQueryValue = await detransformResponse( + restructuredBody.query, + cacheQueryValue, + selectionsArray + ); if (!detransformedCacheQueryValue) { // cache was evicted if any partial cache is missing, which causes detransformResponse to return undefined cacheQueryValue = undefined; @@ -99,12 +116,12 @@ export async function ObsidianRouter({ const t1 = performance.now(); console.log( '%c Obsidian retrieved data from cache and took ' + - (t1 - t0) + - ' milliseconds.', "background: #222; color: #00FF00" + (t1 - t0) + + ' milliseconds.', + 'background: #222; color: #00FF00' ); } - - }; // If not in cache: + } // If not in cache: if (useCache && useQueryCache && !cacheQueryValue) { const gqlResponse = await (graphql as any)( schema, @@ -114,14 +131,27 @@ export async function ObsidianRouter({ body.variables || undefined, body.operationName || undefined ); - const normalizedGQLResponse = normalizeObject(gqlResponse, customIdentifier); - if (isMutation(body)) { + // console.log('gqlResponse raw: ', gqlResponse); + const normalizedGQLResponse = normalizeObject( + gqlResponse, + customIdentifier + ); + // console.log('normalized: ', normalizedGQLResponse); + if (isMutation(restructuredBody)) { + // cache.cacheClear(); const queryString = await request.body().value; - invalidateCache(normalizedGQLResponse, queryString.query); + invalidateCache( + normalizedGQLResponse, + queryString.query, + mutationTableMap + ); } // If read query: run query, normalize GQL response, transform GQL response, write to cache, and write pieces of normalized GQL response objects else { - const transformedGQLResponse = transformResponse(gqlResponse, customIdentifier); + const transformedGQLResponse = transformResponse( + gqlResponse, + customIdentifier + ); await cache.write(body.query, transformedGQLResponse, false); for (const key in normalizedGQLResponse) { await cache.cacheWriteObject(key, normalizedGQLResponse[key]); @@ -132,8 +162,9 @@ export async function ObsidianRouter({ const t1 = performance.now(); console.log( '%c Obsidian received new data and took ' + - (t1 - t0) + - ' milliseconds', 'background: #222; color: #FFFF00' + (t1 - t0) + + ' milliseconds', + 'background: #222; color: #FFFF00' ); } } catch (error) { @@ -151,20 +182,20 @@ export async function ObsidianRouter({ }); // serve graphql playground + // deno-lint-ignore require-await await router.get(path, async (ctx: any) => { const { request, response } = ctx; if (usePlayground) { const prefersHTML = request.accepts('text/html'); const optionsObj: any = { 'schema.polling.enable': false, // enables automatic schema polling - } + }; if (prefersHTML) { - const playground = renderPlaygroundPage({ endpoint: request.url.origin + path, subscriptionEndpoint: request.url.origin, - settings: optionsObj + settings: optionsObj, }); response.status = 200; response.body = playground; diff --git a/src/invalidateCacheCheck.ts b/src/invalidateCacheCheck.ts index ed9824b..52b1833 100644 --- a/src/invalidateCacheCheck.ts +++ b/src/invalidateCacheCheck.ts @@ -1,8 +1,8 @@ /** @format */ -import { gql } from "https://deno.land/x/oak_graphql/mod.ts"; -import { visit } from "https://deno.land/x/graphql_deno/mod.ts"; -import { Cache } from "./quickCache.js"; -import { deepEqual } from "./utils.js"; +import { gql } from 'https://deno.land/x/oak_graphql/mod.ts'; +import { visit } from 'https://deno.land/x/graphql_deno/mod.ts'; +import { redisdb, Cache } from './quickCache.js'; +import { deepEqual } from './utils.js'; const cache = new Cache(); @@ -11,13 +11,13 @@ const cache = new Cache(); * @param {boolean} isMutation - Boolean indicating if it's a mutation query * @return {boolean} isMutation */ -export function isMutation(gqlQuery: { query: any; }): boolean { +export function isMutation(gqlQuery: { query: any }): boolean { let isMutation: boolean = false; let ast: any = gql(gqlQuery.query); const checkMutationVisitor: object = { - OperationDefinition: (node: { operation: string; }) => { - if (node.operation === "mutation") { + OperationDefinition: (node: { operation: string }) => { + if (node.operation === 'mutation') { isMutation = true; } }, @@ -25,8 +25,8 @@ export function isMutation(gqlQuery: { query: any; }): boolean { // left this piece of code in case someone decides to build upon subscriptions, but for now obsidian doesn't do anything with subscriptions const subscriptionTunnelVisitor = { - OperationDefinition: (node: { operation: string; }) => { - if (node.operation === "subscription") { + OperationDefinition: (node: { operation: string }) => { + if (node.operation === 'subscription') { } }, }; @@ -37,7 +37,7 @@ export function isMutation(gqlQuery: { query: any; }): boolean { /** * Invalidates cache in redis based on the mutation type. - * @param {object} normalizedMutation - Object containing hash val in redis as key and normalized object as value. + * @param {object} normalizedMutation - Object containing hash val in redis as key and normalized object as value. * Ex: { * ~7~Movie: {id: 7, __typename: Movie, title: Ad Astra, releaseYear: 2019} * } @@ -45,7 +45,11 @@ export function isMutation(gqlQuery: { query: any; }): boolean { * Ex: 'mutation { addMovie(input: {title: "sdfsdg", releaseYear: 1234, genre: ACTION }) { __typename id ti...' * @return {void} */ -export async function invalidateCache(normalizedMutation: { [key: string]: object; }, queryString: string) { +export async function invalidateCache( + normalizedMutation: { [key: string]: object }, + queryString: string, + mutationTableMap +) { let normalizedData: object; let cachedVal: any; @@ -56,25 +60,50 @@ export async function invalidateCache(normalizedMutation: { [key: string]: objec cachedVal = await cache.cacheReadObject(redisKey); // if response objects from mutation and cache are deeply equal then we delete it from cache because it infers that it's a delete mutation - if (cachedVal !== undefined && deepEqual(normalizedData, cachedVal) || isDelete(queryString)) { + if ( + (cachedVal !== undefined && deepEqual(normalizedData, cachedVal)) || + isDelete(queryString) + ) { await cache.cacheDelete(redisKey); } else { // otherwise it's an update or add mutation because response objects from mutation and cache don't match so we overwrite the existing cache value or write new data if cache at that key doesn't exist // Edge case: update is done without changing any values... cache will be deleted from redis because the response obj and cached obj will be equal // we put it in the backburner because it doesn't make our cache stale, we would just perform an extra operation to re-cache the missing value when a request comes in + // mutationTableMap.mutationType + console.log('normalizedMutation is: ', normalizedMutation); + console.log('queryString is: ', queryString); + let ast = gql(queryString); + const mutationType = + ast.definitions[0].selectionSet.selections[0].name.value; + console.log('mutationType is: ', mutationType); + console.log('mutationTableMap is: ', mutationTableMap); + const staleRefs = mutationTableMap[mutationType]; + console.log('staleRefs is: ', staleRefs); + + //loop through refs in ROOT_QUERY hash in redis + const rootQueryContents = await redisdb.hgetall('ROOT_QUERY'); + console.log('rootQueryContents is: ', rootQueryContents); + //loop through rootQueryContents, checking if + //staleRef === rootQueryContents[i].slice(0, staleRef.length). + //if they're equal, delete from ROOT_QUERY hash (redisdb.hdel(ROOTQUERY, rootQueryContents[i])) + for (let i = 0; i < rootQueryContents.length; i += 2) { + if (staleRefs === rootQueryContents[i].slice(0, staleRefs.length)) { + redisdb.hdel('ROOT_QUERY', rootQueryContents[i]); + } + } await cache.cacheWriteObject(redisKey, normalizedData); } } } /** - * Returns a boolean that's used to decide on deleting a value from cache + * Returns a boolean that's used to decide on deleting a value from cache * @param {string} queryString - raw mutation query. * Ex: 'mutation { addMovie(input: {title: "sdfsdg", releaseYear: 1234, genre: ACTION }) { __typename id ti...' * @return {boolean} isDeleteFlag */ export function isDelete(queryString: string) { - // Because we check if response object from delete mutation equals to cached object to determine if it's a delete mutation + // Because we check if response object from delete mutation equals to cached object to determine if it's a delete mutation // but there may be instances that the object is evicted from cache or never cached previously which would be treated as add or update mutation // if we find any keywords we're looking for in the mutation query that infer deletion we force the deletion const deleteKeywords: Array = ['delete', 'remove']; @@ -88,5 +117,5 @@ export function isDelete(queryString: string) { break; } } - return isDeleteFlag -} \ No newline at end of file + return isDeleteFlag; +} diff --git a/src/mapSelections.js b/src/mapSelections.js index f235c13..c43f4ff 100644 --- a/src/mapSelections.js +++ b/src/mapSelections.js @@ -1,11 +1,13 @@ /** @format */ -import { gql } from "https://deno.land/x/oak_graphql/mod.ts"; +import { gql } from 'https://deno.land/x/oak_graphql/mod.ts'; export function mapSelectionSet(query) { - let selectionKeysMap = { data: "data" }; + // Gets fields from query and stores all in an array - used to selectively query cache + let selectionKeysMap = {}; let ast = gql(query); let selections = ast.definitions[0].selectionSet.selections; + const tableName = selections[0].name.value; const recursiveMap = (recurseSelections) => { for (const selection of recurseSelections) { @@ -22,5 +24,10 @@ export function mapSelectionSet(query) { } }; recursiveMap(selections); - return selectionKeysMap; + + // filter out object name from array, leaving only fields + const fieldsArray = Object.keys(selectionKeysMap).filter( + (key) => key !== tableName + ); + return fieldsArray; } diff --git a/src/normalize.ts b/src/normalize.ts index 2e61dfa..467341d 100644 --- a/src/normalize.ts +++ b/src/normalize.ts @@ -10,18 +10,23 @@ * @return {boolean} Boolean indicating if objectInQuestion is hashable or not */ - export const containsHashableObject = (objectInQuestion: any, hashableKeys: Array):boolean => { - if(typeof objectInQuestion !== 'object' || - Array.isArray(objectInQuestion) || - !objectInQuestion - ) return false; - const objectInQuestionKeysSet = new Set(Object.keys(objectInQuestion)); - return hashableKeys.every(key => objectInQuestionKeysSet.has(key)) -} +export const containsHashableObject = ( + objectInQuestion: any, + hashableKeys: Array +): boolean => { + if ( + typeof objectInQuestion !== 'object' || + Array.isArray(objectInQuestion) || + !objectInQuestion + ) + return false; + const objectInQuestionKeysSet = new Set(Object.keys(objectInQuestion)); + return hashableKeys.every((key) => objectInQuestionKeysSet.has(key)); +}; /* ----------------------------------------------------------------*/ /* ----------------------------------------------------------------*/ -/** isHashableObject - +/** isHashableObject - * Returns a boolean indicating that the passed in value is hashable. It must: * 1) Contain hashable object * 2) Does not have any nesting (i.e., contains no objects or array values) @@ -30,85 +35,109 @@ * @param {Array} hashableKeys Array of hashable keys * @return {boolean} Boolean indicating if objectInQuestion is hashable or not */ -export const isHashableObject = (objectInQuestion: any, hashableKeys: Array): boolean => { +export const isHashableObject = ( + objectInQuestion: any, + hashableKeys: Array +): boolean => { if (!containsHashableObject(objectInQuestion, hashableKeys)) return false; for (const key in objectInQuestion) { if (typeof objectInQuestion[key] === 'object') return false; } return true; -} +}; /* ----------------------------------------------------------------*/ - /* ----------------------------------------------------------------*/ -export type GenericObject = { [key:string]: any}; -type FlatObject = { [key:string]: (string | number | boolean)}; +export type GenericObject = { [key: string]: any }; +type FlatObject = { [key: string]: string | number | boolean }; /** hashMaker - * Creates unique hash string for an object with hashable keys with hashable object passed in * - * @param {FlatObject} hashableObject Object that is hashable + * @param {FlatObject} hashableObject Object that is hashable * @param {Array} hashableKeys Array of hashable keys * @return {string} Hash string */ -export const hashMaker = (hashableObject: FlatObject, hashableKeys:Array):string => { - let hash = ''; - let value = ''; - for(const hashableKey of hashableKeys){ - value = '~'; - value += hashableObject[hashableKey] - hash += value; - } - return hash; -} +export const hashMaker = ( + hashableObject: FlatObject, + hashableKeys: Array +): string => { + let hash = ''; + let value = ''; + for (const hashableKey of hashableKeys) { + value = '~'; + value += hashableObject[hashableKey]; + hash += value; + } + return hash; +}; /* ----------------------------------------------------------------*/ /* ----------------------------------------------------------------*/ /** printHashableObject - - * Creates a hashable object from an object that contains a hashable object. Does not print hashable object - * - * @param {FlatObject} containsHashableObject Object that is hashable + * Creates a hashable object from an object that contains a hashable object. Does not print hashable object + * + * @param {FlatObject} containsHashableObject Object that is hashable * @return {GenericObject} A hashable object */ -export const printHashableObject = (containsHashableObject: GenericObject):GenericObject => { - const hashObj:GenericObject = {}; - for(const key in containsHashableObject){ - if(typeof containsHashableObject[key] !== 'object' && !hashObj.hasOwnProperty(key)) hashObj[key] = containsHashableObject[key]; - } - return hashObj; -} +export const printHashableObject = ( + containsHashableObject: GenericObject +): GenericObject => { + const hashObj: GenericObject = {}; + for (const key in containsHashableObject) { + if ( + typeof containsHashableObject[key] !== 'object' && + !hashObj.hasOwnProperty(key) + ) + hashObj[key] = containsHashableObject[key]; + } + return hashObj; +}; /* ----------------------------------------------------------------*/ /* ----------------------------------------------------------------*/ /** * Recursively flattens an arbitrarily nested object into an objects with hash key and hashable object pairs - * + * * For each key in object (typeof === 'object', meaning it can be array): - * + * * 1) If current object contains hashable object and if it hasn't printed already, * it prints a hashable object and makes its associated hash. If hash doesn't exist in normalizedHashableObjects, * it adds hash key and hashable object pair. - * + * * 2) If the value at the current key is an object (typeof === 'object', meaning it can be array), it recursively * calls normalizeObject with the value passed in. This recursive calls goes inside arbitrary nesting. - * + * * 3) Return normalizedHashableObjects. In the outer most execution context, this will return the output of the function. * In inner execution contexts, this will return that execution context's normalizedHashableObjects. - * + * * @param {GenericObject} nestedObject Nested object - * @param {Array} hashableKeys Array of hashable keys + * @param {Array} hashableKeys Array of hashable keys * @return {FlatObject} Normalized object with hash keys and hashable object pairs */ -export const normalizeObject = (nestedObject: GenericObject, hashableKeys:Array, normalizedHashableObjects:GenericObject = {}):GenericObject => { - let hasAlreadyPrinted = false; - for(const key in nestedObject){ - if(containsHashableObject(nestedObject, hashableKeys) && hasAlreadyPrinted === false){ - hasAlreadyPrinted = true; - const hashableObject = printHashableObject(nestedObject); - const hash = hashMaker(hashableObject, hashableKeys); - if(!normalizedHashableObjects.hasOwnProperty(hash)) normalizedHashableObjects[hash] = hashableObject; - } - if(typeof nestedObject[key] === 'object') normalizeObject(nestedObject[key], hashableKeys, normalizedHashableObjects); +export const normalizeObject = ( + nestedObject: GenericObject, + hashableKeys: Array, + normalizedHashableObjects: GenericObject = {} +): GenericObject => { + let hasAlreadyPrinted = false; + for (const key in nestedObject) { + if ( + containsHashableObject(nestedObject, hashableKeys) && + hasAlreadyPrinted === false + ) { + hasAlreadyPrinted = true; + const hashableObject = printHashableObject(nestedObject); + const hash = hashMaker(hashableObject, hashableKeys); + if (!normalizedHashableObjects.hasOwnProperty(hash)) + normalizedHashableObjects[hash] = hashableObject; } - return normalizedHashableObjects; -} \ No newline at end of file + if (typeof nestedObject[key] === 'object') + normalizeObject( + nestedObject[key], + hashableKeys, + normalizedHashableObjects + ); + } + return normalizedHashableObjects; +}; diff --git a/src/quickCache.js b/src/quickCache.js index 7ed5b06..6dad283 100644 --- a/src/quickCache.js +++ b/src/quickCache.js @@ -1,16 +1,16 @@ /** @format */ -import "https://deno.land/x/dotenv/load.ts"; -import { connect } from "https://deno.land/x/redis/mod.ts"; -import { gql } from "https://deno.land/x/oak_graphql/mod.ts"; -import { print, visit } from "https://deno.land/x/graphql_deno/mod.ts"; +import 'https://deno.land/x/dotenv/load.ts'; +import { connect } from 'https://deno.land/x/redis/mod.ts'; +import { gql } from 'https://deno.land/x/oak_graphql/mod.ts'; +import { print, visit } from 'https://deno.land/x/graphql_deno/mod.ts'; let redis; -const context = window.Deno ? "server" : "client"; +const context = window.Deno ? 'server' : 'client'; -if (context === "server") { +if (context === 'server') { redis = await connect({ - hostname: Deno.env.get("REDIS_HOST"), + hostname: Deno.env.get('REDIS_HOST'), port: 6379, }); } @@ -25,7 +25,7 @@ export class Cache { } ) { this.storage = initialCache; - this.context = window.Deno ? "server" : "client"; + this.context = window.Deno ? 'server' : 'client'; } // set cache configurations @@ -33,13 +33,13 @@ export class Cache { return await redis.configSet(parameter, value); } - // Main functionality methods + // Main functionality methods below // for reading the inital query async read(queryStr) { //the queryStr it gets is the JSON stringified const returnedValue = await this.cacheRead(queryStr); - if (("returnedValue", returnedValue)) { + if (('returnedValue', returnedValue)) { return JSON.parse(returnedValue); } else { return undefined; @@ -47,7 +47,9 @@ export class Cache { } async write(queryStr, respObj, deleteFlag) { // update the original cache with same reference - await this.cacheWrite(queryStr, JSON.stringify(respObj)); + const cacheHash = this.createQueryKey(queryStr); + // console.log('write cacheHash: ', cacheHash); + await this.cacheWrite(cacheHash, JSON.stringify(respObj)); } //will overwrite a list at the given hash by default @@ -72,16 +74,22 @@ export class Cache { cacheWriteObject = async (hash, obj) => { let entries = Object.entries(obj).flat(); entries = entries.map((entry) => JSON.stringify(entry)); - + // console.log('entries: ', entries); + // adding as nested strings? take out one layer for clarity. await redis.hset(hash, ...entries); }; - cacheReadObject = async (hash, field = false) => { - if (field) { - let returnValue = await redisdb.hget(hash, JSON.stringify(field)); - - if (returnValue === undefined) return undefined; - return JSON.parse(returnValue); + cacheReadObject = async (hash, fields = []) => { + // Checks for the fields requested, then queries cache for those specific keys in the hashes + if (fields.length !== 0) { + const fieldObj = {}; + for (const field of fields) { + const rawCacheValue = await redisdb.hget(hash, JSON.stringify(field)); + fieldObj[field] = JSON.parse(rawCacheValue); + } + // if (returnValue === undefined) return undefined; + console.log('fieldObj: ', fieldObj); + return fieldObj; } else { let objArray = await redisdb.hgetall(hash); if (objArray.length == 0) return undefined; @@ -107,36 +115,64 @@ export class Cache { return JSON.stringify(finalReturn); } - async cacheRead(hash) { - if (this.context === "client") { - return this.storage[hash]; + async cacheRead(queryStr) { + if (this.context === 'client') { + return this.storage[queryStr]; } else { - if (hash === "ROOT_QUERY" || hash === "ROOT_MUTATION") { - const hasRootQuery = await redis.get("ROOT_QUERY"); + if (queryStr === 'ROOT_QUERY' || queryStr === 'ROOT_MUTATION') { + const hasRootQuery = await redis.get('ROOT_QUERY'); if (!hasRootQuery) { - await redis.set("ROOT_QUERY", JSON.stringify({})); + await redis.set('ROOT_QUERY', JSON.stringify({})); } - const hasRootMutation = await redis.get("ROOT_MUTATION"); + const hasRootMutation = await redis.get('ROOT_MUTATION'); if (!hasRootMutation) { - await redis.set("ROOT_MUTATION", JSON.stringify({})); + await redis.set('ROOT_MUTATION', JSON.stringify({})); } } - let hashedQuery = await redis.get(hash); + console.log(queryStr); + // use cacheQueryKey to create a key with object name and inputs to save in cache + const queryKey = this.createQueryKey(queryStr); + const cacheResponse = await redis.hget('ROOT_QUERY', queryKey); - if (hashedQuery === undefined) return undefined; - return JSON.parse(hashedQuery); + if (!cacheResponse === undefined) return; + return JSON.parse(cacheResponse); + } + } + + createQueryKey(queryStr) { + // traverses AST and gets object name ("plants"), and any filter keys in the query ("maintenance:Low") + const ast = gql(queryStr); + const tableName = ast.definitions[0].selectionSet.selections[0].name.value; + let queryKey = `${tableName}`; + + if (ast.definitions[0].operation === 'mutation') return queryKey; + if (ast.definitions[0].selectionSet.selections[0].arguments.length) { + const fieldsArray = + ast.definitions[0].selectionSet.selections[0].arguments[0].value.fields; + const resultsObj = {}; + fieldsArray.forEach((el) => { + const name = el.name.value; + const value = el.value.value; + resultsObj[name] = value; + }); + + for (let key in resultsObj) { + queryKey += `:${key}:${resultsObj[key]}`; + } } + // console.log('finished getCacheHash'); + return queryKey; } async cacheWrite(hash, value) { // writes value to object cache or JSON.stringified value to redis cache - if (this.context === "client") { + if (this.context === 'client') { this.storage[hash] = value; } else { value = JSON.stringify(value); - await redis.setex(hash, 6000, value); - let hashedQuery = await redis.get(hash); + await redis.hset('ROOT_QUERY', hash, value); + // let hashedQuery = await redis.hget('ROOT_QUERY', hash); } } @@ -151,21 +187,21 @@ export class Cache { async cacheDelete(hash) { // deletes the hash/value pair on either object cache or redis cache - if (this.context === "client") { + if (this.context === 'client') { delete this.storage[hash]; } else await redis.del(hash); } async cacheClear() { // erases either object cache or redis cache - if (this.context === "client") { + if (this.context === 'client') { this.storage = { ROOT_QUERY: {}, ROOT_MUTATION: {} }; } else { await redis.flushdb((err, successful) => { - if (err) console.log("redis error", err); - console.log(successful, "clear"); + if (err) console.log('redis error', err); + console.log(successful, 'clear'); }); - await redis.set("ROOT_QUERY", JSON.stringify({})); - await redis.set("ROOT_MUTATION", JSON.stringify({})); + await redis.hset('ROOT_QUERY', 'blank', JSON.stringify({})); + await redis.set('ROOT_MUTATION', 'blank', JSON.stringify({})); } } diff --git a/src/restructure.ts b/src/restructure.ts index c99f584..070e591 100644 --- a/src/restructure.ts +++ b/src/restructure.ts @@ -1,6 +1,6 @@ import { gql } from 'https://deno.land/x/oak_graphql/mod.ts'; -import {print, visit} from "https://deno.land/x/graphql_deno/mod.ts"; +import { print, visit } from 'https://deno.land/x/graphql_deno/mod.ts'; /** * The restructure function: @@ -11,139 +11,151 @@ import {print, visit} from "https://deno.land/x/graphql_deno/mod.ts"; * @param {any} value - Query string * @return {string} string */ -export function restructure (value:any){ - - - const variables = value.variables || {}; - const operationName = value.operationName; - - let ast = gql(value.query); - - - let fragments: {[key:string]:any} = {}; - let containsFrags:boolean = false; - let existingFrags: {[key:string]:any}={}; - let existingVars: {[key:string]:any}={}; - - const buildFragsVisitor = { - FragmentDefinition:(node:any)=>{ - fragments[node.name.value]=node.selectionSet.selections; - - } - }; - const buildDefaultVarsVisitor = { - VariableDefinition:(node:any)=>{ - - if (node.defaultValue){ - - if(!variables[node.variable.name.value]){ - variables[node.variable.name.value] = node.defaultValue.value; - } - +export function restructure(value: any) { + const variables = value.variables || {}; + const operationName = value.operationName; + + let ast = gql(value.query); + // console.log('ast from restructure', ast); + // console.log( + // ast.definitions[0].selectionSet.selections[0].name, + // ast.definitions[0].selectionSet.selections[0].name.value + // ); + // const name = ast.definitions[0].selectionSet.selections[0].name.value; + // const fieldsArray = + // ast.definitions[0].selectionSet.selections[0].arguments[0].value.fields; + // const resultsObj = {}; + // fieldsArray.forEach((el) => { + // const name = el.name.value; + // const value = el.value.value; + // resultsObj[name] = value; + // }); + // console.log(resultsObj); + // let cacheHash = `${name}`; + // for (let key in resultsObj) { + // cacheHash += `:${key}:${resultsObj[key]}`; + // } + // console.log('cacheHash: ', cacheHash); + // console.log( + // ast.definitions[0].selectionSet.selections[0].arguments[0].value.fields[0], + // ast.definitions[0].selectionSet.selections[0].arguments[0].value.fields[0] + // .name.value, + // ast.definitions[0].selectionSet.selections[0].arguments[0].value.fields[0] + // .value.value + // ); + + let fragments: { [key: string]: any } = {}; + let containsFrags: boolean = false; + let existingFrags: { [key: string]: any } = {}; + let existingVars: { [key: string]: any } = {}; + + const buildFragsVisitor = { + FragmentDefinition: (node: any) => { + fragments[node.name.value] = node.selectionSet.selections; + }, + }; + const buildDefaultVarsVisitor = { + VariableDefinition: (node: any) => { + if (node.defaultValue) { + if (!variables[node.variable.name.value]) { + variables[node.variable.name.value] = node.defaultValue.value; } - - } + } + }, }; const rewriteVarsVistor = { VariableDefinition: (node: any) => { return null; }, - Variable:(node:any)=>{ - if(variables.hasOwnProperty(node.name.value)){ - return {kind: "EnumValue", value: variables[node.name.value]}; - } + Variable: (node: any) => { + if (variables.hasOwnProperty(node.name.value)) { + return { kind: 'EnumValue', value: variables[node.name.value] }; } - + }, }; - - - const rewriteVisitor = { - FragmentSpread:(node:any)=>{ - if(fragments.hasOwnProperty(node.name.value)){ - - - return fragments[node.name.value]; - } - }, - }; - - const clearFragVisitor = { - FragmentDefinition:(node:any)=>{ - if(fragments.hasOwnProperty(node.name.value)){ - return null; - } + + const rewriteVisitor = { + FragmentSpread: (node: any) => { + if (fragments.hasOwnProperty(node.name.value)) { + return fragments[node.name.value]; } - } - const checkFragmentationVisitor = { - FragmentSpread:(node:any)=>{ - - containsFrags = true; - existingFrags[node.name.value]=true - }, - Variable:(node:any)=>{ - containsFrags = true; - existingVars[node.name.value]=true + }, + }; + + const clearFragVisitor = { + FragmentDefinition: (node: any) => { + if (fragments.hasOwnProperty(node.name.value)) { + return null; } - } - - const firstBuildVisitor = { - ...buildFragsVisitor, - ...buildDefaultVarsVisitor - }; - - - const firstRewriteVisitor={ + }, + }; + const checkFragmentationVisitor = { + FragmentSpread: (node: any) => { + containsFrags = true; + existingFrags[node.name.value] = true; + }, + Variable: (node: any) => { + containsFrags = true; + existingVars[node.name.value] = true; + }, + }; + + const firstBuildVisitor = { + ...buildFragsVisitor, + ...buildDefaultVarsVisitor, + }; + + const firstRewriteVisitor = { ...rewriteVisitor, ...rewriteVarsVistor, - OperationDefinition:(node:any)=>{ - if(operationName&&node.name.value!=operationName){return null}}, - InlineFragment:(node:any)=>{ - return [{ - kind: "Field", - alias: undefined, - name: { kind: "Name", value: "__typename" }, - arguments: [], - directives: [], - selectionSet: undefined - },node] + OperationDefinition: (node: any) => { + if (operationName && node.name.value != operationName) { + return null; + } + }, + InlineFragment: (node: any) => { + return [ + { + kind: 'Field', + alias: undefined, + name: { kind: 'Name', value: '__typename' }, + arguments: [], + directives: [], + selectionSet: undefined, + }, + node, + ]; + }, + }; + visit(ast, { leave: firstBuildVisitor }); + + ast = gql(print(visit(ast, { leave: firstRewriteVisitor }))); + visit(ast, { leave: checkFragmentationVisitor }); + while (containsFrags) { + containsFrags = false; + fragments = {}; + visit(ast, { enter: buildFragsVisitor }); + + ast = gql(print(visit(ast, { leave: firstRewriteVisitor }))); + visit(ast, { leave: checkFragmentationVisitor }); - } - }; - - visit(ast, {leave:firstBuildVisitor}); - - - ast = gql(print(visit(ast,{leave:firstRewriteVisitor}))); - visit(ast,{leave:checkFragmentationVisitor}); - - while(containsFrags){ - containsFrags=false; - fragments={}; - visit(ast, {enter:buildFragsVisitor}); - - ast = gql(print(visit(ast,{leave:firstRewriteVisitor}))); - visit(ast,{leave:checkFragmentationVisitor}); - //if existingFrags has a key that fragments does not - const exfragskeys=Object.keys(existingFrags); - const fragskeys=Object.keys(fragments); - const exvarsskeys=Object.keys(existingVars); - const varkeys =Object.keys(variables); + const exfragskeys = Object.keys(existingFrags); + const fragskeys = Object.keys(fragments); + const exvarsskeys = Object.keys(existingVars); + const varkeys = Object.keys(variables); //exfragskeys.every(key=>fragskeys.includes(key)) - if (!exfragskeys.every(key=>fragskeys.includes(key))){ - - return console.log({error: 'missing fragment definitions'}) + if (!exfragskeys.every((key) => fragskeys.includes(key))) { + return console.log({ error: 'missing fragment definitions' }); } - if (!exvarsskeys.every(key=>varkeys.includes(key))){ - - return console.log({error: 'missing variable definitions'}) + if (!exvarsskeys.every((key) => varkeys.includes(key))) { + return console.log({ error: 'missing variable definitions' }); } } - ast = visit(ast, { leave: clearFragVisitor }); - + return print(ast); } diff --git a/src/transformResponse.ts b/src/transformResponse.ts index 7dfca02..0dd2c4c 100644 --- a/src/transformResponse.ts +++ b/src/transformResponse.ts @@ -1,29 +1,39 @@ -import { isHashableObject, containsHashableObject, hashMaker } from './normalize.ts'; +import { + isHashableObject, + containsHashableObject, + hashMaker, +} from './normalize.ts'; import { GenericObject } from './normalize.ts'; -import { Cache } from './quickCache.js' -const cache = new Cache; +import { Cache } from './quickCache.js'; +const cache = new Cache(); -const isArrayOfHashableObjects = (arrayOfObjects: Array, hashableKeys: Array):boolean => { +const isArrayOfHashableObjects = ( + arrayOfObjects: Array, + hashableKeys: Array +): boolean => { if (Array.isArray(arrayOfObjects)) { - return arrayOfObjects.every(object => { + return arrayOfObjects.every((object) => { return containsHashableObject(object, hashableKeys); - }) + }); } return false; -} +}; /* ----------------------------------------------------------------*/ -/** transformResponse -* Returns a nested object representing an object of references, where the references are hashes in Redis. The responseObject input must: -* 1) Contain hashable object(s) -* 2) have a first key of 'data', as should all GraphQL response objects -* 3) have an inner array of data response objects corresponding to the GraphQL fields -* -* @param {GenericObject} responseObject GraphQL response Object for large read query -* @param {array} hashableKeys Array of hashable keys -* @return {GenericObject} Nested object representing an object of references, where the references are hashes in Redis -*/ -export const transformResponse = (responseObject: any, hashableKeys: Array):GenericObject => { +/** transformResponse + * Returns a nested object representing an object of references, where the references are hashes in Redis. The responseObject input must: + * 1) Contain hashable object(s) + * 2) have a first key of 'data', as should all GraphQL response objects + * 3) have an inner array of data response objects corresponding to the GraphQL fields + * + * @param {GenericObject} responseObject GraphQL response Object for large read query + * @param {array} hashableKeys Array of hashable keys + * @return {GenericObject} Nested object representing an object of references, where the references are hashes in Redis + */ +export const transformResponse = ( + responseObject: any, + hashableKeys: Array +): GenericObject => { const result: GenericObject = {}; if (responseObject.data) { @@ -41,74 +51,101 @@ export const transformResponse = (responseObject: any, hashableKeys: Array => { + * Returns a nested object representing the original graphQL response object for a given queryKey + * @param {String} queryKey String representing the stringified GraphQL query for a big read query, which should have been saved as a key in Redis + * @param {GenericObject} transformedValue Nested object representing of references, where the references are hashes in Redis + * @return {GenericObject} Nested object representing the original graphQL response object for a given queryKey + */ +export const detransformResponse = async ( + queryString: String, + transformedValue: any, + selectionsArray: Array +): Promise => { // remove all text within parentheses aka '(input: ...)' - queryKey = queryKey.replace(/\(([^)]+)\)/, ''); + queryString = queryString.replace(/\(([^)]+)\)/, ''); // save Regex matches for line break followed by '{' - const matches = [...queryKey.matchAll(/\n([^\n]+)\{/g)]; + const matches = [...queryString.matchAll(/\n([^\n]+)\{/g)]; // get fields of query - const fields: Array = []; - matches.forEach(match => { - fields.push(match[1].trim()); + const tableNames: Array = []; + matches.forEach((match) => { + tableNames.push(match[1].trim()); }); - + // fields ends up as array of just the fields ("plants" in the demo case); // define recursiveDetransform function body for use later - const recursiveDetransform = async (transformedValue: any, fields: Array, depth: number = 0):Promise => { - let result: any = {}; + const recursiveDetransform = async ( + transformedValue: any, + tableNames: Array, + selectionsArray: Array, + depth: number = 0 + ): Promise => { + const keys = Object.keys(transformedValue); + let result: any = {}; let currDepth = depth; - // base case: transformedValue is innermost object aka empty object + // base case: transformedValue is innermost object aka empty object if (Object.keys(transformedValue).length === 0) { return result; } else { - let currField: string = fields[currDepth]; - result[currField] = []; - - for (let hash in transformedValue) { - const redisValue: GenericObject = await cache.cacheReadObject(hash); + let currTable: string = tableNames[currDepth]; + result[currTable] = []; + + for (let hash in transformedValue) { + const redisValue: GenericObject = await cache.cacheReadObject( + hash, + selectionsArray + ); // edge case in which our eviction strategy has pushed partial Cache data out of Redis if (!redisValue) { return 'cacheEvicted'; } - result[currField].push(redisValue); - - let recursiveResult = await recursiveDetransform(transformedValue[hash], fields, depth = currDepth + 1) + result[currTable].push(redisValue); + + let recursiveResult = await recursiveDetransform( + transformedValue[hash], + tableNames, + selectionsArray, + (depth = currDepth + 1) + ); // edge case in which our eviction strategy has pushed partial Cache data out of Redis, for recursive call if (recursiveResult === 'cacheEvicted') { return 'cacheEvicted'; - // normal case with no cache eviction + // normal case with no cache eviction } else { - result[currField][result[currField].length - 1] = Object.assign( - result[currField][result[currField].length - 1], recursiveResult); + result[currTable][result[currTable].length - 1] = Object.assign( + result[currTable][result[currTable].length - 1], + recursiveResult + ); } - } return result; } - } + }; // actually call recursiveDetransform - let detransformedResult: any = {'data' : {}}; - const detransformedSubresult = await recursiveDetransform(transformedValue, fields) + let detransformedResult: any = { data: {} }; + const detransformedSubresult = await recursiveDetransform( + transformedValue, + tableNames, + selectionsArray + ); + // console.log('detransformedSubresult: ', detransformedSubresult); if (detransformedSubresult === 'cacheEvicted') { detransformedResult = undefined; } else { - detransformedResult.data = await recursiveDetransform(transformedValue, fields); + detransformedResult.data = await recursiveDetransform( + transformedValue, + tableNames, + selectionsArray + ); } return detransformedResult; -} \ No newline at end of file +}; From 47bca3a9bbd31ddf4144b7613a3f532e96e0ed7c Mon Sep 17 00:00:00 2001 From: Josh Reed Date: Fri, 23 Dec 2022 15:00:43 -0800 Subject: [PATCH 2/6] added ability to invalidate only relevant cache on a mutation --- src/invalidateCacheCheck.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/invalidateCacheCheck.ts b/src/invalidateCacheCheck.ts index 52b1833..ba6952e 100644 --- a/src/invalidateCacheCheck.ts +++ b/src/invalidateCacheCheck.ts @@ -86,9 +86,13 @@ export async function invalidateCache( //loop through rootQueryContents, checking if //staleRef === rootQueryContents[i].slice(0, staleRef.length). //if they're equal, delete from ROOT_QUERY hash (redisdb.hdel(ROOTQUERY, rootQueryContents[i])) - for (let i = 0; i < rootQueryContents.length; i += 2) { - if (staleRefs === rootQueryContents[i].slice(0, staleRefs.length)) { - redisdb.hdel('ROOT_QUERY', rootQueryContents[i]); + for (let j = 0; j < staleRefs.length; j++) { + for (let i = 0; i < rootQueryContents.length; i += 2) { + if ( + staleRefs[j] === rootQueryContents[i].slice(0, staleRefs[j].length) + ) { + redisdb.hdel('ROOT_QUERY', rootQueryContents[i]); + } } } await cache.cacheWriteObject(redisKey, normalizedData); From eef118e5dfbe366ad5156215aad52bbdbac91eb4 Mon Sep 17 00:00:00 2001 From: OkunoD <108437247+OkunoD@users.noreply.github.com> Date: Fri, 23 Dec 2022 16:35:03 -0800 Subject: [PATCH 3/6] Added comments, cleaned consolelogs and refactored Co-authored-by: Jonathan Fangon Co-authored-by: Liam Johnson Co-authored-by: Derek Okuno Co-authored-by: Liam Jeon --- src/Obsidian.ts | 36 +++++++++++++------------- src/invalidateCacheCheck.ts | 50 ++++++++++++++++--------------------- src/mapSelections.js | 6 ++--- src/transformResponse.ts | 18 ++++++------- 4 files changed, 52 insertions(+), 58 deletions(-) diff --git a/src/Obsidian.ts b/src/Obsidian.ts index bafff40..08f9de2 100644 --- a/src/Obsidian.ts +++ b/src/Obsidian.ts @@ -68,7 +68,7 @@ export async function ObsidianRouter({ useQueryCache = true, useRebuildCache = true, customIdentifier = ['id', '__typename'], - mutationTableMap = {}, + mutationTableMap = {}, // Developer passes in object where keys are add mutations and values are arrays of affected tables }: ObsidianRouterOptions): Promise { redisPortExport = redisPort; const router = new Router(); @@ -84,35 +84,36 @@ export async function ObsidianRouter({ //post await router.post(path, async (ctx: any) => { - const t0 = performance.now(); + + const t0 = performance.now(); // Used for demonstration of cache vs. db performance times + const { response, request } = ctx; if (!request.hasBody) return; try { const contextResult = context ? await context(ctx) : undefined; let body = await request.body().value; - // Gets requested data point from query and saves into an array - const selectionsArray = mapSelectionSet(body.query); + const selectedFields = mapSelectionSet(body.query); // Gets requested fields from query and saves into an array if (maxQueryDepth) queryDepthLimiter(body.query, maxQueryDepth); // If a securty limit is set for maxQueryDepth, invoke queryDepthLimiter, which throws error if query depth exceeds maximum - let restructuredBody = { query: restructure(body) }; // Restructre gets rid of variables and fragments from the query + let restructuredBody = { query: restructure(body) }; // Restructure gets rid of variables and fragments from the query - // Parses query string into query key and checks cach for that key - let cacheQueryValue = await cache.read(body.query); + let cacheQueryValue = await cache.read(body.query); // Parses query string into query key and checks cache for that key // Is query in cache? if (useCache && useQueryCache && cacheQueryValue) { - let detransformedCacheQueryValue = await detransformResponse( + let detransformedCacheQueryValue = await detransformResponse( // Returns a nested object representing the original graphQL response object for a given queryKey restructuredBody.query, cacheQueryValue, - selectionsArray + selectedFields ); if (!detransformedCacheQueryValue) { // cache was evicted if any partial cache is missing, which causes detransformResponse to return undefined cacheQueryValue = undefined; - } else { + + } else { // Successful cache hit response.status = 200; - response.body = detransformedCacheQueryValue; + response.body = detransformedCacheQueryValue; // Returns response from cache const t1 = performance.now(); console.log( '%c Obsidian retrieved data from cache and took ' + @@ -131,15 +132,14 @@ export async function ObsidianRouter({ body.variables || undefined, body.operationName || undefined ); - // console.log('gqlResponse raw: ', gqlResponse); - const normalizedGQLResponse = normalizeObject( + + const normalizedGQLResponse = normalizeObject( // Recursively flattens an arbitrarily nested object into an objects with hash key and hashable object pairs gqlResponse, customIdentifier ); - // console.log('normalized: ', normalizedGQLResponse); - if (isMutation(restructuredBody)) { - // cache.cacheClear(); - const queryString = await request.body().value; + + if (isMutation(restructuredBody)) { // If operation is mutation, invalidate relevant responses in cache + const queryString = body; invalidateCache( normalizedGQLResponse, queryString.query, @@ -158,7 +158,7 @@ export async function ObsidianRouter({ } } response.status = 200; - response.body = gqlResponse; + response.body = gqlResponse; // Returns response from database const t1 = performance.now(); console.log( '%c Obsidian received new data and took ' + diff --git a/src/invalidateCacheCheck.ts b/src/invalidateCacheCheck.ts index ba6952e..abffc24 100644 --- a/src/invalidateCacheCheck.ts +++ b/src/invalidateCacheCheck.ts @@ -11,7 +11,7 @@ const cache = new Cache(); * @param {boolean} isMutation - Boolean indicating if it's a mutation query * @return {boolean} isMutation */ -export function isMutation(gqlQuery: { query: any }): boolean { +export function isMutation(gqlQuery: { query: string }): boolean { let isMutation: boolean = false; let ast: any = gql(gqlQuery.query); @@ -48,7 +48,7 @@ export function isMutation(gqlQuery: { query: any }): boolean { export async function invalidateCache( normalizedMutation: { [key: string]: object }, queryString: string, - mutationTableMap + mutationTableMap: Record ) { let normalizedData: object; let cachedVal: any; @@ -65,37 +65,31 @@ export async function invalidateCache( isDelete(queryString) ) { await cache.cacheDelete(redisKey); - } else { - // otherwise it's an update or add mutation because response objects from mutation and cache don't match so we overwrite the existing cache value or write new data if cache at that key doesn't exist + } + else { + // Otherwise it's an update or add mutation because response objects from mutation and cache don't match. + // We overwrite the existing cache value or write new data if cache at that key doesn't exist // Edge case: update is done without changing any values... cache will be deleted from redis because the response obj and cached obj will be equal - // we put it in the backburner because it doesn't make our cache stale, we would just perform an extra operation to re-cache the missing value when a request comes in - // mutationTableMap.mutationType - console.log('normalizedMutation is: ', normalizedMutation); - console.log('queryString is: ', queryString); - let ast = gql(queryString); - const mutationType = - ast.definitions[0].selectionSet.selections[0].name.value; - console.log('mutationType is: ', mutationType); - console.log('mutationTableMap is: ', mutationTableMap); - const staleRefs = mutationTableMap[mutationType]; - console.log('staleRefs is: ', staleRefs); + if (cachedVal === undefined) { // checks if add mutation + let ast = gql(queryString); + const mutationType = + ast.definitions[0].selectionSet.selections[0].name.value; // Extracts mutationType from query string - //loop through refs in ROOT_QUERY hash in redis - const rootQueryContents = await redisdb.hgetall('ROOT_QUERY'); - console.log('rootQueryContents is: ', rootQueryContents); - //loop through rootQueryContents, checking if - //staleRef === rootQueryContents[i].slice(0, staleRef.length). - //if they're equal, delete from ROOT_QUERY hash (redisdb.hdel(ROOTQUERY, rootQueryContents[i])) - for (let j = 0; j < staleRefs.length; j++) { - for (let i = 0; i < rootQueryContents.length; i += 2) { - if ( - staleRefs[j] === rootQueryContents[i].slice(0, staleRefs[j].length) - ) { - redisdb.hdel('ROOT_QUERY', rootQueryContents[i]); + const staleRefs: Array = mutationTableMap[mutationType]; // Grabs array of affected data tables from dev specified mutationTableMap + + const rootQueryContents = await redisdb.hgetall('ROOT_QUERY'); // Creates array of all query keys and values in ROOT_QUERY from Redis + + for (let j = 0; j < staleRefs.length; j++) { // Checks for all query keys that refer to the affected tables and deletes them from Redis + for (let i = 0; i < rootQueryContents.length; i += 2) { + if ( + staleRefs[j] === rootQueryContents[i].slice(0, staleRefs[j].length) + ) { + redisdb.hdel('ROOT_QUERY', rootQueryContents[i]); + } } } } - await cache.cacheWriteObject(redisKey, normalizedData); + await cache.cacheWriteObject(redisKey, normalizedData); // Adds or updates reference in redis cache } } } diff --git a/src/mapSelections.js b/src/mapSelections.js index c43f4ff..1229dbd 100644 --- a/src/mapSelections.js +++ b/src/mapSelections.js @@ -25,9 +25,9 @@ export function mapSelectionSet(query) { }; recursiveMap(selections); - // filter out object name from array, leaving only fields - const fieldsArray = Object.keys(selectionKeysMap).filter( + // filter out table name from array, leaving only fields + const selectedFields = Object.keys(selectionKeysMap).filter( (key) => key !== tableName ); - return fieldsArray; + return selectedFields; } diff --git a/src/transformResponse.ts b/src/transformResponse.ts index 0dd2c4c..545ea70 100644 --- a/src/transformResponse.ts +++ b/src/transformResponse.ts @@ -2,8 +2,8 @@ import { isHashableObject, containsHashableObject, hashMaker, -} from './normalize.ts'; -import { GenericObject } from './normalize.ts'; +} from './normalize'; +import { GenericObject } from './normalize'; import { Cache } from './quickCache.js'; const cache = new Cache(); @@ -63,7 +63,7 @@ export const transformResponse = ( export const detransformResponse = async ( queryString: String, transformedValue: any, - selectionsArray: Array + selectedFields: Array ): Promise => { // remove all text within parentheses aka '(input: ...)' queryString = queryString.replace(/\(([^)]+)\)/, ''); @@ -80,7 +80,7 @@ export const detransformResponse = async ( const recursiveDetransform = async ( transformedValue: any, tableNames: Array, - selectionsArray: Array, + selectedFields: Array, depth: number = 0 ): Promise => { const keys = Object.keys(transformedValue); @@ -97,7 +97,7 @@ export const detransformResponse = async ( for (let hash in transformedValue) { const redisValue: GenericObject = await cache.cacheReadObject( hash, - selectionsArray + selectedFields ); // edge case in which our eviction strategy has pushed partial Cache data out of Redis @@ -110,7 +110,7 @@ export const detransformResponse = async ( let recursiveResult = await recursiveDetransform( transformedValue[hash], tableNames, - selectionsArray, + selectedFields, (depth = currDepth + 1) ); @@ -130,20 +130,20 @@ export const detransformResponse = async ( }; // actually call recursiveDetransform + // Formats Redis cache value into GraphQL response syntax. cacheReadObject is called and returns only fields that are present in selectedFields let detransformedResult: any = { data: {} }; const detransformedSubresult = await recursiveDetransform( transformedValue, tableNames, - selectionsArray + selectedFields ); - // console.log('detransformedSubresult: ', detransformedSubresult); if (detransformedSubresult === 'cacheEvicted') { detransformedResult = undefined; } else { detransformedResult.data = await recursiveDetransform( transformedValue, tableNames, - selectionsArray + selectedFields ); } From 94ee5fd6ec37ae162acd528993c37575af761f93 Mon Sep 17 00:00:00 2001 From: Josh Reed Date: Tue, 3 Jan 2023 16:46:40 -0800 Subject: [PATCH 4/6] Updated readme. Deleted some console logs and added comments. Co-authored-by: Josh Reed Co-authored-by: Jonathan Fangon Co-authored-by: Liam Johnson Co-authored-by: Derek Okuno Co-authored-by: Liam Jeon --- README.md | 92 ++++++++++++++++++++++++++-------------------- src/quickCache.js | 9 +++-- src/rebuild.js | 8 ++-- src/restructure.ts | 27 -------------- 4 files changed, 63 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index c61b289..cfde703 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,9 @@ ## Features +- (New!) Server-side cache invalidation only on affected entries +- (New!) Flexible cache responds with only data requested from selected fields +- (New!) Developer tool for Obsidian is now updated to Manifest version 3 and invalid Bootstrap module imports were also fixed along with CodeMirror dependencies - GraphQL query abstraction and caching improving the performance of your app - SSR React wrapper, allowing you to cache in browser - Configurable caching options, giving you complete control over your cache @@ -36,7 +39,6 @@ Obsidian is Deno's first native GraphQL caching client and server module. Boasti With additional support for use in server-side rendered React apps built with Deno, full stack integration of Obsidian enables a fast and flexible caching solution. - ## Installation
QUICK START
@@ -47,29 +49,31 @@ With additional support for use in server-side rendered React apps built with De ```javascript import { Application, Router } from 'https://deno.land/x/oak@v6.0.1/mod.ts'; import { ObsidianRouter, gql } from 'https://deno.land/x/obsidian/mod.ts'; -import { resolvers } from './ import from your resolvers file' -import { types } from './ import your schema/types from schema/types file' - +import { resolvers } from './ import from your resolvers file'; +import { types } from './ import your schema/types from schema/types file'; interface ObsRouter extends Router { obsidianSchema?: any; } -const GraphQLRouter = await ObsidianRouter({ - Router, - typeDefs: types, - resolvers: resolvers, - redisPort: 6379, //Desired redis port - useCache: true, //Boolean to toggle all cache functionality - usePlayground: true, //Boolean to allow for graphQL playground - useQueryCache: true, //Boolean to toogle full query cache - useRebuildCache: true, //Boolean to toggle rebuilding from normalized data - customIdentifier: ["id", "__typename"] - -}); +const GraphQLRouter = + (await ObsidianRouter) < + ObsRouter > + { + Router, + typeDefs: types, + resolvers: resolvers, + redisPort: 6379, //Desired redis port + useCache: true, //Boolean to toggle all cache functionality + usePlayground: true, //Boolean to allow for graphQL playground + useQueryCache: true, //Boolean to toogle full query cache + useRebuildCache: true, //Boolean to toggle rebuilding from normalized data + customIdentifier: ['id', '__typename'], + mutationTableMap = {}, //Object where keys are add mutation types and value is an array of affected tables (e.g. {addPlants: ['plants'], addMovie: ['movies']}) + }; // attach the graphql routers routes to our app - app.use(GraphQLRouter.routes(), GraphQLRouter.allowedMethods()); +app.use(GraphQLRouter.routes(), GraphQLRouter.allowedMethods()); ``` ## Creating the Wrapper @@ -151,46 +155,56 @@ const MovieApp = () => { ``` ## Documentation + [obsidian.land](http://obsidian.land) ## Developer Tool -information and instructions on how to use our developer tool can be found here
+ +Information and instructions on how to use our developer tool can be found here
works with Obsidian 5.0
[oslabs-beta/obsidian-developer-tool](https://github.com/oslabs-beta/obsidian-developer-tool) ## Obsidian 5.0 Demo -github for a demo with some example code to play with:
+ +Github for a demo with some example code to play with:
[oslabs-beta/obsidian-demo-5.0](https://github.com/oslabs-beta/obsidian-demo-5.0) ## Dockerized Demo -working demo to install locally in docker: + +Working demo to install locally in docker: [oslabs-beta/obsidian-demo-docker](https://github.com/oslabs-beta/obsidian-demo-docker) -## Working Example Demo Code -github for a demo with some example code to play with: -[oslabs-beta/obsidian-demo-3.2](https://github.com/oslabs-beta/obsidian-demo-3.2) +## Features In Progress +- Ability to query the database for only those fields missing from the cache +- Developer Tool Settings component, fully functioning Playground component ## Authors -[Yurii Shchyrba](https://github.com/YuriiShchyrba) -[Linda Zhao](https://github.com/lzhao15) -[Ali Fay](https://github.com/ali-fay) -[Anthony Guan](https://github.com/guananthony) -[Yasir Choudhury](https://github.com/Yasir-Choudhury) -[Yogi Paturu](https://github.com/YogiPaturu) -[Michael Chin](https://github.com/mikechin37) -[Dana Flury](https://github.com/dmflury) -[Sardor Akhmedov](https://github.com/sarkamedo) -[Christopher Berry](https://github.com/cjamesb) + +[Derek Okuno](https://github.com/okunod) +[Liam Johnson](https://github.com/liamdimitri) +[Josh Reed](https://github.com/joshreed104) +[Jonathan Fangon](https://github.com/jonathanfangon) +[Liam Jeon](https://github.com/laj52) +[Yurii Shchyrba](https://github.com/YuriiShchyrba) +[Linda Zhao](https://github.com/lzhao15) +[Ali Fay](https://github.com/ali-fay) +[Anthony Guan](https://github.com/guananthony) +[Yasir Choudhury](https://github.com/Yasir-Choudhury) +[Yogi Paturu](https://github.com/YogiPaturu) +[Michael Chin](https://github.com/mikechin37) +[Dana Flury](https://github.com/dmflury) +[Sardor Akhmedov](https://github.com/sarkamedo) +[Christopher Berry](https://github.com/cjamesb) [Olivia Yeghiazarian](https://github.com/Olivia-code) -[Michael Melville](https://github.com/meekle) -[John Wong](https://github.com/johnwongfc) -[Kyung Lee](https://github.com/kyunglee1) -[Justin McKay](https://github.com/justinwmckay) +[Michael Melville](https://github.com/meekle) +[John Wong](https://github.com/johnwongfc) +[Kyung Lee](https://github.com/kyunglee1) +[Justin McKay](https://github.com/justinwmckay) [Patrick Sullivan](https://github.com/pjmsullivan) [Cameron Simmons](https://github.com/cssim22) [Raymond Ahn](https://github.com/raymondcodes) -[Alonso Garza](https://github.com/Alonsog66) +[Alonso Garza](https://github.com/Alonsog66) [Burak Caliskan](https://github.com/CaliskanBurak) [Matt Meigs](https://github.com/mmeigs) [Travis Frank](https://github.com/TravisFrankMTG/) @@ -198,4 +212,4 @@ github for a demo with some example code to play with: [Esma Sahraoui](https://github.com/EsmaShr) [Derek Miller](https://github.com/dsymiller) [Eric Marcatoma](https://github.com/ericmarc159) -[Spencer Stockton](https://github.com/tonstock) +[Spencer Stockton](https://github.com/tonstock) diff --git a/src/quickCache.js b/src/quickCache.js index 6dad283..ece6cb3 100644 --- a/src/quickCache.js +++ b/src/quickCache.js @@ -141,8 +141,13 @@ export class Cache { } } + /* + Creates a string to search the cache or add as a key in the cache. + If GraphQL query string is query{plants(input:{maintenance:"Low"}) name id ...} + returned queryKey will be plants:maintenance:Low + */ createQueryKey(queryStr) { - // traverses AST and gets object name ("plants"), and any filter keys in the query ("maintenance:Low") + // traverses AST and gets object name, and any filter keys in the query const ast = gql(queryStr); const tableName = ast.definitions[0].selectionSet.selections[0].name.value; let queryKey = `${tableName}`; @@ -162,7 +167,6 @@ export class Cache { queryKey += `:${key}:${resultsObj[key]}`; } } - // console.log('finished getCacheHash'); return queryKey; } async cacheWrite(hash, value) { @@ -172,7 +176,6 @@ export class Cache { } else { value = JSON.stringify(value); await redis.hset('ROOT_QUERY', hash, value); - // let hashedQuery = await redis.hget('ROOT_QUERY', hash); } } diff --git a/src/rebuild.js b/src/rebuild.js index 16290b5..4766a63 100644 --- a/src/rebuild.js +++ b/src/rebuild.js @@ -1,7 +1,7 @@ /** @format */ -import { redisdb } from "./quickCache.js"; -import { gql } from "https://deno.land/x/oak_graphql/mod.ts"; +import { redisdb } from './quickCache.js'; +import { gql } from 'https://deno.land/x/oak_graphql/mod.ts'; let localCacheObject = {}; const cacheReadList = async (hash) => { @@ -93,8 +93,8 @@ const rebuildArrays = async (cachedArray, queryArray) => { for (const queryField of queryArray) { let objKey; let nameyName; - if (queryField.kind == "InlineFragment") { - let __typeof = await cacheReadObject(cachedHash, "__typeof"); + if (queryField.kind == 'InlineFragment') { + let __typeof = await cacheReadObject(cachedHash, '__typeof'); if (__typeof == queryField.typeCondition.name.value) { } } diff --git a/src/restructure.ts b/src/restructure.ts index 070e591..d89de51 100644 --- a/src/restructure.ts +++ b/src/restructure.ts @@ -16,33 +16,6 @@ export function restructure(value: any) { const operationName = value.operationName; let ast = gql(value.query); - // console.log('ast from restructure', ast); - // console.log( - // ast.definitions[0].selectionSet.selections[0].name, - // ast.definitions[0].selectionSet.selections[0].name.value - // ); - // const name = ast.definitions[0].selectionSet.selections[0].name.value; - // const fieldsArray = - // ast.definitions[0].selectionSet.selections[0].arguments[0].value.fields; - // const resultsObj = {}; - // fieldsArray.forEach((el) => { - // const name = el.name.value; - // const value = el.value.value; - // resultsObj[name] = value; - // }); - // console.log(resultsObj); - // let cacheHash = `${name}`; - // for (let key in resultsObj) { - // cacheHash += `:${key}:${resultsObj[key]}`; - // } - // console.log('cacheHash: ', cacheHash); - // console.log( - // ast.definitions[0].selectionSet.selections[0].arguments[0].value.fields[0], - // ast.definitions[0].selectionSet.selections[0].arguments[0].value.fields[0] - // .name.value, - // ast.definitions[0].selectionSet.selections[0].arguments[0].value.fields[0] - // .value.value - // ); let fragments: { [key: string]: any } = {}; let containsFrags: boolean = false; From d3411d48f6a5c7a58f91dfec03ed99063bfd6af0 Mon Sep 17 00:00:00 2001 From: Josh Reed Date: Wed, 4 Jan 2023 10:18:03 -0800 Subject: [PATCH 5/6] Cleaned up some console logs and comments. Co-authored-by: Josh Reed Co-authored-by: Jonathan Fangon Co-authored-by: Liam Johnson Co-authored-by: Derek Okuno Co-authored-by: Liam Jeon --- ObsidianWrapper/ObsidianWrapper.jsx | 1 - src/quickCache.js | 5 ----- 2 files changed, 6 deletions(-) diff --git a/ObsidianWrapper/ObsidianWrapper.jsx b/ObsidianWrapper/ObsidianWrapper.jsx index 77b0ed8..14e51f9 100644 --- a/ObsidianWrapper/ObsidianWrapper.jsx +++ b/ObsidianWrapper/ObsidianWrapper.jsx @@ -50,7 +50,6 @@ function ObsidianWrapper(props) { // when the developer decides to only utilize whole query for cache if (wholeQuery) resObj = await cache.readWholeQuery(query); else resObj = await cache.read(query); - console.log('query function resObj: ', resObj); // check if query is stored in cache if (resObj) { // returning cached response as a promise diff --git a/src/quickCache.js b/src/quickCache.js index ece6cb3..d5c8e70 100644 --- a/src/quickCache.js +++ b/src/quickCache.js @@ -48,7 +48,6 @@ export class Cache { async write(queryStr, respObj, deleteFlag) { // update the original cache with same reference const cacheHash = this.createQueryKey(queryStr); - // console.log('write cacheHash: ', cacheHash); await this.cacheWrite(cacheHash, JSON.stringify(respObj)); } @@ -74,7 +73,6 @@ export class Cache { cacheWriteObject = async (hash, obj) => { let entries = Object.entries(obj).flat(); entries = entries.map((entry) => JSON.stringify(entry)); - // console.log('entries: ', entries); // adding as nested strings? take out one layer for clarity. await redis.hset(hash, ...entries); }; @@ -87,8 +85,6 @@ export class Cache { const rawCacheValue = await redisdb.hget(hash, JSON.stringify(field)); fieldObj[field] = JSON.parse(rawCacheValue); } - // if (returnValue === undefined) return undefined; - console.log('fieldObj: ', fieldObj); return fieldObj; } else { let objArray = await redisdb.hgetall(hash); @@ -131,7 +127,6 @@ export class Cache { await redis.set('ROOT_MUTATION', JSON.stringify({})); } } - console.log(queryStr); // use cacheQueryKey to create a key with object name and inputs to save in cache const queryKey = this.createQueryKey(queryStr); const cacheResponse = await redis.hget('ROOT_QUERY', queryKey); From f51fb1d3f4189b220092ecddb9042ff08fbdb454 Mon Sep 17 00:00:00 2001 From: Josh Reed Date: Wed, 4 Jan 2023 10:51:55 -0800 Subject: [PATCH 6/6] fixed authors in readme --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cfde703..f37bd16 100644 --- a/README.md +++ b/README.md @@ -181,11 +181,11 @@ Working demo to install locally in docker: ## Authors -[Derek Okuno](https://github.com/okunod) -[Liam Johnson](https://github.com/liamdimitri) -[Josh Reed](https://github.com/joshreed104) -[Jonathan Fangon](https://github.com/jonathanfangon) -[Liam Jeon](https://github.com/laj52) +[Derek Okuno](https://github.com/okunod) +[Liam Johnson](https://github.com/liamdimitri) +[Josh Reed](https://github.com/joshreed104) +[Jonathan Fangon](https://github.com/jonathanfangon) +[Liam Jeon](https://github.com/laj52) [Yurii Shchyrba](https://github.com/YuriiShchyrba) [Linda Zhao](https://github.com/lzhao15) [Ali Fay](https://github.com/ali-fay)