Skip to content

Commit

Permalink
Merge branch 'wyszynski/rtdb-emulator-functions' of https://github.co…
Browse files Browse the repository at this point in the history
…m/firebase/firebase-tools into wyszynski/rtdb-emulator-functions
  • Loading branch information
jmwski committed May 30, 2019
2 parents 0485229 + 75375b5 commit cad56b2
Show file tree
Hide file tree
Showing 8 changed files with 236 additions and 91 deletions.
84 changes: 84 additions & 0 deletions src/database/listRemote.ts
@@ -0,0 +1,84 @@
import * as request from "request";
import { Response } from "request";
import * as responseToError from "../responseToError";
import * as utils from "../utils";
import * as FirebaseError from "../error";
import * as logger from "../logger";
import * as api from "../api";

export interface ListRemote {
/**
* Call the shallow get API with limitToFirst=numSubPath.
* @param path the path to list
* @param numSubPath the number of subPaths to fetch.
* @param startAfter omit list entries comparing lower than `startAfter`
* @param timeout milliseconds after which to timeout the request
* @return the list of sub pathes found.
*/
listPath(
path: string,
numSubPath: number,
startAfter?: string,
timeout?: number
): Promise<string[]>;
}

export class RTDBListRemote implements ListRemote {
constructor(private instance: string) {}

async listPath(
path: string,
numSubPath: number,
startAfter?: string,
timeout?: number
): Promise<string[]> {
const url = `${utils.addSubdomain(api.realtimeOrigin, this.instance)}${path}.json`;

const params: any = {
shallow: true,
limitToFirst: numSubPath,
};
if (startAfter) {
params.startAfter = startAfter;
}
if (timeout) {
params.timeout = `${timeout}ms`;
}

const t0 = Date.now();
const reqOptionsWithToken = await api.addRequestHeaders({ url });
reqOptionsWithToken.qs = params;
const paths = await new Promise<string[]>((resolve, reject) => {
request.get(reqOptionsWithToken, (err: Error, res: Response, body: any) => {
if (err) {
return reject(
new FirebaseError("Unexpected error while listing subtrees", {
exit: 2,
original: err,
})
);
} else if (res.statusCode >= 400) {
return reject(responseToError(res, body));
}
let data;
try {
data = JSON.parse(body);
} catch (e) {
return reject(
new FirebaseError("Malformed JSON response in shallow get ", {
exit: 2,
original: e,
})
);
}
if (data) {
return resolve(Object.keys(data));
}
return resolve([]);
});
});
const dt = Date.now() - t0;
logger.debug(`[database] sucessfully fetched ${paths.length} path at ${path} ${dt}`);
return paths;
}
}
5 changes: 4 additions & 1 deletion src/database/remove.ts
@@ -1,6 +1,7 @@
import * as pathLib from "path";

import { RemoveRemote, RTDBRemoveRemote } from "./removeRemote";
import { ListRemote, RTDBListRemote } from "./listRemote";
import { Stack } from "../throttler/stack";

function chunkList<T>(ls: T[], chunkSize: number): T[][] {
Expand All @@ -18,6 +19,7 @@ const MAX_LIST_NUM_SUB_PATH = 204800;
export default class DatabaseRemove {
path: string;
remote: RemoveRemote;
listRemote: ListRemote;
private deleteJobStack: Stack<() => Promise<boolean>, boolean>;
private listStack: Stack<() => Promise<string[]>, string[]>;

Expand All @@ -36,6 +38,7 @@ export default class DatabaseRemove {
concurrency: 1,
retries: 3,
});
this.listRemote = new RTDBListRemote(instance);
this.listStack = new Stack({
name: "list stack",
concurrency: 1,
Expand Down Expand Up @@ -68,7 +71,7 @@ export default class DatabaseRemove {
let batchSize = INITIAL_DELETE_BATCH_SIZE;
while (true) {
const subPathList = await this.listStack.run(() =>
this.remote.listPath(path, listNumSubPath)
this.listRemote.listPath(path, listNumSubPath)
);
if (subPathList.length === 0) {
return Promise.resolve(false);
Expand Down
57 changes: 0 additions & 57 deletions src/database/removeRemote.ts
Expand Up @@ -19,14 +19,6 @@ export interface RemoveRemote {
* @return false if the deleteion failed because the the total size of subpaths exceeds the writeSizeLimit.
*/
deleteSubPath(path: string, subPaths: string[]): Promise<boolean>;

/**
* Call the shallow get API with limitToFirst=numSubPath.
* @param path the path to list
* @param numSubPath the number of subPaths to fetch.
* @return the list of sub pathes found.
*/
listPath(path: string, numSubPath: number): Promise<string[]>;
}

export class RTDBRemoveRemote implements RemoveRemote {
Expand All @@ -48,55 +40,6 @@ export class RTDBRemoveRemote implements RemoveRemote {
return this.patch(path, body, `${subPaths.length} subpaths`);
}

listPath(path: string, numSubPath: number): Promise<string[]> {
const url =
utils.addSubdomain(api.realtimeOrigin, this.instance) +
path +
`.json?shallow=true&limitToFirst=${numSubPath}`;
const t0 = Date.now();
return api
.addRequestHeaders({
url,
})
.then((reqOptionsWithToken) => {
return new Promise<string[]>((resolve, reject) => {
request.get(reqOptionsWithToken, (err: Error, res: Response, body: any) => {
if (err) {
return reject(
new FirebaseError("Unexpected error while listing subtrees", {
exit: 2,
original: err,
})
);
} else if (res.statusCode >= 400) {
return reject(responseToError(res, body));
}
let data = {};
try {
data = JSON.parse(body);
} catch (e) {
return reject(
new FirebaseError("Malformed JSON response in shallow get ", {
exit: 2,
original: e,
})
);
}
if (data) {
const keyList = Object.keys(data);
return resolve(keyList);
}
resolve([]);
});
});
})
.then((paths: string[]) => {
const dt = Date.now() - t0;
logger.debug(`[database] Sucessfully fetched ${paths.length} path at ${path} ${dt}`);
return paths;
});
}

private patch(path: string, body: any, note: string): Promise<boolean> {
const t0 = Date.now();
return new Promise((resolve, reject) => {
Expand Down
101 changes: 101 additions & 0 deletions src/test/database/fakeListRemote.spec.ts
@@ -0,0 +1,101 @@
import * as pathLib from "path";
import * as chai from "chai";

import { ListRemote } from "../../database/listRemote";

const expect = chai.expect;

/**
* `FakeListRemote` is a test fixture for verifying logic lives in the
* `DatabaseRemove` class. It is essentially a mock for the Realtime Database
* that accepts a JSON tree to serve upon construction.
*/
export class FakeListRemote implements ListRemote {
data: any;
delay: number;

/**
* @param data the fake database structure. Each leaf is an integer
* representing the subtree's size.
*/
constructor(data: any) {
this.data = data;
this.delay = 0;
}

async listPath(
path: string,
numChildren: number,
startAfter?: string,
timeout?: number
): Promise<string[]> {
if (timeout === 0) {
throw new Error("timeout");
}
const d = this.dataAtPath(path);
if (d) {
let keys = Object.keys(d);
/*
* We mirror a critical implementation detail of here. Namely, the
* `startAfter` option (if it exists) is applied to the resulting key set
* before the `limitToFirst` option.
*/
if (startAfter) {
keys = keys.filter((key) => key > startAfter);
}
keys = keys.slice(0, numChildren);
return keys;
}
return [];
}

private size(data: any): number {
if (typeof data === "number") {
return data;
}
let size = 0;
for (const key of Object.keys(data)) {
size += this.size(data[key]);
}
return size;
}

private dataAtPath(path: string): any {
const splitedPath = path.slice(1).split("/");
let d = this.data;
for (const p of splitedPath) {
if (d && p !== "") {
if (typeof d === "number") {
d = null;
} else {
d = d[p];
}
}
}
return d;
}
}

describe("FakeListRemote", () => {
it("should return limit the number of subpaths returned", async () => {
const fakeDb = new FakeListRemote({ 1: 1, 2: 2, 3: 3, 4: 4 });
await expect(fakeDb.listPath("/", 4)).to.eventually.eql(["1", "2", "3", "4"]);
await expect(fakeDb.listPath("/", 3)).to.eventually.eql(["1", "2", "3"]);
await expect(fakeDb.listPath("/", 2)).to.eventually.eql(["1", "2"]);
await expect(fakeDb.listPath("/", 1)).to.eventually.eql(["1"]);
await expect(fakeDb.listPath("/", 4, "1")).to.eventually.eql(["2", "3", "4"]);
await expect(fakeDb.listPath("/", 4, "2")).to.eventually.eql(["3", "4"]);
await expect(fakeDb.listPath("/", 4, "3")).to.eventually.eql(["4"]);
await expect(fakeDb.listPath("/", 4, "4")).to.eventually.eql([]);
await expect(fakeDb.listPath("/", 3, "1")).to.eventually.eql(["2", "3", "4"]);
await expect(fakeDb.listPath("/", 3, "2")).to.eventually.eql(["3", "4"]);
await expect(fakeDb.listPath("/", 3, "3")).to.eventually.eql(["4"]);
await expect(fakeDb.listPath("/", 3, "3")).to.eventually.eql(["4"]);
await expect(fakeDb.listPath("/", 3, "4")).to.eventually.eql([]);
await expect(fakeDb.listPath("/", 1, "1")).to.eventually.eql(["2"]);
await expect(fakeDb.listPath("/", 1, "2")).to.eventually.eql(["3"]);
await expect(fakeDb.listPath("/", 1, "3")).to.eventually.eql(["4"]);
await expect(fakeDb.listPath("/", 1, "4")).to.eventually.eql([]);
await expect(fakeDb.listPath("/", 1, "1", 0)).to.be.rejected;
});
});
17 changes: 0 additions & 17 deletions src/test/database/fakeRemoveRemote.spec.ts
Expand Up @@ -19,15 +19,6 @@ export class FakeRemoveRemote implements RemoveRemote {
this.largeThreshold = largeThreshold;
}

listPath(path: string, numChildren: number): Promise<string[]> {
const d = this._dataAtpath(path);
if (d) {
const keys = Object.keys(d);
return Promise.resolve(keys.slice(0, numChildren));
}
return Promise.resolve([]);
}

deletePath(path: string): Promise<boolean> {
const d = this._dataAtpath(path);
const size = this._size(d);
Expand Down Expand Up @@ -94,14 +85,6 @@ export class FakeRemoveRemote implements RemoveRemote {
}

describe("FakeRemoveRemote", () => {
it("should return limit the number of subpaths returned", async () => {
const fakeDb = new FakeRemoveRemote({ 1: 1, 2: 2, 3: 3, 4: 4 });
await expect(fakeDb.listPath("/", 4)).to.eventually.eql(["1", "2", "3", "4"]);
await expect(fakeDb.listPath("/", 3)).to.eventually.eql(["1", "2", "3"]);
await expect(fakeDb.listPath("/", 2)).to.eventually.eql(["1", "2"]);
await expect(fakeDb.listPath("/", 1)).to.eventually.eql(["1"]);
});

it("should failed to delete large path /", async () => {
const data = { 1: 11 };
const fakeDb = new FakeRemoveRemote(data);
Expand Down
29 changes: 29 additions & 0 deletions src/test/database/listRemote.spec.ts
@@ -0,0 +1,29 @@
import { expect } from "chai";
import * as nock from "nock";

import * as utils from "../../utils";
import * as api from "../../api";
import * as helpers from "../helpers";
import { RTDBListRemote } from "../../database/listRemote";

describe("ListRemote", () => {
const instance = "fake-db";
const remote = new RTDBListRemote(instance);
const serverUrl = utils.addSubdomain(api.realtimeOrigin, instance);

afterEach(() => {
nock.cleanAll();
});

it("should return subpaths from shallow get request", async () => {
nock(serverUrl)
.get("/.json")
.query({ shallow: true, limitToFirst: "1234" })
.reply(200, {
a: true,
x: true,
f: true,
});
await expect(remote.listPath("/", 1234)).to.eventually.eql(["a", "x", "f"]);
});
});

0 comments on commit cad56b2

Please sign in to comment.