Skip to content

Commit

Permalink
Merge branch 'master' into escape-comma
Browse files Browse the repository at this point in the history
  • Loading branch information
@bojeil-google
bojeil-google committed May 31, 2019
2 parents c90fb4d + dd14e5e commit fa275c9
Show file tree
Hide file tree
Showing 12 changed files with 286 additions and 148 deletions.
6 changes: 5 additions & 1 deletion changelog.txt
View file
Expand Up @@ -3,4 +3,8 @@
* Fixed bug where collection group query listeners would cause errors in the Firestore emulator.
* Fixed bug where query parameters were not sent to HTTP functions.
* Fixed bug where some HTTP methods (like DELETE) were not allowed.
* Fixed bug where CSV exporting users with commas in displayName causes columns to no longer align.
* Fixed bug where CORS headers were too restrictive.
* Fixed bug where CSV exporting users with commas in displayName causes columns to no longer align.
* Fixed bug where environment variables are unset for script in `emulators:exec`.
* `emulators:exec` script now inherits stdout and stderr.
* Improved reliability and performance of project listing for `firebase use` command.
12 changes: 2 additions & 10 deletions src/commands/emulators-exec.ts
View file
Expand Up @@ -16,7 +16,7 @@ import { FirestoreEmulator } from "../emulator/firestoreEmulator";
async function runScript(script: string): Promise<void> {
utils.logBullet(`Running script: ${clc.bold(script)}`);

const env: NodeJS.ProcessEnv = {};
const env: NodeJS.ProcessEnv = { ...process.env };

const firestoreInstance = EmulatorRegistry.get(Emulators.FIRESTORE);
if (firestoreInstance) {
Expand All @@ -26,22 +26,14 @@ async function runScript(script: string): Promise<void> {
}

const proc = childProcess.spawn(script, {
stdio: ["inherit", "pipe", "pipe"] as StdioOptions,
stdio: ["inherit", "inherit", "inherit"] as StdioOptions,
shell: true,
windowsHide: true,
env,
});

logger.debug(`Running ${script} with environment ${JSON.stringify(env)}`);

proc.stdout.on("data", (data) => {
process.stdout.write(data.toString());
});

proc.stderr.on("data", (data) => {
process.stderr.write(data.toString());
});

return new Promise((resolve, reject) => {
proc.on("error", (err: any) => {
utils.logWarning(`There was an error running the script: ${JSON.stringify(err)}`);
Expand Down
76 changes: 43 additions & 33 deletions src/commands/use.js
View file
Expand Up @@ -3,7 +3,7 @@
var Command = require("../command");
var logger = require("../logger");
var requireAuth = require("../requireAuth");
var api = require("../api");
var firebaseApi = require("../firebaseApi");
var clc = require("cli-color");
var utils = require("../utils");
var _ = require("lodash");
Expand Down Expand Up @@ -58,39 +58,49 @@ module.exports = new Command("use [alias_or_project_id]")
if (newActive) {
// firebase use [alias_or_project]
var aliasedProject = options.rc.get(["projects", newActive]);
return api.getProjects().then(function(projects) {
if (aliasOpt) {
// firebase use [project] --alias [alias]
if (!projects[newActive]) {
return utils.reject(
"Cannot create alias " + clc.bold(aliasOpt) + ", " + verifyMessage(newActive)
);
var project = null;
const lookupProject = aliasedProject || newActive;
return firebaseApi
.getProject(lookupProject)
.then((foundProject) => {
project = foundProject;
})
.catch(() => {
return utils.reject("Invalid project selection, " + verifyMessage(newActive));
})
.then(() => {
if (aliasOpt) {
// firebase use [project] --alias [alias]
if (!project) {
return utils.reject(
"Cannot create alias " + clc.bold(aliasOpt) + ", " + verifyMessage(newActive)
);
}
options.rc.addProjectAlias(aliasOpt, newActive);
aliasedProject = newActive;
logger.info("Created alias", clc.bold(aliasOpt), "for", aliasedProject + ".");
}
options.rc.addProjectAlias(aliasOpt, newActive);
aliasedProject = newActive;
logger.info("Created alias", clc.bold(aliasOpt), "for", aliasedProject + ".");
}

if (aliasedProject) {
// found alias
if (!projects[aliasedProject]) {
// found alias, but not in project list
return utils.reject(
"Unable to use alias " + clc.bold(newActive) + ", " + verifyMessage(aliasedProject)
);
}
if (aliasedProject) {
// found alias
if (!project) {
// found alias, but not in project list
return utils.reject(
"Unable to use alias " + clc.bold(newActive) + ", " + verifyMessage(aliasedProject)
);
}

utils.makeActiveProject(options.projectRoot, newActive);
logger.info("Now using alias", clc.bold(newActive), "(" + aliasedProject + ")");
} else if (projects[newActive]) {
// exact project id specified
utils.makeActiveProject(options.projectRoot, newActive);
logger.info("Now using project", clc.bold(newActive));
} else {
// no alias or project recognized
return utils.reject("Invalid project selection, " + verifyMessage(newActive));
}
});
utils.makeActiveProject(options.projectRoot, newActive);
logger.info("Now using alias", clc.bold(newActive), "(" + aliasedProject + ")");
} else if (project) {
// exact project id specified
utils.makeActiveProject(options.projectRoot, newActive);
logger.info("Now using project", clc.bold(newActive));
} else {
// no alias or project recognized
return utils.reject("Invalid project selection, " + verifyMessage(newActive));
}
});
} else if (options.unalias) {
// firebase use --unalias [alias]
if (_.has(options.rc.data, ["projects", options.unalias])) {
Expand All @@ -110,14 +120,14 @@ module.exports = new Command("use [alias_or_project_id]")
" instead."
);
}
return api.getProjects().then(function(projects) {
return firebaseApi.listProjects().then(function(projects) {
var results = {};
return prompt(results, [
{
type: "list",
name: "project",
message: "Which project do you want to add?",
choices: Object.keys(projects).sort(),
choices: projects.map((p) => p.projectId).sort(),
},
{
type: "input",
Expand Down
84 changes: 84 additions & 0 deletions src/database/listRemote.ts
View file
@@ -0,0 +1,84 @@
import * as request from "request";
import { Response } from "request";
import * as responseToError from "../responseToError";
import * as utils from "../utils";
import * as FirebaseError from "../error";
import * as logger from "../logger";
import * as api from "../api";

export interface ListRemote {
/**
* Call the shallow get API with limitToFirst=numSubPath.
* @param path the path to list
* @param numSubPath the number of subPaths to fetch.
* @param startAfter omit list entries comparing lower than `startAfter`
* @param timeout milliseconds after which to timeout the request
* @return the list of sub pathes found.
*/
listPath(
path: string,
numSubPath: number,
startAfter?: string,
timeout?: number
): Promise<string[]>;
}

export class RTDBListRemote implements ListRemote {
constructor(private instance: string) {}

async listPath(
path: string,
numSubPath: number,
startAfter?: string,
timeout?: number
): Promise<string[]> {
const url = `${utils.addSubdomain(api.realtimeOrigin, this.instance)}${path}.json`;

const params: any = {
shallow: true,
limitToFirst: numSubPath,
};
if (startAfter) {
params.startAfter = startAfter;
}
if (timeout) {
params.timeout = `${timeout}ms`;
}

const t0 = Date.now();
const reqOptionsWithToken = await api.addRequestHeaders({ url });
reqOptionsWithToken.qs = params;
const paths = await new Promise<string[]>((resolve, reject) => {
request.get(reqOptionsWithToken, (err: Error, res: Response, body: any) => {
if (err) {
return reject(
new FirebaseError("Unexpected error while listing subtrees", {
exit: 2,
original: err,
})
);
} else if (res.statusCode >= 400) {
return reject(responseToError(res, body));
}
let data;
try {
data = JSON.parse(body);
} catch (e) {
return reject(
new FirebaseError("Malformed JSON response in shallow get ", {
exit: 2,
original: e,
})
);
}
if (data) {
return resolve(Object.keys(data));
}
return resolve([]);
});
});
const dt = Date.now() - t0;
logger.debug(`[database] sucessfully fetched ${paths.length} path at ${path} ${dt}`);
return paths;
}
}
5 changes: 4 additions & 1 deletion src/database/remove.ts
View file
@@ -1,6 +1,7 @@
import * as pathLib from "path";

import { RemoveRemote, RTDBRemoveRemote } from "./removeRemote";
import { ListRemote, RTDBListRemote } from "./listRemote";
import { Stack } from "../throttler/stack";

function chunkList<T>(ls: T[], chunkSize: number): T[][] {
Expand All @@ -18,6 +19,7 @@ const MAX_LIST_NUM_SUB_PATH = 204800;
export default class DatabaseRemove {
path: string;
remote: RemoveRemote;
listRemote: ListRemote;
private deleteJobStack: Stack<() => Promise<boolean>, boolean>;
private listStack: Stack<() => Promise<string[]>, string[]>;

Expand All @@ -36,6 +38,7 @@ export default class DatabaseRemove {
concurrency: 1,
retries: 3,
});
this.listRemote = new RTDBListRemote(instance);
this.listStack = new Stack({
name: "list stack",
concurrency: 1,
Expand Down Expand Up @@ -68,7 +71,7 @@ export default class DatabaseRemove {
let batchSize = INITIAL_DELETE_BATCH_SIZE;
while (true) {
const subPathList = await this.listStack.run(() =>
this.remote.listPath(path, listNumSubPath)
this.listRemote.listPath(path, listNumSubPath)
);
if (subPathList.length === 0) {
return Promise.resolve(false);
Expand Down
57 changes: 0 additions & 57 deletions src/database/removeRemote.ts
View file
Expand Up @@ -19,14 +19,6 @@ export interface RemoveRemote {
* @return false if the deleteion failed because the the total size of subpaths exceeds the writeSizeLimit.
*/
deleteSubPath(path: string, subPaths: string[]): Promise<boolean>;

/**
* Call the shallow get API with limitToFirst=numSubPath.
* @param path the path to list
* @param numSubPath the number of subPaths to fetch.
* @return the list of sub pathes found.
*/
listPath(path: string, numSubPath: number): Promise<string[]>;
}

export class RTDBRemoveRemote implements RemoveRemote {
Expand All @@ -48,55 +40,6 @@ export class RTDBRemoveRemote implements RemoveRemote {
return this.patch(path, body, `${subPaths.length} subpaths`);
}

listPath(path: string, numSubPath: number): Promise<string[]> {
const url =
utils.addSubdomain(api.realtimeOrigin, this.instance) +
path +
`.json?shallow=true&limitToFirst=${numSubPath}`;
const t0 = Date.now();
return api
.addRequestHeaders({
url,
})
.then((reqOptionsWithToken) => {
return new Promise<string[]>((resolve, reject) => {
request.get(reqOptionsWithToken, (err: Error, res: Response, body: any) => {
if (err) {
return reject(
new FirebaseError("Unexpected error while listing subtrees", {
exit: 2,
original: err,
})
);
} else if (res.statusCode >= 400) {
return reject(responseToError(res, body));
}
let data = {};
try {
data = JSON.parse(body);
} catch (e) {
return reject(
new FirebaseError("Malformed JSON response in shallow get ", {
exit: 2,
original: e,
})
);
}
if (data) {
const keyList = Object.keys(data);
return resolve(keyList);
}
resolve([]);
});
});
})
.then((paths: string[]) => {
const dt = Date.now() - t0;
logger.debug(`[database] Sucessfully fetched ${paths.length} path at ${path} ${dt}`);
return paths;
});
}

private patch(path: string, body: any, note: string): Promise<boolean> {
const t0 = Date.now();
return new Promise((resolve, reject) => {
Expand Down
13 changes: 0 additions & 13 deletions src/emulator/functionsEmulator.ts
View file
Expand Up @@ -69,19 +69,6 @@ export class FunctionsEmulator implements EmulatorInstance {
const hub = express();

hub.use((req, res, next) => {
// Allow CORS to facilitate easier testing.
// Sources:
// * https://enable-cors.org/server_expressjCannot understand what targets to deploys.html
// * https://stackoverflow.com/a/37228330/324977
res.header("Access-Control-Allow-Origin", "*");

// Callable functions send "Authorization" and "Content-Type".
res.header(
"Access-Control-Allow-Headers",
"Origin, X-Requested-With, Content-Type, Authorization, Accept"
);
res.header("Access-Control-Allow-Methods", "GET,OPTIONS,POST");

let data = "";
req.on("data", (chunk: any) => {
data += chunk;
Expand Down

0 comments on commit fa275c9

Please sign in to comment.