Skip to content

Commit

Permalink
fix: doctor errors when backfilling +100 notes
Browse files Browse the repository at this point in the history
  • Loading branch information
@kevinslin
kevinslin committed Dec 6, 2020
1 parent a485570 commit 862e917
Show file tree
Hide file tree
Showing 4 changed files with 215 additions and 5 deletions.
8 changes: 5 additions & 3 deletions packages/dendron-cli/src/commands/backfillV2.ts
View file
Expand Up @@ -15,18 +15,20 @@ export class BackfillV2Command extends BaseCommand<CommandOpts, CommandOutput> {
const { engine, overwriteFields } = _.defaults(opts, {
overwriteFields: [],
});
await Promise.all(
_.values(engine.notes).map((n) => {
const notes = await Promise.all(
_.values(engine.notes).map(async (n) => {
overwriteFields.forEach((f) => {
if (f === "title") {
n.title = NoteUtilsV2.genTitle(n.fname);
} else {
throw Error(`unknown overwrite field: ${f}`);
}
});
return engine.writeNote(n);
return n;
})
);
// @ts-ignore
await engine.store.bulkAddNotes({ notes });
return;
}
}
15 changes: 15 additions & 0 deletions packages/engine-server/src/drivers/file/storev2.ts
View file
Expand Up @@ -41,6 +41,9 @@ import YAML from "yamljs";
import { ParserUtilsV2 } from "../../topics/markdown/utilsv2";
import { loc2Path } from "../../utils";

type BulkAddNoteOpts = {
notes: NotePropsV2[];
};
type FileMetaV2 = {
// file name: eg. foo.md, name = foo
prefix: string;
Expand Down Expand Up @@ -543,6 +546,18 @@ export class FileStorageV2 implements DStoreV2 {
return notes;
}

async bulkAddNotes(opts: BulkAddNoteOpts) {
return Promise.all(
opts.notes.map(async (note) => {
await note2File({
note,
vault: note.vault,
wsRoot: this.wsRoot,
});
})
);
}

async renameNote(opts: RenameNoteOptsV2): Promise<RenameNotePayload> {
const ctx = "Store:renameNote";
const { oldLoc, newLoc } = opts;
Expand Down
21 changes: 20 additions & 1 deletion packages/engine-server/src/engineClient.ts
View file
Expand Up @@ -30,9 +30,15 @@ import {
SchemaUtilsV2,
WriteNoteResp,
} from "@dendronhq/common-all";
import { DendronAPI, VaultUtils } from "@dendronhq/common-server";
import {
createLogger,
DendronAPI,
DLogger,
VaultUtils,
} from "@dendronhq/common-server";
import fs from "fs-extra";
import _ from "lodash";
import { FileStorageV2 } from "../lib";
import { FuseEngine } from "./fuseEngine";
import { HistoryService } from "./history";
import { getPortFilePath } from "./utils";
Expand All @@ -53,6 +59,8 @@ export class DendronEngineClient implements DEngineClientV2 {
public vaultsv3: DVault[];
public configRoot: string;
public history?: HistoryService;
public logger: DLogger;
public store: FileStorageV2;

static create({
port,
Expand Down Expand Up @@ -83,11 +91,13 @@ export class DendronEngineClient implements DEngineClientV2 {
vaults,
ws,
history,
logger,
}: {
api: DendronAPI;
vaults: string[];
ws: string;
history?: HistoryService;
logger?: DLogger;
}) {
this.api = api;
this.notes = {};
Expand All @@ -100,6 +110,12 @@ export class DendronEngineClient implements DEngineClientV2 {
this.ws = ws;
this.configRoot = this.wsRoot;
this.history = history;
this.logger = logger || createLogger();
this.store = new FileStorageV2({
wsRoot: this.ws,
vaultsv3: this.vaultsv3,
logger: this.logger,
});
}

/**
Expand All @@ -110,6 +126,7 @@ export class DendronEngineClient implements DEngineClientV2 {
uri: this.ws,
config: { vaults: this.vaults },
});

if (resp.error && resp.error.code !== ERROR_CODES.MINOR) {
return {
error: resp.error,
Expand All @@ -124,6 +141,8 @@ export class DendronEngineClient implements DEngineClientV2 {
this.schemas = schemas;
await this.fuseEngine.updateNotesIndex(notes);
await this.fuseEngine.updateSchemaIndex(schemas);
this.store.notes = notes;
this.store.schemas = schemas;
return {
error: resp.error,
data: { notes, schemas },
Expand Down
176 changes: 175 additions & 1 deletion packages/plugin-core/src/test/suite-integ/DoctorCommand.test.ts
View file
Expand Up @@ -29,7 +29,7 @@ suite("notes", function () {
fs.writeFileSync(testFile, "bond", { encoding: "utf8" });
await new ReloadIndexCommand().run();
await new DoctorCommand().run();
// cehck taht frontmatter is added
// cehck that frontmatter is added
const resp = fs.readFileSync(testFile, { encoding: "utf8" });
assert.ok(NoteUtilsV2.RE_FM.exec(resp));
assert.ok(NoteUtilsV2.RE_FM_UPDATED.exec(resp));
Expand All @@ -44,6 +44,180 @@ suite("notes", function () {
});
});

// test.only("lots of files", (done) => {
// runLegacySingleWorkspaceTest({
// ctx,
// onInit: async ({ engine, wsRoot, vaults }) => {
// console.log(wsRoot);
// const vault = vaults[0];
// await new DoctorCommand().run();
// expect(_.size(getEngine().notes) > 100).toBeTruthy();
// },
// postSetupHook: async ({ wsRoot, vaults }) => {
// const names = [
// "30x500",
// "root",
// "article",
// "auth",
// "books",
// "bio-and-blurb",
// "business",
// "clojure",
// "daily",
// "companies",
// "ocb",
// "mozfest",
// "people",
// "p",
// "platypush",
// "podcast",
// "scratch",
// "rpg",
// "speaking_at_a_conference",
// "spinup",
// "thinking",
// "taking_smart_notes_with_org_roam",
// "30x500.assets",
// "30x500.amazon_scoop_poop_safari",
// "30x500.exercise_practice_collecting_safari_gold_jargon_worldview_and_recommendations",
// "30x500.audience",
// "30x500.freelance_reddit",
// "30x500.exercise_practice_getting_keywords_and_themes_from_topic_titles",
// "30x500.interview_amy_hoy_and_alex_hillman",
// "30x500.infinite_ebombs_exercise",
// "30x500.rotton_eggs_safari",
// "article.advice_to_building_saas_on_aws",
// "article.grow_your_business_with_email_marketing_done_right",
// "article.encouraging_a_culture_of_written_communication_mcls",
// "article.how_is_design_important_as_a_developer_and_what_can_you_do_to_level_up_colby_fayock",
// "article.hacking_your_keyboard_with_karabiner_kaushik_gopal_s_blog",
// "article.how_to_use_egghead_io_to_level_up_as_a_web_developer",
// "article.how_to_maximize_serendipity",
// "article.naked_brands",
// "article.knowledge_hydrant_a_pattern_language_for_study_groups",
// "article.roads_for_success_navigating_docs",
// "article.one_big_idea_david_perell",
// "article.scraping_recipe_websites_ben_awad_blog",
// "article.second_chances",
// "article.the_framing_of_the_developer",
// "article.the_nature_of_code",
// "article.the_notation_i_use_to_manage_my_macros_tips_tutorials_keyboard_maestro_discourse",
// "article.the_ultimate_guide_to_writing_online",
// "auth.auth0",
// "auth.authentication",
// "auth.authentication_blogpost",
// "auth.smile-multi-factor-authentication",
// "books.7_languages_in_7_weeks",
// "books.To Read",
// "books.amy_hoy_just_fucking_ship",
// "books.anti_racist",
// "books.books_to_read",
// "books.building_a_blog_with_django",
// "books.building_micro_frontends",
// "books.communicate-with-mastery",
// "books.end_of_jobs",
// "books.erling_kagge_walking",
// "books.how_to_take_smart_notes_one_simple_technique_to_boost_writing_learning_and_thinking_for_students_academics_and_nonfiction_book_writers",
// "books.how_to_win_friends_and_influence_people",
// "books.microcopy",
// "books.obviously_awesome",
// "books.on-the-clock",
// "books.template",
// "books.to-read",
// "books.think_do_say",
// "books.word_hero",
// "books.women_fire_and_dangerous_things",
// "clojure.4clojure",
// "business.metric",
// "clojure.clojure_10_source_code",
// "clojure.clojure",
// "clojure.clojure_12_reading_and_problems",
// "clojure.clojure_11_4clojure",
// "clojure.clojure_14_project_structure",
// "clojure.clojure_13_going_camping_again",
// "clojure.clojure_19_office_hours",
// "clojure.clojure_15_zombies_and_videos",
// "clojure.clojure_20_building_a_web_app",
// "clojure.clojure_21_quick_build_with_api",
// "clojure.clojure_7_emacs_and_elisp",
// "clojure.clojure_8_core_and_problems",
// "clojure.clojure_9_recursion_again",
// "clojure.clojure_for_the_brave_and_true",
// "clojure.clojure_from_the_ground_up_checkpoints",
// "companies.truva",
// "daily.affirmations",
// "ocb.florence_road_live_setup",
// "p.accessibility",
// "p.airtable",
// "p.animation",
// "p.aws",
// "p.blog",
// "p.css",
// "p.dendron",
// "p.dotfiles",
// "p.dropbox",
// "p.education",
// "p.egghead",
// "p.electronics",
// "p.emacs",
// "p.express",
// "p.git_submodules",
// "p.framework_training",
// "p.gradual",
// "p.graphql",
// "p.gridsome",
// "p.hwp",
// "p.javascript",
// "p.icons",
// "p.mastermind",
// "p.karabiner",
// "p.nextjs",
// "p.phpsussex",
// "p.pkm",
// "p.programming",
// "p.pushpad",
// "p.shopify",
// "p.testing",
// "p.wordpress",
// "p.writing",
// "people.MartinEdwards",
// "people.TomCritchlow",
// "people.adam_lee",
// ];

// await Promise.all(
// names.map((name) => {
// FileTestUtils.createFiles(
// "/Users/kevinlin/Dendron/vault/",
// // vault2Path({ wsRoot, vault: vaults[0] }),
// [{ path: `${name}.md`, body: `${name}` }]
// );
// })
// );
// },
// });

// // onWSInit(async () => {
// // FileTestUtils.createFiles()
// // const testFile = path.join(root.name, "vault", "bond2.md");
// // fs.writeFileSync(testFile, "bond", { encoding: "utf8" });
// // await new ReloadIndexCommand().run();
// // await new DoctorCommand().run();
// // // cehck that frontmatter is added
// // const resp = fs.readFileSync(testFile, { encoding: "utf8" });
// // assert.ok(NoteUtilsV2.RE_FM.exec(resp));
// // assert.ok(NoteUtilsV2.RE_FM_UPDATED.exec(resp));
// // assert.ok(NoteUtilsV2.RE_FM_CREATED.exec(resp));
// // done();
// // });
// // setupDendronWorkspace(root.name, ctx, {
// // lsp: true,
// // useCb: async (vaultDir) => {
// // await NodeTestPresetsV2.createOneNoteOneSchemaPreset({ vaultDir });
// // },
// // });
// });

test("missing doc folder", (done) => {
onWSInit(async () => {
const testFile = path.join(root.name, "vault", "bond2.md");
Expand Down

0 comments on commit 862e917

Please sign in to comment.