Skip to content

Commit

Permalink
move find&replace to backend
Browse files Browse the repository at this point in the history
  • Loading branch information
dae committed May 12, 2020
1 parent c51ca66 commit 8b557ec
Show file tree
Hide file tree
Showing 9 changed files with 295 additions and 65 deletions.
20 changes: 20 additions & 0 deletions proto/backend.proto
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,8 @@ message BackendInput {
bool new_deck_legacy = 75;
int64 remove_deck = 76;
Empty deck_tree_legacy = 77;
FieldNamesForNotesIn field_names_for_notes = 78;
FindAndReplaceIn find_and_replace = 79;
}
}

Expand Down Expand Up @@ -159,6 +161,8 @@ message BackendOutput {
bytes new_deck_legacy = 75;
Empty remove_deck = 76;
bytes deck_tree_legacy = 77;
FieldNamesForNotesOut field_names_for_notes = 78;
uint32 find_and_replace = 79;

BackendError error = 2047;
}
Expand Down Expand Up @@ -712,3 +716,19 @@ message AddOrUpdateDeckLegacyIn {
bool preserve_usn_and_mtime = 2;
}

message FieldNamesForNotesIn {
repeated int64 nids = 1;
}

message FieldNamesForNotesOut {
repeated string fields = 1;
}

message FindAndReplaceIn {
repeated int64 nids = 1;
string search = 2;
string replacement = 3;
bool regex = 4;
bool match_case = 5;
string field_name = 6;
}
76 changes: 11 additions & 65 deletions pylib/anki/find.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,10 @@

from __future__ import annotations

import re
from typing import TYPE_CHECKING, Optional, Set

from anki.hooks import *
from anki.utils import ids2str, intTime, joinFields, splitFields, stripHTMLMedia
from anki.utils import ids2str, splitFields, stripHTMLMedia

if TYPE_CHECKING:
from anki.collection import _Collection
Expand Down Expand Up @@ -38,56 +37,16 @@ def findReplace(
field: Optional[str] = None,
fold: bool = True,
) -> int:
"Find and replace fields in a note."
mmap: Dict[str, Any] = {}
if field:
for m in col.models.all():
for f in m["flds"]:
if f["name"].lower() == field.lower():
mmap[str(m["id"])] = f["ord"]
if not mmap:
return 0
# find and gather replacements
if not regex:
src = re.escape(src)
dst = dst.replace("\\", "\\\\")
if fold:
src = "(?i)" + src
compiled_re = re.compile(src)

def repl(s: str):
return compiled_re.sub(dst, s)

d = []
snids = ids2str(nids)
nids = []
for nid, mid, flds in col.db.execute(
"select id, mid, flds from notes where id in " + snids
):
origFlds = flds
# does it match?
sflds = splitFields(flds)
if field:
try:
ord = mmap[str(mid)]
sflds[ord] = repl(sflds[ord])
except KeyError:
# note doesn't have that field
continue
else:
for c in range(len(sflds)):
sflds[c] = repl(sflds[c])
flds = joinFields(sflds)
if flds != origFlds:
nids.append(nid)
d.append((flds, intTime(), col.usn(), nid))
if not d:
return 0
# replace
col.db.executemany("update notes set flds=?,mod=?,usn=? where id=?", d)
col.updateFieldCache(nids)
col.genCards(nids)
return len(d)
"Find and replace fields in a note. Returns changed note count."
return col.backend.find_and_replace(nids, src, dst, regex, fold, field)


def fieldNamesForNotes(col: _Collection, nids: List[int]) -> List[str]:
return list(col.backend.field_names_for_note_ids(nids))


# Find duplicates
##########################################################################


def fieldNames(col, downcase=True) -> List:
Expand All @@ -100,19 +59,6 @@ def fieldNames(col, downcase=True) -> List:
return list(fields)


def fieldNamesForNotes(col, nids) -> List:
fields: Set[str] = set()
mids = col.db.list("select distinct mid from notes where id in %s" % ids2str(nids))
for mid in mids:
model = col.models.get(mid)
for name in col.models.fieldNames(model):
if name not in fields: # slower w/o
fields.add(name)
return sorted(fields, key=lambda x: x.lower())


# Find duplicates
##########################################################################
# returns array of ("dupestr", [nids])
def findDupes(
col: _Collection, fieldName: str, search: str = ""
Expand Down
27 changes: 27 additions & 0 deletions pylib/anki/rsbackend.py
Original file line number Diff line number Diff line change
Expand Up @@ -745,6 +745,33 @@ def legacy_deck_tree(self) -> Sequence:
).deck_tree_legacy
return orjson.loads(bytes)[5]

def field_names_for_note_ids(self, nids: List[int]) -> Sequence[str]:
return self._run_command(
pb.BackendInput(field_names_for_notes=pb.FieldNamesForNotesIn(nids=nids))
).field_names_for_notes.fields

def find_and_replace(
self,
nids: List[int],
search: str,
repl: str,
re: bool,
nocase: bool,
field_name: Optional[str],
) -> int:
return self._run_command(
pb.BackendInput(
find_and_replace=pb.FindAndReplaceIn(
nids=nids,
search=search,
replacement=repl,
regex=re,
match_case=not nocase,
field_name=field_name,
)
)
).find_and_replace


def translate_string_in(
key: TR, **kwargs: Union[str, int, float]
Expand Down
38 changes: 38 additions & 0 deletions rslib/src/backend/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use crate::{
deckconf::{DeckConf, DeckConfID},
decks::{Deck, DeckID, DeckSchema11},
err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind},
findreplace::FindReplaceContext,
i18n::{tr_args, I18n, TR},
latex::{extract_latex, extract_latex_expanding_clozes, ExtractedLatex},
log,
Expand Down Expand Up @@ -361,6 +362,10 @@ impl Backend {
OValue::CheckDatabase(pb::Empty {})
}
Value::DeckTreeLegacy(_) => OValue::DeckTreeLegacy(self.deck_tree_legacy()?),
Value::FieldNamesForNotes(input) => {
OValue::FieldNamesForNotes(self.field_names_for_notes(input)?)
}
Value::FindAndReplace(input) => OValue::FindAndReplace(self.find_and_replace(input)?),
})
}

Expand Down Expand Up @@ -1056,6 +1061,39 @@ impl Backend {
serde_json::to_vec(&tree).map_err(Into::into)
})
}

fn field_names_for_notes(
&self,
input: pb::FieldNamesForNotesIn,
) -> Result<pb::FieldNamesForNotesOut> {
self.with_col(|col| {
let nids: Vec<_> = input.nids.into_iter().map(NoteID).collect();
col.storage
.field_names_for_notes(&nids)
.map(|fields| pb::FieldNamesForNotesOut { fields })
})
}

fn find_and_replace(&self, input: pb::FindAndReplaceIn) -> Result<u32> {
let mut search = if input.regex {
input.search
} else {
regex::escape(&input.search)
};
if !input.match_case {
search = format!("(?i){}", search);
}
let nids = input.nids.into_iter().map(NoteID).collect();
let field_name = if input.field_name.is_empty() {
None
} else {
Some(input.field_name)
};
let repl = input.replacement;
self.with_col(|col| {
col.find_and_replace(FindReplaceContext::new(nids, &search, &repl, field_name)?)
})
}
}

fn translate_arg_to_fluent_val(arg: &pb::TranslateArgValue) -> FluentValue {
Expand Down
146 changes: 146 additions & 0 deletions rslib/src/findreplace.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
// Copyright: Ankitects Pty Ltd and contributors
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html

use crate::{
collection::Collection,
err::{AnkiError, Result},
notes::NoteID,
notetype::CardGenContext,
types::Usn,
};
use itertools::Itertools;
use regex::Regex;
use std::borrow::Cow;

pub struct FindReplaceContext {
nids: Vec<NoteID>,
search: Regex,
replacement: String,
field_name: Option<String>,
}

impl FindReplaceContext {
pub fn new(
nids: Vec<NoteID>,
search_re: &str,
repl: impl Into<String>,
field_name: Option<String>,
) -> Result<Self> {
Ok(FindReplaceContext {
nids,
search: Regex::new(search_re).map_err(|_| AnkiError::invalid_input("invalid regex"))?,
replacement: repl.into(),
field_name,
})
}

fn replace_text<'a>(&self, text: &'a str) -> Cow<'a, str> {
self.search.replace_all(text, self.replacement.as_str())
}
}

impl Collection {
pub fn find_and_replace(&mut self, ctx: FindReplaceContext) -> Result<u32> {
self.transact(None, |col| col.find_and_replace_inner(ctx, col.usn()?))
}

fn find_and_replace_inner(&mut self, ctx: FindReplaceContext, usn: Usn) -> Result<u32> {
let mut total_changed = 0;
let nids_by_notetype = self.storage.note_ids_by_notetype(&ctx.nids)?;
for (ntid, group) in &nids_by_notetype.into_iter().group_by(|tup| tup.0) {
let nt = self
.get_notetype(ntid)?
.ok_or_else(|| AnkiError::invalid_input("missing note type"))?;
let genctx = CardGenContext::new(&nt, usn);
let field_ord = ctx.field_name.as_ref().and_then(|n| nt.get_field_ord(n));
for (_, nid) in group {
let mut note = self.storage.get_note(nid)?.unwrap();
let mut changed = false;
match field_ord {
None => {
// all fields
for txt in &mut note.fields {
if let Cow::Owned(otxt) = ctx.replace_text(txt) {
changed = true;
*txt = otxt;
}
}
}
Some(ord) => {
// single field
if let Some(txt) = note.fields.get_mut(ord) {
if let Cow::Owned(otxt) = ctx.replace_text(txt) {
changed = true;
*txt = otxt;
}
}
}
}
if changed {
self.update_note_inner(&genctx, &mut note)?;
total_changed += 1;
}
}
}

Ok(total_changed)
}
}

#[cfg(test)]
mod test {
use super::*;
use crate::{collection::open_test_collection, decks::DeckID};

#[test]
fn findreplace() -> Result<()> {
let mut col = open_test_collection();

let nt = col.get_notetype_by_name("Basic")?.unwrap();
let mut note = nt.new_note();
note.fields[0] = "one aaa".into();
note.fields[1] = "two aaa".into();
col.add_note(&mut note, DeckID(1))?;

let nt = col.get_notetype_by_name("Cloze")?.unwrap();
let mut note2 = nt.new_note();
note2.fields[0] = "three aaa".into();
col.add_note(&mut note2, DeckID(1))?;

let nids = col.search_notes_only("")?;
let cnt = col.find_and_replace(FindReplaceContext::new(
nids.clone(),
"(?i)AAA",
"BBB",
None,
)?)?;
assert_eq!(cnt, 2);

let note = col.storage.get_note(note.id)?.unwrap();
// but the update should be limited to the specified field when it was available
assert_eq!(&note.fields, &["one BBB", "two BBB"]);

let note2 = col.storage.get_note(note2.id)?.unwrap();
assert_eq!(&note2.fields, &["three BBB"]);

assert_eq!(
col.storage.field_names_for_notes(&nids)?,
vec!["Back".to_string(), "Front".into(), "Text".into()]
);
let cnt = col.find_and_replace(FindReplaceContext::new(
nids.clone(),
"BBB",
"ccc",
Some("Front".into()),
)?)?;
// still 2, as the caller is expected to provide only note ids that have
// that field, and if we can't find the field we fall back on all fields
assert_eq!(cnt, 2);

let note = col.storage.get_note(note.id)?.unwrap();
// but the update should be limited to the specified field when it was available
assert_eq!(&note.fields, &["one ccc", "two BBB"]);

Ok(())
}
}
1 change: 1 addition & 0 deletions rslib/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ pub mod dbcheck;
pub mod deckconf;
pub mod decks;
pub mod err;
pub mod findreplace;
pub mod i18n;
pub mod latex;
pub mod log;
Expand Down
Loading

0 comments on commit 8b557ec

Please sign in to comment.