From 9752de5aaadd59e1cca77453eacd9fa7735aa60e Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 18 Mar 2020 13:51:19 +1000 Subject: [PATCH] finish the remaining searches Searches that require multiple deck or note type lookups won't perform very well at the moment - it either needs caching or to be split up at the DB level. Nothing tested yet. --- rslib/src/lib.rs | 1 + rslib/src/media/check.rs | 4 +- rslib/src/notes.rs | 40 ++---------- rslib/src/notetypes.rs | 39 +++++++++++ rslib/src/search/searcher.rs | 123 +++++++++++++++++++++++++---------- rslib/src/storage/sqlite.rs | 23 ++++++- 6 files changed, 156 insertions(+), 74 deletions(-) create mode 100644 rslib/src/notetypes.rs diff --git a/rslib/src/lib.rs b/rslib/src/lib.rs index 37a8db7f6..da4d91107 100644 --- a/rslib/src/lib.rs +++ b/rslib/src/lib.rs @@ -21,6 +21,7 @@ pub mod latex; pub mod log; pub mod media; pub mod notes; +pub mod notetypes; pub mod sched; pub mod search; pub mod storage; diff --git a/rslib/src/media/check.rs b/rslib/src/media/check.rs index 2273a4f97..87726d458 100644 --- a/rslib/src/media/check.rs +++ b/rslib/src/media/check.rs @@ -10,7 +10,7 @@ use crate::media::database::MediaDatabaseContext; use crate::media::files::{ data_for_file, filename_if_normalized, trash_folder, MEDIA_SYNC_FILESIZE_LIMIT, }; -use crate::notes::{for_every_note, get_note_types, set_note, Note}; +use crate::notes::{for_every_note, set_note, Note}; use crate::text::{normalize_to_nfc, MediaRef}; use crate::{media::MediaManager, text::extract_media_refs}; use coarsetime::Instant; @@ -379,7 +379,7 @@ where renamed: &HashMap, ) -> Result> { let mut referenced_files = HashSet::new(); - let note_types = get_note_types(&self.ctx.storage.db)?; + let note_types = self.ctx.storage.all_note_types()?; let mut collection_modified = false; for_every_note(&self.ctx.storage.db, |note| { diff --git a/rslib/src/notes.rs b/rslib/src/notes.rs index 99efb9410..cad1f614c 100644 --- a/rslib/src/notes.rs +++ b/rslib/src/notes.rs @@ -6,11 +6,11 @@ use crate::err::{AnkiError, DBErrorKind, Result}; use crate::text::strip_html_preserving_image_filenames; use crate::time::i64_unix_secs; -use crate::types::{ObjID, Timestamp, Usn}; +use crate::{ + notetypes::NoteType, + types::{ObjID, Timestamp, Usn}, +}; use rusqlite::{params, Connection, Row, NO_PARAMS}; -use serde_aux::field_attributes::deserialize_number_from_string; -use serde_derive::Deserialize; -use std::collections::HashMap; use std::convert::TryInto; #[derive(Debug)] @@ -47,38 +47,6 @@ pub(crate) fn field_checksum(text: &str) -> u32 { u32::from_be_bytes(digest[..4].try_into().unwrap()) } -#[derive(Deserialize, Debug)] -pub(super) struct NoteType { - #[serde(deserialize_with = "deserialize_number_from_string")] - id: ObjID, - #[serde(rename = "sortf")] - sort_field_idx: u16, - - #[serde(rename = "latexsvg", default)] - latex_svg: bool, -} - -impl NoteType { - pub fn latex_uses_svg(&self) -> bool { - self.latex_svg - } -} - -pub(super) fn get_note_types(db: &Connection) -> Result> { - let mut stmt = db.prepare("select models from col")?; - let note_types = stmt - .query_and_then(NO_PARAMS, |row| -> Result> { - let v: HashMap = serde_json::from_str(row.get_raw(0).as_str()?)?; - Ok(v) - })? - .next() - .ok_or_else(|| AnkiError::DBError { - info: "col table empty".to_string(), - kind: DBErrorKind::MissingEntity, - })??; - Ok(note_types) -} - #[allow(dead_code)] fn get_note(db: &Connection, nid: ObjID) -> Result> { let mut stmt = db.prepare_cached("select id, mid, mod, usn, flds from notes where id=?")?; diff --git a/rslib/src/notetypes.rs b/rslib/src/notetypes.rs new file mode 100644 index 000000000..4b9295939 --- /dev/null +++ b/rslib/src/notetypes.rs @@ -0,0 +1,39 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::types::ObjID; +use serde_aux::field_attributes::deserialize_number_from_string; +use serde_derive::Deserialize; + +#[derive(Deserialize, Debug)] +pub(crate) struct NoteType { + #[serde(deserialize_with = "deserialize_number_from_string")] + pub id: ObjID, + pub name: String, + #[serde(rename = "sortf")] + pub sort_field_idx: u16, + #[serde(rename = "latexsvg", default)] + pub latex_svg: bool, + #[serde(rename = "tmpls")] + pub templates: Vec, + #[serde(rename = "flds")] + pub fields: Vec, +} + +#[derive(Deserialize, Debug)] +pub(crate) struct CardTemplate { + pub name: String, + pub ord: u16, +} + +#[derive(Deserialize, Debug)] +pub(crate) struct NoteField { + pub name: String, + pub ord: u16, +} + +impl NoteType { + pub fn latex_uses_svg(&self) -> bool { + self.latex_svg + } +} diff --git a/rslib/src/search/searcher.rs b/rslib/src/search/searcher.rs index 5f7f419b8..35229a21d 100644 --- a/rslib/src/search/searcher.rs +++ b/rslib/src/search/searcher.rs @@ -57,17 +57,17 @@ fn write_search_node_to_sql(ctx: &mut SearchContext, node: &SearchNode) -> Resul match node { SearchNode::UnqualifiedText(text) => write_unqualified(ctx, text), SearchNode::SingleField { field, text } => { - write_single_field(ctx, field.as_ref(), text.as_ref()) + write_single_field(ctx, field.as_ref(), text.as_ref())? } SearchNode::AddedInDays(days) => { write!(ctx.sql, "c.id > {}", days).unwrap(); } - SearchNode::CardTemplate(template) => write_template(ctx, template), + SearchNode::CardTemplate(template) => write_template(ctx, template)?, SearchNode::Deck(deck) => write_deck(ctx, deck.as_ref())?, SearchNode::NoteTypeID(ntid) => { write!(ctx.sql, "n.mid = {}", ntid).unwrap(); } - SearchNode::NoteType(notetype) => write_note_type(ctx, notetype.as_ref()), + SearchNode::NoteType(notetype) => write_note_type(ctx, notetype.as_ref())?, SearchNode::Rated { days, ease } => write_rated(ctx, *days, *ease)?, SearchNode::Tag(tag) => write_tag(ctx, tag), SearchNode::Duplicates { note_type_id, text } => write_dupes(ctx, *note_type_id, text), @@ -211,55 +211,74 @@ fn write_deck(ctx: &mut SearchContext, deck: &str) -> Result<()> { dids_with_children }; - if dids_with_children.is_empty() { - write!(ctx.sql, "false") - } else { - let did_strings: Vec = - dids_with_children.iter().map(ToString::to_string).collect(); - write!(ctx.sql, "c.did in ({})", did_strings.join(",")) - } - .unwrap(); + ctx.sql.push_str("c.did in "); + ids_to_string(&mut ctx.sql, &dids_with_children); } }; Ok(()) } -// fixme: need note type manager -fn write_template(ctx: &mut SearchContext, template: &TemplateKind) { +fn write_template(ctx: &mut SearchContext, template: &TemplateKind) -> Result<()> { match template { TemplateKind::Ordinal(n) => { write!(ctx.sql, "c.ord = {}", n).unwrap(); } - TemplateKind::Name(_name) => { - // fixme: search through note types loooking for template name + TemplateKind::Name(name) => { + let note_types = ctx.req.storage.all_note_types()?; + let mut id_ords = vec![]; + for nt in note_types.values() { + for tmpl in &nt.templates { + if matches_wildcard(&tmpl.name, name) { + id_ords.push(format!("(n.mid = {} and c.ord = {})", nt.id, tmpl.ord)); + } + } + } + + if id_ords.is_empty() { + ctx.sql.push_str("false"); + } else { + write!(ctx.sql, "({})", id_ords.join(",")).unwrap(); + } + } + }; + Ok(()) +} + +fn write_note_type(ctx: &mut SearchContext, nt_name: &str) -> Result<()> { + let ntids: Vec<_> = ctx + .req + .storage + .all_note_types()? + .values() + .filter(|nt| matches_wildcard(&nt.name, nt_name)) + .map(|nt| nt.id) + .collect(); + ctx.sql.push_str("n.mid in "); + ids_to_string(&mut ctx.sql, &ntids); + Ok(()) +} + +fn write_single_field(ctx: &mut SearchContext, field_name: &str, val: &str) -> Result<()> { + let note_types = ctx.req.storage.all_note_types()?; + + let mut field_map = vec![]; + for nt in note_types.values() { + for field in &nt.fields { + if field.name.eq_ignore_ascii_case(field_name) { + field_map.push((nt.id, field.ord)); + } } } -} -// fixme: need note type manager -fn write_note_type(ctx: &mut SearchContext, _notetype: &str) { - let ntid: Option = None; // fixme: get id via name search - if let Some(ntid) = ntid { - write!(ctx.sql, "n.mid = {}", ntid).unwrap(); - } else { + if field_map.is_empty() { write!(ctx.sql, "false").unwrap(); - } -} - -// fixme: need note type manager -fn write_single_field(ctx: &mut SearchContext, field: &str, val: &str) { - let _ = field; - let fields = vec![(0, 0)]; // fixme: get list of (ntid, ordinal) - - if fields.is_empty() { - write!(ctx.sql, "false").unwrap(); - return; + return Ok(()); } write!(ctx.sql, "(").unwrap(); ctx.args.push(val.to_string().into()); let arg_idx = ctx.args.len(); - for (ntid, ord) in fields { + for (ntid, ord) in field_map { write!( ctx.sql, "(n.mid = {} and field_at_index(n.flds, {}) like ?{})", @@ -268,6 +287,8 @@ fn write_single_field(ctx: &mut SearchContext, field: &str, val: &str) { .unwrap(); } write!(ctx.sql, ")").unwrap(); + + Ok(()) } fn write_dupes(ctx: &mut SearchContext, ntid: ObjID, text: &str) { @@ -282,8 +303,42 @@ fn write_dupes(ctx: &mut SearchContext, ntid: ObjID, text: &str) { ctx.args.push(text.to_string().into()) } +// Write a list of IDs as '(x,y,...)' into the provided string. +fn ids_to_string(buf: &mut String, ids: &[T]) +where + T: std::fmt::Display, +{ + buf.push('('); + if !ids.is_empty() { + for id in ids.iter().skip(1) { + write!(buf, "{},", id).unwrap(); + } + write!(buf, "{}", ids[0]).unwrap(); + } + buf.push(')'); +} + #[cfg(test)] mod test { + use super::ids_to_string; + + #[test] + fn ids_string() { + let mut s = String::new(); + ids_to_string::(&mut s, &[]); + assert_eq!(s, "()"); + s.clear(); + ids_to_string(&mut s, &[7]); + assert_eq!(s, "(7)"); + s.clear(); + ids_to_string(&mut s, &[7, 6]); + assert_eq!(s, "(6,7)"); + s.clear(); + ids_to_string(&mut s, &[7, 6, 5]); + assert_eq!(s, "(6,5,7)"); + s.clear(); + } + // use super::super::parser::parse; // use super::*; diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index bcf702110..a36497fa1 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -8,11 +8,15 @@ use crate::err::{AnkiError, DBErrorKind}; use crate::time::{i64_unix_millis, i64_unix_secs}; use crate::{ decks::Deck, + notetypes::NoteType, sched::cutoff::{sched_timing_today, SchedTimingToday}, - types::Usn, + types::{ObjID, Usn}, }; use rusqlite::{params, Connection, NO_PARAMS}; -use std::path::{Path, PathBuf}; +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; const SCHEMA_MIN_VERSION: u8 = 11; const SCHEMA_MAX_VERSION: u8 = 11; @@ -233,6 +237,21 @@ impl StorageContext<'_> { }) } + pub(crate) fn all_note_types(&self) -> Result> { + let mut stmt = self.db.prepare("select models from col")?; + let note_types = stmt + .query_and_then(NO_PARAMS, |row| -> Result> { + let v: HashMap = serde_json::from_str(row.get_raw(0).as_str()?)?; + Ok(v) + })? + .next() + .ok_or_else(|| AnkiError::DBError { + info: "col table empty".to_string(), + kind: DBErrorKind::MissingEntity, + })??; + Ok(note_types) + } + #[allow(dead_code)] pub(crate) fn timing_today(&mut self) -> Result { if self.timing_today.is_none() {