From 97c9dd4c18970dc8365ba9315bac10739b780b3e Mon Sep 17 00:00:00 2001 From: RumovZ Date: Thu, 31 Mar 2022 10:15:25 +0200 Subject: [PATCH] Use temporary tables for gathering export_apkg() now accepts a search instead of a deck id. Decks are gathered according to the matched notes' cards. --- rslib/src/decks/name.rs | 4 + rslib/src/import_export/gather.rs | 143 ++++++------------ .../src/import_export/package/apkg/export.rs | 8 +- rslib/src/stats/graphs.rs | 2 +- rslib/src/storage/card/mod.rs | 14 ++ rslib/src/storage/deck/mod.rs | 20 +++ rslib/src/storage/note/mod.rs | 10 ++ rslib/src/storage/notetype/mod.rs | 17 +++ rslib/src/storage/revlog/mod.rs | 9 +- 9 files changed, 123 insertions(+), 104 deletions(-) diff --git a/rslib/src/decks/name.rs b/rslib/src/decks/name.rs index 2f8a30086..fa22f6a50 100644 --- a/rslib/src/decks/name.rs +++ b/rslib/src/decks/name.rs @@ -82,6 +82,10 @@ impl NativeDeckName { .chain(self.components().skip(old_parent.components().count())) .join("\x1f") } + + pub(crate) fn immediate_parent_name(&self) -> Option<&str> { + immediate_parent_name(self.as_native_str()) + } } impl std::fmt::Display for NativeDeckName { diff --git a/rslib/src/import_export/gather.rs b/rslib/src/import_export/gather.rs index 1bc4dae99..c298fcb68 100644 --- a/rslib/src/import_export/gather.rs +++ b/rslib/src/import_export/gather.rs @@ -15,8 +15,6 @@ use crate::{ latex::extract_latex, prelude::*, revlog::RevlogEntry, - search::{Negated, SearchNode, SortMode}, - storage::ids_to_string, text::{extract_media_refs, extract_underscored_css_imports, extract_underscored_references}, }; @@ -31,43 +29,27 @@ pub(super) struct ExportData { pub(super) media_paths: HashSet, } -fn sibling_search(notes: &[Note], cards: &[Card]) -> SearchBuilder { - let mut nids = String::new(); - ids_to_string(&mut nids, notes.iter().map(|note| note.id)); - let mut cids = String::new(); - ids_to_string(&mut cids, cards.iter().map(|card| card.id)); - SearchBuilder::from(SearchNode::NoteIds(nids)).and(SearchNode::CardIds(cids).negated()) -} - -fn optional_deck_search(deck_id: Option) -> SearchNode { - if let Some(did) = deck_id { - SearchNode::from_deck_id(did, true) - } else { - SearchNode::WholeCollection - } -} - impl ExportData { pub(super) fn gather_data( &mut self, col: &mut Collection, - deck_id: Option, + search: impl TryIntoSearch, with_scheduling: bool, ) -> Result<()> { - self.decks = col.gather_decks(deck_id)?; - let search = optional_deck_search(deck_id); - self.notes = col.gather_notes(search.clone())?; - self.cards = col.gather_cards(search, &self.notes, deck_id)?; - self.notetypes = col.gather_notetypes(&self.notes)?; + self.notes = col.gather_notes(search)?; + self.cards = col.gather_cards()?; + self.decks = col.gather_decks()?; + self.notetypes = col.gather_notetypes()?; if with_scheduling { - self.revlog = col.gather_revlog(&self.cards)?; + self.revlog = col.gather_revlog()?; self.decks_configs = col.gather_deck_configs(&self.decks)?; } else { self.remove_scheduling_information(col); }; - Ok(()) + col.storage.clear_searched_notes_table()?; + col.storage.clear_searched_cards_table() } pub(super) fn gather_media_paths(&mut self) { @@ -176,81 +158,50 @@ fn svg_getter(notetypes: &[Notetype]) -> impl Fn(NotetypeId) -> bool { } impl Collection { - fn gather_decks(&mut self, deck_id: Option) -> Result> { - if let Some(did) = deck_id { - let deck = self.get_deck(did)?.ok_or(AnkiError::NotFound)?; - self.storage - .deck_id_with_children(&deck)? - .iter() - .filter(|did| **did != DeckId(1)) - .map(|did| self.storage.get_deck(*did)?.ok_or(AnkiError::NotFound)) - .collect() - } else { - Ok(self - .storage - .get_all_decks()? - .into_iter() - .filter(|deck| deck.id != DeckId(1)) - .collect()) + fn gather_notes(&mut self, search: impl TryIntoSearch) -> Result> { + self.search_notes_into_table(search)?; + self.storage.all_searched_notes() + } + + fn gather_cards(&mut self) -> Result> { + self.storage.search_notes_cards_into_table()?; + self.storage.all_searched_cards() + } + + fn gather_decks(&mut self) -> Result> { + let decks = self.storage.get_decks_for_search_cards()?; + let parents = self.get_parent_decks(&decks)?; + Ok(decks + .into_iter() + .filter(|deck| deck.id != DeckId(1)) + .chain(parents) + .collect()) + } + + fn get_parent_decks(&mut self, decks: &[Deck]) -> Result> { + let mut parent_names: HashSet<&str> = + decks.iter().map(|deck| deck.name.as_native_str()).collect(); + let mut parents = Vec::new(); + for deck in decks { + while let Some(parent_name) = deck.name.immediate_parent_name() { + if parent_names.insert(parent_name) { + parents.push( + self.storage + .get_deck_by_name(parent_name)? + .ok_or(AnkiError::DatabaseCheckRequired)?, + ) + } + } } + Ok(parents) } - fn gather_notes(&mut self, search: SearchNode) -> Result> { - self.search_notes(search, SortMode::NoOrder)? - .iter() - .map(|nid| self.storage.get_note(*nid)?.ok_or(AnkiError::NotFound)) - .collect() + fn gather_notetypes(&mut self) -> Result> { + self.storage.get_notetypes_for_search_notes() } - fn gather_cards( - &mut self, - search: SearchNode, - notes: &[Note], - deck_id: Option, - ) -> Result> { - let mut cards: Vec<_> = self - .search_cards(search, SortMode::NoOrder)? - .iter() - .map(|cid| self.storage.get_card(*cid)?.ok_or(AnkiError::NotFound)) - .collect::>()?; - - if let Some(did) = deck_id { - let mut siblings = self.gather_siblings(notes, &cards, did)?; - cards.append(&mut siblings); - } - - Ok(cards) - } - - fn gather_siblings( - &mut self, - notes: &[Note], - cards: &[Card], - deck_id: DeckId, - ) -> Result> { - self.search_cards(sibling_search(notes, cards), SortMode::NoOrder)? - .iter() - .map(|cid| { - let mut card = self.storage.get_card(*cid)?.ok_or(AnkiError::NotFound)?; - card.deck_id = deck_id; - Ok(card) - }) - .collect() - } - - fn gather_notetypes(&mut self, notes: &[Note]) -> Result> { - notes - .iter() - .map(|note| note.notetype_id) - .unique() - .map(|ntid| self.storage.get_notetype(ntid)?.ok_or(AnkiError::NotFound)) - .collect() - } - - fn gather_revlog(&mut self, cards: &[Card]) -> Result> { - let mut cids = String::new(); - ids_to_string(&mut cids, cards.iter().map(|card| card.id)); - self.storage.get_revlog_entries_for_card_ids(cids) + fn gather_revlog(&mut self) -> Result> { + self.storage.get_revlog_entries_for_searched_cards() } fn gather_deck_configs(&mut self, decks: &[Deck]) -> Result> { diff --git a/rslib/src/import_export/package/apkg/export.rs b/rslib/src/import_export/package/apkg/export.rs index e3a8be855..918cdbea7 100644 --- a/rslib/src/import_export/package/apkg/export.rs +++ b/rslib/src/import_export/package/apkg/export.rs @@ -26,7 +26,7 @@ impl Collection { pub fn export_apkg( &mut self, out_path: impl AsRef, - deck_id: Option, + search: impl TryIntoSearch, with_scheduling: bool, with_media: bool, media_fn: Option) -> MediaIter>, @@ -40,7 +40,7 @@ impl Collection { .ok_or_else(|| AnkiError::IoError("tempfile with non-unicode name".into()))?; let media = self.export_collection_extracting_media( temp_col_path, - deck_id, + search, with_scheduling, with_media, )?; @@ -66,12 +66,12 @@ impl Collection { fn export_collection_extracting_media( &mut self, path: &str, - deck_id: Option, + search: impl TryIntoSearch, with_scheduling: bool, with_media: bool, ) -> Result> { let mut data = ExportData::default(); - data.gather_data(self, deck_id, with_scheduling)?; + data.gather_data(self, search, with_scheduling)?; if with_media { data.gather_media_paths(); } diff --git a/rslib/src/stats/graphs.rs b/rslib/src/stats/graphs.rs index 0a158eb66..eb5236e66 100644 --- a/rslib/src/stats/graphs.rs +++ b/rslib/src/stats/graphs.rs @@ -38,7 +38,7 @@ impl Collection { self.storage.get_all_revlog_entries(revlog_start)? } else { self.storage - .get_revlog_entries_for_searched_cards(revlog_start)? + .get_pb_revlog_entries_for_searched_cards(revlog_start)? }; self.storage.clear_searched_cards_table()?; diff --git a/rslib/src/storage/card/mod.rs b/rslib/src/storage/card/mod.rs index 8d07d61e6..f0eb7e282 100644 --- a/rslib/src/storage/card/mod.rs +++ b/rslib/src/storage/card/mod.rs @@ -455,6 +455,20 @@ impl super::SqliteStorage { Ok(nids) } + /// Place the ids of cards with notes in 'search_nids' into 'search_cids'. + /// Returns number of added cards. + pub(crate) fn search_notes_cards_into_table(&self) -> Result { + self.setup_searched_cards_table()?; + self.db + .prepare(concat!( + "INSERT INTO search_cids", + " (SELECT id FROM cards WHERE nid IN", + " (SELECT nid FROM search_nids))", + ))? + .execute([]) + .map_err(Into::into) + } + pub(crate) fn all_searched_cards(&self) -> Result> { self.db .prepare_cached(concat!( diff --git a/rslib/src/storage/deck/mod.rs b/rslib/src/storage/deck/mod.rs index da1c5d0f7..f99467143 100644 --- a/rslib/src/storage/deck/mod.rs +++ b/rslib/src/storage/deck/mod.rs @@ -74,6 +74,15 @@ impl SqliteStorage { .transpose() } + pub(crate) fn get_deck_by_name(&self, machine_name: &str) -> Result> { + self.db + .prepare_cached(concat!(include_str!("get_deck.sql"), " WHERE name = ?"))? + .query_and_then([machine_name], row_to_deck)? + .next() + .transpose() + .map_err(Into::into) + } + pub(crate) fn get_all_decks(&self) -> Result> { self.db .prepare(include_str!("get_deck.sql"))? @@ -111,6 +120,17 @@ impl SqliteStorage { .map_err(Into::into) } + pub(crate) fn get_decks_for_search_cards(&self) -> Result> { + self.db + .prepare_cached(concat!( + include_str!("get_deck.sql"), + " WHERE id IN (SELECT DISTINCT did FROM cards WHERE id IN", + " (SELECT cid FROM search_cids))", + ))? + .query_and_then([], row_to_deck)? + .collect() + } + // caller should ensure name unique pub(crate) fn add_deck(&self, deck: &mut Deck) -> Result<()> { assert!(deck.id.0 == 0); diff --git a/rslib/src/storage/note/mod.rs b/rslib/src/storage/note/mod.rs index 4c7c20bc8..1322eec71 100644 --- a/rslib/src/storage/note/mod.rs +++ b/rslib/src/storage/note/mod.rs @@ -210,6 +210,16 @@ impl super::SqliteStorage { Ok(()) } + pub(crate) fn all_searched_notes(&self) -> Result> { + self.db + .prepare_cached(concat!( + include_str!("get.sql"), + " WHERE id IN (SELECT nid FROM search_nids)" + ))? + .query_and_then([], |r| row_to_note(r).map_err(Into::into))? + .collect() + } + pub(crate) fn get_note_tags_by_predicate(&mut self, want: F) -> Result> where F: Fn(&str) -> bool, diff --git a/rslib/src/storage/notetype/mod.rs b/rslib/src/storage/notetype/mod.rs index 94b551a0a..3ada340a8 100644 --- a/rslib/src/storage/notetype/mod.rs +++ b/rslib/src/storage/notetype/mod.rs @@ -99,6 +99,23 @@ impl SqliteStorage { .map_err(Into::into) } + pub(crate) fn get_notetypes_for_search_notes(&self) -> Result> { + self.db + .prepare_cached(concat!( + include_str!("get_notetype.sql"), + " WHERE id IN (SELECT DISTINCT mid FROM notes WHERE id IN", + " (SELECT nid FROM search_nids))", + ))? + .query_and_then([], |r| { + row_to_notetype_core(r).and_then(|mut nt| { + nt.fields = self.get_notetype_fields(nt.id)?; + nt.templates = self.get_notetype_templates(nt.id)?; + Ok(nt) + }) + })? + .collect() + } + pub fn get_all_notetype_names(&self) -> Result> { self.db .prepare_cached(include_str!("get_notetype_names.sql"))? diff --git a/rslib/src/storage/revlog/mod.rs b/rslib/src/storage/revlog/mod.rs index c70b56cda..27ee78ed8 100644 --- a/rslib/src/storage/revlog/mod.rs +++ b/rslib/src/storage/revlog/mod.rs @@ -107,7 +107,7 @@ impl SqliteStorage { .collect() } - pub(crate) fn get_revlog_entries_for_searched_cards( + pub(crate) fn get_pb_revlog_entries_for_searched_cards( &self, after: TimestampSecs, ) -> Result> { @@ -120,9 +120,12 @@ impl SqliteStorage { .collect() } - pub(crate) fn get_revlog_entries_for_card_ids(&self, cids: String) -> Result> { + pub(crate) fn get_revlog_entries_for_searched_cards(&self) -> Result> { self.db - .prepare_cached(&format!("{} where cid in {cids}", include_str!("get.sql"),))? + .prepare_cached(concat!( + include_str!("get.sql"), + " where cid in (select cid from search_cids)" + ))? .query_and_then([], row_to_revlog_entry)? .collect() }