Use temporary tables for gathering

export_apkg() now accepts a search instead of a deck id. Decks are
gathered according to the matched notes' cards.

rslib/src/decks/name.rs

@@ -82,6 +82,10 @@ impl NativeDeckName {
             .chain(self.components().skip(old_parent.components().count()))
             .join("\x1f")
     }
+
+    pub(crate) fn immediate_parent_name(&self) -> Option<&str> {
+        immediate_parent_name(self.as_native_str())
+    }
 }
 
 impl std::fmt::Display for NativeDeckName {

rslib/src/import_export/gather.rs

@@ -15,8 +15,6 @@ use crate::{
     latex::extract_latex,
     prelude::*,
     revlog::RevlogEntry,
-    search::{Negated, SearchNode, SortMode},
-    storage::ids_to_string,
     text::{extract_media_refs, extract_underscored_css_imports, extract_underscored_references},
 };
 
@@ -31,43 +29,27 @@ pub(super) struct ExportData {
     pub(super) media_paths: HashSet<PathBuf>,
 }
 
-fn sibling_search(notes: &[Note], cards: &[Card]) -> SearchBuilder {
-    let mut nids = String::new();
-    ids_to_string(&mut nids, notes.iter().map(|note| note.id));
-    let mut cids = String::new();
-    ids_to_string(&mut cids, cards.iter().map(|card| card.id));
-    SearchBuilder::from(SearchNode::NoteIds(nids)).and(SearchNode::CardIds(cids).negated())
-}
-
-fn optional_deck_search(deck_id: Option<DeckId>) -> SearchNode {
-    if let Some(did) = deck_id {
-        SearchNode::from_deck_id(did, true)
-    } else {
-        SearchNode::WholeCollection
-    }
-}
-
 impl ExportData {
     pub(super) fn gather_data(
         &mut self,
         col: &mut Collection,
-        deck_id: Option<DeckId>,
+        search: impl TryIntoSearch,
         with_scheduling: bool,
     ) -> Result<()> {
-        self.decks = col.gather_decks(deck_id)?;
+        self.notes = col.gather_notes(search)?;
-        let search = optional_deck_search(deck_id);
+        self.cards = col.gather_cards()?;
-        self.notes = col.gather_notes(search.clone())?;
+        self.decks = col.gather_decks()?;
-        self.cards = col.gather_cards(search, &self.notes, deck_id)?;
+        self.notetypes = col.gather_notetypes()?;
-        self.notetypes = col.gather_notetypes(&self.notes)?;
 
         if with_scheduling {
-            self.revlog = col.gather_revlog(&self.cards)?;
+            self.revlog = col.gather_revlog()?;
             self.decks_configs = col.gather_deck_configs(&self.decks)?;
         } else {
             self.remove_scheduling_information(col);
         };
 
-        Ok(())
+        col.storage.clear_searched_notes_table()?;
+        col.storage.clear_searched_cards_table()
     }
 
     pub(super) fn gather_media_paths(&mut self) {
@@ -176,81 +158,50 @@ fn svg_getter(notetypes: &[Notetype]) -> impl Fn(NotetypeId) -> bool {
 }
 
 impl Collection {
-    fn gather_decks(&mut self, deck_id: Option<DeckId>) -> Result<Vec<Deck>> {
+    fn gather_notes(&mut self, search: impl TryIntoSearch) -> Result<Vec<Note>> {
-        if let Some(did) = deck_id {
+        self.search_notes_into_table(search)?;
-            let deck = self.get_deck(did)?.ok_or(AnkiError::NotFound)?;
+        self.storage.all_searched_notes()
-            self.storage
+    }
-                .deck_id_with_children(&deck)?
+
-                .iter()
+    fn gather_cards(&mut self) -> Result<Vec<Card>> {
-                .filter(|did| **did != DeckId(1))
+        self.storage.search_notes_cards_into_table()?;
-                .map(|did| self.storage.get_deck(*did)?.ok_or(AnkiError::NotFound))
+        self.storage.all_searched_cards()
-                .collect()
+    }
-        } else {
+
-            Ok(self
+    fn gather_decks(&mut self) -> Result<Vec<Deck>> {
-                .storage
+        let decks = self.storage.get_decks_for_search_cards()?;
-                .get_all_decks()?
+        let parents = self.get_parent_decks(&decks)?;
-                .into_iter()
+        Ok(decks
-                .filter(|deck| deck.id != DeckId(1))
+            .into_iter()
-                .collect())
+            .filter(|deck| deck.id != DeckId(1))
+            .chain(parents)
+            .collect())
+    }
+
+    fn get_parent_decks(&mut self, decks: &[Deck]) -> Result<Vec<Deck>> {
+        let mut parent_names: HashSet<&str> =
+            decks.iter().map(|deck| deck.name.as_native_str()).collect();
+        let mut parents = Vec::new();
+        for deck in decks {
+            while let Some(parent_name) = deck.name.immediate_parent_name() {
+                if parent_names.insert(parent_name) {
+                    parents.push(
+                        self.storage
+                            .get_deck_by_name(parent_name)?
+                            .ok_or(AnkiError::DatabaseCheckRequired)?,
+                    )
+                }
+            }
         }
+        Ok(parents)
     }
 
-    fn gather_notes(&mut self, search: SearchNode) -> Result<Vec<Note>> {
+    fn gather_notetypes(&mut self) -> Result<Vec<Notetype>> {
-        self.search_notes(search, SortMode::NoOrder)?
+        self.storage.get_notetypes_for_search_notes()
-            .iter()
-            .map(|nid| self.storage.get_note(*nid)?.ok_or(AnkiError::NotFound))
-            .collect()
     }
 
-    fn gather_cards(
+    fn gather_revlog(&mut self) -> Result<Vec<RevlogEntry>> {
-        &mut self,
+        self.storage.get_revlog_entries_for_searched_cards()
-        search: SearchNode,
-        notes: &[Note],
-        deck_id: Option<DeckId>,
-    ) -> Result<Vec<Card>> {
-        let mut cards: Vec<_> = self
-            .search_cards(search, SortMode::NoOrder)?
-            .iter()
-            .map(|cid| self.storage.get_card(*cid)?.ok_or(AnkiError::NotFound))
-            .collect::<Result<_>>()?;
-
-        if let Some(did) = deck_id {
-            let mut siblings = self.gather_siblings(notes, &cards, did)?;
-            cards.append(&mut siblings);
-        }
-
-        Ok(cards)
-    }
-
-    fn gather_siblings(
-        &mut self,
-        notes: &[Note],
-        cards: &[Card],
-        deck_id: DeckId,
-    ) -> Result<Vec<Card>> {
-        self.search_cards(sibling_search(notes, cards), SortMode::NoOrder)?
-            .iter()
-            .map(|cid| {
-                let mut card = self.storage.get_card(*cid)?.ok_or(AnkiError::NotFound)?;
-                card.deck_id = deck_id;
-                Ok(card)
-            })
-            .collect()
-    }
-
-    fn gather_notetypes(&mut self, notes: &[Note]) -> Result<Vec<Notetype>> {
-        notes
-            .iter()
-            .map(|note| note.notetype_id)
-            .unique()
-            .map(|ntid| self.storage.get_notetype(ntid)?.ok_or(AnkiError::NotFound))
-            .collect()
-    }
-
-    fn gather_revlog(&mut self, cards: &[Card]) -> Result<Vec<RevlogEntry>> {
-        let mut cids = String::new();
-        ids_to_string(&mut cids, cards.iter().map(|card| card.id));
-        self.storage.get_revlog_entries_for_card_ids(cids)
     }
 
     fn gather_deck_configs(&mut self, decks: &[Deck]) -> Result<Vec<DeckConfig>> {

rslib/src/import_export/package/apkg/export.rs

@@ -26,7 +26,7 @@ impl Collection {
     pub fn export_apkg(
         &mut self,
         out_path: impl AsRef<Path>,
-        deck_id: Option<DeckId>,
+        search: impl TryIntoSearch,
         with_scheduling: bool,
         with_media: bool,
         media_fn: Option<impl FnOnce(HashSet<PathBuf>) -> MediaIter>,
@@ -40,7 +40,7 @@ impl Collection {
             .ok_or_else(|| AnkiError::IoError("tempfile with non-unicode name".into()))?;
         let media = self.export_collection_extracting_media(
             temp_col_path,
-            deck_id,
+            search,
             with_scheduling,
             with_media,
         )?;
@@ -66,12 +66,12 @@ impl Collection {
     fn export_collection_extracting_media(
         &mut self,
         path: &str,
-        deck_id: Option<DeckId>,
+        search: impl TryIntoSearch,
         with_scheduling: bool,
         with_media: bool,
     ) -> Result<HashSet<PathBuf>> {
         let mut data = ExportData::default();
-        data.gather_data(self, deck_id, with_scheduling)?;
+        data.gather_data(self, search, with_scheduling)?;
         if with_media {
             data.gather_media_paths();
         }

rslib/src/stats/graphs.rs

@@ -38,7 +38,7 @@ impl Collection {
             self.storage.get_all_revlog_entries(revlog_start)?
         } else {
             self.storage
-                .get_revlog_entries_for_searched_cards(revlog_start)?
+                .get_pb_revlog_entries_for_searched_cards(revlog_start)?
         };
 
         self.storage.clear_searched_cards_table()?;

rslib/src/storage/card/mod.rs

@@ -455,6 +455,20 @@ impl super::SqliteStorage {
         Ok(nids)
     }
 
+    /// Place the ids of cards with notes in 'search_nids' into 'search_cids'.
+    /// Returns number of added cards.
+    pub(crate) fn search_notes_cards_into_table(&self) -> Result<usize> {
+        self.setup_searched_cards_table()?;
+        self.db
+            .prepare(concat!(
+                "INSERT INTO search_cids",
+                " (SELECT id FROM cards WHERE nid IN",
+                " (SELECT nid FROM search_nids))",
+            ))?
+            .execute([])
+            .map_err(Into::into)
+    }
+
     pub(crate) fn all_searched_cards(&self) -> Result<Vec<Card>> {
         self.db
             .prepare_cached(concat!(

rslib/src/storage/deck/mod.rs

@@ -74,6 +74,15 @@ impl SqliteStorage {
             .transpose()
     }
 
+    pub(crate) fn get_deck_by_name(&self, machine_name: &str) -> Result<Option<Deck>> {
+        self.db
+            .prepare_cached(concat!(include_str!("get_deck.sql"), " WHERE name = ?"))?
+            .query_and_then([machine_name], row_to_deck)?
+            .next()
+            .transpose()
+            .map_err(Into::into)
+    }
+
     pub(crate) fn get_all_decks(&self) -> Result<Vec<Deck>> {
         self.db
             .prepare(include_str!("get_deck.sql"))?
@@ -111,6 +120,17 @@ impl SqliteStorage {
             .map_err(Into::into)
     }
 
+    pub(crate) fn get_decks_for_search_cards(&self) -> Result<Vec<Deck>> {
+        self.db
+            .prepare_cached(concat!(
+                include_str!("get_deck.sql"),
+                " WHERE id IN (SELECT DISTINCT did FROM cards WHERE id IN",
+                " (SELECT cid FROM search_cids))",
+            ))?
+            .query_and_then([], row_to_deck)?
+            .collect()
+    }
+
     // caller should ensure name unique
     pub(crate) fn add_deck(&self, deck: &mut Deck) -> Result<()> {
         assert!(deck.id.0 == 0);

rslib/src/storage/note/mod.rs

@@ -210,6 +210,16 @@ impl super::SqliteStorage {
         Ok(())
     }
 
+    pub(crate) fn all_searched_notes(&self) -> Result<Vec<Note>> {
+        self.db
+            .prepare_cached(concat!(
+                include_str!("get.sql"),
+                " WHERE id IN (SELECT nid FROM search_nids)"
+            ))?
+            .query_and_then([], |r| row_to_note(r).map_err(Into::into))?
+            .collect()
+    }
+
     pub(crate) fn get_note_tags_by_predicate<F>(&mut self, want: F) -> Result<Vec<NoteTags>>
     where
         F: Fn(&str) -> bool,

rslib/src/storage/notetype/mod.rs

@@ -99,6 +99,23 @@ impl SqliteStorage {
             .map_err(Into::into)
     }
 
+    pub(crate) fn get_notetypes_for_search_notes(&self) -> Result<Vec<Notetype>> {
+        self.db
+            .prepare_cached(concat!(
+                include_str!("get_notetype.sql"),
+                " WHERE id IN (SELECT DISTINCT mid FROM notes WHERE id IN",
+                " (SELECT nid FROM search_nids))",
+            ))?
+            .query_and_then([], |r| {
+                row_to_notetype_core(r).and_then(|mut nt| {
+                    nt.fields = self.get_notetype_fields(nt.id)?;
+                    nt.templates = self.get_notetype_templates(nt.id)?;
+                    Ok(nt)
+                })
+            })?
+            .collect()
+    }
+
     pub fn get_all_notetype_names(&self) -> Result<Vec<(NotetypeId, String)>> {
         self.db
             .prepare_cached(include_str!("get_notetype_names.sql"))?

rslib/src/storage/revlog/mod.rs

@@ -107,7 +107,7 @@ impl SqliteStorage {
             .collect()
     }
 
-    pub(crate) fn get_revlog_entries_for_searched_cards(
+    pub(crate) fn get_pb_revlog_entries_for_searched_cards(
         &self,
         after: TimestampSecs,
     ) -> Result<Vec<pb::RevlogEntry>> {
@@ -120,9 +120,12 @@ impl SqliteStorage {
             .collect()
     }
 
-    pub(crate) fn get_revlog_entries_for_card_ids(&self, cids: String) -> Result<Vec<RevlogEntry>> {
+    pub(crate) fn get_revlog_entries_for_searched_cards(&self) -> Result<Vec<RevlogEntry>> {
         self.db
-            .prepare_cached(&format!("{} where cid in {cids}", include_str!("get.sql"),))?
+            .prepare_cached(concat!(
+                include_str!("get.sql"),
+                " where cid in (select cid from search_cids)"
+            ))?
             .query_and_then([], row_to_revlog_entry)?
             .collect()
     }