From 6cd10e858e5308e06ef09f4c27a658961877ca03 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Sat, 4 Jun 2022 11:01:04 +0200 Subject: [PATCH] Optionally export deck and notetype with CSV --- ftl/core/exporting.ftl | 2 + proto/anki/import_export.proto | 4 +- pylib/anki/collection.py | 4 + qt/aqt/exporting.py | 3 + qt/aqt/forms/exporting.ui | 24 ++- qt/aqt/import_export/exporting.py | 12 ++ rslib/src/backend/import_export.rs | 12 +- rslib/src/import_export/text/csv/export.rs | 152 +++++++++++++++--- rslib/src/notetype/mod.rs | 15 ++ .../deck/all_decks_of_search_notes.sql | 8 + rslib/src/storage/deck/mod.rs | 8 + rslib/src/storage/notetype/mod.rs | 9 ++ 12 files changed, 215 insertions(+), 38 deletions(-) create mode 100644 rslib/src/storage/deck/all_decks_of_search_notes.sql diff --git a/ftl/core/exporting.ftl b/ftl/core/exporting.ftl index 4c3e30477..66f876a35 100644 --- a/ftl/core/exporting.ftl +++ b/ftl/core/exporting.ftl @@ -38,3 +38,5 @@ exporting-processed-media-files = [one] Processed { $count } media file... *[other] Processed { $count } media files... } +exporting-include-deck = Include deck +exporting-include-notetype = Include notetype diff --git a/proto/anki/import_export.proto b/proto/anki/import_export.proto index 41dc02652..35c54f5fe 100644 --- a/proto/anki/import_export.proto +++ b/proto/anki/import_export.proto @@ -176,7 +176,9 @@ message ExportNoteCsvRequest { string out_path = 1; bool with_html = 2; bool with_tags = 3; - ExportLimit limit = 4; + bool with_deck = 4; + bool with_notetype = 5; + ExportLimit limit = 6; } message ExportLimit { diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index b5c4e79e1..4b2a318f9 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -423,11 +423,15 @@ class Collection(DeprecatedNamesMixin): limit: ExportLimit, with_html: bool, with_tags: bool, + with_deck: bool, + with_notetype: bool, ) -> int: return self._backend.export_note_csv( out_path=out_path, with_html=with_html, with_tags=with_tags, + with_deck=with_deck, + with_notetype=with_notetype, limit=pb_export_limit(limit), ) diff --git a/qt/aqt/exporting.py b/qt/aqt/exporting.py index f17c5310d..9d5a32b98 100644 --- a/qt/aqt/exporting.py +++ b/qt/aqt/exporting.py @@ -98,6 +98,9 @@ class ExportDialog(QDialog): self.frm.includeHTML.setVisible(False) # show deck list? self.frm.deck.setVisible(not self.isVerbatim) + # used by the new export screen + self.frm.includeDeck.setVisible(False) + self.frm.includeNotetype.setVisible(False) def accept(self) -> None: self.exporter.includeSched = self.frm.includeSched.isChecked() diff --git a/qt/aqt/forms/exporting.ui b/qt/aqt/forms/exporting.ui index 3d39e9416..a317c957e 100644 --- a/qt/aqt/forms/exporting.ui +++ b/qt/aqt/forms/exporting.ui @@ -6,8 +6,8 @@ 0 0 - 563 - 245 + 610 + 348 @@ -87,6 +87,26 @@ + + + + true + + + exporting_include_deck + + + + + + + true + + + exporting_include_notetype + + + diff --git a/qt/aqt/import_export/exporting.py b/qt/aqt/import_export/exporting.py index 669a2ddc8..97c2b5539 100644 --- a/qt/aqt/import_export/exporting.py +++ b/qt/aqt/import_export/exporting.py @@ -91,6 +91,8 @@ class ExportDialog(QDialog): self.frm.includeMedia.setVisible(self.exporter.show_include_media) self.frm.includeTags.setVisible(self.exporter.show_include_tags) self.frm.includeHTML.setVisible(self.exporter.show_include_html) + self.frm.includeDeck.setVisible(self.exporter.show_include_deck) + self.frm.includeNotetype.setVisible(self.exporter.show_include_notetype) self.frm.legacy_support.setVisible(self.exporter.show_legacy_support) self.frm.deck.setVisible(self.exporter.show_deck_list) @@ -135,6 +137,8 @@ class ExportDialog(QDialog): include_media=self.frm.includeMedia.isChecked(), include_tags=self.frm.includeTags.isChecked(), include_html=self.frm.includeHTML.isChecked(), + include_deck=self.frm.includeDeck.isChecked(), + include_notetype=self.frm.includeNotetype.isChecked(), legacy_support=self.frm.legacy_support.isChecked(), limit=limit, ) @@ -165,6 +169,8 @@ class Options: include_media: bool include_tags: bool include_html: bool + include_deck: bool + include_notetype: bool legacy_support: bool limit: ExportLimit @@ -177,6 +183,8 @@ class Exporter(ABC): show_include_tags = False show_include_html = False show_legacy_support = False + show_include_deck = False + show_include_notetype = False @staticmethod @abstractmethod @@ -255,6 +263,8 @@ class NoteCsvExporter(Exporter): show_deck_list = True show_include_html = True show_include_tags = True + show_include_deck = True + show_include_notetype = True @staticmethod def name() -> str: @@ -269,6 +279,8 @@ class NoteCsvExporter(Exporter): limit=options.limit, with_html=options.include_html, with_tags=options.include_tags, + with_deck=options.include_deck, + with_notetype=options.include_notetype, ), success=lambda count: tooltip( tr.exporting_note_exported(count=count), parent=mw diff --git a/rslib/src/backend/import_export.rs b/rslib/src/backend/import_export.rs index 56773441b..57c529181 100644 --- a/rslib/src/backend/import_export.rs +++ b/rslib/src/backend/import_export.rs @@ -93,16 +93,8 @@ impl ImportExportService for Backend { } fn export_note_csv(&self, input: pb::ExportNoteCsvRequest) -> Result { - self.with_col(|col| { - col.export_note_csv( - &input.out_path, - SearchNode::from(input.limit.unwrap_or_default()), - input.with_html, - input.with_tags, - self.export_progress_fn(), - ) - }) - .map(Into::into) + self.with_col(|col| col.export_note_csv(input, self.export_progress_fn())) + .map(Into::into) } fn export_card_csv(&self, input: pb::ExportCardCsvRequest) -> Result { diff --git a/rslib/src/import_export/text/csv/export.rs b/rslib/src/import_export/text/csv/export.rs index e1c702be7..f11b12e2c 100644 --- a/rslib/src/import_export/text/csv/export.rs +++ b/rslib/src/import_export/text/csv/export.rs @@ -1,7 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use std::{borrow::Cow, fs::File, io::Write}; +use std::{borrow::Cow, collections::HashMap, fs::File, io::Write, sync::Arc}; use itertools::Itertools; use lazy_static::lazy_static; @@ -9,10 +9,11 @@ use regex::Regex; use super::metadata::Delimiter; use crate::{ + backend_proto::ExportNoteCsvRequest, import_export::{ExportProgress, IncrementableProgress}, notetype::RenderCardOutput, prelude::*, - search::SortMode, + search::{SearchNode, SortMode}, template::RenderedNode, text::{html_to_text_line, CowMapping}, }; @@ -45,21 +46,19 @@ impl Collection { pub fn export_note_csv( &mut self, - path: &str, - search: impl TryIntoSearch, - with_html: bool, - with_tags: bool, + mut request: ExportNoteCsvRequest, progress_fn: impl 'static + FnMut(ExportProgress, bool) -> bool, ) -> Result { let mut progress = IncrementableProgress::new(progress_fn); progress.call(ExportProgress::File)?; let mut incrementor = progress.incrementor(ExportProgress::Notes); - let mut writer = file_writer_with_header(path)?; - self.search_notes_into_table(search)?; + self.search_notes_into_table(request.search_node())?; + let ctx = NoteContext::new(&request, self)?; + let mut writer = note_file_writer_with_header(&request.out_path, &ctx)?; self.storage.for_each_note_in_search(|note| { incrementor.increment()?; - writer.write_record(note_record(¬e, with_html, with_tags))?; + writer.write_record(ctx.record(¬e))?; Ok(()) })?; writer.flush()?; @@ -79,15 +78,41 @@ impl Collection { fn file_writer_with_header(path: &str) -> Result> { let mut file = File::create(path)?; - write_header(&mut file)?; + write_file_header(&mut file)?; Ok(csv::WriterBuilder::new() .delimiter(DELIMITER.byte()) - .flexible(true) .from_writer(file)) } -fn write_header(writer: &mut impl Write) -> Result<()> { - write!(writer, "#separator:{}\n#html:true\n", DELIMITER.name())?; +fn write_file_header(writer: &mut impl Write) -> Result<()> { + writeln!(writer, "#separator:{}", DELIMITER.name())?; + writeln!(writer, "#html:true")?; + Ok(()) +} + +fn note_file_writer_with_header(path: &str, ctx: &NoteContext) -> Result> { + let mut file = File::create(path)?; + write_note_file_header(&mut file, ctx)?; + Ok(csv::WriterBuilder::new() + .delimiter(DELIMITER.byte()) + .from_writer(file)) +} + +fn write_note_file_header(writer: &mut impl Write, ctx: &NoteContext) -> Result<()> { + write_file_header(writer)?; + write_column_header(ctx, writer) +} + +fn write_column_header(ctx: &NoteContext, writer: &mut impl Write) -> Result<()> { + for (name, column) in [ + ("notetype", ctx.notetype_column()), + ("deck", ctx.deck_column()), + ("tags", ctx.tags_column()), + ] { + if let Some(index) = column { + writeln!(writer, "#{name} column:{index}")?; + } + } Ok(()) } @@ -117,18 +142,6 @@ fn rendered_nodes_to_str(nodes: &[RenderedNode]) -> String { .join("") } -fn note_record(note: &Note, with_html: bool, with_tags: bool) -> Vec { - let mut fields: Vec<_> = note - .fields() - .iter() - .map(|f| field_to_record_field(f, with_html)) - .collect(); - if with_tags { - fields.push(note.tags.join(" ")); - } - fields -} - fn field_to_record_field(field: &str, with_html: bool) -> String { let mut text = strip_redundant_sections(field); if !with_html { @@ -157,3 +170,92 @@ fn strip_answer_side_question(text: &str) -> Cow { } RE.replace_all(text.as_ref(), "") } + +struct NoteContext { + with_html: bool, + with_tags: bool, + with_deck: bool, + with_notetype: bool, + notetypes: HashMap>, + deck_ids: HashMap, + deck_names: HashMap, + field_columns: usize, +} + +impl NoteContext { + /// Caller must have searched notes into table. + fn new(request: &ExportNoteCsvRequest, col: &mut Collection) -> Result { + let notetypes = col.get_all_notetypes_of_search_notes()?; + let field_columns = notetypes + .values() + .map(|nt| nt.fields.len()) + .max() + .unwrap_or_default(); + let deck_ids = col.storage.all_decks_of_search_notes()?; + let deck_names = HashMap::from_iter(col.storage.get_all_deck_names()?.into_iter()); + + Ok(Self { + with_html: request.with_html, + with_tags: request.with_tags, + with_deck: request.with_deck, + with_notetype: request.with_notetype, + notetypes, + field_columns, + deck_ids, + deck_names, + }) + } + + fn notetype_column(&self) -> Option { + self.with_notetype.then(|| 1) + } + + fn deck_column(&self) -> Option { + self.with_deck + .then(|| 1 + self.notetype_column().unwrap_or_default()) + } + + fn tags_column(&self) -> Option { + self.with_tags + .then(|| 1 + self.deck_column().unwrap_or_default() + self.field_columns) + } + + fn record<'i, 's: 'i, 'n: 'i>(&'s self, note: &'n Note) -> impl Iterator + 'i { + self.notetype_name(note) + .into_iter() + .chain(self.deck_name(note).into_iter()) + .chain(self.note_fields(note).into_iter()) + .chain(self.with_tags.then(|| note.tags.join(" ")).into_iter()) + } + + fn notetype_name(&self, note: &Note) -> Option { + self.with_notetype.then(|| { + self.notetypes + .get(¬e.notetype_id) + .map_or(String::new(), |nt| nt.name.clone()) + }) + } + + fn deck_name(&self, note: &Note) -> Option { + self.with_deck.then(|| { + self.deck_ids + .get(¬e.id) + .and_then(|did| self.deck_names.get(did)) + .map_or(String::new(), |name| name.clone()) + }) + } + + fn note_fields<'n>(&self, note: &'n Note) -> impl Iterator + 'n { + let with_html = self.with_html; + note.fields() + .iter() + .map(move |f| field_to_record_field(f, with_html)) + .pad_using(self.field_columns, |_| String::new()) + } +} + +impl ExportNoteCsvRequest { + fn search_node(&mut self) -> SearchNode { + SearchNode::from(self.limit.take().unwrap_or_default()) + } +} diff --git a/rslib/src/notetype/mod.rs b/rslib/src/notetype/mod.rs index 5cc68c70c..c8e5922a1 100644 --- a/rslib/src/notetype/mod.rs +++ b/rslib/src/notetype/mod.rs @@ -218,6 +218,21 @@ impl Collection { .collect() } + pub fn get_all_notetypes_of_search_notes( + &mut self, + ) -> Result>> { + self.storage + .all_notetypes_of_search_notes()? + .into_iter() + .map(|ntid| { + self.get_notetype(ntid) + .transpose() + .unwrap() + .map(|nt| (ntid, nt)) + }) + .collect() + } + pub fn remove_notetype(&mut self, ntid: NotetypeId) -> Result> { self.transact(Op::RemoveNotetype, |col| col.remove_notetype_inner(ntid)) } diff --git a/rslib/src/storage/deck/all_decks_of_search_notes.sql b/rslib/src/storage/deck/all_decks_of_search_notes.sql new file mode 100644 index 000000000..0bf183f15 --- /dev/null +++ b/rslib/src/storage/deck/all_decks_of_search_notes.sql @@ -0,0 +1,8 @@ +SELECT nid, + did +FROM cards +WHERE ord = 0 + AND nid IN ( + SELECT nid + FROM search_nids + ) \ No newline at end of file diff --git a/rslib/src/storage/deck/mod.rs b/rslib/src/storage/deck/mod.rs index f99467143..d31846fbb 100644 --- a/rslib/src/storage/deck/mod.rs +++ b/rslib/src/storage/deck/mod.rs @@ -131,6 +131,14 @@ impl SqliteStorage { .collect() } + /// Returns the deck id of the first card of every searched note. + pub(crate) fn all_decks_of_search_notes(&self) -> Result> { + self.db + .prepare_cached(include_str!("all_decks_of_search_notes.sql"))? + .query_and_then([], |r| Ok((r.get(0)?, r.get(1)?)))? + .collect() + } + // caller should ensure name unique pub(crate) fn add_deck(&self, deck: &mut Deck) -> Result<()> { assert!(deck.id.0 == 0); diff --git a/rslib/src/storage/notetype/mod.rs b/rslib/src/storage/notetype/mod.rs index f68e636f8..cd73ab0d4 100644 --- a/rslib/src/storage/notetype/mod.rs +++ b/rslib/src/storage/notetype/mod.rs @@ -116,6 +116,15 @@ impl SqliteStorage { .collect() } + pub(crate) fn all_notetypes_of_search_notes(&self) -> Result> { + self.db + .prepare_cached( + "SELECT DISTINCT mid FROM notes WHERE id IN (SELECT nid FROM search_nids)", + )? + .query_and_then([], |r| Ok(r.get(0)?))? + .collect() + } + pub fn get_all_notetype_names(&self) -> Result> { self.db .prepare_cached(include_str!("get_notetype_names.sql"))?