From 9f0f4e61595b32d7d035f6414447c4721d6aed10 Mon Sep 17 00:00:00 2001 From: RumovZ Date: Sat, 7 May 2022 10:04:17 +0200 Subject: [PATCH] Add csv and json importing on backend --- proto/anki/import_export.proto | 12 +- rslib/src/backend/import_export.rs | 47 ++-- rslib/src/import_export/mod.rs | 29 ++- .../import_export/package/apkg/import/mod.rs | 4 +- .../package/apkg/import/notes.rs | 28 +-- rslib/src/import_export/package/mod.rs | 1 - rslib/src/import_export/text/csv.rs | 21 +- rslib/src/import_export/text/import.rs | 216 ++++++++++++++++++ rslib/src/import_export/text/json.rs | 14 ++ rslib/src/import_export/text/mod.rs | 45 +++- rslib/src/notetype/cloze_styling.css | 7 + rslib/src/notetype/mod.rs | 26 ++- rslib/src/notetype/stock.rs | 13 +- 13 files changed, 372 insertions(+), 91 deletions(-) create mode 100644 rslib/src/import_export/text/import.rs create mode 100644 rslib/src/import_export/text/json.rs create mode 100644 rslib/src/notetype/cloze_styling.css diff --git a/proto/anki/import_export.proto b/proto/anki/import_export.proto index 05e3df930..54738c113 100644 --- a/proto/anki/import_export.proto +++ b/proto/anki/import_export.proto @@ -14,10 +14,10 @@ service ImportExportService { returns (generic.Empty); rpc ExportCollectionPackage(ExportCollectionPackageRequest) returns (generic.Empty); - rpc ImportAnkiPackage(ImportAnkiPackageRequest) - returns (ImportAnkiPackageResponse); + rpc ImportAnkiPackage(ImportAnkiPackageRequest) returns (ImportResponse); rpc ExportAnkiPackage(ExportAnkiPackageRequest) returns (generic.UInt32); - rpc ImportCsv(ImportCsvRequest) returns (generic.Empty); + rpc ImportCsv(ImportCsvRequest) returns (ImportResponse); + rpc ImportJson(generic.String) returns (ImportResponse); } message ImportCollectionPackageRequest { @@ -37,7 +37,7 @@ message ImportAnkiPackageRequest { string package_path = 1; } -message ImportAnkiPackageResponse { +message ImportResponse { message Note { notes.NoteId id = 1; repeated string fields = 2; @@ -95,7 +95,7 @@ message MediaEntries { } message ImportCsvRequest { - message Column { + message CsvColumn { enum Other { IGNORE = 0; TAGS = 1; @@ -108,7 +108,7 @@ message ImportCsvRequest { string path = 1; int64 deck_id = 2; int64 notetype_id = 3; - repeated Column columns = 4; + repeated CsvColumn columns = 4; string delimiter = 5; bool allow_html = 6; } diff --git a/rslib/src/backend/import_export.rs b/rslib/src/backend/import_export.rs index 36a93f828..105deb0ca 100644 --- a/rslib/src/backend/import_export.rs +++ b/rslib/src/backend/import_export.rs @@ -9,15 +9,10 @@ use crate::{ backend_proto::{ self as pb, export_anki_package_request::Selector, - import_csv_request::{ - column::{Other as OtherColumn, Variant as ColumnVariant}, - Column as ProtoColumn, - }, + import_csv_request::{csv_column, CsvColumn}, }, import_export::{ - package::{import_colpkg, NoteLog}, - text::csv::Column, - ExportProgress, ImportProgress, + package::import_colpkg, text::csv::Column, ExportProgress, ImportProgress, NoteLog, }, prelude::*, search::SearchNode, @@ -63,7 +58,7 @@ impl ImportExportService for Backend { fn import_anki_package( &self, input: pb::ImportAnkiPackageRequest, - ) -> Result { + ) -> Result { self.with_col(|col| col.import_apkg(&input.package_path, self.import_progress_fn())) .map(Into::into) } @@ -86,19 +81,23 @@ impl ImportExportService for Backend { .map(Into::into) } - fn import_csv(&self, input: pb::ImportCsvRequest) -> Result { - let out = self.with_col(|col| { + fn import_csv(&self, input: pb::ImportCsvRequest) -> Result { + self.with_col(|col| { col.import_csv( &input.path, input.deck_id.into(), input.notetype_id.into(), input.columns.into_iter().map(Into::into).collect(), byte_from_string(&input.delimiter)?, - input.allow_html, + //input.allow_html, ) - })?; - println!("{:?}", out); - Ok(pb::Empty {}) + }) + .map(Into::into) + } + + fn import_json(&self, input: pb::String) -> Result { + self.with_col(|col| col.import_json(&input.val)) + .map(Into::into) } } @@ -124,7 +123,7 @@ impl Backend { } } -impl From> for pb::ImportAnkiPackageResponse { +impl From> for pb::ImportResponse { fn from(output: OpOutput) -> Self { Self { changes: Some(output.changes.into()), @@ -133,14 +132,16 @@ impl From> for pb::ImportAnkiPackageResponse { } } -impl From for Column { - fn from(column: ProtoColumn) -> Self { - match column.variant.unwrap_or(ColumnVariant::Other(0)) { - ColumnVariant::Field(idx) => Column::Field(idx as usize), - ColumnVariant::Other(i) => match OtherColumn::from_i32(i).unwrap_or_default() { - OtherColumn::Tags => Column::Tags, - OtherColumn::Ignore => Column::Ignore, - }, +impl From for Column { + fn from(column: CsvColumn) -> Self { + match column.variant.unwrap_or(csv_column::Variant::Other(0)) { + csv_column::Variant::Field(idx) => Column::Field(idx as usize), + csv_column::Variant::Other(i) => { + match csv_column::Other::from_i32(i).unwrap_or_default() { + csv_column::Other::Tags => Column::Tags, + csv_column::Other::Ignore => Column::Ignore, + } + } } } } diff --git a/rslib/src/import_export/mod.rs b/rslib/src/import_export/mod.rs index d76d2b351..5b49312c7 100644 --- a/rslib/src/import_export/mod.rs +++ b/rslib/src/import_export/mod.rs @@ -8,7 +8,14 @@ pub mod text; use std::marker::PhantomData; -use crate::prelude::*; +pub use crate::backend_proto::import_response::{Log as NoteLog, Note as LogNote}; +use crate::{ + prelude::*, + text::{ + newlines_to_spaces, strip_html_preserving_media_filenames, truncate_to_char_boundary, + CowMapping, + }, +}; #[derive(Debug, Clone, Copy, PartialEq)] pub enum ImportProgress { @@ -96,3 +103,23 @@ impl<'f, F: 'f + FnMut(usize) -> Result<()>> Incrementor<'f, F> { (self.update_fn)(self.count) } } + +impl Note { + pub(crate) fn into_log_note(self) -> LogNote { + LogNote { + id: Some(self.id.into()), + fields: self + .into_fields() + .into_iter() + .map(|field| { + let mut reduced = strip_html_preserving_media_filenames(&field) + .map_cow(newlines_to_spaces) + .get_owned() + .unwrap_or(field); + truncate_to_char_boundary(&mut reduced, 80); + reduced + }) + .collect(), + } + } +} diff --git a/rslib/src/import_export/package/apkg/import/mod.rs b/rslib/src/import_export/package/apkg/import/mod.rs index c25bb1bb3..9581c7909 100644 --- a/rslib/src/import_export/package/apkg/import/mod.rs +++ b/rslib/src/import_export/package/apkg/import/mod.rs @@ -17,9 +17,7 @@ use zstd::stream::copy_decode; use crate::{ collection::CollectionBuilder, import_export::{ - gather::ExchangeData, - package::{Meta, NoteLog}, - ImportProgress, IncrementableProgress, + gather::ExchangeData, package::Meta, ImportProgress, IncrementableProgress, NoteLog, }, prelude::*, search::SearchNode, diff --git a/rslib/src/import_export/package/apkg/import/notes.rs b/rslib/src/import_export/package/apkg/import/notes.rs index 4fb8431b9..a260a5a5d 100644 --- a/rslib/src/import_export/package/apkg/import/notes.rs +++ b/rslib/src/import_export/package/apkg/import/notes.rs @@ -13,14 +13,10 @@ use sha1::Sha1; use super::{media::MediaUseMap, Context}; use crate::{ import_export::{ - package::{media::safe_normalized_file_name, LogNote, NoteLog}, - ImportProgress, IncrementableProgress, + package::media::safe_normalized_file_name, ImportProgress, IncrementableProgress, NoteLog, }, prelude::*, - text::{ - newlines_to_spaces, replace_media_refs, strip_html_preserving_media_filenames, - truncate_to_char_boundary, CowMapping, - }, + text::replace_media_refs, }; struct NoteContext<'a> { @@ -65,26 +61,6 @@ impl NoteImports { } } -impl Note { - fn into_log_note(self) -> LogNote { - LogNote { - id: Some(self.id.into()), - fields: self - .into_fields() - .into_iter() - .map(|field| { - let mut reduced = strip_html_preserving_media_filenames(&field) - .map_cow(newlines_to_spaces) - .get_owned() - .unwrap_or(field); - truncate_to_char_boundary(&mut reduced, 80); - reduced - }) - .collect(), - } - } -} - #[derive(Debug, Clone, Copy)] pub(crate) struct NoteMeta { id: NoteId, diff --git a/rslib/src/import_export/package/mod.rs b/rslib/src/import_export/package/mod.rs index f999d84ce..70b8bfb14 100644 --- a/rslib/src/import_export/package/mod.rs +++ b/rslib/src/import_export/package/mod.rs @@ -11,5 +11,4 @@ pub(crate) use colpkg::export::export_colpkg_from_data; pub use colpkg::import::import_colpkg; pub(self) use meta::{Meta, Version}; -pub use crate::backend_proto::import_anki_package_response::{Log as NoteLog, Note as LogNote}; pub(self) use crate::backend_proto::{media_entries::MediaEntry, MediaEntries}; diff --git a/rslib/src/import_export/text/csv.rs b/rslib/src/import_export/text/csv.rs index 30ff281f8..ee7e2b06f 100644 --- a/rslib/src/import_export/text/csv.rs +++ b/rslib/src/import_export/text/csv.rs @@ -1,6 +1,5 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -#![allow(dead_code, unused_imports, unused_variables)] use std::{ fs::File, @@ -8,7 +7,10 @@ use std::{ }; use crate::{ - import_export::text::{ForeignData, ForeignNote}, + import_export::{ + text::{ForeignData, ForeignNote}, + NoteLog, + }, prelude::*, }; @@ -27,18 +29,21 @@ impl Collection { notetype_id: NotetypeId, columns: Vec, delimiter: u8, - allow_html: bool, - ) -> Result { + //allow_html: bool, + ) -> Result> { let notetype = self.get_notetype(notetype_id)?.ok_or(AnkiError::NotFound)?; let fields_len = notetype.fields.len(); let file = File::open(path)?; let notes = deserialize_csv(file, &columns, fields_len, delimiter)?; - Ok(ForeignData { - default_deck: deck_id, - default_notetype: notetype_id, + ForeignData { + // TODO: refactor to allow passing ids directly + default_deck: deck_id.to_string(), + default_notetype: notetype_id.to_string(), notes, - }) + ..Default::default() + } + .import(self) } } diff --git a/rslib/src/import_export/text/import.rs b/rslib/src/import_export/text/import.rs new file mode 100644 index 000000000..65558451b --- /dev/null +++ b/rslib/src/import_export/text/import.rs @@ -0,0 +1,216 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::{collections::HashMap, sync::Arc}; + +use crate::{ + import_export::{ + text::{ForeignCard, ForeignData, ForeignNote, ForeignNotetype, ForeignTemplate}, + LogNote, NoteLog, + }, + notetype::{CardGenContext, CardTemplate, NoteField, NotetypeConfig}, + prelude::*, +}; + +impl ForeignData { + pub fn import(self, col: &mut Collection) -> Result> { + col.transact(Op::Import, |col| { + let mut ctx = Context::new(&self, col)?; + ctx.import_foreign_notetypes(self.notetypes)?; + ctx.import_foreign_notes(self.notes) + }) + } +} + +struct Context<'a> { + col: &'a mut Collection, + notetypes: HashMap>>, + deck_ids: HashMap>, + usn: Usn, + normalize_notes: bool, + //progress: IncrementableProgress, +} + +impl<'a> Context<'a> { + fn new(data: &ForeignData, col: &'a mut Collection) -> Result { + let usn = col.usn()?; + let normalize_notes = col.get_config_bool(BoolKey::NormalizeNoteText); + let mut notetypes = HashMap::new(); + notetypes.insert( + String::new(), + col.get_notetype_for_string(&data.default_notetype)?, + ); + let mut deck_ids = HashMap::new(); + deck_ids.insert(String::new(), col.deck_id_for_string(&data.default_deck)?); + Ok(Self { + col, + usn, + normalize_notes, + notetypes, + deck_ids, + }) + } + + fn import_foreign_notetypes(&mut self, notetypes: Vec) -> Result<()> { + for foreign in notetypes { + let mut notetype = foreign.into_native(); + notetype.usn = self.usn; + self.col + .add_notetype_inner(&mut notetype, self.usn, false)?; + } + Ok(()) + } + fn notetype_for_note(&mut self, note: &ForeignNote) -> Result>> { + Ok(if let Some(nt) = self.notetypes.get(¬e.notetype) { + nt.clone() + } else { + let nt = self.col.get_notetype_for_string(¬e.notetype)?; + self.notetypes.insert(note.notetype.clone(), nt.clone()); + nt + }) + } + + fn deck_id_for_note(&mut self, note: &ForeignNote) -> Result> { + Ok(if let Some(did) = self.deck_ids.get(¬e.deck) { + *did + } else { + let did = self.col.deck_id_for_string(¬e.deck)?; + self.deck_ids.insert(note.deck.clone(), did); + did + }) + } + + fn import_foreign_notes(&mut self, notes: Vec) -> Result { + let mut log = NoteLog::default(); + for foreign in notes { + if let Some(notetype) = self.notetype_for_note(&foreign)? { + if let Some(deck_id) = self.deck_id_for_note(&foreign)? { + let log_note = self.import_foreign_note(foreign, ¬etype, deck_id)?; + log.new.push(log_note); + } + } + } + Ok(log) + } + + fn import_foreign_note( + &mut self, + foreign: ForeignNote, + notetype: &Notetype, + deck_id: DeckId, + ) -> Result { + let (mut note, mut cards) = foreign.into_native(notetype, deck_id); + self.import_note(&mut note, notetype)?; + self.import_cards(&mut cards, note.id)?; + self.generate_missing_cards(notetype, deck_id, ¬e)?; + Ok(note.into_log_note()) + } + + fn import_note(&mut self, note: &mut Note, notetype: &Notetype) -> Result<()> { + self.col.canonify_note_tags(note, self.usn)?; + note.prepare_for_update(notetype, self.normalize_notes)?; + note.usn = self.usn; + self.col.add_note_only_undoable(note) + } + + fn import_cards(&mut self, cards: &mut [Card], note_id: NoteId) -> Result<()> { + for card in cards { + card.note_id = note_id; + self.col.add_card(card)?; + } + Ok(()) + } + + fn generate_missing_cards( + &mut self, + notetype: &Notetype, + deck_id: DeckId, + note: &Note, + ) -> Result<()> { + let card_gen_context = CardGenContext::new(notetype, Some(deck_id), self.usn); + self.col + .generate_cards_for_existing_note(&card_gen_context, note) + } +} + +impl Collection { + fn deck_id_for_string(&mut self, deck: &str) -> Result> { + if let Ok(did) = deck.parse::() { + if self.get_deck(did)?.is_some() { + return Ok(Some(did)); + } + } + self.get_deck_id(deck) + } + + fn get_notetype_for_string(&mut self, notetype: &str) -> Result>> { + if let Some(nt) = self.get_notetype_for_id_string(notetype)? { + Ok(Some(nt)) + } else { + self.get_notetype_by_name(notetype) + } + } + + fn get_notetype_for_id_string(&mut self, notetype: &str) -> Result>> { + notetype + .parse::() + .ok() + .map(|ntid| self.get_notetype(ntid)) + .unwrap_or(Ok(None)) + } +} + +impl ForeignNote { + fn into_native(self, notetype: &Notetype, deck_id: DeckId) -> (Note, Vec) { + let mut note = Note::new(notetype); + note.tags = self.tags; + note.fields_mut() + .iter_mut() + .zip(self.fields.into_iter()) + .for_each(|(field, value)| *field = value); + let cards = self + .cards + .into_iter() + .enumerate() + .map(|(idx, c)| c.into_native(NoteId(0), idx as u16, deck_id)) + .collect(); + (note, cards) + } +} + +impl ForeignCard { + fn into_native(self, note_id: NoteId, template_idx: u16, deck_id: DeckId) -> Card { + let mut card = Card::new(note_id, template_idx, deck_id, self.due); + card.interval = self.ivl; + card.ease_factor = self.factor; + card.reps = self.reps; + card.lapses = self.lapses; + card + } +} + +impl ForeignNotetype { + fn into_native(self) -> Notetype { + Notetype { + name: self.name, + fields: self.fields.into_iter().map(NoteField::new).collect(), + templates: self + .templates + .into_iter() + .map(ForeignTemplate::into_native) + .collect(), + config: if self.is_cloze { + NotetypeConfig::new_cloze() + } else { + NotetypeConfig::new() + }, + ..Notetype::default() + } + } +} + +impl ForeignTemplate { + fn into_native(self) -> CardTemplate { + CardTemplate::new(self.name, self.qfmt, self.afmt) + } +} diff --git a/rslib/src/import_export/text/json.rs b/rslib/src/import_export/text/json.rs new file mode 100644 index 000000000..8c955436c --- /dev/null +++ b/rslib/src/import_export/text/json.rs @@ -0,0 +1,14 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::{ + import_export::{text::ForeignData, NoteLog}, + prelude::*, +}; + +impl Collection { + pub fn import_json(&mut self, json: &str) -> Result> { + let data: ForeignData = serde_json::from_str(json)?; + data.import(self) + } +} diff --git a/rslib/src/import_export/text/mod.rs b/rslib/src/import_export/text/mod.rs index a0b0d3cbd..e5ca0188b 100644 --- a/rslib/src/import_export/text/mod.rs +++ b/rslib/src/import_export/text/mod.rs @@ -1,20 +1,53 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -#![allow(dead_code, unused_imports, unused_variables)] pub mod csv; +pub mod import; +mod json; -use crate::prelude::*; +use serde_derive::{Deserialize, Serialize}; -#[derive(Debug)] +#[derive(Debug, Default, Serialize, Deserialize)] +#[serde(default)] pub struct ForeignData { - default_deck: DeckId, - default_notetype: NotetypeId, + default_deck: String, + default_notetype: String, notes: Vec, + notetypes: Vec, } -#[derive(Debug, PartialEq, Default)] +#[derive(Debug, PartialEq, Default, Serialize, Deserialize)] +#[serde(default)] pub struct ForeignNote { fields: Vec, tags: Vec, + notetype: String, + deck: String, + cards: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Default, Serialize, Deserialize)] +#[serde(default)] +pub struct ForeignCard { + pub due: i32, + pub ivl: u32, + pub factor: u16, + pub reps: u32, + pub lapses: u32, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct ForeignNotetype { + name: String, + fields: Vec, + templates: Vec, + #[serde(default)] + is_cloze: bool, +} + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +pub struct ForeignTemplate { + name: String, + qfmt: String, + afmt: String, } diff --git a/rslib/src/notetype/cloze_styling.css b/rslib/src/notetype/cloze_styling.css new file mode 100644 index 000000000..335a0bafe --- /dev/null +++ b/rslib/src/notetype/cloze_styling.css @@ -0,0 +1,7 @@ +.cloze { + font-weight: bold; + color: blue; +} +.nightMode .cloze { + color: lightblue; +} diff --git a/rslib/src/notetype/mod.rs b/rslib/src/notetype/mod.rs index 84ebb3e24..5cc68c70c 100644 --- a/rslib/src/notetype/mod.rs +++ b/rslib/src/notetype/mod.rs @@ -53,6 +53,7 @@ use crate::{ define_newtype!(NotetypeId, i64); pub(crate) const DEFAULT_CSS: &str = include_str!("styling.css"); +pub(crate) const DEFAULT_CLOZE_CSS: &str = include_str!("cloze_styling.css"); pub(crate) const DEFAULT_LATEX_HEADER: &str = include_str!("header.tex"); pub(crate) const DEFAULT_LATEX_FOOTER: &str = r"\end{document}"; lazy_static! { @@ -88,16 +89,29 @@ impl Default for Notetype { usn: Usn(0), fields: vec![], templates: vec![], - config: NotetypeConfig { - css: DEFAULT_CSS.into(), - latex_pre: DEFAULT_LATEX_HEADER.into(), - latex_post: DEFAULT_LATEX_FOOTER.into(), - ..Default::default() - }, + config: NotetypeConfig::new(), } } } +impl NotetypeConfig { + pub(crate) fn new() -> Self { + NotetypeConfig { + css: DEFAULT_CSS.into(), + latex_pre: DEFAULT_LATEX_HEADER.into(), + latex_post: DEFAULT_LATEX_FOOTER.into(), + ..Default::default() + } + } + + pub(crate) fn new_cloze() -> Self { + let mut config = Self::new(); + config.css += DEFAULT_CLOZE_CSS; + config.kind = NotetypeKind::Cloze as i32; + config + } +} + impl Notetype { pub fn new_note(&self) -> Note { Note::new(self) diff --git a/rslib/src/notetype/stock.rs b/rslib/src/notetype/stock.rs index 58619889b..32935a546 100644 --- a/rslib/src/notetype/stock.rs +++ b/rslib/src/notetype/stock.rs @@ -1,7 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use super::NotetypeKind; +use super::NotetypeConfig; use crate::{ backend_proto::stock_notetype::Kind, config::{ConfigEntry, ConfigKey}, @@ -112,6 +112,7 @@ pub(crate) fn basic_optional_reverse(tr: &I18n) -> Notetype { pub(crate) fn cloze(tr: &I18n) -> Notetype { let mut nt = Notetype { name: tr.notetypes_cloze_name().into(), + config: NotetypeConfig::new_cloze(), ..Default::default() }; let text = tr.notetypes_text_field(); @@ -121,15 +122,5 @@ pub(crate) fn cloze(tr: &I18n) -> Notetype { let qfmt = format!("{{{{cloze:{}}}}}", text); let afmt = format!("{}
\n{{{{{}}}}}", qfmt, back_extra); nt.add_template(nt.name.clone(), qfmt, afmt); - nt.config.kind = NotetypeKind::Cloze as i32; - nt.config.css += " -.cloze { - font-weight: bold; - color: blue; -} -.nightMode .cloze { - color: lightblue; -} -"; nt }