Add csv and json importing on backend

proto/anki/import_export.proto

@@ -14,10 +14,10 @@ service ImportExportService {
       returns (generic.Empty);
   rpc ExportCollectionPackage(ExportCollectionPackageRequest)
       returns (generic.Empty);
-  rpc ImportAnkiPackage(ImportAnkiPackageRequest)
+  rpc ImportAnkiPackage(ImportAnkiPackageRequest) returns (ImportResponse);
-      returns (ImportAnkiPackageResponse);
   rpc ExportAnkiPackage(ExportAnkiPackageRequest) returns (generic.UInt32);
-  rpc ImportCsv(ImportCsvRequest) returns (generic.Empty);
+  rpc ImportCsv(ImportCsvRequest) returns (ImportResponse);
+  rpc ImportJson(generic.String) returns (ImportResponse);
 }
 
 message ImportCollectionPackageRequest {
@@ -37,7 +37,7 @@ message ImportAnkiPackageRequest {
   string package_path = 1;
 }
 
-message ImportAnkiPackageResponse {
+message ImportResponse {
   message Note {
     notes.NoteId id = 1;
     repeated string fields = 2;
@@ -95,7 +95,7 @@ message MediaEntries {
 }
 
 message ImportCsvRequest {
-  message Column {
+  message CsvColumn {
     enum Other {
       IGNORE = 0;
       TAGS = 1;
@@ -108,7 +108,7 @@ message ImportCsvRequest {
   string path = 1;
   int64 deck_id = 2;
   int64 notetype_id = 3;
-  repeated Column columns = 4;
+  repeated CsvColumn columns = 4;
   string delimiter = 5;
   bool allow_html = 6;
 }

rslib/src/backend/import_export.rs

@@ -9,15 +9,10 @@ use crate::{
     backend_proto::{
         self as pb,
         export_anki_package_request::Selector,
-        import_csv_request::{
+        import_csv_request::{csv_column, CsvColumn},
-            column::{Other as OtherColumn, Variant as ColumnVariant},
-            Column as ProtoColumn,
-        },
     },
     import_export::{
-        package::{import_colpkg, NoteLog},
+        package::import_colpkg, text::csv::Column, ExportProgress, ImportProgress, NoteLog,
-        text::csv::Column,
-        ExportProgress, ImportProgress,
     },
     prelude::*,
     search::SearchNode,
@@ -63,7 +58,7 @@ impl ImportExportService for Backend {
     fn import_anki_package(
         &self,
         input: pb::ImportAnkiPackageRequest,
-    ) -> Result<pb::ImportAnkiPackageResponse> {
+    ) -> Result<pb::ImportResponse> {
         self.with_col(|col| col.import_apkg(&input.package_path, self.import_progress_fn()))
             .map(Into::into)
     }
@@ -86,19 +81,23 @@ impl ImportExportService for Backend {
         .map(Into::into)
     }
 
-    fn import_csv(&self, input: pb::ImportCsvRequest) -> Result<pb::Empty> {
+    fn import_csv(&self, input: pb::ImportCsvRequest) -> Result<pb::ImportResponse> {
-        let out = self.with_col(|col| {
+        self.with_col(|col| {
             col.import_csv(
                 &input.path,
                 input.deck_id.into(),
                 input.notetype_id.into(),
                 input.columns.into_iter().map(Into::into).collect(),
                 byte_from_string(&input.delimiter)?,
-                input.allow_html,
+                //input.allow_html,
             )
-        })?;
+        })
-        println!("{:?}", out);
+        .map(Into::into)
-        Ok(pb::Empty {})
+    }
+
+    fn import_json(&self, input: pb::String) -> Result<pb::ImportResponse> {
+        self.with_col(|col| col.import_json(&input.val))
+            .map(Into::into)
     }
 }
 
@@ -124,7 +123,7 @@ impl Backend {
     }
 }
 
-impl From<OpOutput<NoteLog>> for pb::ImportAnkiPackageResponse {
+impl From<OpOutput<NoteLog>> for pb::ImportResponse {
     fn from(output: OpOutput<NoteLog>) -> Self {
         Self {
             changes: Some(output.changes.into()),
@@ -133,14 +132,16 @@ impl From<OpOutput<NoteLog>> for pb::ImportAnkiPackageResponse {
     }
 }
 
-impl From<ProtoColumn> for Column {
+impl From<CsvColumn> for Column {
-    fn from(column: ProtoColumn) -> Self {
+    fn from(column: CsvColumn) -> Self {
-        match column.variant.unwrap_or(ColumnVariant::Other(0)) {
+        match column.variant.unwrap_or(csv_column::Variant::Other(0)) {
-            ColumnVariant::Field(idx) => Column::Field(idx as usize),
+            csv_column::Variant::Field(idx) => Column::Field(idx as usize),
-            ColumnVariant::Other(i) => match OtherColumn::from_i32(i).unwrap_or_default() {
+            csv_column::Variant::Other(i) => {
-                OtherColumn::Tags => Column::Tags,
+                match csv_column::Other::from_i32(i).unwrap_or_default() {
-                OtherColumn::Ignore => Column::Ignore,
+                    csv_column::Other::Tags => Column::Tags,
-            },
+                    csv_column::Other::Ignore => Column::Ignore,
+                }
+            }
         }
     }
 }

rslib/src/import_export/mod.rs

@@ -8,7 +8,14 @@ pub mod text;
 
 use std::marker::PhantomData;
 
-use crate::prelude::*;
+pub use crate::backend_proto::import_response::{Log as NoteLog, Note as LogNote};
+use crate::{
+    prelude::*,
+    text::{
+        newlines_to_spaces, strip_html_preserving_media_filenames, truncate_to_char_boundary,
+        CowMapping,
+    },
+};
 
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum ImportProgress {
@@ -96,3 +103,23 @@ impl<'f, F: 'f + FnMut(usize) -> Result<()>> Incrementor<'f, F> {
         (self.update_fn)(self.count)
     }
 }
+
+impl Note {
+    pub(crate) fn into_log_note(self) -> LogNote {
+        LogNote {
+            id: Some(self.id.into()),
+            fields: self
+                .into_fields()
+                .into_iter()
+                .map(|field| {
+                    let mut reduced = strip_html_preserving_media_filenames(&field)
+                        .map_cow(newlines_to_spaces)
+                        .get_owned()
+                        .unwrap_or(field);
+                    truncate_to_char_boundary(&mut reduced, 80);
+                    reduced
+                })
+                .collect(),
+        }
+    }
+}

rslib/src/import_export/package/apkg/import/mod.rs

@@ -17,9 +17,7 @@ use zstd::stream::copy_decode;
 use crate::{
     collection::CollectionBuilder,
     import_export::{
-        gather::ExchangeData,
+        gather::ExchangeData, package::Meta, ImportProgress, IncrementableProgress, NoteLog,
-        package::{Meta, NoteLog},
-        ImportProgress, IncrementableProgress,
     },
     prelude::*,
     search::SearchNode,

rslib/src/import_export/package/apkg/import/notes.rs

@@ -13,14 +13,10 @@ use sha1::Sha1;
 use super::{media::MediaUseMap, Context};
 use crate::{
     import_export::{
-        package::{media::safe_normalized_file_name, LogNote, NoteLog},
+        package::media::safe_normalized_file_name, ImportProgress, IncrementableProgress, NoteLog,
-        ImportProgress, IncrementableProgress,
     },
     prelude::*,
-    text::{
+    text::replace_media_refs,
-        newlines_to_spaces, replace_media_refs, strip_html_preserving_media_filenames,
-        truncate_to_char_boundary, CowMapping,
-    },
 };
 
 struct NoteContext<'a> {
@@ -65,26 +61,6 @@ impl NoteImports {
     }
 }
 
-impl Note {
-    fn into_log_note(self) -> LogNote {
-        LogNote {
-            id: Some(self.id.into()),
-            fields: self
-                .into_fields()
-                .into_iter()
-                .map(|field| {
-                    let mut reduced = strip_html_preserving_media_filenames(&field)
-                        .map_cow(newlines_to_spaces)
-                        .get_owned()
-                        .unwrap_or(field);
-                    truncate_to_char_boundary(&mut reduced, 80);
-                    reduced
-                })
-                .collect(),
-        }
-    }
-}
-
 #[derive(Debug, Clone, Copy)]
 pub(crate) struct NoteMeta {
     id: NoteId,

rslib/src/import_export/package/mod.rs

@@ -11,5 +11,4 @@ pub(crate) use colpkg::export::export_colpkg_from_data;
 pub use colpkg::import::import_colpkg;
 pub(self) use meta::{Meta, Version};
 
-pub use crate::backend_proto::import_anki_package_response::{Log as NoteLog, Note as LogNote};
 pub(self) use crate::backend_proto::{media_entries::MediaEntry, MediaEntries};

rslib/src/import_export/text/csv.rs

@@ -1,6 +1,5 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
-#![allow(dead_code, unused_imports, unused_variables)]
 
 use std::{
     fs::File,
@@ -8,7 +7,10 @@ use std::{
 };
 
 use crate::{
-    import_export::text::{ForeignData, ForeignNote},
+    import_export::{
+        text::{ForeignData, ForeignNote},
+        NoteLog,
+    },
     prelude::*,
 };
 
@@ -27,18 +29,21 @@ impl Collection {
         notetype_id: NotetypeId,
         columns: Vec<Column>,
         delimiter: u8,
-        allow_html: bool,
+        //allow_html: bool,
-    ) -> Result<ForeignData> {
+    ) -> Result<OpOutput<NoteLog>> {
         let notetype = self.get_notetype(notetype_id)?.ok_or(AnkiError::NotFound)?;
         let fields_len = notetype.fields.len();
         let file = File::open(path)?;
         let notes = deserialize_csv(file, &columns, fields_len, delimiter)?;
 
-        Ok(ForeignData {
+        ForeignData {
-            default_deck: deck_id,
+            // TODO: refactor to allow passing ids directly
-            default_notetype: notetype_id,
+            default_deck: deck_id.to_string(),
+            default_notetype: notetype_id.to_string(),
             notes,
-        })
+            ..Default::default()
+        }
+        .import(self)
     }
 }
 

rslib/src/import_export/text/import.rs

@@ -0,0 +1,216 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use std::{collections::HashMap, sync::Arc};
+
+use crate::{
+    import_export::{
+        text::{ForeignCard, ForeignData, ForeignNote, ForeignNotetype, ForeignTemplate},
+        LogNote, NoteLog,
+    },
+    notetype::{CardGenContext, CardTemplate, NoteField, NotetypeConfig},
+    prelude::*,
+};
+
+impl ForeignData {
+    pub fn import(self, col: &mut Collection) -> Result<OpOutput<NoteLog>> {
+        col.transact(Op::Import, |col| {
+            let mut ctx = Context::new(&self, col)?;
+            ctx.import_foreign_notetypes(self.notetypes)?;
+            ctx.import_foreign_notes(self.notes)
+        })
+    }
+}
+
+struct Context<'a> {
+    col: &'a mut Collection,
+    notetypes: HashMap<String, Option<Arc<Notetype>>>,
+    deck_ids: HashMap<String, Option<DeckId>>,
+    usn: Usn,
+    normalize_notes: bool,
+    //progress: IncrementableProgress<ImportProgress>,
+}
+
+impl<'a> Context<'a> {
+    fn new(data: &ForeignData, col: &'a mut Collection) -> Result<Self> {
+        let usn = col.usn()?;
+        let normalize_notes = col.get_config_bool(BoolKey::NormalizeNoteText);
+        let mut notetypes = HashMap::new();
+        notetypes.insert(
+            String::new(),
+            col.get_notetype_for_string(&data.default_notetype)?,
+        );
+        let mut deck_ids = HashMap::new();
+        deck_ids.insert(String::new(), col.deck_id_for_string(&data.default_deck)?);
+        Ok(Self {
+            col,
+            usn,
+            normalize_notes,
+            notetypes,
+            deck_ids,
+        })
+    }
+
+    fn import_foreign_notetypes(&mut self, notetypes: Vec<ForeignNotetype>) -> Result<()> {
+        for foreign in notetypes {
+            let mut notetype = foreign.into_native();
+            notetype.usn = self.usn;
+            self.col
+                .add_notetype_inner(&mut notetype, self.usn, false)?;
+        }
+        Ok(())
+    }
+    fn notetype_for_note(&mut self, note: &ForeignNote) -> Result<Option<Arc<Notetype>>> {
+        Ok(if let Some(nt) = self.notetypes.get(&note.notetype) {
+            nt.clone()
+        } else {
+            let nt = self.col.get_notetype_for_string(&note.notetype)?;
+            self.notetypes.insert(note.notetype.clone(), nt.clone());
+            nt
+        })
+    }
+
+    fn deck_id_for_note(&mut self, note: &ForeignNote) -> Result<Option<DeckId>> {
+        Ok(if let Some(did) = self.deck_ids.get(&note.deck) {
+            *did
+        } else {
+            let did = self.col.deck_id_for_string(&note.deck)?;
+            self.deck_ids.insert(note.deck.clone(), did);
+            did
+        })
+    }
+
+    fn import_foreign_notes(&mut self, notes: Vec<ForeignNote>) -> Result<NoteLog> {
+        let mut log = NoteLog::default();
+        for foreign in notes {
+            if let Some(notetype) = self.notetype_for_note(&foreign)? {
+                if let Some(deck_id) = self.deck_id_for_note(&foreign)? {
+                    let log_note = self.import_foreign_note(foreign, &notetype, deck_id)?;
+                    log.new.push(log_note);
+                }
+            }
+        }
+        Ok(log)
+    }
+
+    fn import_foreign_note(
+        &mut self,
+        foreign: ForeignNote,
+        notetype: &Notetype,
+        deck_id: DeckId,
+    ) -> Result<LogNote> {
+        let (mut note, mut cards) = foreign.into_native(notetype, deck_id);
+        self.import_note(&mut note, notetype)?;
+        self.import_cards(&mut cards, note.id)?;
+        self.generate_missing_cards(notetype, deck_id, &note)?;
+        Ok(note.into_log_note())
+    }
+
+    fn import_note(&mut self, note: &mut Note, notetype: &Notetype) -> Result<()> {
+        self.col.canonify_note_tags(note, self.usn)?;
+        note.prepare_for_update(notetype, self.normalize_notes)?;
+        note.usn = self.usn;
+        self.col.add_note_only_undoable(note)
+    }
+
+    fn import_cards(&mut self, cards: &mut [Card], note_id: NoteId) -> Result<()> {
+        for card in cards {
+            card.note_id = note_id;
+            self.col.add_card(card)?;
+        }
+        Ok(())
+    }
+
+    fn generate_missing_cards(
+        &mut self,
+        notetype: &Notetype,
+        deck_id: DeckId,
+        note: &Note,
+    ) -> Result<()> {
+        let card_gen_context = CardGenContext::new(notetype, Some(deck_id), self.usn);
+        self.col
+            .generate_cards_for_existing_note(&card_gen_context, note)
+    }
+}
+
+impl Collection {
+    fn deck_id_for_string(&mut self, deck: &str) -> Result<Option<DeckId>> {
+        if let Ok(did) = deck.parse::<DeckId>() {
+            if self.get_deck(did)?.is_some() {
+                return Ok(Some(did));
+            }
+        }
+        self.get_deck_id(deck)
+    }
+
+    fn get_notetype_for_string(&mut self, notetype: &str) -> Result<Option<Arc<Notetype>>> {
+        if let Some(nt) = self.get_notetype_for_id_string(notetype)? {
+            Ok(Some(nt))
+        } else {
+            self.get_notetype_by_name(notetype)
+        }
+    }
+
+    fn get_notetype_for_id_string(&mut self, notetype: &str) -> Result<Option<Arc<Notetype>>> {
+        notetype
+            .parse::<NotetypeId>()
+            .ok()
+            .map(|ntid| self.get_notetype(ntid))
+            .unwrap_or(Ok(None))
+    }
+}
+
+impl ForeignNote {
+    fn into_native(self, notetype: &Notetype, deck_id: DeckId) -> (Note, Vec<Card>) {
+        let mut note = Note::new(notetype);
+        note.tags = self.tags;
+        note.fields_mut()
+            .iter_mut()
+            .zip(self.fields.into_iter())
+            .for_each(|(field, value)| *field = value);
+        let cards = self
+            .cards
+            .into_iter()
+            .enumerate()
+            .map(|(idx, c)| c.into_native(NoteId(0), idx as u16, deck_id))
+            .collect();
+        (note, cards)
+    }
+}
+
+impl ForeignCard {
+    fn into_native(self, note_id: NoteId, template_idx: u16, deck_id: DeckId) -> Card {
+        let mut card = Card::new(note_id, template_idx, deck_id, self.due);
+        card.interval = self.ivl;
+        card.ease_factor = self.factor;
+        card.reps = self.reps;
+        card.lapses = self.lapses;
+        card
+    }
+}
+
+impl ForeignNotetype {
+    fn into_native(self) -> Notetype {
+        Notetype {
+            name: self.name,
+            fields: self.fields.into_iter().map(NoteField::new).collect(),
+            templates: self
+                .templates
+                .into_iter()
+                .map(ForeignTemplate::into_native)
+                .collect(),
+            config: if self.is_cloze {
+                NotetypeConfig::new_cloze()
+            } else {
+                NotetypeConfig::new()
+            },
+            ..Notetype::default()
+        }
+    }
+}
+
+impl ForeignTemplate {
+    fn into_native(self) -> CardTemplate {
+        CardTemplate::new(self.name, self.qfmt, self.afmt)
+    }
+}

rslib/src/import_export/text/json.rs

@@ -0,0 +1,14 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use crate::{
+    import_export::{text::ForeignData, NoteLog},
+    prelude::*,
+};
+
+impl Collection {
+    pub fn import_json(&mut self, json: &str) -> Result<OpOutput<NoteLog>> {
+        let data: ForeignData = serde_json::from_str(json)?;
+        data.import(self)
+    }
+}

rslib/src/import_export/text/mod.rs

@@ -1,20 +1,53 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
-#![allow(dead_code, unused_imports, unused_variables)]
 
 pub mod csv;
+pub mod import;
+mod json;
 
-use crate::prelude::*;
+use serde_derive::{Deserialize, Serialize};
 
-#[derive(Debug)]
+#[derive(Debug, Default, Serialize, Deserialize)]
+#[serde(default)]
 pub struct ForeignData {
-    default_deck: DeckId,
+    default_deck: String,
-    default_notetype: NotetypeId,
+    default_notetype: String,
     notes: Vec<ForeignNote>,
+    notetypes: Vec<ForeignNotetype>,
 }
 
-#[derive(Debug, PartialEq, Default)]
+#[derive(Debug, PartialEq, Default, Serialize, Deserialize)]
+#[serde(default)]
 pub struct ForeignNote {
     fields: Vec<String>,
     tags: Vec<String>,
+    notetype: String,
+    deck: String,
+    cards: Vec<ForeignCard>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Default, Serialize, Deserialize)]
+#[serde(default)]
+pub struct ForeignCard {
+    pub due: i32,
+    pub ivl: u32,
+    pub factor: u16,
+    pub reps: u32,
+    pub lapses: u32,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct ForeignNotetype {
+    name: String,
+    fields: Vec<String>,
+    templates: Vec<ForeignTemplate>,
+    #[serde(default)]
+    is_cloze: bool,
+}
+
+#[derive(Debug, PartialEq, Serialize, Deserialize)]
+pub struct ForeignTemplate {
+    name: String,
+    qfmt: String,
+    afmt: String,
 }

rslib/src/notetype/cloze_styling.css

@@ -0,0 +1,7 @@
+.cloze {
+ font-weight: bold;
+ color: blue;
+}
+.nightMode .cloze {
+ color: lightblue;
+}

rslib/src/notetype/mod.rs

@@ -53,6 +53,7 @@ use crate::{
 define_newtype!(NotetypeId, i64);
 
 pub(crate) const DEFAULT_CSS: &str = include_str!("styling.css");
+pub(crate) const DEFAULT_CLOZE_CSS: &str = include_str!("cloze_styling.css");
 pub(crate) const DEFAULT_LATEX_HEADER: &str = include_str!("header.tex");
 pub(crate) const DEFAULT_LATEX_FOOTER: &str = r"\end{document}";
 lazy_static! {
@@ -88,16 +89,29 @@ impl Default for Notetype {
             usn: Usn(0),
             fields: vec![],
             templates: vec![],
-            config: NotetypeConfig {
+            config: NotetypeConfig::new(),
-                css: DEFAULT_CSS.into(),
-                latex_pre: DEFAULT_LATEX_HEADER.into(),
-                latex_post: DEFAULT_LATEX_FOOTER.into(),
-                ..Default::default()
-            },
         }
     }
 }
 
+impl NotetypeConfig {
+    pub(crate) fn new() -> Self {
+        NotetypeConfig {
+            css: DEFAULT_CSS.into(),
+            latex_pre: DEFAULT_LATEX_HEADER.into(),
+            latex_post: DEFAULT_LATEX_FOOTER.into(),
+            ..Default::default()
+        }
+    }
+
+    pub(crate) fn new_cloze() -> Self {
+        let mut config = Self::new();
+        config.css += DEFAULT_CLOZE_CSS;
+        config.kind = NotetypeKind::Cloze as i32;
+        config
+    }
+}
+
 impl Notetype {
     pub fn new_note(&self) -> Note {
         Note::new(self)

rslib/src/notetype/stock.rs

@@ -1,7 +1,7 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-use super::NotetypeKind;
+use super::NotetypeConfig;
 use crate::{
     backend_proto::stock_notetype::Kind,
     config::{ConfigEntry, ConfigKey},
@@ -112,6 +112,7 @@ pub(crate) fn basic_optional_reverse(tr: &I18n) -> Notetype {
 pub(crate) fn cloze(tr: &I18n) -> Notetype {
     let mut nt = Notetype {
         name: tr.notetypes_cloze_name().into(),
+        config: NotetypeConfig::new_cloze(),
         ..Default::default()
     };
     let text = tr.notetypes_text_field();
@@ -121,15 +122,5 @@ pub(crate) fn cloze(tr: &I18n) -> Notetype {
     let qfmt = format!("{{{{cloze:{}}}}}", text);
     let afmt = format!("{}<br>\n{{{{{}}}}}", qfmt, back_extra);
     nt.add_template(nt.name.clone(), qfmt, afmt);
-    nt.config.kind = NotetypeKind::Cloze as i32;
-    nt.config.css += "
-.cloze {
- font-weight: bold;
- color: blue;
-}
-.nightMode .cloze {
- color: lightblue;
-}
-";
     nt
 }