load/save note types in backend

This allows us to normalize bad data, and is the first step towards
splitting note types into separate tables.

proto/backend.proto

@@ -65,6 +65,8 @@ message BackendInput {
         SetConfigJson set_config_json = 53;
         bytes set_all_config = 54;
         Empty get_all_config = 55;
+        Empty get_all_notetypes = 56;
+        bytes set_all_notetypes = 57;
     }
 }
 
@@ -113,6 +115,8 @@ message BackendOutput {
         Empty set_config_json = 53;
         Empty set_all_config = 54;
         bytes get_all_config = 55;
+        bytes get_all_notetypes = 56;
+        Empty set_all_notetypes = 57;
 
         BackendError error = 2047;
     }

pylib/anki/collection.py

@@ -160,15 +160,15 @@ class _Collection:
             self.dty,  # no longer used
             self._usn,
             self.ls,
-            models,
             decks,
         ) = self.db.first(
             """
 select crt, mod, scm, dty, usn, ls,
-models, decks from col"""
+decks from col"""
         )
-        self.models.load(models)
         self.decks.load(decks)
+        self.models.models = self.backend.get_all_notetypes()
+        self.models.changed = False
 
     def setMod(self) -> None:
         """Mark DB modified.

pylib/anki/models.py

@@ -4,7 +4,6 @@
 from __future__ import annotations
 
 import copy
-import json
 import re
 import time
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
@@ -95,11 +94,6 @@ class ModelManager:
         self.models = {}
         self.changed = False
 
-    def load(self, json_: str) -> None:
-        "Load registry from JSON."
-        self.changed = False
-        self.models = json.loads(json_)
-
     def save(
         self,
         m: Optional[NoteType] = None,
@@ -121,7 +115,7 @@ class ModelManager:
         "Flush the registry if any models were changed."
         if self.changed:
             self.ensureNotEmpty()
-            self.col.db.execute("update col set models = ?", json.dumps(self.models))
+            self.col.backend.set_all_notetypes(self.models)
             self.changed = False
 
     def ensureNotEmpty(self) -> Optional[bool]:

pylib/anki/rsbackend.py

@@ -113,6 +113,8 @@ def proto_exception_to_native(err: pb.BackendError) -> Exception:
         return TemplateError(err.localized)
     elif val == "invalid_input":
         return StringError(err.localized)
+    elif val == "json_error":
+        return StringError(err.localized)
     else:
         assert_impossible_literal(val)
 
@@ -606,6 +608,15 @@ class RustBackend:
     def set_all_config(self, conf: Dict[str, Any]):
         self._run_command(pb.BackendInput(set_all_config=orjson.dumps(conf)))
 
+    def get_all_notetypes(self) -> Dict[str, Dict[str, Any]]:
+        jstr = self._run_command(
+            pb.BackendInput(get_all_notetypes=pb.Empty())
+        ).get_all_notetypes
+        return orjson.loads(jstr)
+
+    def set_all_notetypes(self, nts: Dict[str, Dict[str, Any]]):
+        self._run_command(pb.BackendInput(set_all_notetypes=orjson.dumps(nts)))
+
 
 def translate_string_in(
     key: TR, **kwargs: Union[str, int, float]

rslib/src/backend/mod.rs

@@ -1,35 +1,39 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-use crate::backend::dbproxy::db_command_bytes;
+use crate::{
-use crate::backend_proto::{
+    backend::dbproxy::db_command_bytes,
+    backend_proto as pb,
+    backend_proto::{
         AddOrUpdateDeckConfigIn, BuiltinSortKind, Empty, RenderedTemplateReplacement, SyncMediaIn,
+    },
+    card::{Card, CardID},
+    card::{CardQueue, CardType},
+    collection::{open_collection, Collection},
+    config::SortKind,
+    deckconf::{DeckConf, DeckConfID},
+    decks::DeckID,
+    err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind},
+    i18n::{tr_args, I18n, TR},
+    latex::{extract_latex, extract_latex_expanding_clozes, ExtractedLatex},
+    log,
+    log::{default_logger, Logger},
+    media::check::MediaChecker,
+    media::sync::MediaSyncProgress,
+    media::MediaManager,
+    notes::NoteID,
+    notetype::{NoteType, NoteTypeID},
+    sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today},
+    sched::timespan::{answer_button_time, learning_congrats, studied_today, time_span},
+    search::{search_cards, search_notes, SortMode},
+    template::{
+        render_card, without_legacy_template_directives, FieldMap, FieldRequirements,
+        ParsedTemplate, RenderedNode,
+    },
+    text::{extract_av_tags, strip_av_tags, AVTag},
+    timestamp::TimestampSecs,
+    types::Usn,
 };
-use crate::card::{Card, CardID};
-use crate::card::{CardQueue, CardType};
-use crate::collection::{open_collection, Collection};
-use crate::config::SortKind;
-use crate::deckconf::{DeckConf, DeckConfID};
-use crate::decks::DeckID;
-use crate::err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind};
-use crate::i18n::{tr_args, I18n, TR};
-use crate::latex::{extract_latex, extract_latex_expanding_clozes, ExtractedLatex};
-use crate::log::{default_logger, Logger};
-use crate::media::check::MediaChecker;
-use crate::media::sync::MediaSyncProgress;
-use crate::media::MediaManager;
-use crate::notes::NoteID;
-use crate::sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today};
-use crate::sched::timespan::{answer_button_time, learning_congrats, studied_today, time_span};
-use crate::search::{search_cards, search_notes, SortMode};
-use crate::template::{
-    render_card, without_legacy_template_directives, FieldMap, FieldRequirements, ParsedTemplate,
-    RenderedNode,
-};
-use crate::text::{extract_av_tags, strip_av_tags, AVTag};
-use crate::timestamp::TimestampSecs;
-use crate::types::Usn;
-use crate::{backend_proto as pb, log};
 use fluent::FluentValue;
 use futures::future::{AbortHandle, Abortable};
 use log::error;
@@ -304,6 +308,11 @@ impl Backend {
                 pb::Empty {}
             }),
             Value::GetAllConfig(_) => OValue::GetAllConfig(self.get_all_config()?),
+            Value::GetAllNotetypes(_) => OValue::GetAllNotetypes(self.get_all_notetypes()?),
+            Value::SetAllNotetypes(bytes) => {
+                self.set_all_notetypes(&bytes)?;
+                OValue::SetAllNotetypes(pb::Empty {})
+            }
         })
     }
 
@@ -840,6 +849,23 @@ impl Backend {
             serde_json::to_vec(&conf).map_err(Into::into)
         })
     }
+
+    fn set_all_notetypes(&self, json: &[u8]) -> Result<()> {
+        let val: HashMap<NoteTypeID, NoteType> = serde_json::from_slice(json)?;
+        self.with_col(|col| {
+            col.transact(None, |col| {
+                col.storage
+                    .set_all_notetypes(val, col.usn()?, TimestampSecs::now())
+            })
+        })
+    }
+
+    fn get_all_notetypes(&self) -> Result<Vec<u8>> {
+        self.with_col(|col| {
+            let nts = col.storage.get_all_notetypes()?;
+            serde_json::to_vec(&nts).map_err(Into::into)
+        })
+    }
 }
 
 fn translate_arg_to_fluent_val(arg: &pb::TranslateArgValue) -> FluentValue {

rslib/src/lib.rs

@@ -22,7 +22,7 @@ pub mod latex;
 pub mod log;
 pub mod media;
 pub mod notes;
-pub mod notetypes;
+pub mod notetype;
 pub mod sched;
 pub mod search;
 pub mod serde;

rslib/src/media/check.rs

@@ -380,7 +380,7 @@ where
         renamed: &HashMap<String, String>,
     ) -> Result<HashSet<String>> {
         let mut referenced_files = HashSet::new();
-        let note_types = self.ctx.storage.all_note_types()?;
+        let note_types = self.ctx.storage.get_all_notetypes()?;
         let mut collection_modified = false;
 
         for_every_note(&self.ctx.storage.db, |note| {

rslib/src/notes.rs

@@ -4,10 +4,10 @@
 /// At the moment, this is just basic note reading/updating functionality for
 /// the media DB check.
 use crate::err::{AnkiError, DBErrorKind, Result};
-use crate::notetypes::NoteTypeID;
+use crate::notetype::NoteTypeID;
 use crate::text::strip_html_preserving_image_filenames;
 use crate::timestamp::TimestampSecs;
-use crate::{define_newtype, notetypes::NoteType, types::Usn};
+use crate::{define_newtype, notetype::NoteType, types::Usn};
 use rusqlite::{params, Connection, Row, NO_PARAMS};
 use std::convert::TryInto;
 

rslib/src/notetype/field.rs

@@ -0,0 +1,35 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use crate::serde::deserialize_bool_from_anything;
+use serde_derive::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::HashMap;
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct NoteField {
+    pub(crate) name: String,
+    pub(crate) ord: u16,
+    #[serde(deserialize_with = "deserialize_bool_from_anything")]
+    pub(crate) sticky: bool,
+    #[serde(deserialize_with = "deserialize_bool_from_anything")]
+    pub(crate) rtl: bool,
+    pub(crate) font: String,
+    pub(crate) size: u16,
+    #[serde(flatten)]
+    pub(crate) other: HashMap<String, Value>,
+}
+
+impl Default for NoteField {
+    fn default() -> Self {
+        Self {
+            name: String::new(),
+            ord: 0,
+            sticky: false,
+            rtl: false,
+            font: "Arial".to_string(),
+            size: 20,
+            other: Default::default(),
+        }
+    }
+}

rslib/src/notetype/mod.rs

@@ -0,0 +1,139 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+mod field;
+mod template;
+
+pub use field::NoteField;
+pub use template::CardTemplate;
+
+use crate::{
+    decks::DeckID,
+    define_newtype,
+    serde::{default_on_invalid, deserialize_number_from_string},
+    timestamp::TimestampSecs,
+    types::Usn,
+};
+use serde_derive::{Deserialize, Serialize};
+use serde_json::Value;
+use serde_repr::{Deserialize_repr, Serialize_repr};
+use serde_tuple::Serialize_tuple;
+use std::collections::HashMap;
+
+define_newtype!(NoteTypeID, i64);
+
+pub(crate) const DEFAULT_CSS: &str = "\
+.card {
+ font-family: arial;
+ font-size: 20px;
+ text-align: center;
+ color: black;
+ background-color: white;
+}
+";
+
+pub(crate) const DEFAULT_LATEX_HEADER: &str = r#"\documentclass[12pt]{article}
+\special{papersize=3in,5in}
+\usepackage[utf8]{inputenc}
+\usepackage{amssymb,amsmath}
+\pagestyle{empty}
+\setlength{\parindent}{0in}
+\begin{document}
+"#;
+
+pub(crate) const DEFAULT_LATEX_FOOTER: &str = r#"\end{document}"#;
+
+#[derive(Serialize_repr, Deserialize_repr, PartialEq, Debug, Clone)]
+#[repr(u8)]
+pub enum NoteTypeKind {
+    Standard = 0,
+    Cloze = 1,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct NoteType {
+    #[serde(deserialize_with = "deserialize_number_from_string")]
+    pub(crate) id: NoteTypeID,
+    pub(crate) name: String,
+    #[serde(rename = "type")]
+    pub(crate) kind: NoteTypeKind,
+    #[serde(rename = "mod")]
+    pub(crate) mtime: TimestampSecs,
+    pub(crate) usn: Usn,
+    #[serde(rename = "sortf")]
+    pub(crate) sort_field_idx: u16,
+    #[serde(rename = "did", deserialize_with = "default_on_invalid")]
+    pub(crate) deck_id_for_adding: Option<DeckID>,
+    #[serde(rename = "tmpls")]
+    pub(crate) templates: Vec<CardTemplate>,
+    #[serde(rename = "flds")]
+    pub(crate) fields: Vec<NoteField>,
+    #[serde(deserialize_with = "default_on_invalid")]
+    pub(crate) css: String,
+    #[serde(default)]
+    pub(crate) latex_pre: String,
+    #[serde(default)]
+    pub(crate) latex_post: String,
+    #[serde(rename = "latexsvg", default)]
+    pub latex_svg: bool,
+    #[serde(default, deserialize_with = "default_on_invalid")]
+    pub(crate) req: CardRequirements,
+    #[serde(default, deserialize_with = "default_on_invalid")]
+    pub(crate) tags: Vec<String>,
+    #[serde(flatten)]
+    pub(crate) other: HashMap<String, Value>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub(crate) struct CardRequirements(pub(crate) Vec<CardRequirement>);
+
+impl Default for CardRequirements {
+    fn default() -> Self {
+        CardRequirements(vec![])
+    }
+}
+
+#[derive(Serialize_tuple, Deserialize, Debug, Clone)]
+pub(crate) struct CardRequirement {
+    pub(crate) card_ord: u16,
+    pub(crate) kind: FieldRequirementKind,
+    pub(crate) field_ords: Vec<u16>,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "lowercase")]
+pub enum FieldRequirementKind {
+    Any,
+    All,
+    None,
+}
+
+impl Default for NoteType {
+    fn default() -> Self {
+        Self {
+            id: NoteTypeID(0),
+            name: String::new(),
+            kind: NoteTypeKind::Standard,
+            mtime: TimestampSecs(0),
+            usn: Usn(0),
+            sort_field_idx: 0,
+            deck_id_for_adding: None,
+            fields: vec![],
+            templates: vec![],
+            css: DEFAULT_CSS.to_owned(),
+            latex_pre: DEFAULT_LATEX_HEADER.to_owned(),
+            latex_post: DEFAULT_LATEX_FOOTER.to_owned(),
+            req: Default::default(),
+            tags: vec![],
+            latex_svg: false,
+            other: Default::default(),
+        }
+    }
+}
+
+impl NoteType {
+    pub fn latex_uses_svg(&self) -> bool {
+        self.latex_svg
+    }
+}

rslib/src/notetype/template.rs

@@ -0,0 +1,28 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use crate::{decks::DeckID, serde::default_on_invalid};
+use serde_derive::{Deserialize, Serialize};
+use serde_json::Value;
+use std::collections::HashMap;
+
+#[derive(Serialize, Deserialize, Debug, Default, Clone)]
+pub struct CardTemplate {
+    pub(crate) name: String,
+    pub(crate) ord: u16,
+    pub(crate) qfmt: String,
+    #[serde(default)]
+    pub(crate) afmt: String,
+    #[serde(default)]
+    pub(crate) bqfmt: String,
+    #[serde(default)]
+    pub(crate) bafmt: String,
+    #[serde(rename = "did", deserialize_with = "default_on_invalid")]
+    pub(crate) override_did: Option<DeckID>,
+    #[serde(default, deserialize_with = "default_on_invalid")]
+    pub(crate) bfont: String,
+    #[serde(default, deserialize_with = "default_on_invalid")]
+    pub(crate) bsize: u8,
+    #[serde(flatten)]
+    pub(crate) other: HashMap<String, Value>,
+}

rslib/src/notetypes.rs

@@ -1,41 +0,0 @@
-// Copyright: Ankitects Pty Ltd and contributors
-// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
-
-use crate::define_newtype;
-use serde_aux::field_attributes::deserialize_number_from_string;
-use serde_derive::Deserialize;
-
-define_newtype!(NoteTypeID, i64);
-
-#[derive(Deserialize, Debug)]
-pub(crate) struct NoteType {
-    #[serde(deserialize_with = "deserialize_number_from_string")]
-    pub id: NoteTypeID,
-    pub name: String,
-    #[serde(rename = "sortf")]
-    pub sort_field_idx: u16,
-    #[serde(rename = "latexsvg", default)]
-    pub latex_svg: bool,
-    #[serde(rename = "tmpls")]
-    pub templates: Vec<CardTemplate>,
-    #[serde(rename = "flds")]
-    pub fields: Vec<NoteField>,
-}
-
-#[derive(Deserialize, Debug)]
-pub(crate) struct CardTemplate {
-    pub name: String,
-    pub ord: u16,
-}
-
-#[derive(Deserialize, Debug)]
-pub(crate) struct NoteField {
-    pub name: String,
-    pub ord: u16,
-}
-
-impl NoteType {
-    pub fn latex_uses_svg(&self) -> bool {
-        self.latex_svg
-    }
-}

rslib/src/search/cards.rs

@@ -113,7 +113,7 @@ fn prepare_sort(req: &mut Collection, kind: &SortKind) -> Result<()> {
                     }
                 }
                 NoteType => {
-                    for (k, v) in req.storage.all_note_types()? {
+                    for (k, v) in req.storage.get_all_notetypes()? {
                         stmt.execute(params![k, v.name])?;
                     }
                 }
@@ -127,7 +127,7 @@ fn prepare_sort(req: &mut Collection, kind: &SortKind) -> Result<()> {
                 .db
                 .prepare("insert into sort_order (k1,k2,v) values (?,?,?)")?;
 
-            for (ntid, nt) in req.storage.all_note_types()? {
+            for (ntid, nt) in req.storage.get_all_notetypes()? {
                 for tmpl in nt.templates {
                     stmt.execute(params![ntid, tmpl.ord, tmpl.name])?;
                 }

rslib/src/search/parser.rs

@@ -2,7 +2,7 @@
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
 use crate::err::{AnkiError, Result};
-use crate::notetypes::NoteTypeID;
+use crate::notetype::NoteTypeID;
 use nom::branch::alt;
 use nom::bytes::complete::{escaped, is_not, tag, take_while1};
 use nom::character::complete::{anychar, char, one_of};

rslib/src/search/sqlwriter.rs

@@ -7,7 +7,7 @@ use crate::decks::child_ids;
 use crate::decks::get_deck;
 use crate::err::{AnkiError, Result};
 use crate::notes::field_checksum;
-use crate::notetypes::NoteTypeID;
+use crate::notetype::NoteTypeID;
 use crate::text::matches_wildcard;
 use crate::text::without_combining;
 use crate::{collection::Collection, text::strip_html_preserving_image_filenames};
@@ -263,7 +263,7 @@ impl SqlWriter<'_> {
                 write!(self.sql, "c.ord = {}", n).unwrap();
             }
             TemplateKind::Name(name) => {
-                let note_types = self.col.storage.all_note_types()?;
+                let note_types = self.col.storage.get_all_notetypes()?;
                 let mut id_ords = vec![];
                 for nt in note_types.values() {
                     for tmpl in &nt.templates {
@@ -294,7 +294,7 @@ impl SqlWriter<'_> {
         let mut ntids: Vec<_> = self
             .col
             .storage
-            .all_note_types()?
+            .get_all_notetypes()?
             .values()
             .filter(|nt| matches_wildcard(&nt.name, nt_name))
             .map(|nt| nt.id)
@@ -307,7 +307,7 @@ impl SqlWriter<'_> {
     }
 
     fn write_single_field(&mut self, field_name: &str, val: &str, is_re: bool) -> Result<()> {
-        let note_types = self.col.storage.all_note_types()?;
+        let note_types = self.col.storage.get_all_notetypes()?;
 
         let mut field_map = vec![];
         for nt in note_types.values() {

rslib/src/serde.rs

@@ -2,6 +2,9 @@
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
 use serde::Deserialize as DeTrait;
+pub(crate) use serde_aux::field_attributes::{
+    deserialize_bool_from_anything, deserialize_number_from_string,
+};
 use serde_json::Value;
 
 pub(crate) fn default_on_invalid<'de, T, D>(deserializer: D) -> Result<T, D::Error>

rslib/src/storage/mod.rs

@@ -4,6 +4,7 @@
 mod card;
 mod config;
 mod deckconf;
+mod notetype;
 mod sqlite;
 mod tag;
 mod upgrades;

rslib/src/storage/notetype/mod.rs

@@ -0,0 +1,41 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use super::SqliteStorage;
+use crate::{
+    err::{AnkiError, DBErrorKind, Result},
+    notetype::{NoteType, NoteTypeID},
+    timestamp::TimestampSecs,
+    types::Usn,
+};
+use rusqlite::NO_PARAMS;
+use std::collections::HashMap;
+
+impl SqliteStorage {
+    pub(crate) fn get_all_notetypes(&self) -> Result<HashMap<NoteTypeID, NoteType>> {
+        let mut stmt = self.db.prepare("select models from col")?;
+        let note_types = stmt
+            .query_and_then(NO_PARAMS, |row| -> Result<HashMap<NoteTypeID, NoteType>> {
+                let v: HashMap<NoteTypeID, NoteType> =
+                    serde_json::from_str(row.get_raw(0).as_str()?)?;
+                Ok(v)
+            })?
+            .next()
+            .ok_or_else(|| AnkiError::DBError {
+                info: "col table empty".to_string(),
+                kind: DBErrorKind::MissingEntity,
+            })??;
+        Ok(note_types)
+    }
+
+    pub(crate) fn set_all_notetypes(
+        &self,
+        notetypes: HashMap<NoteTypeID, NoteType>,
+        _usn: Usn,
+        _mtime: TimestampSecs,
+    ) -> Result<()> {
+        let json = serde_json::to_string(&notetypes)?;
+        self.db.execute("update col set models = ?", &[json])?;
+        Ok(())
+    }
+}

rslib/src/storage/sqlite.rs

@@ -5,9 +5,8 @@ use crate::config::schema11_config_as_string;
 use crate::decks::DeckID;
 use crate::err::Result;
 use crate::err::{AnkiError, DBErrorKind};
-use crate::notetypes::NoteTypeID;
 use crate::timestamp::{TimestampMillis, TimestampSecs};
-use crate::{decks::Deck, i18n::I18n, notetypes::NoteType, text::without_combining, types::Usn};
+use crate::{decks::Deck, i18n::I18n, text::without_combining, types::Usn};
 use regex::Regex;
 use rusqlite::{functions::FunctionFlags, params, Connection, NO_PARAMS};
 use std::cmp::Ordering;
@@ -292,22 +291,6 @@ impl SqliteStorage {
             })
     }
 
-    pub(crate) fn all_note_types(&self) -> Result<HashMap<NoteTypeID, NoteType>> {
-        let mut stmt = self.db.prepare("select models from col")?;
-        let note_types = stmt
-            .query_and_then(NO_PARAMS, |row| -> Result<HashMap<NoteTypeID, NoteType>> {
-                let v: HashMap<NoteTypeID, NoteType> =
-                    serde_json::from_str(row.get_raw(0).as_str()?)?;
-                Ok(v)
-            })?
-            .next()
-            .ok_or_else(|| AnkiError::DBError {
-                info: "col table empty".to_string(),
-                kind: DBErrorKind::MissingEntity,
-            })??;
-        Ok(note_types)
-    }
-
     pub(crate) fn creation_stamp(&self) -> Result<TimestampSecs> {
         self.db
             .prepare_cached("select crt from col")?