split note types into separate tables

- store the config in protobuf instead of json
- still loading+saving in bulk for now
- code using the schema11 structs needs to be migrated

proto/backend.proto

@@ -140,6 +140,7 @@ message BackendError {
         // user interrupted operation
         Empty interrupted = 8;
         string json_error = 9;
+        string proto_error = 10;
     }
 }
 
@@ -479,3 +480,70 @@ message SetConfigJson {
         Empty remove = 3;
     }
 }
+
+message NoteFieldConfig {
+    bool sticky = 1;
+    bool rtl = 2;
+    string font_name = 3;
+    uint32 font_size = 4;
+    bytes other = 5;
+}
+
+message NoteField {
+    uint32 ord = 1;
+    string name = 2;
+    NoteFieldConfig config = 5;
+}
+
+message CardTemplateConfig {
+    string q_format = 1;
+    string a_format = 2;
+    string q_format_browser = 3;
+    string a_format_browser= 4;
+    int64 target_deck_id = 5;
+    string browser_font_name = 6;
+    uint32 browser_font_size = 7;
+    bytes other = 8;
+}
+
+message CardTemplate {
+    uint32 ord = 1;
+    string name = 2;
+    uint32 mtime_secs = 3;
+    sint32 usn = 4;
+    CardTemplateConfig config = 5;
+}
+
+enum NoteTypeKind {
+    NORMAL = 0;
+    CLOZE = 1;
+}
+
+message NoteTypeConfig {
+    NoteTypeKind kind = 1;
+    uint32 sort_field_idx = 2;
+    string css = 3;
+    // fixme: anki currently sets this without flushing
+    int64 target_deck_id = 4;
+    string latex_pre = 5;
+    string latex_post = 6;
+    bool latex_svg = 7;
+    repeated CardRequirement reqs = 8;
+    bytes other = 9;
+}
+
+message CardRequirement {
+    uint32 card_ord = 1;
+    uint32 kind = 2;
+    repeated uint32 field_ords = 3;
+}
+
+message NoteType {
+    int64 id = 1;
+    string name = 2;
+    uint32 mtime_secs = 3;
+    sint32 usn = 4;
+    NoteTypeConfig config = 7;
+    repeated NoteField fields = 8;
+    repeated CardTemplate templates = 9;
+}

rslib/src/backend/mod.rs

@@ -21,7 +21,7 @@ use crate::{
     media::sync::MediaSyncProgress,
     media::MediaManager,
     notes::NoteID,
-    notetype::{NoteType, NoteTypeID},
+    notetype::{NoteTypeID, NoteTypeSchema11},
     sched::cutoff::{local_minutes_west_for_stamp, sched_timing_today},
     sched::timespan::{answer_button_time, learning_congrats, studied_today, time_span},
     search::{search_cards, search_notes, SortMode},
@@ -80,6 +80,7 @@ fn anki_error_to_proto_error(err: AnkiError, i18n: &I18n) -> pb::BackendError {
         AnkiError::CollectionAlreadyOpen => V::InvalidInput(pb::Empty {}),
         AnkiError::SchemaChange => V::InvalidInput(pb::Empty {}),
         AnkiError::JSONError { info } => V::JsonError(info),
+        AnkiError::ProtoError { info } => V::ProtoError(info),
     };
 
     pb::BackendError {
@@ -855,18 +856,18 @@ impl Backend {
     }
 
     fn set_all_notetypes(&self, json: &[u8]) -> Result<()> {
-        let val: HashMap<NoteTypeID, NoteType> = serde_json::from_slice(json)?;
+        let val: HashMap<NoteTypeID, NoteTypeSchema11> = serde_json::from_slice(json)?;
         self.with_col(|col| {
             col.transact(None, |col| {
-                col.storage
+                col.storage.set_schema11_notetypes(val)?;
-                    .set_all_notetypes(val, col.usn()?, TimestampSecs::now())
+                col.storage.upgrade_notetypes_to_schema15()
             })
         })
     }
 
     fn get_all_notetypes(&self) -> Result<Vec<u8>> {
         self.with_col(|col| {
-            let nts = col.storage.get_all_notetypes()?;
+            let nts = col.storage.get_all_notetypes_as_schema11()?;
             serde_json::to_vec(&nts).map_err(Into::into)
         })
     }

rslib/src/err.rs

@@ -34,6 +34,9 @@ pub enum AnkiError {
     #[fail(display = "JSON encode/decode error: {}", info)]
     JSONError { info: String },
 
+    #[fail(display = "Protobuf encode/decode error: {}", info)]
+    ProtoError { info: String },
+
     #[fail(display = "The user interrupted the operation.")]
     Interrupted,
 
@@ -232,6 +235,22 @@ impl From<serde_json::Error> for AnkiError {
     }
 }
 
+impl From<prost::EncodeError> for AnkiError {
+    fn from(err: prost::EncodeError) -> Self {
+        AnkiError::ProtoError {
+            info: err.to_string(),
+        }
+    }
+}
+
+impl From<prost::DecodeError> for AnkiError {
+    fn from(err: prost::DecodeError) -> Self {
+        AnkiError::ProtoError {
+            info: err.to_string(),
+        }
+    }
+}
+
 #[derive(Debug, PartialEq)]
 pub enum DBErrorKind {
     FileTooNew,

rslib/src/media/check.rs

@@ -380,7 +380,7 @@ where
         renamed: &HashMap<String, String>,
     ) -> Result<HashSet<String>> {
         let mut referenced_files = HashSet::new();
-        let note_types = self.ctx.storage.get_all_notetypes()?;
+        let note_types = self.ctx.storage.get_all_notetypes_as_schema11()?;
         let mut collection_modified = false;
 
         for_every_note(&self.ctx.storage.db, |note| {

rslib/src/notes.rs

@@ -7,7 +7,7 @@ use crate::err::{AnkiError, DBErrorKind, Result};
 use crate::notetype::NoteTypeID;
 use crate::text::strip_html_preserving_image_filenames;
 use crate::timestamp::TimestampSecs;
-use crate::{define_newtype, notetype::NoteType, types::Usn};
+use crate::{define_newtype, notetype::NoteTypeSchema11, types::Usn};
 use rusqlite::{params, Connection, Row, NO_PARAMS};
 use std::convert::TryInto;
 
@@ -84,7 +84,11 @@ fn row_to_note(row: &Row) -> Result<Note> {
     })
 }
 
-pub(super) fn set_note(db: &Connection, note: &mut Note, note_type: &NoteType) -> Result<()> {
+pub(super) fn set_note(
+    db: &Connection,
+    note: &mut Note,
+    note_type: &NoteTypeSchema11,
+) -> Result<()> {
     note.mtime = TimestampSecs::now();
     // hard-coded for now
     note.usn = Usn(-1);

rslib/src/notetype/field.rs

@@ -1,13 +1,16 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-use crate::serde::deserialize_bool_from_anything;
+use crate::{
+    backend_proto::{NoteField as NoteFieldProto, NoteFieldConfig},
+    serde::deserialize_bool_from_anything,
+};
 use serde_derive::{Deserialize, Serialize};
 use serde_json::Value;
 use std::collections::HashMap;
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
-pub struct NoteField {
+pub struct NoteFieldSchema11 {
     pub(crate) name: String,
     pub(crate) ord: u16,
     #[serde(deserialize_with = "deserialize_bool_from_anything")]
@@ -20,7 +23,7 @@ pub struct NoteField {
     pub(crate) other: HashMap<String, Value>,
 }
 
-impl Default for NoteField {
+impl Default for NoteFieldSchema11 {
     fn default() -> Self {
         Self {
             name: String::new(),
@@ -33,3 +36,34 @@ impl Default for NoteField {
         }
     }
 }
+
+impl From<NoteFieldSchema11> for NoteFieldProto {
+    fn from(f: NoteFieldSchema11) -> Self {
+        NoteFieldProto {
+            ord: f.ord as u32,
+            name: f.name,
+            config: Some(NoteFieldConfig {
+                sticky: f.sticky,
+                rtl: f.rtl,
+                font_name: f.font,
+                font_size: f.size as u32,
+                other: serde_json::to_vec(&f.other).unwrap(),
+            }),
+        }
+    }
+}
+
+impl From<NoteFieldProto> for NoteFieldSchema11 {
+    fn from(p: NoteFieldProto) -> Self {
+        let conf = p.config.unwrap();
+        NoteFieldSchema11 {
+            name: p.name,
+            ord: p.ord as u16,
+            sticky: conf.sticky,
+            rtl: conf.rtl,
+            font: conf.font_name,
+            size: conf.font_size as u16,
+            other: serde_json::from_slice(&conf.other).unwrap(),
+        }
+    }
+}

rslib/src/notetype/mod.rs

@@ -4,13 +4,17 @@
 mod field;
 mod template;
 
-pub use field::NoteField;
+pub use field::NoteFieldSchema11;
-pub use template::CardTemplate;
+pub use template::CardTemplateSchema11;
 
 use crate::{
+    backend_proto::{
+        CardRequirement as CardRequirementProto, NoteType as NoteTypeProto, NoteTypeConfig,
+    },
     decks::DeckID,
     define_newtype,
     serde::{default_on_invalid, deserialize_number_from_string},
+    text::ensure_string_in_nfc,
     timestamp::TimestampSecs,
     types::Usn,
 };
@@ -18,7 +22,8 @@ use serde_derive::{Deserialize, Serialize};
 use serde_json::Value;
 use serde_repr::{Deserialize_repr, Serialize_repr};
 use serde_tuple::Serialize_tuple;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
+use unicase::UniCase;
 
 define_newtype!(NoteTypeID, i64);
 
@@ -52,7 +57,7 @@ pub enum NoteTypeKind {
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(rename_all = "camelCase")]
-pub struct NoteType {
+pub struct NoteTypeSchema11 {
     #[serde(deserialize_with = "deserialize_number_from_string")]
     pub(crate) id: NoteTypeID,
     pub(crate) name: String,
@@ -66,9 +71,9 @@ pub struct NoteType {
     #[serde(rename = "did", deserialize_with = "default_on_invalid")]
     pub(crate) deck_id_for_adding: Option<DeckID>,
     #[serde(rename = "tmpls")]
-    pub(crate) templates: Vec<CardTemplate>,
+    pub(crate) templates: Vec<CardTemplateSchema11>,
     #[serde(rename = "flds")]
-    pub(crate) fields: Vec<NoteField>,
+    pub(crate) fields: Vec<NoteFieldSchema11>,
     #[serde(deserialize_with = "default_on_invalid")]
     pub(crate) css: String,
     #[serde(default)]
@@ -79,8 +84,6 @@ pub struct NoteType {
     pub latex_svg: bool,
     #[serde(default, deserialize_with = "default_on_invalid")]
     pub(crate) req: CardRequirements,
-    #[serde(default, deserialize_with = "default_on_invalid")]
-    pub(crate) tags: Vec<String>,
     #[serde(flatten)]
     pub(crate) other: HashMap<String, Value>,
 }
@@ -109,7 +112,7 @@ pub enum FieldRequirementKind {
     None,
 }
 
-impl Default for NoteType {
+impl Default for NoteTypeSchema11 {
     fn default() -> Self {
         Self {
             id: NoteTypeID(0),
@@ -125,15 +128,130 @@ impl Default for NoteType {
             latex_pre: DEFAULT_LATEX_HEADER.to_owned(),
             latex_post: DEFAULT_LATEX_FOOTER.to_owned(),
             req: Default::default(),
-            tags: vec![],
             latex_svg: false,
             other: Default::default(),
         }
     }
 }
 
-impl NoteType {
+impl NoteTypeSchema11 {
     pub fn latex_uses_svg(&self) -> bool {
         self.latex_svg
     }
 }
+
+impl From<NoteTypeSchema11> for NoteTypeProto {
+    fn from(nt: NoteTypeSchema11) -> Self {
+        NoteTypeProto {
+            id: nt.id.0,
+            name: nt.name,
+            mtime_secs: nt.mtime.0 as u32,
+            usn: nt.usn.0,
+            config: Some(NoteTypeConfig {
+                kind: nt.kind as i32,
+                sort_field_idx: nt.sort_field_idx as u32,
+                css: nt.css,
+                target_deck_id: nt.deck_id_for_adding.unwrap_or(DeckID(0)).0,
+                latex_pre: nt.latex_pre,
+                latex_post: nt.latex_post,
+                latex_svg: nt.latex_svg,
+                reqs: nt.req.0.into_iter().map(Into::into).collect(),
+                other: serde_json::to_vec(&nt.other).unwrap(),
+            }),
+            fields: nt.fields.into_iter().map(Into::into).collect(),
+            templates: nt.templates.into_iter().map(Into::into).collect(),
+        }
+    }
+}
+
+impl From<NoteTypeProto> for NoteTypeSchema11 {
+    fn from(p: NoteTypeProto) -> Self {
+        let c = p.config.unwrap();
+        NoteTypeSchema11 {
+            id: NoteTypeID(p.id),
+            name: p.name,
+            kind: if c.kind == 1 {
+                NoteTypeKind::Cloze
+            } else {
+                NoteTypeKind::Standard
+            },
+            mtime: TimestampSecs(p.mtime_secs as i64),
+            usn: Usn(p.usn),
+            sort_field_idx: c.sort_field_idx as u16,
+            deck_id_for_adding: if c.target_deck_id == 0 {
+                None
+            } else {
+                Some(DeckID(c.target_deck_id))
+            },
+            templates: p.templates.into_iter().map(Into::into).collect(),
+            fields: p.fields.into_iter().map(Into::into).collect(),
+            css: c.css,
+            latex_pre: c.latex_pre,
+            latex_post: c.latex_post,
+            latex_svg: c.latex_svg,
+            req: CardRequirements(c.reqs.into_iter().map(Into::into).collect()),
+            other: serde_json::from_slice(&c.other).unwrap_or_default(),
+        }
+    }
+}
+
+impl From<CardRequirement> for CardRequirementProto {
+    fn from(r: CardRequirement) -> Self {
+        CardRequirementProto {
+            card_ord: r.card_ord as u32,
+            kind: r.kind as u32,
+            field_ords: r.field_ords.into_iter().map(|n| n as u32).collect(),
+        }
+    }
+}
+
+impl From<CardRequirementProto> for CardRequirement {
+    fn from(p: CardRequirementProto) -> Self {
+        CardRequirement {
+            card_ord: p.card_ord as u16,
+            kind: match p.kind {
+                0 => FieldRequirementKind::Any,
+                1 => FieldRequirementKind::All,
+                _ => FieldRequirementKind::None,
+            },
+            field_ords: p.field_ords.into_iter().map(|n| n as u16).collect(),
+        }
+    }
+}
+
+impl NoteTypeProto {
+    pub(crate) fn ensure_names_unique(&mut self) {
+        let mut names = HashSet::new();
+        for t in &mut self.templates {
+            loop {
+                let name = UniCase::new(t.name.clone());
+                if !names.contains(&name) {
+                    names.insert(name);
+                    break;
+                }
+                t.name.push('_');
+            }
+        }
+        names.clear();
+        for t in &mut self.fields {
+            loop {
+                let name = UniCase::new(t.name.clone());
+                if !names.contains(&name) {
+                    names.insert(name);
+                    break;
+                }
+                t.name.push('_');
+            }
+        }
+    }
+
+    pub(crate) fn normalize_names(&mut self) {
+        ensure_string_in_nfc(&mut self.name);
+        for f in &mut self.fields {
+            ensure_string_in_nfc(&mut f.name);
+        }
+        for t in &mut self.templates {
+            ensure_string_in_nfc(&mut t.name);
+        }
+    }
+}

rslib/src/notetype/template.rs

@@ -1,13 +1,17 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-use crate::{decks::DeckID, serde::default_on_invalid};
+use crate::{
+    backend_proto::{CardTemplate as CardTemplateProto, CardTemplateConfig},
+    decks::DeckID,
+    serde::default_on_invalid,
+};
 use serde_derive::{Deserialize, Serialize};
 use serde_json::Value;
 use std::collections::HashMap;
 
 #[derive(Serialize, Deserialize, Debug, Default, Clone)]
-pub struct CardTemplate {
+pub struct CardTemplateSchema11 {
     pub(crate) name: String,
     pub(crate) ord: u16,
     pub(crate) qfmt: String,
@@ -26,3 +30,46 @@ pub struct CardTemplate {
     #[serde(flatten)]
     pub(crate) other: HashMap<String, Value>,
 }
+
+impl From<CardTemplateSchema11> for CardTemplateProto {
+    fn from(t: CardTemplateSchema11) -> Self {
+        CardTemplateProto {
+            ord: t.ord as u32,
+            name: t.name,
+            mtime_secs: 0,
+            usn: 0,
+            config: Some(CardTemplateConfig {
+                q_format: t.qfmt,
+                a_format: t.afmt,
+                q_format_browser: t.bqfmt,
+                a_format_browser: t.bafmt,
+                target_deck_id: t.override_did.unwrap_or(DeckID(0)).0,
+                browser_font_name: t.bfont,
+                browser_font_size: t.bsize as u32,
+                other: serde_json::to_vec(&t.other).unwrap(),
+            }),
+        }
+    }
+}
+
+impl From<CardTemplateProto> for CardTemplateSchema11 {
+    fn from(p: CardTemplateProto) -> Self {
+        let conf = p.config.unwrap();
+        CardTemplateSchema11 {
+            name: p.name,
+            ord: p.ord as u16,
+            qfmt: conf.q_format,
+            afmt: conf.a_format,
+            bqfmt: conf.q_format_browser,
+            bafmt: conf.a_format_browser,
+            override_did: if conf.target_deck_id > 0 {
+                Some(DeckID(conf.target_deck_id))
+            } else {
+                None
+            },
+            bfont: conf.browser_font_name,
+            bsize: conf.browser_font_size as u8,
+            other: serde_json::from_slice(&conf.other).unwrap(),
+        }
+    }
+}

rslib/src/search/cards.rs

@@ -113,7 +113,7 @@ fn prepare_sort(req: &mut Collection, kind: &SortKind) -> Result<()> {
                     }
                 }
                 NoteType => {
-                    for (k, v) in req.storage.get_all_notetypes()? {
+                    for (k, v) in req.storage.get_all_notetypes_as_schema11()? {
                         stmt.execute(params![k, v.name])?;
                     }
                 }
@@ -127,7 +127,7 @@ fn prepare_sort(req: &mut Collection, kind: &SortKind) -> Result<()> {
                 .db
                 .prepare("insert into sort_order (k1,k2,v) values (?,?,?)")?;
 
-            for (ntid, nt) in req.storage.get_all_notetypes()? {
+            for (ntid, nt) in req.storage.get_all_notetypes_as_schema11()? {
                 for tmpl in nt.templates {
                     stmt.execute(params![ntid, tmpl.ord, tmpl.name])?;
                 }

rslib/src/search/sqlwriter.rs

@@ -267,7 +267,7 @@ impl SqlWriter<'_> {
                 write!(self.sql, "c.ord = {}", n).unwrap();
             }
             TemplateKind::Name(name) => {
-                let note_types = self.col.storage.get_all_notetypes()?;
+                let note_types = self.col.storage.get_all_notetypes_as_schema11()?;
                 let mut id_ords = vec![];
                 for nt in note_types.values() {
                     for tmpl in &nt.templates {
@@ -298,7 +298,7 @@ impl SqlWriter<'_> {
         let mut ntids: Vec<_> = self
             .col
             .storage
-            .get_all_notetypes()?
+            .get_all_notetypes_as_schema11()?
             .values()
             .filter(|nt| matches_wildcard(&nt.name, nt_name))
             .map(|nt| nt.id)
@@ -311,7 +311,7 @@ impl SqlWriter<'_> {
     }
 
     fn write_single_field(&mut self, field_name: &str, val: &str, is_re: bool) -> Result<()> {
-        let note_types = self.col.storage.get_all_notetypes()?;
+        let note_types = self.col.storage.get_all_notetypes_as_schema11()?;
 
         let mut field_map = vec![];
         for nt in note_types.values() {

rslib/src/storage/notetype/add_notetype.sql

@@ -0,0 +1,3 @@
+insert into notetypes (id, name, mtime_secs, usn, config)
+values
+  (?, ?, ?, ?, ?)

rslib/src/storage/notetype/get_fields.sql

@@ -0,0 +1,7 @@
+select
+  ord,
+  name,
+  config
+from fields
+where
+  ntid = ?

rslib/src/storage/notetype/get_notetype.sql

@@ -0,0 +1,8 @@
+select
+  name,
+  mtime_secs,
+  usn,
+  config
+from notetypes
+where
+  id = ?

rslib/src/storage/notetype/get_notetype_names.sql

@@ -0,0 +1,4 @@
+select
+  id,
+  name
+from notetypes

rslib/src/storage/notetype/get_templates.sql

@@ -0,0 +1,9 @@
+select
+  ord,
+  name,
+  mtime_secs,
+  usn,
+  config
+from templates
+where
+  ntid = ?

rslib/src/storage/notetype/mod.rs

@@ -3,23 +3,194 @@
 
 use super::SqliteStorage;
 use crate::{
+    backend_proto::{
+        CardTemplate as CardTemplateProto, CardTemplateConfig, NoteField as NoteFieldProto,
+        NoteFieldConfig, NoteType as NoteTypeProto, NoteTypeConfig,
+    },
     err::{AnkiError, DBErrorKind, Result},
-    notetype::{NoteType, NoteTypeID},
+    notetype::{NoteTypeID, NoteTypeSchema11},
-    timestamp::TimestampSecs,
-    types::Usn,
 };
-use rusqlite::NO_PARAMS;
+use prost::Message;
-use std::collections::HashMap;
+use rusqlite::{params, NO_PARAMS};
+use std::collections::{HashMap, HashSet};
+use unicase::UniCase;
 
 impl SqliteStorage {
-    pub(crate) fn get_all_notetypes(&self) -> Result<HashMap<NoteTypeID, NoteType>> {
+    fn get_notetype_core(&self, ntid: NoteTypeID) -> Result<Option<NoteTypeProto>> {
+        self.db
+            .prepare_cached(include_str!("get_notetype.sql"))?
+            .query_and_then(&[ntid], |row| {
+                let config = NoteTypeConfig::decode(row.get_raw(3).as_blob()?)?;
+                Ok(NoteTypeProto {
+                    id: ntid.0,
+                    name: row.get(0)?,
+                    mtime_secs: row.get(1)?,
+                    usn: row.get(2)?,
+                    config: Some(config),
+                    fields: vec![],
+                    templates: vec![],
+                })
+            })?
+            .next()
+            .transpose()
+    }
+
+    fn get_notetype_fields(&self, ntid: NoteTypeID) -> Result<Vec<NoteFieldProto>> {
+        self.db
+            .prepare_cached(include_str!("get_fields.sql"))?
+            .query_and_then(&[ntid], |row| {
+                let config = NoteFieldConfig::decode(row.get_raw(2).as_blob()?)?;
+                Ok(NoteFieldProto {
+                    ord: row.get(0)?,
+                    name: row.get(1)?,
+                    config: Some(config),
+                })
+            })?
+            .collect()
+    }
+
+    fn get_notetype_templates(&self, ntid: NoteTypeID) -> Result<Vec<CardTemplateProto>> {
+        self.db
+            .prepare_cached(include_str!("get_templates.sql"))?
+            .query_and_then(&[ntid], |row| {
+                let config = CardTemplateConfig::decode(row.get_raw(4).as_blob()?)?;
+                Ok(CardTemplateProto {
+                    ord: row.get(0)?,
+                    name: row.get(1)?,
+                    mtime_secs: row.get(2)?,
+                    usn: row.get(3)?,
+                    config: Some(config),
+                })
+            })?
+            .collect()
+    }
+
+    fn get_full_notetype(&self, ntid: NoteTypeID) -> Result<Option<NoteTypeProto>> {
+        match self.get_notetype_core(ntid)? {
+            Some(mut nt) => {
+                nt.fields = self.get_notetype_fields(ntid)?;
+                nt.templates = self.get_notetype_templates(ntid)?;
+                Ok(Some(nt))
+            }
+            None => Ok(None),
+        }
+    }
+
+    fn get_all_notetype_meta(&self) -> Result<Vec<(NoteTypeID, String)>> {
+        self.db
+            .prepare_cached(include_str!("get_notetype_names.sql"))?
+            .query_and_then(NO_PARAMS, |row| Ok((row.get(0)?, row.get(1)?)))?
+            .collect()
+    }
+
+    pub(crate) fn get_all_notetypes_as_schema11(
+        &self,
+    ) -> Result<HashMap<NoteTypeID, NoteTypeSchema11>> {
+        let mut nts = HashMap::new();
+        for (ntid, _name) in self.get_all_notetype_meta()? {
+            let full = self.get_full_notetype(ntid)?.unwrap();
+            nts.insert(ntid, full.into());
+        }
+        Ok(nts)
+    }
+
+    fn update_notetype_fields(&self, ntid: NoteTypeID, fields: &[NoteFieldProto]) -> Result<()> {
+        self.db
+            .prepare_cached("delete from fields where ntid=?")?
+            .execute(&[ntid])?;
+        let mut stmt = self.db.prepare_cached(include_str!("update_fields.sql"))?;
+        for (ord, field) in fields.iter().enumerate() {
+            let mut config_bytes = vec![];
+            field.config.as_ref().unwrap().encode(&mut config_bytes)?;
+            stmt.execute(params![ntid, ord as u32, field.name, config_bytes,])?;
+        }
+
+        Ok(())
+    }
+
+    fn update_notetype_templates(
+        &self,
+        ntid: NoteTypeID,
+        templates: &[CardTemplateProto],
+    ) -> Result<()> {
+        self.db
+            .prepare_cached("delete from templates where ntid=?")?
+            .execute(&[ntid])?;
+        let mut stmt = self
+            .db
+            .prepare_cached(include_str!("update_templates.sql"))?;
+        for (ord, template) in templates.iter().enumerate() {
+            let mut config_bytes = vec![];
+            template
+                .config
+                .as_ref()
+                .unwrap()
+                .encode(&mut config_bytes)?;
+            stmt.execute(params![
+                ntid,
+                ord as u32,
+                template.name,
+                template.mtime_secs,
+                template.usn,
+                config_bytes,
+            ])?;
+        }
+
+        Ok(())
+    }
+
+    fn update_notetype_meta(&self, nt: &NoteTypeProto) -> Result<()> {
+        assert!(nt.id != 0);
+        let mut stmt = self
+            .db
+            .prepare_cached(include_str!("update_notetype_meta.sql"))?;
+        let mut config_bytes = vec![];
+        nt.config.as_ref().unwrap().encode(&mut config_bytes)?;
+        stmt.execute(params![nt.id, nt.name, nt.mtime_secs, nt.usn, config_bytes])?;
+
+        Ok(())
+    }
+
+    // Upgrading/downgrading
+
+    pub(crate) fn upgrade_notetypes_to_schema15(&self) -> Result<()> {
+        let nts = self.get_schema11_notetypes()?;
+        let mut names = HashSet::new();
+        for (ntid, nt) in nts {
+            let mut nt = NoteTypeProto::from(nt);
+            nt.normalize_names();
+            nt.ensure_names_unique();
+            loop {
+                let name = UniCase::new(nt.name.clone());
+                if !names.contains(&name) {
+                    names.insert(name);
+                    break;
+                }
+                nt.name.push('_');
+            }
+            self.update_notetype_meta(&nt)?;
+            self.update_notetype_fields(ntid, &nt.fields)?;
+            self.update_notetype_templates(ntid, &nt.templates)?;
+        }
+        Ok(())
+    }
+
+    pub(crate) fn downgrade_notetypes_from_schema15(&self) -> Result<()> {
+        let nts = self.get_all_notetypes_as_schema11()?;
+        self.set_schema11_notetypes(nts)
+    }
+
+    fn get_schema11_notetypes(&self) -> Result<HashMap<NoteTypeID, NoteTypeSchema11>> {
         let mut stmt = self.db.prepare("select models from col")?;
         let note_types = stmt
-            .query_and_then(NO_PARAMS, |row| -> Result<HashMap<NoteTypeID, NoteType>> {
+            .query_and_then(
-                let v: HashMap<NoteTypeID, NoteType> =
+                NO_PARAMS,
-                    serde_json::from_str(row.get_raw(0).as_str()?)?;
+                |row| -> Result<HashMap<NoteTypeID, NoteTypeSchema11>> {
-                Ok(v)
+                    let v: HashMap<NoteTypeID, NoteTypeSchema11> =
-            })?
+                        serde_json::from_str(row.get_raw(0).as_str()?)?;
+                    Ok(v)
+                },
+            )?
             .next()
             .ok_or_else(|| AnkiError::DBError {
                 info: "col table empty".to_string(),
@@ -28,11 +199,9 @@ impl SqliteStorage {
         Ok(note_types)
     }
 
-    pub(crate) fn set_all_notetypes(
+    pub(crate) fn set_schema11_notetypes(
         &self,
-        notetypes: HashMap<NoteTypeID, NoteType>,
+        notetypes: HashMap<NoteTypeID, NoteTypeSchema11>,
-        _usn: Usn,
-        _mtime: TimestampSecs,
     ) -> Result<()> {
         let json = serde_json::to_string(&notetypes)?;
         self.db.execute("update col set models = ?", &[json])?;

rslib/src/storage/notetype/update_fields.sql

@@ -0,0 +1,3 @@
+insert into fields (ntid, ord, name, config)
+values
+  (?, ?, ?, ?);

rslib/src/storage/notetype/update_notetype_config.sql

@@ -0,0 +1,4 @@
+insert
+  or replace into notetype_config (ntid, config)
+values
+  (?, ?)

rslib/src/storage/notetype/update_notetype_meta.sql

@@ -0,0 +1,4 @@
+insert
+  or replace into notetypes (id, name, mtime_secs, usn, config)
+values
+  (?, ?, ?, ?, ?)

rslib/src/storage/notetype/update_templates.sql

@@ -0,0 +1,3 @@
+insert into templates (ntid, ord, name, mtime_secs, usn, config)
+values
+  (?, ?, ?, ?, ?, ?)

rslib/src/storage/sqlite.rs

@@ -14,7 +14,7 @@ use unicase::UniCase;
 
 const SCHEMA_MIN_VERSION: u8 = 11;
 const SCHEMA_STARTING_VERSION: u8 = 11;
-const SCHEMA_MAX_VERSION: u8 = 14;
+const SCHEMA_MAX_VERSION: u8 = 15;
 
 fn unicase_compare(s1: &str, s2: &str) -> Ordering {
     UniCase::new(s1).cmp(&UniCase::new(s2))

rslib/src/storage/upgrades/mod.rs

@@ -13,6 +13,11 @@ impl SqliteStorage {
             self.upgrade_tags_to_schema14()?;
             self.upgrade_config_to_schema14()?;
         }
+        if ver < 15 {
+            self.db
+                .execute_batch(include_str!("schema15_upgrade.sql"))?;
+            self.upgrade_notetypes_to_schema15()?;
+        }
 
         Ok(())
     }
@@ -20,6 +25,7 @@ impl SqliteStorage {
     pub(super) fn downgrade_to_schema_11(&self) -> Result<()> {
         self.begin_trx()?;
 
+        self.downgrade_notetypes_from_schema15()?;
         self.downgrade_config_from_schema14()?;
         self.downgrade_tags_from_schema14()?;
         self.downgrade_deck_conf_from_schema14()?;

rslib/src/storage/upgrades/schema11_downgrade.sql

@@ -1,6 +1,9 @@
 drop table config;
 drop table deck_config;
 drop table tags;
+drop table fields;
+drop table templates;
+drop table notetypes;
 update col
 set
   ver = 11;

rslib/src/storage/upgrades/schema15_upgrade.sql

@@ -0,0 +1,32 @@
+create table fields (
+  ntid integer not null,
+  ord integer not null,
+  name text not null collate unicase,
+  config blob not null,
+  primary key (ntid, ord)
+) without rowid;
+create unique index idx_fields_name_ntid on fields (name, ntid);
+create table templates (
+  ntid integer not null,
+  ord integer not null,
+  name text not null collate unicase,
+  mtime_secs integer not null,
+  usn integer not null,
+  config blob not null,
+  primary key (ntid, ord)
+) without rowid;
+create unique index idx_templates_name_ntid on templates (name, ntid);
+create index idx_templates_usn on templates (usn);
+create table notetypes (
+  id integer not null primary key,
+  name text not null collate unicase,
+  mtime_secs integer not null,
+  usn integer not null,
+  config blob not null
+);
+create unique index idx_notetypes_name on notetypes (name);
+create index idx_notetypes_usn on notetypes (usn);
+update col
+set
+  ver = 15;
+analyze;

rslib/src/text.rs

@@ -226,6 +226,12 @@ pub(crate) fn normalize_to_nfc(s: &str) -> Cow<str> {
     }
 }
 
+pub(crate) fn ensure_string_in_nfc(s: &mut String) {
+    if !is_nfc(s) {
+        *s = s.chars().nfc().collect()
+    }
+}
+
 /// True if search is equal to text, folding case.
 /// Supports '*' to match 0 or more characters.
 pub(crate) fn matches_wildcard(text: &str, search: &str) -> bool {