(de)serialize decks in backend

proto/backend.proto

@@ -67,6 +67,8 @@ message BackendInput {
         Empty get_all_config = 55;
         Empty get_all_notetypes = 56;
         bytes set_all_notetypes = 57;
+        Empty get_all_decks = 58;
+        bytes set_all_decks = 59;
     }
 }
 
@@ -117,6 +119,8 @@ message BackendOutput {
         bytes get_all_config = 55;
         bytes get_all_notetypes = 56;
         Empty set_all_notetypes = 57;
+        bytes get_all_decks = 58;
+        Empty set_all_decks = 59;
 
         BackendError error = 2047;
     }

pylib/anki/collection.py

@@ -166,7 +166,8 @@ class _Collection:
 select crt, mod, scm, dty, usn, ls,
 decks from col"""
         )
-        self.decks.load(decks)
+        self.decks.decks = self.backend.get_all_decks()
+        self.decks.changed = False
         self.models.models = self.backend.get_all_notetypes()
         self.models.changed = False
 

pylib/anki/decks.py

@@ -4,7 +4,6 @@
 from __future__ import annotations
 
 import copy
-import json
 import unicodedata
 from typing import Any, Dict, List, Optional, Set, Tuple, Union
 
@@ -64,10 +63,6 @@ class DeckManager:
         self.decks = {}
         self._dconf_cache: Optional[Dict[int, Dict[str, Any]]] = None
 
-    def load(self, decks: str) -> None:
-        self.decks = json.loads(decks)
-        self.changed = False
-
     def save(self, g: Optional[Any] = None) -> None:
         "Can be called with either a deck or a deck configuration."
         if g:
@@ -82,9 +77,7 @@ class DeckManager:
 
     def flush(self) -> None:
         if self.changed:
-            self.col.db.execute(
+            self.col.backend.set_all_decks(self.decks)
-                "update col set decks=?", json.dumps(self.decks),
-            )
             self.changed = False
 
     # Deck save/load

pylib/anki/rsbackend.py

@@ -617,6 +617,15 @@ class RustBackend:
     def set_all_notetypes(self, nts: Dict[str, Dict[str, Any]]):
         self._run_command(pb.BackendInput(set_all_notetypes=orjson.dumps(nts)))
 
+    def get_all_decks(self) -> Dict[str, Dict[str, Any]]:
+        jstr = self._run_command(
+            pb.BackendInput(get_all_decks=pb.Empty())
+        ).get_all_decks
+        return orjson.loads(jstr)
+
+    def set_all_decks(self, nts: Dict[str, Dict[str, Any]]):
+        self._run_command(pb.BackendInput(set_all_decks=orjson.dumps(nts)))
+
 
 def translate_string_in(
     key: TR, **kwargs: Union[str, int, float]

rslib/src/backend/mod.rs

@@ -12,7 +12,7 @@ use crate::{
     collection::{open_collection, Collection},
     config::SortKind,
     deckconf::{DeckConf, DeckConfID},
-    decks::DeckID,
+    decks::{Deck, DeckID},
     err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind},
     i18n::{tr_args, I18n, TR},
     latex::{extract_latex, extract_latex_expanding_clozes, ExtractedLatex},
@@ -313,6 +313,11 @@ impl Backend {
                 self.set_all_notetypes(&bytes)?;
                 OValue::SetAllNotetypes(pb::Empty {})
             }
+            Value::GetAllDecks(_) => OValue::GetAllDecks(self.get_all_decks()?),
+            Value::SetAllDecks(bytes) => {
+                self.set_all_decks(&bytes)?;
+                OValue::SetAllDecks(pb::Empty {})
+            }
         })
     }
 
@@ -866,6 +871,18 @@ impl Backend {
             serde_json::to_vec(&nts).map_err(Into::into)
         })
     }
+
+    fn set_all_decks(&self, json: &[u8]) -> Result<()> {
+        let val: HashMap<DeckID, Deck> = serde_json::from_slice(json)?;
+        self.with_col(|col| col.transact(None, |col| col.storage.set_all_decks(val)))
+    }
+
+    fn get_all_decks(&self) -> Result<Vec<u8>> {
+        self.with_col(|col| {
+            let decks = col.storage.get_all_decks()?;
+            serde_json::to_vec(&decks).map_err(Into::into)
+        })
+    }
 }
 
 fn translate_arg_to_fluent_val(arg: &pb::TranslateArgValue) -> FluentValue {

rslib/src/decks.rs

@@ -1,30 +1,221 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-use crate::define_newtype;
+use crate::{
-use serde_aux::field_attributes::deserialize_number_from_string;
+    define_newtype,
-use serde_derive::Deserialize;
+    serde::{default_on_invalid, deserialize_bool_from_anything, deserialize_number_from_string},
+    timestamp::TimestampSecs,
+    types::Usn,
+};
+use serde_derive::{Deserialize, Serialize};
+use serde_json::Value;
+use serde_tuple::Serialize_tuple;
+use std::collections::HashMap;
 
 define_newtype!(DeckID, i64);
 
-#[derive(Deserialize)]
+#[derive(Serialize, PartialEq, Debug, Clone)]
-pub struct Deck {
+#[serde(untagged)]
+pub enum Deck {
+    Normal(NormalDeck),
+    Filtered(FilteredDeck),
+}
+
+// serde doesn't support integer/bool enum tags, so we manually pick the correct variant
+mod dynfix {
+    use super::{Deck, FilteredDeck, NormalDeck};
+    use serde::de::{self, Deserialize, Deserializer};
+    use serde_json::{Map, Value};
+
+    impl<'de> Deserialize<'de> for Deck {
+        fn deserialize<D>(deserializer: D) -> Result<Deck, D::Error>
+        where
+            D: Deserializer<'de>,
+        {
+            let mut map = Map::deserialize(deserializer)?;
+
+            let is_dyn = map
+                .get("dyn")
+                .ok_or_else(|| de::Error::missing_field("dyn"))
+                .map(|v| {
+                    match v {
+                        Value::Bool(b) => *b,
+                        Value::Number(n) => n.as_i64().unwrap_or(0) == 1,
+                        _ => {
+                            // invalid type, default to normal deck
+                            false
+                        }
+                    }
+                })?;
+
+            // remove some obsolete keys
+            map.remove("separate");
+            map.remove("return");
+
+            let rest = Value::Object(map);
+            if is_dyn {
+                FilteredDeck::deserialize(rest)
+                    .map(Deck::Filtered)
+                    .map_err(de::Error::custom)
+            } else {
+                NormalDeck::deserialize(rest)
+                    .map(Deck::Normal)
+                    .map_err(de::Error::custom)
+            }
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]
+pub struct DeckCommon {
     #[serde(deserialize_with = "deserialize_number_from_string")]
     pub(crate) id: DeckID,
+    #[serde(rename = "mod", deserialize_with = "deserialize_number_from_string")]
+    pub(crate) mtime: TimestampSecs,
     pub(crate) name: String,
+    pub(crate) usn: Usn,
+    #[serde(flatten)]
+    pub(crate) today: DeckToday,
+    collapsed: bool,
+    #[serde(default)]
+    desc: String,
+    #[serde(rename = "dyn", deserialize_with = "deserialize_bool_from_anything")]
+    dynamic: bool,
+    #[serde(flatten)]
+    other: HashMap<String, Value>,
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct NormalDeck {
+    #[serde(flatten)]
+    pub(crate) common: DeckCommon,
+
+    #[serde(deserialize_with = "deserialize_number_from_string")]
+    pub(crate) conf: i64,
+    #[serde(default, deserialize_with = "default_on_invalid")]
+    extend_new: i32,
+    #[serde(default, deserialize_with = "default_on_invalid")]
+    extend_rev: i32,
+}
+
+#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct FilteredDeck {
+    #[serde(flatten)]
+    common: DeckCommon,
+
+    #[serde(deserialize_with = "deserialize_bool_from_anything")]
+    resched: bool,
+    terms: Vec<FilteredSearch>,
+
+    // old scheduler
+    #[serde(default, deserialize_with = "default_on_invalid")]
+    delays: Option<Vec<f32>>,
+
+    // new scheduler
+    #[serde(default)]
+    preview_delay: u16,
+}
+#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)]
+pub struct DeckToday {
+    #[serde(rename = "lrnToday")]
+    pub(crate) lrn: TodayAmount,
+    #[serde(rename = "revToday")]
+    pub(crate) rev: TodayAmount,
+    #[serde(rename = "newToday")]
+    pub(crate) new: TodayAmount,
+    #[serde(rename = "timeToday")]
+    pub(crate) time: TodayAmount,
+}
+
+#[derive(Serialize_tuple, Deserialize, Debug, PartialEq, Default, Clone)]
+#[serde(from = "Vec<Value>")]
+pub struct TodayAmount {
+    day: i32,
+    amount: i32,
+}
+
+impl From<Vec<Value>> for TodayAmount {
+    fn from(mut v: Vec<Value>) -> Self {
+        let amt = v.pop().and_then(|v| v.as_i64()).unwrap_or(0);
+        let day = v.pop().and_then(|v| v.as_i64()).unwrap_or(0);
+        TodayAmount {
+            amount: amt as i32,
+            day: day as i32,
+        }
+    }
+}
+#[derive(Serialize_tuple, Deserialize, Debug, PartialEq, Clone)]
+pub struct FilteredSearch {
+    search: String,
+    #[serde(deserialize_with = "deserialize_number_from_string")]
+    limit: i32,
+    order: i8,
+}
+
+impl Deck {
+    pub fn common(&self) -> &DeckCommon {
+        match self {
+            Deck::Normal(d) => &d.common,
+            Deck::Filtered(d) => &d.common,
+        }
+    }
+
+    // pub(crate) fn common_mut(&mut self) -> &mut DeckCommon {
+    //     match self {
+    //         Deck::Normal(d) => &mut d.common,
+    //         Deck::Filtered(d) => &mut d.common,
+    //     }
+    // }
+
+    pub fn id(&self) -> DeckID {
+        self.common().id
+    }
+
+    pub fn name(&self) -> &str {
+        &self.common().name
+    }
+}
+
+impl Default for Deck {
+    fn default() -> Self {
+        Deck::Normal(NormalDeck::default())
+    }
+}
+
+impl Default for NormalDeck {
+    fn default() -> Self {
+        NormalDeck {
+            common: DeckCommon {
+                id: DeckID(0),
+                mtime: TimestampSecs(0),
+                name: "".to_string(),
+                usn: Usn(0),
+                collapsed: false,
+                desc: "".to_string(),
+                today: Default::default(),
+                other: Default::default(),
+                dynamic: true,
+            },
+            conf: 1,
+            extend_new: 0,
+            extend_rev: 0,
+        }
+    }
 }
 
 pub(crate) fn child_ids<'a>(decks: &'a [Deck], name: &str) -> impl Iterator<Item = DeckID> + 'a {
     let prefix = format!("{}::", name.to_ascii_lowercase());
     decks
         .iter()
-        .filter(move |d| d.name.to_ascii_lowercase().starts_with(&prefix))
+        .filter(move |d| d.name().to_ascii_lowercase().starts_with(&prefix))
-        .map(|d| d.id)
+        .map(|d| d.id())
 }
 
 pub(crate) fn get_deck(decks: &[Deck], id: DeckID) -> Option<&Deck> {
     for d in decks {
-        if d.id == id {
+        if d.id() == id {
             return Some(d);
         }
     }

rslib/src/search/cards.rs

@@ -108,8 +108,8 @@ fn prepare_sort(req: &mut Collection, kind: &SortKind) -> Result<()> {
 
             match kind {
                 CardDeck => {
-                    for (k, v) in req.storage.all_decks()? {
+                    for (k, v) in req.storage.get_all_decks()? {
-                        stmt.execute(params![k, v.name])?;
+                        stmt.execute(params![k, v.name()])?;
                     }
                 }
                 NoteType => {

rslib/src/search/sqlwriter.rs

@@ -226,7 +226,7 @@ impl SqlWriter<'_> {
                 let all_decks: Vec<_> = self
                     .col
                     .storage
-                    .all_decks()?
+                    .get_all_decks()?
                     .into_iter()
                     .map(|(_, v)| v)
                     .collect();
@@ -235,15 +235,18 @@ impl SqlWriter<'_> {
                     let mut dids_with_children = vec![current_id];
                     let current = get_deck(&all_decks, current_id)
                         .ok_or_else(|| AnkiError::invalid_input("invalid current deck"))?;
-                    for child_did in child_ids(&all_decks, &current.name) {
+                    for child_did in child_ids(&all_decks, &current.name()) {
                         dids_with_children.push(child_did);
                     }
                     dids_with_children
                 } else {
                     let mut dids_with_children = vec![];
-                    for deck in all_decks.iter().filter(|d| matches_wildcard(&d.name, deck)) {
+                    for deck in all_decks
-                        dids_with_children.push(deck.id);
+                        .iter()
-                        for child_id in child_ids(&all_decks, &deck.name) {
+                        .filter(|d| matches_wildcard(&d.name(), deck))
+                    {
+                        dids_with_children.push(deck.id());
+                        for child_id in child_ids(&all_decks, &deck.name()) {
                             dids_with_children.push(child_id);
                         }
                     }

rslib/src/storage/deck/mod.rs

@@ -0,0 +1,25 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use super::SqliteStorage;
+use crate::{
+    decks::{Deck, DeckID},
+    err::Result,
+};
+use rusqlite::NO_PARAMS;
+use std::collections::HashMap;
+
+impl SqliteStorage {
+    pub(crate) fn get_all_decks(&self) -> Result<HashMap<DeckID, Deck>> {
+        self.db
+            .query_row_and_then("select decks from col", NO_PARAMS, |row| -> Result<_> {
+                Ok(serde_json::from_str(row.get_raw(0).as_str()?)?)
+            })
+    }
+
+    pub(crate) fn set_all_decks(&self, decks: HashMap<DeckID, Deck>) -> Result<()> {
+        let json = serde_json::to_string(&decks)?;
+        self.db.execute("update col set decks = ?", &[json])?;
+        Ok(())
+    }
+}

rslib/src/storage/mod.rs

@@ -3,6 +3,7 @@
 
 mod card;
 mod config;
+mod deck;
 mod deckconf;
 mod notetype;
 mod sqlite;

rslib/src/storage/sqlite.rs

@@ -2,15 +2,14 @@
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
 use crate::config::schema11_config_as_string;
-use crate::decks::DeckID;
 use crate::err::Result;
 use crate::err::{AnkiError, DBErrorKind};
 use crate::timestamp::{TimestampMillis, TimestampSecs};
-use crate::{decks::Deck, i18n::I18n, text::without_combining, types::Usn};
+use crate::{i18n::I18n, text::without_combining, types::Usn};
 use regex::Regex;
 use rusqlite::{functions::FunctionFlags, params, Connection, NO_PARAMS};
 use std::cmp::Ordering;
-use std::{borrow::Cow, collections::HashMap, path::Path};
+use std::{borrow::Cow, path::Path};
 use unicase::UniCase;
 
 const SCHEMA_MIN_VERSION: u8 = 11;
@@ -284,13 +283,6 @@ impl SqliteStorage {
         }
     }
 
-    pub(crate) fn all_decks(&self) -> Result<HashMap<DeckID, Deck>> {
-        self.db
-            .query_row_and_then("select decks from col", NO_PARAMS, |row| -> Result<_> {
-                Ok(serde_json::from_str(row.get_raw(0).as_str()?)?)
-            })
-    }
-
     pub(crate) fn creation_stamp(&self) -> Result<TimestampSecs> {
         self.db
             .prepare_cached("select crt from col")?