From 4c7210b4309878d9ee5fd63ca488750aec1a1e9c Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 9 Apr 2020 11:23:53 +1000 Subject: [PATCH] (de)serialize decks in backend --- proto/backend.proto | 4 + pylib/anki/collection.py | 3 +- pylib/anki/decks.py | 9 +- pylib/anki/rsbackend.py | 9 ++ rslib/src/backend/mod.rs | 19 +++- rslib/src/decks.rs | 207 ++++++++++++++++++++++++++++++++-- rslib/src/search/cards.rs | 4 +- rslib/src/search/sqlwriter.rs | 13 ++- rslib/src/storage/deck/mod.rs | 25 ++++ rslib/src/storage/mod.rs | 1 + rslib/src/storage/sqlite.rs | 12 +- 11 files changed, 271 insertions(+), 35 deletions(-) create mode 100644 rslib/src/storage/deck/mod.rs diff --git a/proto/backend.proto b/proto/backend.proto index 7f393a1de..1ecd6fb11 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -67,6 +67,8 @@ message BackendInput { Empty get_all_config = 55; Empty get_all_notetypes = 56; bytes set_all_notetypes = 57; + Empty get_all_decks = 58; + bytes set_all_decks = 59; } } @@ -117,6 +119,8 @@ message BackendOutput { bytes get_all_config = 55; bytes get_all_notetypes = 56; Empty set_all_notetypes = 57; + bytes get_all_decks = 58; + Empty set_all_decks = 59; BackendError error = 2047; } diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index a0d2e54a0..3f88ed4fa 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -166,7 +166,8 @@ class _Collection: select crt, mod, scm, dty, usn, ls, decks from col""" ) - self.decks.load(decks) + self.decks.decks = self.backend.get_all_decks() + self.decks.changed = False self.models.models = self.backend.get_all_notetypes() self.models.changed = False diff --git a/pylib/anki/decks.py b/pylib/anki/decks.py index 755bc3ed6..18796f713 100644 --- a/pylib/anki/decks.py +++ b/pylib/anki/decks.py @@ -4,7 +4,6 @@ from __future__ import annotations import copy -import json import unicodedata from typing import Any, Dict, List, Optional, Set, Tuple, Union @@ -64,10 +63,6 @@ class DeckManager: self.decks = {} self._dconf_cache: Optional[Dict[int, Dict[str, Any]]] = None - def load(self, decks: str) -> None: - self.decks = json.loads(decks) - self.changed = False - def save(self, g: Optional[Any] = None) -> None: "Can be called with either a deck or a deck configuration." if g: @@ -82,9 +77,7 @@ class DeckManager: def flush(self) -> None: if self.changed: - self.col.db.execute( - "update col set decks=?", json.dumps(self.decks), - ) + self.col.backend.set_all_decks(self.decks) self.changed = False # Deck save/load diff --git a/pylib/anki/rsbackend.py b/pylib/anki/rsbackend.py index bbef05bec..c0cdcbd3d 100644 --- a/pylib/anki/rsbackend.py +++ b/pylib/anki/rsbackend.py @@ -617,6 +617,15 @@ class RustBackend: def set_all_notetypes(self, nts: Dict[str, Dict[str, Any]]): self._run_command(pb.BackendInput(set_all_notetypes=orjson.dumps(nts))) + def get_all_decks(self) -> Dict[str, Dict[str, Any]]: + jstr = self._run_command( + pb.BackendInput(get_all_decks=pb.Empty()) + ).get_all_decks + return orjson.loads(jstr) + + def set_all_decks(self, nts: Dict[str, Dict[str, Any]]): + self._run_command(pb.BackendInput(set_all_decks=orjson.dumps(nts))) + def translate_string_in( key: TR, **kwargs: Union[str, int, float] diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index a36b24dda..bd838bf41 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -12,7 +12,7 @@ use crate::{ collection::{open_collection, Collection}, config::SortKind, deckconf::{DeckConf, DeckConfID}, - decks::DeckID, + decks::{Deck, DeckID}, err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind}, i18n::{tr_args, I18n, TR}, latex::{extract_latex, extract_latex_expanding_clozes, ExtractedLatex}, @@ -313,6 +313,11 @@ impl Backend { self.set_all_notetypes(&bytes)?; OValue::SetAllNotetypes(pb::Empty {}) } + Value::GetAllDecks(_) => OValue::GetAllDecks(self.get_all_decks()?), + Value::SetAllDecks(bytes) => { + self.set_all_decks(&bytes)?; + OValue::SetAllDecks(pb::Empty {}) + } }) } @@ -866,6 +871,18 @@ impl Backend { serde_json::to_vec(&nts).map_err(Into::into) }) } + + fn set_all_decks(&self, json: &[u8]) -> Result<()> { + let val: HashMap = serde_json::from_slice(json)?; + self.with_col(|col| col.transact(None, |col| col.storage.set_all_decks(val))) + } + + fn get_all_decks(&self) -> Result> { + self.with_col(|col| { + let decks = col.storage.get_all_decks()?; + serde_json::to_vec(&decks).map_err(Into::into) + }) + } } fn translate_arg_to_fluent_val(arg: &pb::TranslateArgValue) -> FluentValue { diff --git a/rslib/src/decks.rs b/rslib/src/decks.rs index 1f1524418..932d94650 100644 --- a/rslib/src/decks.rs +++ b/rslib/src/decks.rs @@ -1,30 +1,221 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::define_newtype; -use serde_aux::field_attributes::deserialize_number_from_string; -use serde_derive::Deserialize; +use crate::{ + define_newtype, + serde::{default_on_invalid, deserialize_bool_from_anything, deserialize_number_from_string}, + timestamp::TimestampSecs, + types::Usn, +}; +use serde_derive::{Deserialize, Serialize}; +use serde_json::Value; +use serde_tuple::Serialize_tuple; +use std::collections::HashMap; define_newtype!(DeckID, i64); -#[derive(Deserialize)] -pub struct Deck { +#[derive(Serialize, PartialEq, Debug, Clone)] +#[serde(untagged)] +pub enum Deck { + Normal(NormalDeck), + Filtered(FilteredDeck), +} + +// serde doesn't support integer/bool enum tags, so we manually pick the correct variant +mod dynfix { + use super::{Deck, FilteredDeck, NormalDeck}; + use serde::de::{self, Deserialize, Deserializer}; + use serde_json::{Map, Value}; + + impl<'de> Deserialize<'de> for Deck { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let mut map = Map::deserialize(deserializer)?; + + let is_dyn = map + .get("dyn") + .ok_or_else(|| de::Error::missing_field("dyn")) + .map(|v| { + match v { + Value::Bool(b) => *b, + Value::Number(n) => n.as_i64().unwrap_or(0) == 1, + _ => { + // invalid type, default to normal deck + false + } + } + })?; + + // remove some obsolete keys + map.remove("separate"); + map.remove("return"); + + let rest = Value::Object(map); + if is_dyn { + FilteredDeck::deserialize(rest) + .map(Deck::Filtered) + .map_err(de::Error::custom) + } else { + NormalDeck::deserialize(rest) + .map(Deck::Normal) + .map_err(de::Error::custom) + } + } + } +} + +#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)] +pub struct DeckCommon { #[serde(deserialize_with = "deserialize_number_from_string")] pub(crate) id: DeckID, + #[serde(rename = "mod", deserialize_with = "deserialize_number_from_string")] + pub(crate) mtime: TimestampSecs, pub(crate) name: String, + pub(crate) usn: Usn, + #[serde(flatten)] + pub(crate) today: DeckToday, + collapsed: bool, + #[serde(default)] + desc: String, + #[serde(rename = "dyn", deserialize_with = "deserialize_bool_from_anything")] + dynamic: bool, + #[serde(flatten)] + other: HashMap, +} + +#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct NormalDeck { + #[serde(flatten)] + pub(crate) common: DeckCommon, + + #[serde(deserialize_with = "deserialize_number_from_string")] + pub(crate) conf: i64, + #[serde(default, deserialize_with = "default_on_invalid")] + extend_new: i32, + #[serde(default, deserialize_with = "default_on_invalid")] + extend_rev: i32, +} + +#[derive(Serialize, Deserialize, PartialEq, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct FilteredDeck { + #[serde(flatten)] + common: DeckCommon, + + #[serde(deserialize_with = "deserialize_bool_from_anything")] + resched: bool, + terms: Vec, + + // old scheduler + #[serde(default, deserialize_with = "default_on_invalid")] + delays: Option>, + + // new scheduler + #[serde(default)] + preview_delay: u16, +} +#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone)] +pub struct DeckToday { + #[serde(rename = "lrnToday")] + pub(crate) lrn: TodayAmount, + #[serde(rename = "revToday")] + pub(crate) rev: TodayAmount, + #[serde(rename = "newToday")] + pub(crate) new: TodayAmount, + #[serde(rename = "timeToday")] + pub(crate) time: TodayAmount, +} + +#[derive(Serialize_tuple, Deserialize, Debug, PartialEq, Default, Clone)] +#[serde(from = "Vec")] +pub struct TodayAmount { + day: i32, + amount: i32, +} + +impl From> for TodayAmount { + fn from(mut v: Vec) -> Self { + let amt = v.pop().and_then(|v| v.as_i64()).unwrap_or(0); + let day = v.pop().and_then(|v| v.as_i64()).unwrap_or(0); + TodayAmount { + amount: amt as i32, + day: day as i32, + } + } +} +#[derive(Serialize_tuple, Deserialize, Debug, PartialEq, Clone)] +pub struct FilteredSearch { + search: String, + #[serde(deserialize_with = "deserialize_number_from_string")] + limit: i32, + order: i8, +} + +impl Deck { + pub fn common(&self) -> &DeckCommon { + match self { + Deck::Normal(d) => &d.common, + Deck::Filtered(d) => &d.common, + } + } + + // pub(crate) fn common_mut(&mut self) -> &mut DeckCommon { + // match self { + // Deck::Normal(d) => &mut d.common, + // Deck::Filtered(d) => &mut d.common, + // } + // } + + pub fn id(&self) -> DeckID { + self.common().id + } + + pub fn name(&self) -> &str { + &self.common().name + } +} + +impl Default for Deck { + fn default() -> Self { + Deck::Normal(NormalDeck::default()) + } +} + +impl Default for NormalDeck { + fn default() -> Self { + NormalDeck { + common: DeckCommon { + id: DeckID(0), + mtime: TimestampSecs(0), + name: "".to_string(), + usn: Usn(0), + collapsed: false, + desc: "".to_string(), + today: Default::default(), + other: Default::default(), + dynamic: true, + }, + conf: 1, + extend_new: 0, + extend_rev: 0, + } + } } pub(crate) fn child_ids<'a>(decks: &'a [Deck], name: &str) -> impl Iterator + 'a { let prefix = format!("{}::", name.to_ascii_lowercase()); decks .iter() - .filter(move |d| d.name.to_ascii_lowercase().starts_with(&prefix)) - .map(|d| d.id) + .filter(move |d| d.name().to_ascii_lowercase().starts_with(&prefix)) + .map(|d| d.id()) } pub(crate) fn get_deck(decks: &[Deck], id: DeckID) -> Option<&Deck> { for d in decks { - if d.id == id { + if d.id() == id { return Some(d); } } diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index 68169ea51..2679f7ad3 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -108,8 +108,8 @@ fn prepare_sort(req: &mut Collection, kind: &SortKind) -> Result<()> { match kind { CardDeck => { - for (k, v) in req.storage.all_decks()? { - stmt.execute(params![k, v.name])?; + for (k, v) in req.storage.get_all_decks()? { + stmt.execute(params![k, v.name()])?; } } NoteType => { diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs index 0214b0b7b..b2017fd4c 100644 --- a/rslib/src/search/sqlwriter.rs +++ b/rslib/src/search/sqlwriter.rs @@ -226,7 +226,7 @@ impl SqlWriter<'_> { let all_decks: Vec<_> = self .col .storage - .all_decks()? + .get_all_decks()? .into_iter() .map(|(_, v)| v) .collect(); @@ -235,15 +235,18 @@ impl SqlWriter<'_> { let mut dids_with_children = vec![current_id]; let current = get_deck(&all_decks, current_id) .ok_or_else(|| AnkiError::invalid_input("invalid current deck"))?; - for child_did in child_ids(&all_decks, ¤t.name) { + for child_did in child_ids(&all_decks, ¤t.name()) { dids_with_children.push(child_did); } dids_with_children } else { let mut dids_with_children = vec![]; - for deck in all_decks.iter().filter(|d| matches_wildcard(&d.name, deck)) { - dids_with_children.push(deck.id); - for child_id in child_ids(&all_decks, &deck.name) { + for deck in all_decks + .iter() + .filter(|d| matches_wildcard(&d.name(), deck)) + { + dids_with_children.push(deck.id()); + for child_id in child_ids(&all_decks, &deck.name()) { dids_with_children.push(child_id); } } diff --git a/rslib/src/storage/deck/mod.rs b/rslib/src/storage/deck/mod.rs new file mode 100644 index 000000000..577d59dfd --- /dev/null +++ b/rslib/src/storage/deck/mod.rs @@ -0,0 +1,25 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use super::SqliteStorage; +use crate::{ + decks::{Deck, DeckID}, + err::Result, +}; +use rusqlite::NO_PARAMS; +use std::collections::HashMap; + +impl SqliteStorage { + pub(crate) fn get_all_decks(&self) -> Result> { + self.db + .query_row_and_then("select decks from col", NO_PARAMS, |row| -> Result<_> { + Ok(serde_json::from_str(row.get_raw(0).as_str()?)?) + }) + } + + pub(crate) fn set_all_decks(&self, decks: HashMap) -> Result<()> { + let json = serde_json::to_string(&decks)?; + self.db.execute("update col set decks = ?", &[json])?; + Ok(()) + } +} diff --git a/rslib/src/storage/mod.rs b/rslib/src/storage/mod.rs index b8f2d19cd..681182028 100644 --- a/rslib/src/storage/mod.rs +++ b/rslib/src/storage/mod.rs @@ -3,6 +3,7 @@ mod card; mod config; +mod deck; mod deckconf; mod notetype; mod sqlite; diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 57eb5a20c..b92951347 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -2,15 +2,14 @@ // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html use crate::config::schema11_config_as_string; -use crate::decks::DeckID; use crate::err::Result; use crate::err::{AnkiError, DBErrorKind}; use crate::timestamp::{TimestampMillis, TimestampSecs}; -use crate::{decks::Deck, i18n::I18n, text::without_combining, types::Usn}; +use crate::{i18n::I18n, text::without_combining, types::Usn}; use regex::Regex; use rusqlite::{functions::FunctionFlags, params, Connection, NO_PARAMS}; use std::cmp::Ordering; -use std::{borrow::Cow, collections::HashMap, path::Path}; +use std::{borrow::Cow, path::Path}; use unicase::UniCase; const SCHEMA_MIN_VERSION: u8 = 11; @@ -284,13 +283,6 @@ impl SqliteStorage { } } - pub(crate) fn all_decks(&self) -> Result> { - self.db - .query_row_and_then("select decks from col", NO_PARAMS, |row| -> Result<_> { - Ok(serde_json::from_str(row.get_raw(0).as_str()?)?) - }) - } - pub(crate) fn creation_stamp(&self) -> Result { self.db .prepare_cached("select crt from col")?