experiment with exposing raw card/revlog data to frontend

proto/backend.proto

@@ -992,6 +992,8 @@ message GraphsOut {
     repeated HourGraphData hours = 2;
     TodayGraphData today = 3;
     ButtonsGraphData buttons = 4;
+    repeated Card cards2 = 5;
+    repeated RevlogEntry revlog = 6;
 }
 
 message CardsGraphData {
@@ -1029,3 +1031,15 @@ message ButtonsGraphData {
     repeated uint32 young = 2;
     repeated uint32 mature = 3;
 }
+
+message RevlogEntry {
+    int64 id = 1;
+    int64 cid = 2;
+    int32 usn = 3;
+    uint32 button_chosen = 4;
+    int32 interval = 5;
+    int32 last_interval = 6;
+    uint32 ease_factor = 7;
+    uint32 taken_millis = 8;
+    uint32 review_kind = 9;
+}

pylib/tools/genbackend.py

@@ -31,6 +31,7 @@ LABEL_REPEATED = 3
 
 # messages we don't want to unroll in codegen
 SKIP_UNROLL_INPUT = {"TranslateString"}
+SKIP_DECODE = {"Graphs"}
 
 
 def python_type(field):
@@ -118,13 +119,25 @@ def render_method(method, idx):
     else:
         single_field = ""
         return_type = f"pb.{method.output_type.name}"
-    return f"""\
+
+    if method.name in SKIP_DECODE:
+        return_type = "bytes"
+
+    buf = f"""\
     def {name}({input_args}) -> {return_type}:
-        {input_assign_outer}output = pb.{method.output_type.name}()
+        {input_assign_outer}"""
+
+    if method.name in SKIP_DECODE:
+        buf += f"""return self._run_command({idx+1}, input)
+"""
+    else:
+        buf += f"""output = pb.{method.output_type.name}()
         output.ParseFromString(self._run_command({idx+1}, input))
         return output{single_field}
 """
 
+    return buf
+
 
 out = []
 for idx, method in enumerate(pb._BACKENDSERVICE.methods):

qt/aqt/mediasrv.py

@@ -1,6 +1,9 @@
 # Copyright: Ankitects Pty Ltd and contributors
 # -*- coding: utf-8 -*-
 # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+from __future__ import annotations
+
 import http.server
 import re
 import socket
@@ -9,6 +12,7 @@ import threading
 from http import HTTPStatus
 from typing import Optional
 
+import aqt
 from anki.collection import Collection
 from anki.utils import devMode
 from aqt.qt import *
@@ -59,7 +63,8 @@ class MediaServer(threading.Thread):
 
     def run(self):
         RequestHandler.mw = self.mw
-        self.server = ThreadedHTTPServer(("127.0.0.1", 0), RequestHandler)
+        desired_port = int(os.getenv("ANKI_API_PORT", 0))
+        self.server = ThreadedHTTPServer(("127.0.0.1", desired_port), RequestHandler)
         self._ready.set()
         self.server.serve_forever()
 
@@ -74,7 +79,7 @@ class MediaServer(threading.Thread):
 class RequestHandler(http.server.SimpleHTTPRequestHandler):
 
     timeout = 1
-    mw: Optional[Collection] = None
+    mw: Optional[aqt.main.AnkiQt] = None
 
     def do_GET(self):
         f = self.send_head()
@@ -168,6 +173,32 @@ class RequestHandler(http.server.SimpleHTTPRequestHandler):
 
         return path
 
+    def do_POST(self):
+        if not self.path.startswith("/_anki/"):
+            self.send_error(HTTPStatus.NOT_FOUND, "Method not found")
+            return
+
+        cmd = self.path[len("/_anki/") :]
+
+        if cmd == "graphData":
+            data = graph_data(self.mw.col)
+        else:
+            self.send_error(HTTPStatus.NOT_FOUND, "Method not found")
+            return
+
+        self.send_response(HTTPStatus.OK)
+        self.send_header("Content-Type", "application/binary")
+        self.send_header("Content-Length", str(len(data)))
+        self.send_header("Access-Control-Allow-Origin", "*")
+        self.end_headers()
+
+        self.wfile.write(data)
+
+
+def graph_data(col: Collection) -> bytes:
+    graphs = col.backend.graphs(search="", days=0)
+    return graphs
+
 
 # work around Windows machines with incorrect mime type
 RequestHandler.extensions_map[".css"] = "text/css"

rslib/src/backend/mod.rs

@@ -685,7 +685,7 @@ impl BackendService for Backend {
             col.storage
                 .get_card(input.into())
                 .and_then(|opt| opt.ok_or(AnkiError::NotFound))
-                .map(card_to_pb)
+                .map(Into::into)
         })
     }
 
@@ -1700,7 +1700,8 @@ fn sort_kind_from_pb(kind: i32) -> SortKind {
     }
 }
 
-fn card_to_pb(c: Card) -> pb::Card {
+impl From<Card> for pb::Card {
+    fn from(c: Card) -> Self {
         pb::Card {
             id: c.id.0,
             nid: c.nid.0,
@@ -1722,6 +1723,7 @@ fn card_to_pb(c: Card) -> pb::Card {
             data: c.data,
         }
     }
+}
 
 fn pbcard_to_native(c: pb::Card) -> Result<Card> {
     let ctype = CardType::try_from(c.ctype as u8)

rslib/src/revlog.rs

@@ -1,6 +1,7 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
+use crate::serde::default_on_invalid;
 use crate::{define_newtype, prelude::*};
 use num_enum::TryFromPrimitive;
 use serde::Deserialize;
@@ -30,7 +31,7 @@ pub struct RevlogEntry {
     /// Amount of milliseconds taken to answer the card.
     #[serde(rename = "time")]
     pub taken_millis: u32,
-    #[serde(rename = "type")]
+    #[serde(rename = "type", default, deserialize_with = "default_on_invalid")]
     pub review_kind: RevlogReviewKind,
 }
 

rslib/src/search/cards.rs

@@ -11,6 +11,7 @@ use crate::collection::Collection;
 use crate::config::SortKind;
 use crate::err::Result;
 use crate::search::parser::parse;
+use rusqlite::NO_PARAMS;
 
 #[derive(Debug, PartialEq, Clone)]
 pub enum SortMode {
@@ -87,6 +88,31 @@ impl Collection {
         Ok(ids)
     }
 
+    /// Place the matched card ids into a temporary 'search_cids' table
+    /// instead of returning them. Use clear_searched_cards() to remove it.
+    pub(crate) fn search_cards_into_table(&mut self, search: &str) -> Result<()> {
+        let top_node = Node::Group(parse(search)?);
+        let writer = SqlWriter::new(self);
+
+        let (mut sql, args) = writer.build_cards_query(&top_node, RequiredTable::Cards)?;
+        self.storage.db.execute_batch(concat!(
+            "drop table if exists search_cids;",
+            "create temporary table search_cids (id integer primary key not null);"
+        ))?;
+        let sql = format!("insert into search_cids {}", sql);
+
+        self.storage.db.prepare(&sql)?.execute(&args)?;
+
+        Ok(())
+    }
+
+    pub(crate) fn clear_searched_cards(&mut self) -> Result<()> {
+        self.storage
+            .db
+            .execute("drop table if exists search_cids", NO_PARAMS)?;
+        Ok(())
+    }
+
     /// If the sort mode is based on a config setting, look it up.
     fn resolve_config_sort(&self, mode: &mut SortMode) {
         if mode == &SortMode::FromConfig {

rslib/src/stats/graphs.rs

@@ -8,7 +8,6 @@ use crate::{
     prelude::*,
     revlog::{RevlogEntry, RevlogReviewKind},
     sched::cutoff::SchedTimingToday,
-    search::SortMode,
 };
 
 struct GraphsContext {
@@ -27,6 +26,8 @@ struct AllStats {
     buttons: pb::ButtonsGraphData,
     hours: Vec<pb::HourGraphData>,
     cards: pb::CardsGraphData,
+    cards2: Vec<Card>,
+    revlog: Vec<pb::RevlogEntry>,
 }
 
 impl Default for AllStats {
@@ -41,6 +42,8 @@ impl Default for AllStats {
             buttons,
             hours: vec![Default::default(); 24],
             cards: Default::default(),
+            cards2: vec![],
+            revlog: vec![],
         }
     }
 }
@@ -52,6 +55,8 @@ impl From<AllStats> for pb::GraphsOut {
             hours: s.hours,
             today: Some(s.today),
             buttons: Some(s.buttons),
+            cards2: s.cards2.into_iter().map(Into::into).collect(),
+            revlog: s.revlog,
         }
     }
 }
@@ -217,13 +222,15 @@ impl Collection {
         search: &str,
         days: u32,
     ) -> Result<pb::GraphsOut> {
-        let cids = self.search_cards(search, SortMode::NoOrder)?;
+        self.search_cards_into_table(search)?;
-        let stats = self.graph_data(&cids, days)?;
+        let i = std::time::Instant::now();
-        println!("{:#?}", stats);
+        let all = search.trim().is_empty();
-        Ok(stats.into())
+        let stats = self.graph_data(all, days)?;
+        let stats = stats.into();
+        Ok(stats)
     }
 
-    fn graph_data(&self, cids: &[CardID], days: u32) -> Result<AllStats> {
+    fn graph_data(&self, all: bool, days: u32) -> Result<AllStats> {
         let timing = self.timing_today()?;
         let revlog_start = TimestampSecs(if days > 0 {
             timing.next_day_at - (((days as i64) + 1) * 86_400)
@@ -242,17 +249,35 @@ impl Collection {
             stats: AllStats::default(),
         };
 
-        for cid in cids {
+        let cards = self.storage.all_searched_cards()?;
-            let card = self.storage.get_card(*cid)?.ok_or(AnkiError::NotFound)?;
+        let revlog = if all {
-            ctx.observe_card(&card);
+            self.storage.get_all_revlog_entries(revlog_start)?
+        } else {
             self.storage
-                .for_each_revlog_entry_of_card(*cid, revlog_start, |entry| {
+                .get_revlog_entries_for_searched_cards(revlog_start)?
-                    Ok(ctx.observe_review(entry))
+        };
-                })?;
-        }
 
-        ctx.stats.cards.note_count = self.storage.note_ids_of_cards(cids)?.len() as u32;
+        ctx.stats.cards2 = cards;
+        ctx.stats.revlog = revlog;
+
+        // ctx.stats.cards.note_count = self.storage.note_ids_of_cards(cids)?.len() as u32;
 
         Ok(ctx.stats)
     }
 }
+
+impl From<RevlogEntry> for pb::RevlogEntry {
+    fn from(e: RevlogEntry) -> Self {
+        pb::RevlogEntry {
+            id: e.id.0,
+            cid: e.cid.0,
+            usn: e.usn.0,
+            button_chosen: e.button_chosen as u32,
+            interval: e.interval,
+            last_interval: e.last_interval,
+            ease_factor: e.ease_factor,
+            taken_millis: e.taken_millis,
+            review_kind: e.review_kind as u32,
+        }
+    }
+}

rslib/src/storage/card/mod.rs

@@ -250,6 +250,33 @@ impl super::SqliteStorage {
         }
         Ok(nids)
     }
+
+    pub(crate) fn for_each_card_in_search<F>(&self, mut func: F) -> Result<()>
+    where
+        F: FnMut(&Card) -> Result<()>,
+    {
+        let mut stmt = self.db.prepare_cached(concat!(
+            include_str!("get_card.sql"),
+            " where id in (select id from search_cids)"
+        ))?;
+        let mut rows = stmt.query(NO_PARAMS)?;
+        while let Some(row) = rows.next()? {
+            let entry = row_to_card(row)?;
+            func(&entry)?
+        }
+
+        Ok(())
+    }
+
+    pub(crate) fn all_searched_cards(&self) -> Result<Vec<Card>> {
+        self.db
+            .prepare_cached(concat!(
+                include_str!("get_card.sql"),
+                " where id in (select id from search_cids)"
+            ))?
+            .query_and_then(NO_PARAMS, |r| row_to_card(r).map_err(Into::into))?
+            .collect()
+    }
 }
 
 #[cfg(test)]

rslib/src/storage/revlog/mod.rs

@@ -4,6 +4,7 @@
 use super::SqliteStorage;
 use crate::err::Result;
 use crate::{
+    backend_proto as pb,
     prelude::*,
     revlog::{RevlogEntry, RevlogReviewKind},
 };
@@ -34,7 +35,7 @@ fn row_to_revlog_entry(row: &Row) -> Result<RevlogEntry> {
         last_interval: row.get(5)?,
         ease_factor: row.get(6)?,
         taken_millis: row.get(7)?,
-        review_kind: row.get(8)?,
+        review_kind: row.get(8).unwrap_or_default(),
     })
 }
 
@@ -85,24 +86,31 @@ impl SqliteStorage {
             .collect()
     }
 
-    pub(crate) fn for_each_revlog_entry_of_card<F>(
+    pub(crate) fn get_revlog_entries_for_searched_cards(
         &self,
-        cid: CardID,
+        after: TimestampSecs,
-        from: TimestampSecs,
+    ) -> Result<Vec<pb::RevlogEntry>> {
-        mut func: F,
+        self.db
-    ) -> Result<()>
+            .prepare_cached(concat!(
-    where
+                include_str!("get.sql"),
-        F: FnMut(&RevlogEntry) -> Result<()>,
+                " where cid in (select id from search_cids) and id >= ?"
-    {
+            ))?
-        let mut stmt = self
+            .query_and_then(&[after.0 * 1000], |r| {
-            .db
+                row_to_revlog_entry(r).map(Into::into)
-            .prepare_cached(concat!(include_str!("get.sql"), " where cid=? and id>=?"))?;
+            })?
-        let mut rows = stmt.query(&[cid.0, from.0 * 1000])?;
+            .collect()
-        while let Some(row) = rows.next()? {
-            let entry = row_to_revlog_entry(row)?;
-            func(&entry)?
     }
 
-        Ok(())
+    /// This includes entries from deleted cards.
+    pub(crate) fn get_all_revlog_entries(
+        &self,
+        after: TimestampSecs,
+    ) -> Result<Vec<pb::RevlogEntry>> {
+        self.db
+            .prepare_cached(concat!(include_str!("get.sql"), " where id >= ?"))?
+            .query_and_then(&[after.0 * 1000], |r| {
+                row_to_revlog_entry(r).map(Into::into)
+            })?
+            .collect()
     }
 }