diff --git a/proto/backend.proto b/proto/backend.proto index f8278cfae..ea27a12ad 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -992,6 +992,8 @@ message GraphsOut { repeated HourGraphData hours = 2; TodayGraphData today = 3; ButtonsGraphData buttons = 4; + repeated Card cards2 = 5; + repeated RevlogEntry revlog = 6; } message CardsGraphData { @@ -1029,3 +1031,15 @@ message ButtonsGraphData { repeated uint32 young = 2; repeated uint32 mature = 3; } + +message RevlogEntry { + int64 id = 1; + int64 cid = 2; + int32 usn = 3; + uint32 button_chosen = 4; + int32 interval = 5; + int32 last_interval = 6; + uint32 ease_factor = 7; + uint32 taken_millis = 8; + uint32 review_kind = 9; +} diff --git a/pylib/tools/genbackend.py b/pylib/tools/genbackend.py index ce419f1ee..25d63ce62 100755 --- a/pylib/tools/genbackend.py +++ b/pylib/tools/genbackend.py @@ -31,6 +31,7 @@ LABEL_REPEATED = 3 # messages we don't want to unroll in codegen SKIP_UNROLL_INPUT = {"TranslateString"} +SKIP_DECODE = {"Graphs"} def python_type(field): @@ -118,13 +119,25 @@ def render_method(method, idx): else: single_field = "" return_type = f"pb.{method.output_type.name}" - return f"""\ + + if method.name in SKIP_DECODE: + return_type = "bytes" + + buf = f"""\ def {name}({input_args}) -> {return_type}: - {input_assign_outer}output = pb.{method.output_type.name}() + {input_assign_outer}""" + + if method.name in SKIP_DECODE: + buf += f"""return self._run_command({idx+1}, input) +""" + else: + buf += f"""output = pb.{method.output_type.name}() output.ParseFromString(self._run_command({idx+1}, input)) return output{single_field} """ + return buf + out = [] for idx, method in enumerate(pb._BACKENDSERVICE.methods): diff --git a/qt/aqt/mediasrv.py b/qt/aqt/mediasrv.py index 6705ce4d9..a3a38d1e2 100644 --- a/qt/aqt/mediasrv.py +++ b/qt/aqt/mediasrv.py @@ -1,6 +1,9 @@ # Copyright: Ankitects Pty Ltd and contributors # -*- coding: utf-8 -*- # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +from __future__ import annotations + import http.server import re import socket @@ -9,6 +12,7 @@ import threading from http import HTTPStatus from typing import Optional +import aqt from anki.collection import Collection from anki.utils import devMode from aqt.qt import * @@ -59,7 +63,8 @@ class MediaServer(threading.Thread): def run(self): RequestHandler.mw = self.mw - self.server = ThreadedHTTPServer(("127.0.0.1", 0), RequestHandler) + desired_port = int(os.getenv("ANKI_API_PORT", 0)) + self.server = ThreadedHTTPServer(("127.0.0.1", desired_port), RequestHandler) self._ready.set() self.server.serve_forever() @@ -74,7 +79,7 @@ class MediaServer(threading.Thread): class RequestHandler(http.server.SimpleHTTPRequestHandler): timeout = 1 - mw: Optional[Collection] = None + mw: Optional[aqt.main.AnkiQt] = None def do_GET(self): f = self.send_head() @@ -168,6 +173,32 @@ class RequestHandler(http.server.SimpleHTTPRequestHandler): return path + def do_POST(self): + if not self.path.startswith("/_anki/"): + self.send_error(HTTPStatus.NOT_FOUND, "Method not found") + return + + cmd = self.path[len("/_anki/") :] + + if cmd == "graphData": + data = graph_data(self.mw.col) + else: + self.send_error(HTTPStatus.NOT_FOUND, "Method not found") + return + + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "application/binary") + self.send_header("Content-Length", str(len(data))) + self.send_header("Access-Control-Allow-Origin", "*") + self.end_headers() + + self.wfile.write(data) + + +def graph_data(col: Collection) -> bytes: + graphs = col.backend.graphs(search="", days=0) + return graphs + # work around Windows machines with incorrect mime type RequestHandler.extensions_map[".css"] = "text/css" diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 5f22fcd8d..850e32648 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -685,7 +685,7 @@ impl BackendService for Backend { col.storage .get_card(input.into()) .and_then(|opt| opt.ok_or(AnkiError::NotFound)) - .map(card_to_pb) + .map(Into::into) }) } @@ -1700,26 +1700,28 @@ fn sort_kind_from_pb(kind: i32) -> SortKind { } } -fn card_to_pb(c: Card) -> pb::Card { - pb::Card { - id: c.id.0, - nid: c.nid.0, - did: c.did.0, - ord: c.ord as u32, - mtime: c.mtime.0, - usn: c.usn.0, - ctype: c.ctype as u32, - queue: c.queue as i32, - due: c.due, - ivl: c.ivl, - factor: c.factor as u32, - reps: c.reps, - lapses: c.lapses, - left: c.left, - odue: c.odue, - odid: c.odid.0, - flags: c.flags as u32, - data: c.data, +impl From for pb::Card { + fn from(c: Card) -> Self { + pb::Card { + id: c.id.0, + nid: c.nid.0, + did: c.did.0, + ord: c.ord as u32, + mtime: c.mtime.0, + usn: c.usn.0, + ctype: c.ctype as u32, + queue: c.queue as i32, + due: c.due, + ivl: c.ivl, + factor: c.factor as u32, + reps: c.reps, + lapses: c.lapses, + left: c.left, + odue: c.odue, + odid: c.odid.0, + flags: c.flags as u32, + data: c.data, + } } } diff --git a/rslib/src/revlog.rs b/rslib/src/revlog.rs index 37d78767c..73caec66c 100644 --- a/rslib/src/revlog.rs +++ b/rslib/src/revlog.rs @@ -1,6 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use crate::serde::default_on_invalid; use crate::{define_newtype, prelude::*}; use num_enum::TryFromPrimitive; use serde::Deserialize; @@ -30,7 +31,7 @@ pub struct RevlogEntry { /// Amount of milliseconds taken to answer the card. #[serde(rename = "time")] pub taken_millis: u32, - #[serde(rename = "type")] + #[serde(rename = "type", default, deserialize_with = "default_on_invalid")] pub review_kind: RevlogReviewKind, } diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index e4afa8ea2..224838671 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -11,6 +11,7 @@ use crate::collection::Collection; use crate::config::SortKind; use crate::err::Result; use crate::search::parser::parse; +use rusqlite::NO_PARAMS; #[derive(Debug, PartialEq, Clone)] pub enum SortMode { @@ -87,6 +88,31 @@ impl Collection { Ok(ids) } + /// Place the matched card ids into a temporary 'search_cids' table + /// instead of returning them. Use clear_searched_cards() to remove it. + pub(crate) fn search_cards_into_table(&mut self, search: &str) -> Result<()> { + let top_node = Node::Group(parse(search)?); + let writer = SqlWriter::new(self); + + let (mut sql, args) = writer.build_cards_query(&top_node, RequiredTable::Cards)?; + self.storage.db.execute_batch(concat!( + "drop table if exists search_cids;", + "create temporary table search_cids (id integer primary key not null);" + ))?; + let sql = format!("insert into search_cids {}", sql); + + self.storage.db.prepare(&sql)?.execute(&args)?; + + Ok(()) + } + + pub(crate) fn clear_searched_cards(&mut self) -> Result<()> { + self.storage + .db + .execute("drop table if exists search_cids", NO_PARAMS)?; + Ok(()) + } + /// If the sort mode is based on a config setting, look it up. fn resolve_config_sort(&self, mode: &mut SortMode) { if mode == &SortMode::FromConfig { diff --git a/rslib/src/stats/graphs.rs b/rslib/src/stats/graphs.rs index 7d8456da7..14d6d166d 100644 --- a/rslib/src/stats/graphs.rs +++ b/rslib/src/stats/graphs.rs @@ -8,7 +8,6 @@ use crate::{ prelude::*, revlog::{RevlogEntry, RevlogReviewKind}, sched::cutoff::SchedTimingToday, - search::SortMode, }; struct GraphsContext { @@ -27,6 +26,8 @@ struct AllStats { buttons: pb::ButtonsGraphData, hours: Vec, cards: pb::CardsGraphData, + cards2: Vec, + revlog: Vec, } impl Default for AllStats { @@ -41,6 +42,8 @@ impl Default for AllStats { buttons, hours: vec![Default::default(); 24], cards: Default::default(), + cards2: vec![], + revlog: vec![], } } } @@ -52,6 +55,8 @@ impl From for pb::GraphsOut { hours: s.hours, today: Some(s.today), buttons: Some(s.buttons), + cards2: s.cards2.into_iter().map(Into::into).collect(), + revlog: s.revlog, } } } @@ -217,13 +222,15 @@ impl Collection { search: &str, days: u32, ) -> Result { - let cids = self.search_cards(search, SortMode::NoOrder)?; - let stats = self.graph_data(&cids, days)?; - println!("{:#?}", stats); - Ok(stats.into()) + self.search_cards_into_table(search)?; + let i = std::time::Instant::now(); + let all = search.trim().is_empty(); + let stats = self.graph_data(all, days)?; + let stats = stats.into(); + Ok(stats) } - fn graph_data(&self, cids: &[CardID], days: u32) -> Result { + fn graph_data(&self, all: bool, days: u32) -> Result { let timing = self.timing_today()?; let revlog_start = TimestampSecs(if days > 0 { timing.next_day_at - (((days as i64) + 1) * 86_400) @@ -242,17 +249,35 @@ impl Collection { stats: AllStats::default(), }; - for cid in cids { - let card = self.storage.get_card(*cid)?.ok_or(AnkiError::NotFound)?; - ctx.observe_card(&card); + let cards = self.storage.all_searched_cards()?; + let revlog = if all { + self.storage.get_all_revlog_entries(revlog_start)? + } else { self.storage - .for_each_revlog_entry_of_card(*cid, revlog_start, |entry| { - Ok(ctx.observe_review(entry)) - })?; - } + .get_revlog_entries_for_searched_cards(revlog_start)? + }; - ctx.stats.cards.note_count = self.storage.note_ids_of_cards(cids)?.len() as u32; + ctx.stats.cards2 = cards; + ctx.stats.revlog = revlog; + + // ctx.stats.cards.note_count = self.storage.note_ids_of_cards(cids)?.len() as u32; Ok(ctx.stats) } } + +impl From for pb::RevlogEntry { + fn from(e: RevlogEntry) -> Self { + pb::RevlogEntry { + id: e.id.0, + cid: e.cid.0, + usn: e.usn.0, + button_chosen: e.button_chosen as u32, + interval: e.interval, + last_interval: e.last_interval, + ease_factor: e.ease_factor, + taken_millis: e.taken_millis, + review_kind: e.review_kind as u32, + } + } +} diff --git a/rslib/src/storage/card/mod.rs b/rslib/src/storage/card/mod.rs index e60158777..379083d8f 100644 --- a/rslib/src/storage/card/mod.rs +++ b/rslib/src/storage/card/mod.rs @@ -250,6 +250,33 @@ impl super::SqliteStorage { } Ok(nids) } + + pub(crate) fn for_each_card_in_search(&self, mut func: F) -> Result<()> + where + F: FnMut(&Card) -> Result<()>, + { + let mut stmt = self.db.prepare_cached(concat!( + include_str!("get_card.sql"), + " where id in (select id from search_cids)" + ))?; + let mut rows = stmt.query(NO_PARAMS)?; + while let Some(row) = rows.next()? { + let entry = row_to_card(row)?; + func(&entry)? + } + + Ok(()) + } + + pub(crate) fn all_searched_cards(&self) -> Result> { + self.db + .prepare_cached(concat!( + include_str!("get_card.sql"), + " where id in (select id from search_cids)" + ))? + .query_and_then(NO_PARAMS, |r| row_to_card(r).map_err(Into::into))? + .collect() + } } #[cfg(test)] diff --git a/rslib/src/storage/revlog/mod.rs b/rslib/src/storage/revlog/mod.rs index 9976d11d8..9c751c8bc 100644 --- a/rslib/src/storage/revlog/mod.rs +++ b/rslib/src/storage/revlog/mod.rs @@ -4,6 +4,7 @@ use super::SqliteStorage; use crate::err::Result; use crate::{ + backend_proto as pb, prelude::*, revlog::{RevlogEntry, RevlogReviewKind}, }; @@ -34,7 +35,7 @@ fn row_to_revlog_entry(row: &Row) -> Result { last_interval: row.get(5)?, ease_factor: row.get(6)?, taken_millis: row.get(7)?, - review_kind: row.get(8)?, + review_kind: row.get(8).unwrap_or_default(), }) } @@ -85,24 +86,31 @@ impl SqliteStorage { .collect() } - pub(crate) fn for_each_revlog_entry_of_card( + pub(crate) fn get_revlog_entries_for_searched_cards( &self, - cid: CardID, - from: TimestampSecs, - mut func: F, - ) -> Result<()> - where - F: FnMut(&RevlogEntry) -> Result<()>, - { - let mut stmt = self - .db - .prepare_cached(concat!(include_str!("get.sql"), " where cid=? and id>=?"))?; - let mut rows = stmt.query(&[cid.0, from.0 * 1000])?; - while let Some(row) = rows.next()? { - let entry = row_to_revlog_entry(row)?; - func(&entry)? - } + after: TimestampSecs, + ) -> Result> { + self.db + .prepare_cached(concat!( + include_str!("get.sql"), + " where cid in (select id from search_cids) and id >= ?" + ))? + .query_and_then(&[after.0 * 1000], |r| { + row_to_revlog_entry(r).map(Into::into) + })? + .collect() + } - Ok(()) + /// This includes entries from deleted cards. + pub(crate) fn get_all_revlog_entries( + &self, + after: TimestampSecs, + ) -> Result> { + self.db + .prepare_cached(concat!(include_str!("get.sql"), " where id >= ?"))? + .query_and_then(&[after.0 * 1000], |r| { + row_to_revlog_entry(r).map(Into::into) + })? + .collect() } }