diff --git a/proto/backend.proto b/proto/backend.proto index ea27a12ad..552a499d9 100644 --- a/proto/backend.proto +++ b/proto/backend.proto @@ -988,21 +988,25 @@ message GraphsIn { } message GraphsOut { - CardsGraphData cards = 1; - repeated HourGraphData hours = 2; - TodayGraphData today = 3; - ButtonsGraphData buttons = 4; - repeated Card cards2 = 5; - repeated RevlogEntry revlog = 6; +// CardsGraphData cards = 1; +// TodayGraphData today = 3; +// ButtonsGraphData buttons = 4; +// repeated HourGraphData hours = 2; + repeated Card cards = 1; + repeated RevlogEntry revlog = 2; + + uint32 days_elapsed = 3; + // Based on rollover hour + uint32 next_day_at_secs = 4; + uint32 scheduler_version = 5; + /// Seconds to add to UTC timestamps to get local time. + uint32 local_offset_secs = 7; + + uint32 note_count = 10; + } message CardsGraphData { - uint32 card_count = 1; - uint32 note_count = 2; - float ease_factor_min = 3; - float ease_factor_max = 4; - float ease_factor_sum = 5; - uint32 ease_factor_count = 6; uint32 mature_count = 7; uint32 young_or_learning_count = 8; uint32 new_count = 9; diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs index 224838671..3ce336944 100644 --- a/rslib/src/search/cards.rs +++ b/rslib/src/search/cards.rs @@ -94,7 +94,7 @@ impl Collection { let top_node = Node::Group(parse(search)?); let writer = SqlWriter::new(self); - let (mut sql, args) = writer.build_cards_query(&top_node, RequiredTable::Cards)?; + let (sql, args) = writer.build_cards_query(&top_node, RequiredTable::Cards)?; self.storage.db.execute_batch(concat!( "drop table if exists search_cids;", "create temporary table search_cids (id integer primary key not null);" @@ -106,7 +106,7 @@ impl Collection { Ok(()) } - pub(crate) fn clear_searched_cards(&mut self) -> Result<()> { + pub(crate) fn clear_searched_cards(&self) -> Result<()> { self.storage .db .execute("drop table if exists search_cids", NO_PARAMS)?; diff --git a/rslib/src/stats/graphs.rs b/rslib/src/stats/graphs.rs index 14d6d166d..738154f38 100644 --- a/rslib/src/stats/graphs.rs +++ b/rslib/src/stats/graphs.rs @@ -1,220 +1,104 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use crate::{ - backend_proto as pb, - card::{CardQueue, CardType}, - config::SchedulerVersion, - prelude::*, - revlog::{RevlogEntry, RevlogReviewKind}, - sched::cutoff::SchedTimingToday, -}; +use crate::{backend_proto as pb, prelude::*, revlog::RevlogEntry}; -struct GraphsContext { - scheduler: SchedulerVersion, - timing: SchedTimingToday, - /// Based on the set rollover hour. - today_rolled_over_at_millis: i64, - /// Seconds to add to UTC timestamps to get local time. - local_offset_secs: i64, - stats: AllStats, -} +// impl GraphsContext { +// fn observe_button_stats_for_review(&mut self, review: &RevlogEntry) { +// let mut button_num = review.button_chosen as usize; +// if button_num == 0 { +// return; +// } -#[derive(Debug)] -struct AllStats { - today: pb::TodayGraphData, - buttons: pb::ButtonsGraphData, - hours: Vec, - cards: pb::CardsGraphData, - cards2: Vec, - revlog: Vec, -} +// let buttons = &mut self.stats.buttons; +// let category = match review.review_kind { +// RevlogReviewKind::Learning | RevlogReviewKind::Relearning => { +// // V1 scheduler only had 3 buttons in learning +// if button_num == 4 && self.scheduler == SchedulerVersion::V1 { +// button_num = 3; +// } -impl Default for AllStats { - fn default() -> Self { - let buttons = pb::ButtonsGraphData { - learn: vec![0; 4], - young: vec![0; 4], - mature: vec![0; 4], - }; - AllStats { - today: Default::default(), - buttons, - hours: vec![Default::default(); 24], - cards: Default::default(), - cards2: vec![], - revlog: vec![], - } - } -} +// &mut buttons.learn +// } +// RevlogReviewKind::Review | RevlogReviewKind::EarlyReview => { +// if review.last_interval < 21 { +// &mut buttons.young +// } else { +// &mut buttons.mature +// } +// } +// }; -impl From for pb::GraphsOut { - fn from(s: AllStats) -> Self { - pb::GraphsOut { - cards: Some(s.cards), - hours: s.hours, - today: Some(s.today), - buttons: Some(s.buttons), - cards2: s.cards2.into_iter().map(Into::into).collect(), - revlog: s.revlog, - } - } -} +// if let Some(count) = category.get_mut(button_num - 1) { +// *count += 1; +// } +// } -#[derive(Default, Debug)] -struct ButtonStats { - /// In V1 scheduler, 4th element is ignored - learn: [u32; 4], - young: [u32; 4], - mature: [u32; 4], -} +// fn observe_hour_stats_for_review(&mut self, review: &RevlogEntry) { +// match review.review_kind { +// RevlogReviewKind::Learning +// | RevlogReviewKind::Review +// | RevlogReviewKind::Relearning => { +// let hour_idx = (((review.id.0 / 1000) + self.local_offset_secs) / 3600) % 24; +// let hour = &mut self.stats.hours[hour_idx as usize]; -#[derive(Default, Debug)] -struct HourStats { - review_count: u32, - correct_count: u32, -} +// hour.review_count += 1; +// if review.button_chosen != 1 { +// hour.correct_count += 1; +// } +// } +// RevlogReviewKind::EarlyReview => {} +// } +// } -#[derive(Default, Debug)] -struct CardStats { - card_count: u32, - note_count: u32, - ease_factor_min: f32, - ease_factor_max: f32, - ease_factor_sum: f32, - ease_factor_count: u32, - mature_count: u32, - young_or_learning_count: u32, - new_count: u32, - suspended_or_buried_count: u32, -} +// fn observe_today_stats_for_review(&mut self, review: &RevlogEntry) { +// if review.id.0 < self.today_rolled_over_at_millis { +// return; +// } -impl GraphsContext { - fn observe_card(&mut self, card: &Card) { - self.observe_card_stats_for_card(card); - } +// let today = &mut self.stats.today; - fn observe_review(&mut self, entry: &RevlogEntry) { - self.observe_button_stats_for_review(entry); - self.observe_hour_stats_for_review(entry); - self.observe_today_stats_for_review(entry); - } +// // total +// today.answer_count += 1; +// today.answer_millis += review.taken_millis; - fn observe_button_stats_for_review(&mut self, review: &RevlogEntry) { - let mut button_num = review.button_chosen as usize; - if button_num == 0 { - return; - } +// // correct +// if review.button_chosen > 1 { +// today.correct_count += 1; +// } - let buttons = &mut self.stats.buttons; - let category = match review.review_kind { - RevlogReviewKind::Learning | RevlogReviewKind::Relearning => { - // V1 scheduler only had 3 buttons in learning - if button_num == 4 && self.scheduler == SchedulerVersion::V1 { - button_num = 3; - } +// // mature +// if review.last_interval >= 21 { +// today.mature_count += 1; +// if review.button_chosen > 1 { +// today.mature_correct += 1; +// } +// } - &mut buttons.learn - } - RevlogReviewKind::Review | RevlogReviewKind::EarlyReview => { - if review.last_interval < 21 { - &mut buttons.young - } else { - &mut buttons.mature - } - } - }; +// // type counts +// match review.review_kind { +// RevlogReviewKind::Learning => today.learn_count += 1, +// RevlogReviewKind::Review => today.review_count += 1, +// RevlogReviewKind::Relearning => today.relearn_count += 1, +// RevlogReviewKind::EarlyReview => today.early_review_count += 1, +// } +// } - if let Some(count) = category.get_mut(button_num - 1) { - *count += 1; - } - } - - fn observe_hour_stats_for_review(&mut self, review: &RevlogEntry) { - match review.review_kind { - RevlogReviewKind::Learning - | RevlogReviewKind::Review - | RevlogReviewKind::Relearning => { - let hour_idx = (((review.id.0 / 1000) + self.local_offset_secs) / 3600) % 24; - let hour = &mut self.stats.hours[hour_idx as usize]; - - hour.review_count += 1; - if review.button_chosen != 1 { - hour.correct_count += 1; - } - } - RevlogReviewKind::EarlyReview => {} - } - } - - fn observe_today_stats_for_review(&mut self, review: &RevlogEntry) { - if review.id.0 < self.today_rolled_over_at_millis { - return; - } - - let today = &mut self.stats.today; - - // total - today.answer_count += 1; - today.answer_millis += review.taken_millis; - - // correct - if review.button_chosen > 1 { - today.correct_count += 1; - } - - // mature - if review.last_interval >= 21 { - today.mature_count += 1; - if review.button_chosen > 1 { - today.mature_correct += 1; - } - } - - // type counts - match review.review_kind { - RevlogReviewKind::Learning => today.learn_count += 1, - RevlogReviewKind::Review => today.review_count += 1, - RevlogReviewKind::Relearning => today.relearn_count += 1, - RevlogReviewKind::EarlyReview => today.early_review_count += 1, - } - } - - fn observe_card_stats_for_card(&mut self, card: &Card) { - let cstats = &mut self.stats.cards; - - cstats.card_count += 1; - - // counts by type - match card.queue { - CardQueue::New => cstats.new_count += 1, - CardQueue::Review if card.ivl >= 21 => cstats.mature_count += 1, - CardQueue::Review | CardQueue::Learn | CardQueue::DayLearn => { - cstats.young_or_learning_count += 1 - } - CardQueue::Suspended | CardQueue::UserBuried | CardQueue::SchedBuried => { - cstats.suspended_or_buried_count += 1 - } - CardQueue::PreviewRepeat => {} - } - - // ease factor - if card.ctype == CardType::Review { - let ease_factor = (card.factor as f32) / 1000.0; - - cstats.ease_factor_count += 1; - cstats.ease_factor_sum += ease_factor; - - if ease_factor < cstats.ease_factor_min || cstats.ease_factor_min == 0.0 { - cstats.ease_factor_min = ease_factor; - } - - if ease_factor > cstats.ease_factor_max { - cstats.ease_factor_max = ease_factor; - } - } - } -} +// fn observe_card_stats_for_card(&mut self, card: &Card) { +// counts by type +// match card.queue { +// CardQueue::New => cstats.new_count += 1, +// CardQueue::Review if card.ivl >= 21 => cstats.mature_count += 1, +// CardQueue::Review | CardQueue::Learn | CardQueue::DayLearn => { +// cstats.young_or_learning_count += 1 +// } +// CardQueue::Suspended | CardQueue::UserBuried | CardQueue::SchedBuried => { +// cstats.suspended_or_buried_count += 1 +// } +// CardQueue::PreviewRepeat => {} +// } +// } +// } impl Collection { pub(crate) fn graph_data_for_search( @@ -223,14 +107,11 @@ impl Collection { days: u32, ) -> Result { self.search_cards_into_table(search)?; - let i = std::time::Instant::now(); let all = search.trim().is_empty(); - let stats = self.graph_data(all, days)?; - let stats = stats.into(); - Ok(stats) + self.graph_data(all, days) } - fn graph_data(&self, all: bool, days: u32) -> Result { + fn graph_data(&self, all: bool, days: u32) -> Result { let timing = self.timing_today()?; let revlog_start = TimestampSecs(if days > 0 { timing.next_day_at - (((days as i64) + 1) * 86_400) @@ -241,14 +122,6 @@ impl Collection { let offset = self.local_offset(); let local_offset_secs = offset.local_minus_utc() as i64; - let mut ctx = GraphsContext { - scheduler: self.sched_ver(), - today_rolled_over_at_millis: (timing.next_day_at - 86_400) * 1000, - timing, - local_offset_secs, - stats: AllStats::default(), - }; - let cards = self.storage.all_searched_cards()?; let revlog = if all { self.storage.get_all_revlog_entries(revlog_start)? @@ -257,12 +130,17 @@ impl Collection { .get_revlog_entries_for_searched_cards(revlog_start)? }; - ctx.stats.cards2 = cards; - ctx.stats.revlog = revlog; + self.clear_searched_cards()?; - // ctx.stats.cards.note_count = self.storage.note_ids_of_cards(cids)?.len() as u32; - - Ok(ctx.stats) + Ok(pb::GraphsOut { + cards: cards.into_iter().map(Into::into).collect(), + revlog, + days_elapsed: timing.days_elapsed, + next_day_at_secs: timing.next_day_at as u32, + scheduler_version: self.sched_ver() as u32, + local_offset_secs: local_offset_secs as u32, + note_count: 0, + }) } } diff --git a/rslib/src/storage/card/mod.rs b/rslib/src/storage/card/mod.rs index 379083d8f..8ca5ad79d 100644 --- a/rslib/src/storage/card/mod.rs +++ b/rslib/src/storage/card/mod.rs @@ -251,23 +251,6 @@ impl super::SqliteStorage { Ok(nids) } - pub(crate) fn for_each_card_in_search(&self, mut func: F) -> Result<()> - where - F: FnMut(&Card) -> Result<()>, - { - let mut stmt = self.db.prepare_cached(concat!( - include_str!("get_card.sql"), - " where id in (select id from search_cids)" - ))?; - let mut rows = stmt.query(NO_PARAMS)?; - while let Some(row) = rows.next()? { - let entry = row_to_card(row)?; - func(&entry)? - } - - Ok(()) - } - pub(crate) fn all_searched_cards(&self) -> Result> { self.db .prepare_cached(concat!( diff --git a/ts/.eslintrc.js b/ts/.eslintrc.js index 9fde61347..871397b23 100644 --- a/ts/.eslintrc.js +++ b/ts/.eslintrc.js @@ -9,6 +9,10 @@ module.exports = { rules: { "prefer-const": "warn", "@typescript-eslint/ban-ts-ignore": "warn", + "@typescript-eslint/no-unused-vars": [ + "error", + { argsIgnorePattern: "^_", varsIgnorePattern: "^_" }, + ], }, overrides: [ { diff --git a/ts/src/stats/AddedGraph.svelte b/ts/src/stats/AddedGraph.svelte new file mode 100644 index 000000000..a398ee470 --- /dev/null +++ b/ts/src/stats/AddedGraph.svelte @@ -0,0 +1,50 @@ + + +{#if histogramData} +
+

Added

+ +
+ + + + +
+ + +
+{/if} diff --git a/ts/src/stats/GraphsPage.svelte b/ts/src/stats/GraphsPage.svelte index 6e8ca2305..431f51bbf 100644 --- a/ts/src/stats/GraphsPage.svelte +++ b/ts/src/stats/GraphsPage.svelte @@ -10,6 +10,7 @@ import { getGraphData, GraphRange } from "./graphs"; import IntervalsGraph from "./IntervalsGraph.svelte"; import EaseGraph from "./EaseGraph.svelte"; + import AddedGraph from "./AddedGraph.svelte"; let data: pb.BackendProto.GraphsOut | null = null; @@ -107,5 +108,6 @@ + diff --git a/ts/src/stats/added.ts b/ts/src/stats/added.ts new file mode 100644 index 000000000..2dc2e54cb --- /dev/null +++ b/ts/src/stats/added.ts @@ -0,0 +1,82 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +/* eslint +@typescript-eslint/no-non-null-assertion: "off", +@typescript-eslint/no-explicit-any: "off", + */ + +import pb from "../backend/proto"; +import { extent, histogram } from "d3-array"; +import { scaleLinear, scaleSequential } from "d3-scale"; +import { HistogramData } from "./histogram-graph"; +import { interpolateBlues } from "d3-scale-chromatic"; + +export enum AddedRange { + Month = 0, + Quarter = 1, + Year = 2, + AllTime = 3, +} + +export interface GraphData { + daysAdded: number[]; +} + +export function gatherData(data: pb.BackendProto.GraphsOut): GraphData { + const daysAdded = (data.cards as pb.BackendProto.Card[]).map((card) => { + const elapsedSecs = (card.id as number) / 1000 - data.nextDayAtSecs; + return Math.ceil(elapsedSecs / 86400); + }); + return { daysAdded }; +} + +function hoverText( + data: HistogramData, + binIdx: number, + cumulative: number, + _percent: number +): string { + const bin = data.bins[binIdx]; + return ( + `${bin.length} at ${bin.x1! - 1} days.
` + + ` ${cumulative} cards at or below this point.` + ); +} + +export function prepareData(data: GraphData, range: AddedRange): HistogramData | null { + // get min/max + const total = data.daysAdded.length; + if (!total) { + return null; + } + + const [xMinOrig, _xMax] = extent(data.daysAdded); + let xMin = xMinOrig; + + // cap max to selected range + switch (range) { + case AddedRange.Month: + xMin = -31; + break; + case AddedRange.Quarter: + xMin = -90; + break; + case AddedRange.Year: + xMin = -365; + break; + case AddedRange.AllTime: + break; + } + const xMax = 1; + const desiredBars = Math.min(70, Math.abs(xMin!)); + + const scale = scaleLinear().domain([xMin!, xMax]); + const bins = histogram() + .domain(scale.domain() as any) + .thresholds(scale.ticks(desiredBars))(data.daysAdded); + + const colourScale = scaleSequential(interpolateBlues).domain([xMin!, xMax]); + + return { scale, bins, total, hoverText, colourScale, showArea: true }; +} diff --git a/ts/src/stats/ease.ts b/ts/src/stats/ease.ts index 6d8a8171c..41ca71ed9 100644 --- a/ts/src/stats/ease.ts +++ b/ts/src/stats/ease.ts @@ -18,7 +18,7 @@ export interface GraphData { } export function gatherData(data: pb.BackendProto.GraphsOut): GraphData { - const eases = (data.cards2 as pb.BackendProto.Card[]) + const eases = (data.cards as pb.BackendProto.Card[]) .filter((c) => c.queue == CardQueue.Review) .map((c) => c.factor / 10); return { eases }; diff --git a/ts/src/stats/histogram-graph.ts b/ts/src/stats/histogram-graph.ts index 70a5bdc84..bcdfb5e95 100644 --- a/ts/src/stats/histogram-graph.ts +++ b/ts/src/stats/histogram-graph.ts @@ -9,8 +9,7 @@ import "d3-transition"; import { select, mouse } from "d3-selection"; import { cumsum, max, Bin } from "d3-array"; -import { interpolateBlues, interpolateRdYlGn } from "d3-scale-chromatic"; -import { scaleLinear, scaleSequential, ScaleLinear, ScaleSequential } from "d3-scale"; +import { scaleLinear, ScaleLinear, ScaleSequential } from "d3-scale"; import { axisBottom, axisLeft } from "d3-axis"; import { area } from "d3-shape"; import { showTooltip, hideTooltip } from "./tooltip"; @@ -20,7 +19,12 @@ export interface HistogramData { scale: ScaleLinear; bins: Bin[]; total: number; - hoverText: (data: HistogramData, binIdx: number, percent: number) => string; + hoverText: ( + data: HistogramData, + binIdx: number, + cumulative: number, + percent: number + ) => string; showArea: boolean; colourScale: ScaleSequential; } @@ -66,7 +70,7 @@ export function histogramGraph( .attr("x", (d: any) => x(d.x0)) .attr("y", (d: any) => y(d.length)!) .attr("height", (d: any) => y(0) - y(d.length)) - .attr("fill", (d, idx) => data.colourScale(d.x1)); + .attr("fill", (d) => data.colourScale(d.x1)); }; svg.select("g.bars") @@ -125,7 +129,7 @@ export function histogramGraph( .on("mousemove", function (this: any, d: any, idx) { const [x, y] = mouse(document.body); const pct = data.showArea ? (areaData[idx + 1] / data.total) * 100 : 0; - showTooltip(data.hoverText(data, idx, pct), x, y); + showTooltip(data.hoverText(data, idx, areaData[idx + 1], pct), x, y); }) .on("mouseout", hideTooltip); } diff --git a/ts/src/stats/intervals.ts b/ts/src/stats/intervals.ts index b9875787e..24612e093 100644 --- a/ts/src/stats/intervals.ts +++ b/ts/src/stats/intervals.ts @@ -26,13 +26,18 @@ export enum IntervalRange { } export function gatherIntervalData(data: pb.BackendProto.GraphsOut): IntervalGraphData { - const intervals = (data.cards2 as pb.BackendProto.Card[]) + const intervals = (data.cards as pb.BackendProto.Card[]) .filter((c) => c.queue == CardQueue.Review) .map((c) => c.ivl); return { intervals }; } -function hoverText(data: HistogramData, binIdx: number, percent: number): string { +function hoverText( + data: HistogramData, + binIdx: number, + _cumulative: number, + percent: number +): string { const bin = data.bins[binIdx]; const interval = bin.x1! - bin.x0! === 1