cards added graph

proto/backend.proto

@@ -988,21 +988,25 @@ message GraphsIn {
 }
 
 message GraphsOut {
-    CardsGraphData cards = 1;
+//    CardsGraphData cards = 1;
-    repeated HourGraphData hours = 2;
+//    TodayGraphData today = 3;
-    TodayGraphData today = 3;
+//    ButtonsGraphData buttons = 4;
-    ButtonsGraphData buttons = 4;
+//    repeated HourGraphData hours = 2;
-    repeated Card cards2 = 5;
+    repeated Card cards = 1;
-    repeated RevlogEntry revlog = 6;
+    repeated RevlogEntry revlog = 2;
+
+    uint32 days_elapsed = 3;
+    // Based on rollover hour
+    uint32 next_day_at_secs = 4;
+    uint32 scheduler_version = 5;
+    /// Seconds to add to UTC timestamps to get local time.
+    uint32 local_offset_secs = 7;
+
+    uint32 note_count = 10;
+
 }
 
 message CardsGraphData {
-    uint32 card_count = 1;
-    uint32 note_count = 2;
-    float ease_factor_min = 3;
-    float ease_factor_max = 4;
-    float ease_factor_sum = 5;
-    uint32 ease_factor_count = 6;
     uint32 mature_count = 7;
     uint32 young_or_learning_count = 8;
     uint32 new_count = 9;

rslib/src/search/cards.rs

@@ -94,7 +94,7 @@ impl Collection {
         let top_node = Node::Group(parse(search)?);
         let writer = SqlWriter::new(self);
 
-        let (mut sql, args) = writer.build_cards_query(&top_node, RequiredTable::Cards)?;
+        let (sql, args) = writer.build_cards_query(&top_node, RequiredTable::Cards)?;
         self.storage.db.execute_batch(concat!(
             "drop table if exists search_cids;",
             "create temporary table search_cids (id integer primary key not null);"
@@ -106,7 +106,7 @@ impl Collection {
         Ok(())
     }
 
-    pub(crate) fn clear_searched_cards(&mut self) -> Result<()> {
+    pub(crate) fn clear_searched_cards(&self) -> Result<()> {
         self.storage
             .db
             .execute("drop table if exists search_cids", NO_PARAMS)?;

rslib/src/stats/graphs.rs

@@ -1,220 +1,104 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-use crate::{
+use crate::{backend_proto as pb, prelude::*, revlog::RevlogEntry};
-    backend_proto as pb,
-    card::{CardQueue, CardType},
-    config::SchedulerVersion,
-    prelude::*,
-    revlog::{RevlogEntry, RevlogReviewKind},
-    sched::cutoff::SchedTimingToday,
-};
 
-struct GraphsContext {
+// impl GraphsContext {
-    scheduler: SchedulerVersion,
+//     fn observe_button_stats_for_review(&mut self, review: &RevlogEntry) {
-    timing: SchedTimingToday,
+//         let mut button_num = review.button_chosen as usize;
-    /// Based on the set rollover hour.
+//         if button_num == 0 {
-    today_rolled_over_at_millis: i64,
+//             return;
-    /// Seconds to add to UTC timestamps to get local time.
+//         }
-    local_offset_secs: i64,
-    stats: AllStats,
-}
 
-#[derive(Debug)]
+//         let buttons = &mut self.stats.buttons;
-struct AllStats {
+//         let category = match review.review_kind {
-    today: pb::TodayGraphData,
+//             RevlogReviewKind::Learning | RevlogReviewKind::Relearning => {
-    buttons: pb::ButtonsGraphData,
+//                 // V1 scheduler only had 3 buttons in learning
-    hours: Vec<pb::HourGraphData>,
+//                 if button_num == 4 && self.scheduler == SchedulerVersion::V1 {
-    cards: pb::CardsGraphData,
+//                     button_num = 3;
-    cards2: Vec<Card>,
+//                 }
-    revlog: Vec<pb::RevlogEntry>,
-}
 
-impl Default for AllStats {
+//                 &mut buttons.learn
-    fn default() -> Self {
+//             }
-        let buttons = pb::ButtonsGraphData {
+//             RevlogReviewKind::Review | RevlogReviewKind::EarlyReview => {
-            learn: vec![0; 4],
+//                 if review.last_interval < 21 {
-            young: vec![0; 4],
+//                     &mut buttons.young
-            mature: vec![0; 4],
+//                 } else {
-        };
+//                     &mut buttons.mature
-        AllStats {
+//                 }
-            today: Default::default(),
+//             }
-            buttons,
+//         };
-            hours: vec![Default::default(); 24],
-            cards: Default::default(),
-            cards2: vec![],
-            revlog: vec![],
-        }
-    }
-}
 
-impl From<AllStats> for pb::GraphsOut {
+//         if let Some(count) = category.get_mut(button_num - 1) {
-    fn from(s: AllStats) -> Self {
+//             *count += 1;
-        pb::GraphsOut {
+//         }
-            cards: Some(s.cards),
+//     }
-            hours: s.hours,
-            today: Some(s.today),
-            buttons: Some(s.buttons),
-            cards2: s.cards2.into_iter().map(Into::into).collect(),
-            revlog: s.revlog,
-        }
-    }
-}
 
-#[derive(Default, Debug)]
+//     fn observe_hour_stats_for_review(&mut self, review: &RevlogEntry) {
-struct ButtonStats {
+//         match review.review_kind {
-    /// In V1 scheduler, 4th element is ignored
+//             RevlogReviewKind::Learning
-    learn: [u32; 4],
+//             | RevlogReviewKind::Review
-    young: [u32; 4],
+//             | RevlogReviewKind::Relearning => {
-    mature: [u32; 4],
+//                 let hour_idx = (((review.id.0 / 1000) + self.local_offset_secs) / 3600) % 24;
-}
+//                 let hour = &mut self.stats.hours[hour_idx as usize];
 
-#[derive(Default, Debug)]
+//                 hour.review_count += 1;
-struct HourStats {
+//                 if review.button_chosen != 1 {
-    review_count: u32,
+//                     hour.correct_count += 1;
-    correct_count: u32,
+//                 }
-}
+//             }
+//             RevlogReviewKind::EarlyReview => {}
+//         }
+//     }
 
-#[derive(Default, Debug)]
+//     fn observe_today_stats_for_review(&mut self, review: &RevlogEntry) {
-struct CardStats {
+//         if review.id.0 < self.today_rolled_over_at_millis {
-    card_count: u32,
+//             return;
-    note_count: u32,
+//         }
-    ease_factor_min: f32,
-    ease_factor_max: f32,
-    ease_factor_sum: f32,
-    ease_factor_count: u32,
-    mature_count: u32,
-    young_or_learning_count: u32,
-    new_count: u32,
-    suspended_or_buried_count: u32,
-}
 
-impl GraphsContext {
+//         let today = &mut self.stats.today;
-    fn observe_card(&mut self, card: &Card) {
-        self.observe_card_stats_for_card(card);
-    }
 
-    fn observe_review(&mut self, entry: &RevlogEntry) {
+//         // total
-        self.observe_button_stats_for_review(entry);
+//         today.answer_count += 1;
-        self.observe_hour_stats_for_review(entry);
+//         today.answer_millis += review.taken_millis;
-        self.observe_today_stats_for_review(entry);
-    }
 
-    fn observe_button_stats_for_review(&mut self, review: &RevlogEntry) {
+//         // correct
-        let mut button_num = review.button_chosen as usize;
+//         if review.button_chosen > 1 {
-        if button_num == 0 {
+//             today.correct_count += 1;
-            return;
+//         }
-        }
 
-        let buttons = &mut self.stats.buttons;
+//         // mature
-        let category = match review.review_kind {
+//         if review.last_interval >= 21 {
-            RevlogReviewKind::Learning | RevlogReviewKind::Relearning => {
+//             today.mature_count += 1;
-                // V1 scheduler only had 3 buttons in learning
+//             if review.button_chosen > 1 {
-                if button_num == 4 && self.scheduler == SchedulerVersion::V1 {
+//                 today.mature_correct += 1;
-                    button_num = 3;
+//             }
-                }
+//         }
 
-                &mut buttons.learn
+//         // type counts
-            }
+//         match review.review_kind {
-            RevlogReviewKind::Review | RevlogReviewKind::EarlyReview => {
+//             RevlogReviewKind::Learning => today.learn_count += 1,
-                if review.last_interval < 21 {
+//             RevlogReviewKind::Review => today.review_count += 1,
-                    &mut buttons.young
+//             RevlogReviewKind::Relearning => today.relearn_count += 1,
-                } else {
+//             RevlogReviewKind::EarlyReview => today.early_review_count += 1,
-                    &mut buttons.mature
+//         }
-                }
+//     }
-            }
-        };
 
-        if let Some(count) = category.get_mut(button_num - 1) {
+//     fn observe_card_stats_for_card(&mut self, card: &Card) {
-            *count += 1;
+//         counts by type
-        }
+//         match card.queue {
-    }
+//             CardQueue::New => cstats.new_count += 1,
-
+//             CardQueue::Review if card.ivl >= 21 => cstats.mature_count += 1,
-    fn observe_hour_stats_for_review(&mut self, review: &RevlogEntry) {
+//             CardQueue::Review | CardQueue::Learn | CardQueue::DayLearn => {
-        match review.review_kind {
+//                 cstats.young_or_learning_count += 1
-            RevlogReviewKind::Learning
+//             }
-            | RevlogReviewKind::Review
+//             CardQueue::Suspended | CardQueue::UserBuried | CardQueue::SchedBuried => {
-            | RevlogReviewKind::Relearning => {
+//                 cstats.suspended_or_buried_count += 1
-                let hour_idx = (((review.id.0 / 1000) + self.local_offset_secs) / 3600) % 24;
+//             }
-                let hour = &mut self.stats.hours[hour_idx as usize];
+//             CardQueue::PreviewRepeat => {}
-
+//         }
-                hour.review_count += 1;
+//     }
-                if review.button_chosen != 1 {
+// }
-                    hour.correct_count += 1;
-                }
-            }
-            RevlogReviewKind::EarlyReview => {}
-        }
-    }
-
-    fn observe_today_stats_for_review(&mut self, review: &RevlogEntry) {
-        if review.id.0 < self.today_rolled_over_at_millis {
-            return;
-        }
-
-        let today = &mut self.stats.today;
-
-        // total
-        today.answer_count += 1;
-        today.answer_millis += review.taken_millis;
-
-        // correct
-        if review.button_chosen > 1 {
-            today.correct_count += 1;
-        }
-
-        // mature
-        if review.last_interval >= 21 {
-            today.mature_count += 1;
-            if review.button_chosen > 1 {
-                today.mature_correct += 1;
-            }
-        }
-
-        // type counts
-        match review.review_kind {
-            RevlogReviewKind::Learning => today.learn_count += 1,
-            RevlogReviewKind::Review => today.review_count += 1,
-            RevlogReviewKind::Relearning => today.relearn_count += 1,
-            RevlogReviewKind::EarlyReview => today.early_review_count += 1,
-        }
-    }
-
-    fn observe_card_stats_for_card(&mut self, card: &Card) {
-        let cstats = &mut self.stats.cards;
-
-        cstats.card_count += 1;
-
-        // counts by type
-        match card.queue {
-            CardQueue::New => cstats.new_count += 1,
-            CardQueue::Review if card.ivl >= 21 => cstats.mature_count += 1,
-            CardQueue::Review | CardQueue::Learn | CardQueue::DayLearn => {
-                cstats.young_or_learning_count += 1
-            }
-            CardQueue::Suspended | CardQueue::UserBuried | CardQueue::SchedBuried => {
-                cstats.suspended_or_buried_count += 1
-            }
-            CardQueue::PreviewRepeat => {}
-        }
-
-        // ease factor
-        if card.ctype == CardType::Review {
-            let ease_factor = (card.factor as f32) / 1000.0;
-
-            cstats.ease_factor_count += 1;
-            cstats.ease_factor_sum += ease_factor;
-
-            if ease_factor < cstats.ease_factor_min || cstats.ease_factor_min == 0.0 {
-                cstats.ease_factor_min = ease_factor;
-            }
-
-            if ease_factor > cstats.ease_factor_max {
-                cstats.ease_factor_max = ease_factor;
-            }
-        }
-    }
-}
 
 impl Collection {
     pub(crate) fn graph_data_for_search(
@@ -223,14 +107,11 @@ impl Collection {
         days: u32,
     ) -> Result<pb::GraphsOut> {
         self.search_cards_into_table(search)?;
-        let i = std::time::Instant::now();
         let all = search.trim().is_empty();
-        let stats = self.graph_data(all, days)?;
+        self.graph_data(all, days)
-        let stats = stats.into();
-        Ok(stats)
     }
 
-    fn graph_data(&self, all: bool, days: u32) -> Result<AllStats> {
+    fn graph_data(&self, all: bool, days: u32) -> Result<pb::GraphsOut> {
         let timing = self.timing_today()?;
         let revlog_start = TimestampSecs(if days > 0 {
             timing.next_day_at - (((days as i64) + 1) * 86_400)
@@ -241,14 +122,6 @@ impl Collection {
         let offset = self.local_offset();
         let local_offset_secs = offset.local_minus_utc() as i64;
 
-        let mut ctx = GraphsContext {
-            scheduler: self.sched_ver(),
-            today_rolled_over_at_millis: (timing.next_day_at - 86_400) * 1000,
-            timing,
-            local_offset_secs,
-            stats: AllStats::default(),
-        };
-
         let cards = self.storage.all_searched_cards()?;
         let revlog = if all {
             self.storage.get_all_revlog_entries(revlog_start)?
@@ -257,12 +130,17 @@ impl Collection {
                 .get_revlog_entries_for_searched_cards(revlog_start)?
         };
 
-        ctx.stats.cards2 = cards;
+        self.clear_searched_cards()?;
-        ctx.stats.revlog = revlog;
 
-        // ctx.stats.cards.note_count = self.storage.note_ids_of_cards(cids)?.len() as u32;
+        Ok(pb::GraphsOut {
-
+            cards: cards.into_iter().map(Into::into).collect(),
-        Ok(ctx.stats)
+            revlog,
+            days_elapsed: timing.days_elapsed,
+            next_day_at_secs: timing.next_day_at as u32,
+            scheduler_version: self.sched_ver() as u32,
+            local_offset_secs: local_offset_secs as u32,
+            note_count: 0,
+        })
     }
 }
 

rslib/src/storage/card/mod.rs

@@ -251,23 +251,6 @@ impl super::SqliteStorage {
         Ok(nids)
     }
 
-    pub(crate) fn for_each_card_in_search<F>(&self, mut func: F) -> Result<()>
-    where
-        F: FnMut(&Card) -> Result<()>,
-    {
-        let mut stmt = self.db.prepare_cached(concat!(
-            include_str!("get_card.sql"),
-            " where id in (select id from search_cids)"
-        ))?;
-        let mut rows = stmt.query(NO_PARAMS)?;
-        while let Some(row) = rows.next()? {
-            let entry = row_to_card(row)?;
-            func(&entry)?
-        }
-
-        Ok(())
-    }
-
     pub(crate) fn all_searched_cards(&self) -> Result<Vec<Card>> {
         self.db
             .prepare_cached(concat!(

ts/.eslintrc.js

@@ -9,6 +9,10 @@ module.exports = {
     rules: {
         "prefer-const": "warn",
         "@typescript-eslint/ban-ts-ignore": "warn",
+        "@typescript-eslint/no-unused-vars": [
+            "error",
+            { argsIgnorePattern: "^_", varsIgnorePattern: "^_" },
+        ],
     },
     overrides: [
         {

ts/src/stats/AddedGraph.svelte

@@ -0,0 +1,50 @@
+<script lang="typescript">
+    import { HistogramData } from "./histogram-graph";
+    import { gatherData, prepareData, GraphData, AddedRange } from "./added";
+    import pb from "../backend/proto";
+    import HistogramGraph from "./HistogramGraph.svelte";
+
+    export let data: pb.BackendProto.GraphsOut | null = null;
+
+    let svg = null as HTMLElement | SVGElement | null;
+    let histogramData = null as HistogramData | null;
+    let range = AddedRange.Month;
+
+    let addedData: GraphData | null = null;
+    $: if (data) {
+        console.log("gathering data");
+        addedData = gatherData(data);
+    }
+
+    $: if (addedData) {
+        console.log("preparing data");
+        histogramData = prepareData(addedData, range);
+    }
+</script>
+
+{#if histogramData}
+    <div class="graph">
+        <h1>Added</h1>
+
+        <div class="range-box">
+            <label>
+                <input type="radio" bind:group={range} value={AddedRange.Month} />
+                Month
+            </label>
+            <label>
+                <input type="radio" bind:group={range} value={AddedRange.Quarter} />
+                3 months
+            </label>
+            <label>
+                <input type="radio" bind:group={range} value={AddedRange.Year} />
+                Year
+            </label>
+            <label>
+                <input type="radio" bind:group={range} value={AddedRange.AllTime} />
+                All time
+            </label>
+        </div>
+
+        <HistogramGraph data={histogramData} xText="Days" yText="Number of cards" />
+    </div>
+{/if}

ts/src/stats/GraphsPage.svelte

@@ -10,6 +10,7 @@
     import { getGraphData, GraphRange } from "./graphs";
     import IntervalsGraph from "./IntervalsGraph.svelte";
     import EaseGraph from "./EaseGraph.svelte";
+    import AddedGraph from "./AddedGraph.svelte";
 
     let data: pb.BackendProto.GraphsOut | null = null;
 
@@ -107,5 +108,6 @@
     </label>
 </div>
 
+<AddedGraph {data} />
 <IntervalsGraph {data} />
 <EaseGraph {data} />

ts/src/stats/added.ts

@@ -0,0 +1,82 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+/* eslint
+@typescript-eslint/no-non-null-assertion: "off",
+@typescript-eslint/no-explicit-any: "off",
+ */
+
+import pb from "../backend/proto";
+import { extent, histogram } from "d3-array";
+import { scaleLinear, scaleSequential } from "d3-scale";
+import { HistogramData } from "./histogram-graph";
+import { interpolateBlues } from "d3-scale-chromatic";
+
+export enum AddedRange {
+    Month = 0,
+    Quarter = 1,
+    Year = 2,
+    AllTime = 3,
+}
+
+export interface GraphData {
+    daysAdded: number[];
+}
+
+export function gatherData(data: pb.BackendProto.GraphsOut): GraphData {
+    const daysAdded = (data.cards as pb.BackendProto.Card[]).map((card) => {
+        const elapsedSecs = (card.id as number) / 1000 - data.nextDayAtSecs;
+        return Math.ceil(elapsedSecs / 86400);
+    });
+    return { daysAdded };
+}
+
+function hoverText(
+    data: HistogramData,
+    binIdx: number,
+    cumulative: number,
+    _percent: number
+): string {
+    const bin = data.bins[binIdx];
+    return (
+        `${bin.length} at ${bin.x1! - 1} days.<br>` +
+        ` ${cumulative} cards at or below this point.`
+    );
+}
+
+export function prepareData(data: GraphData, range: AddedRange): HistogramData | null {
+    // get min/max
+    const total = data.daysAdded.length;
+    if (!total) {
+        return null;
+    }
+
+    const [xMinOrig, _xMax] = extent(data.daysAdded);
+    let xMin = xMinOrig;
+
+    // cap max to selected range
+    switch (range) {
+        case AddedRange.Month:
+            xMin = -31;
+            break;
+        case AddedRange.Quarter:
+            xMin = -90;
+            break;
+        case AddedRange.Year:
+            xMin = -365;
+            break;
+        case AddedRange.AllTime:
+            break;
+    }
+    const xMax = 1;
+    const desiredBars = Math.min(70, Math.abs(xMin!));
+
+    const scale = scaleLinear().domain([xMin!, xMax]);
+    const bins = histogram()
+        .domain(scale.domain() as any)
+        .thresholds(scale.ticks(desiredBars))(data.daysAdded);
+
+    const colourScale = scaleSequential(interpolateBlues).domain([xMin!, xMax]);
+
+    return { scale, bins, total, hoverText, colourScale, showArea: true };
+}

ts/src/stats/ease.ts

@@ -18,7 +18,7 @@ export interface GraphData {
 }
 
 export function gatherData(data: pb.BackendProto.GraphsOut): GraphData {
-    const eases = (data.cards2 as pb.BackendProto.Card[])
+    const eases = (data.cards as pb.BackendProto.Card[])
         .filter((c) => c.queue == CardQueue.Review)
         .map((c) => c.factor / 10);
     return { eases };

ts/src/stats/histogram-graph.ts

@@ -9,8 +9,7 @@
 import "d3-transition";
 import { select, mouse } from "d3-selection";
 import { cumsum, max, Bin } from "d3-array";
-import { interpolateBlues, interpolateRdYlGn } from "d3-scale-chromatic";
+import { scaleLinear, ScaleLinear, ScaleSequential } from "d3-scale";
-import { scaleLinear, scaleSequential, ScaleLinear, ScaleSequential } from "d3-scale";
 import { axisBottom, axisLeft } from "d3-axis";
 import { area } from "d3-shape";
 import { showTooltip, hideTooltip } from "./tooltip";
@@ -20,7 +19,12 @@ export interface HistogramData {
     scale: ScaleLinear<number, number>;
     bins: Bin<number, number>[];
     total: number;
-    hoverText: (data: HistogramData, binIdx: number, percent: number) => string;
+    hoverText: (
+        data: HistogramData,
+        binIdx: number,
+        cumulative: number,
+        percent: number
+    ) => string;
     showArea: boolean;
     colourScale: ScaleSequential<string>;
 }
@@ -66,7 +70,7 @@ export function histogramGraph(
             .attr("x", (d: any) => x(d.x0))
             .attr("y", (d: any) => y(d.length)!)
             .attr("height", (d: any) => y(0) - y(d.length))
-            .attr("fill", (d, idx) => data.colourScale(d.x1));
+            .attr("fill", (d) => data.colourScale(d.x1));
     };
 
     svg.select("g.bars")
@@ -125,7 +129,7 @@ export function histogramGraph(
         .on("mousemove", function (this: any, d: any, idx) {
             const [x, y] = mouse(document.body);
             const pct = data.showArea ? (areaData[idx + 1] / data.total) * 100 : 0;
-            showTooltip(data.hoverText(data, idx, pct), x, y);
+            showTooltip(data.hoverText(data, idx, areaData[idx + 1], pct), x, y);
         })
         .on("mouseout", hideTooltip);
 }

ts/src/stats/intervals.ts

@@ -26,13 +26,18 @@ export enum IntervalRange {
 }
 
 export function gatherIntervalData(data: pb.BackendProto.GraphsOut): IntervalGraphData {
-    const intervals = (data.cards2 as pb.BackendProto.Card[])
+    const intervals = (data.cards as pb.BackendProto.Card[])
         .filter((c) => c.queue == CardQueue.Review)
         .map((c) => c.ivl);
     return { intervals };
 }
 
-function hoverText(data: HistogramData, binIdx: number, percent: number): string {
+function hoverText(
+    data: HistogramData,
+    binIdx: number,
+    _cumulative: number,
+    percent: number
+): string {
     const bin = data.bins[binIdx];
     const interval =
         bin.x1! - bin.x0! === 1