add a new ts/ folder with a new graph proof of concept

ts/.eslintignore

@@ -0,0 +1,2 @@
+src/backend/proto.d.ts
+webpack.config.js

ts/.eslintrc.js

@@ -0,0 +1,23 @@
+module.exports = {
+    extends: [
+        "eslint:recommended",
+        "plugin:@typescript-eslint/eslint-recommended",
+        "plugin:@typescript-eslint/recommended",
+    ],
+    parser: "@typescript-eslint/parser",
+    plugins: ["@typescript-eslint", "svelte3"],
+    rules: {
+        "prefer-const": "warn",
+        "@typescript-eslint/ban-ts-ignore": "warn",
+    },
+    overrides: [
+        {
+            files: ["*.svelte"],
+            processor: "svelte3/svelte3",
+            rules: {
+                "@typescript-eslint/explicit-function-return-type": "off",
+            },
+        },
+    ],
+    env: { browser: true },
+};

ts/.gitignore

@@ -0,0 +1,5 @@
+node_modules
+.build
+dist
+package-lock.json
+

ts/.prettierignore

@@ -0,0 +1 @@
+src/backend/proto.d.ts

ts/.prettierrc

@@ -0,0 +1,6 @@
+{
+  "trailingComma": "es5",
+  "printWidth": 88,
+  "tabWidth": 4,
+  "semi": true
+}

ts/Makefile

@@ -0,0 +1,51 @@
+SHELL := /bin/bash
+
+ifndef SHELLFLAGS
+	SHELLFLAGS :=
+endif
+
+.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c
+MAKEFLAGS += --warn-undefined-variables
+MAKEFLAGS += --no-builtin-rules
+
+ifndef OS
+	OS := unknown
+endif
+
+.DELETE_ON_ERROR:
+.SUFFIXES:
+
+$(shell mkdir -p .build)
+
+PHONY: all
+all: check
+
+.build/npm: package.json
+	npm i
+	@touch $@
+
+PROTODEPS := ../proto/backend.proto ../proto/fluent.proto
+BUILDDEPS := .build/npm webpack.config.js
+
+.build/proto: $(BUILDDEPS) $(PROTODEPS)
+	npm run proto
+	@touch $@
+
+PHONY: dev
+dev: .build/proto
+	npm run dev
+
+PHONY: build
+build: .build/build
+
+.build/build: .build/proto $(BUILDDEPS) $(wildcard src/*/*.svelte src/*/*.ts)
+	npm run build
+	@touch $@
+
+.PHONY: check
+check:
+	npm run check
+
+.PHONY: fix
+fix:
+	npm run fix

ts/css.d.ts

@@ -0,0 +1 @@
+declare module "*.css";

ts/d3_missing.d.ts

@@ -0,0 +1,4 @@
+import "d3-array";
+declare module "d3-array" {
+    export function cumsum(arg0: number[]): Float64Array;
+}

ts/package.json

@@ -0,0 +1,56 @@
+{
+    "name": "anki",
+    "version": "0.1.0",
+    "devDependencies": {
+        "@pyoner/svelte-types": "^3.4.4-2",
+        "@types/d3-array": "^2.0.0",
+        "@types/d3-axis": "^1.0.12",
+        "@types/d3-scale": "^2.2.0",
+        "@types/d3-scale-chromatic": "^1.5.0",
+        "@types/d3-selection": "^1.4.1",
+        "@types/d3-shape": "^1.3.2",
+        "@types/d3-transition": "^1.1.6",
+        "@types/lodash.throttle": "^4.1.6",
+        "@types/long": "^4.0.1",
+        "@typescript-eslint/eslint-plugin": "^2.11.0",
+        "@typescript-eslint/parser": "^2.11.0",
+        "cross-env": "^7.0.2",
+        "css-loader": "^3.6.0",
+        "eslint": "^6.7.2",
+        "eslint-loader": "^4.0.2",
+        "eslint-plugin-svelte3": "^2.7.3",
+        "html-webpack-plugin": "^4.3.0",
+        "prettier": "^2.0.0",
+        "prettier-plugin-svelte": "^1.1.0",
+        "style-loader": "^1.2.1",
+        "svelte": "^3.23.2",
+        "svelte-loader": "^2.13.6",
+        "svelte-preprocess": "^3.9.9",
+        "ts-loader": "^7.0.5",
+        "typescript": "^3.7.3",
+        "webpack": "^4.43.0",
+        "webpack-cli": "^3.3.11",
+        "webpack-dev-server": "^3.11.0"
+    },
+    "scripts": {
+        "proto": "pbjs -t json-module -w es6 ../proto/backend.proto -o src/backend/proto.js; pbjs -t static-module ../proto/backend.proto | pbts -o src/backend/proto.d.ts -",
+        "fix": "prettier --write src/*/*.ts",
+        "check": "prettier --check src/*/*.ts && eslint --max-warnings=0 --ext .ts src",
+        "build": "cross-env NODE_ENV=production webpack",
+        "dev": "webpack-dev-server"
+    },
+    "dependencies": {
+        "d3-array": "^2.4.0",
+        "d3-axis": "^1.0.12",
+        "d3-scale": "^3.2.1",
+        "d3-scale-chromatic": "^1.5.0",
+        "d3-selection": "^1.4.1",
+        "d3-shape": "^1.3.7",
+        "d3-transition": "^1.3.2",
+        "lodash.throttle": "^4.1.1",
+        "protobufjs": "^6.9.0"
+    },
+    "files": [
+        "dist/*"
+    ]
+}

ts/src/backend/.gitignore

@@ -0,0 +1,2 @@
+proto.js
+proto.d.ts

ts/src/cards.ts

@@ -0,0 +1,19 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+export enum CardQueue {
+    /// due is the order cards are shown in
+    New = 0,
+    /// due is a unix timestamp
+    Learn = 1,
+    /// due is days since creation date
+    Review = 2,
+    DayLearn = 3,
+    /// due is a unix timestamp.
+    /// preview cards only placed here when failed.
+    PreviewRepeat = 4,
+    /// cards are not due in these states
+    Suspended = -1,
+    UserBuried = -2,
+    SchedBuried = -3,
+}

ts/src/html/graphs.html

@@ -0,0 +1,12 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <meta charset="utf-8" />
+    </head>
+    <body>
+        <div id="svelte"></div>
+    </body>
+    <script>
+        anki.renderGraphs();
+    </script>
+</html>

ts/src/stats/IntervalsGraph.svelte

@@ -0,0 +1,62 @@
+<script lang="typescript">
+    import {
+        gatherIntervalData,
+        intervalGraph,
+        IntervalUpdateFn,
+        IntervalRange,
+    } from "./intervals";
+    import type { IntervalGraphData } from "./intervals";
+    import { onMount } from "svelte";
+    import pb from "../backend/proto";
+
+    export let cards: pb.BackendProto.Card[] | null = null;
+
+    let svg = null as HTMLElement | SVGElement | null;
+    let updater = null as IntervalUpdateFn | null;
+
+    onMount(() => {
+        updater = intervalGraph(svg as SVGElement);
+    });
+
+    let range = IntervalRange.Percentile95;
+
+    let graphData: IntervalGraphData;
+    $: graphData = gatherIntervalData(cards!);
+    $: if (updater) {
+        updater(graphData, range);
+    }
+</script>
+
+<div class="graph intervals">
+    <h1>Review Intervals</h1>
+
+    <div class="range-box">
+        <label>
+            <input type="radio" bind:group={range} value={IntervalRange.Month} />
+            Month
+        </label>
+        <label>
+            <input type="radio" bind:group={range} value={IntervalRange.Percentile50} />
+            50th percentile
+        </label>
+        <label>
+            <input type="radio" bind:group={range} value={IntervalRange.Percentile95} />
+            95th percentile
+        </label>
+        <label>
+            <input
+                type="radio"
+                bind:group={range}
+                value={IntervalRange.Percentile999} />
+            99.9th percentile
+        </label>
+        <label>
+            <input type="radio" bind:group={range} value={IntervalRange.All} />
+            All
+        </label>
+    </div>
+
+    <svg bind:this={svg}>
+        <path class="area" />
+    </svg>
+</div>

ts/src/stats/graphs.css

@@ -0,0 +1,58 @@
+* {
+    box-sizing: border-box;
+}
+
+.graph-tooltip {
+    position: absolute;
+    background-color: white;
+    padding: 15px;
+    border-radius: 5px;
+    font-size: 20px;
+    opacity: 0;
+    pointer-events: none;
+    transition: opacity 0.3s;
+    background: #fff;
+}
+
+.graph {
+    margin-left: auto;
+    margin-right: auto;
+}
+
+.graph h1 {
+    text-align: center;
+}
+
+/* .cards-graph-grid {
+    display: flex;
+    flex-wrap: wrap;
+    padding-left: 0;
+    padding-right: 1em;
+    align-items: center;
+}
+
+.cards-graph-grid svg {
+    flex: 3 0 0;
+    flex-direction: column;
+    min-width: 500px;
+}
+
+.cards-graph-grid div.graph-description {
+    flex: 0 0 10em;
+    font-family: "Arial";
+} */
+
+.no-domain-line .domain {
+    display: none;
+}
+
+.range-box {
+    display: flex;
+    justify-content: center;
+}
+
+.intervals .area {
+    opacity: 0.05;
+    pointer-events: none;
+    stroke: black;
+}

ts/src/stats/graphs.ts

@@ -0,0 +1,46 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+/* eslint
+@typescript-eslint/no-non-null-assertion: "off",
+@typescript-eslint/no-explicit-any: "off",
+@typescript-eslint/ban-ts-ignore: "off" */
+
+import pb from "../backend/proto";
+import style from "./graphs.css";
+
+async function fetchData(): Promise<Uint8Array> {
+    let t = performance.now();
+    const resp = await fetch("/_anki/graphData", {
+        method: "POST",
+    });
+    if (!resp.ok) {
+        throw Error(`unexpected reply: ${resp.statusText}`);
+    }
+    console.log(`fetch in ${performance.now() - t}ms`);
+    t = performance.now();
+    // get returned bytes
+    const respBlob = await resp.blob();
+    const respBuf = await new Response(respBlob).arrayBuffer();
+    const bytes = new Uint8Array(respBuf);
+    console.log(`bytes in ${performance.now() - t}ms`);
+    t = performance.now();
+    return bytes;
+}
+
+import IntervalGraph from "./IntervalsGraph.svelte";
+
+export async function renderGraphs(): Promise<void> {
+    const bytes = await fetchData();
+    let t = performance.now();
+    const data = pb.BackendProto.GraphsOut.decode(bytes);
+    console.log(`decode in ${performance.now() - t}ms`);
+    t = performance.now();
+
+    document.head.append(style);
+
+    new IntervalGraph({
+        target: document.getElementById("svelte")!,
+        props: { cards: data.cards2 },
+    });
+}

ts/src/stats/intervals.ts

@@ -0,0 +1,231 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+/* eslint
+@typescript-eslint/no-non-null-assertion: "off",
+@typescript-eslint/no-explicit-any: "off",
+@typescript-eslint/ban-ts-ignore: "off",
+@typescript-eslint/explicit-function-return-type: "off" */
+
+import { select, mouse, Selection } from "d3-selection";
+import { cumsum, extent, max, histogram, quantile } from "d3-array";
+import { interpolateBlues } from "d3-scale-chromatic";
+import { scaleLinear, scaleSequential } from "d3-scale";
+import { axisBottom, axisLeft } from "d3-axis";
+import { area } from "d3-shape";
+import "d3-transition";
+import { CardQueue } from "../cards";
+import { showTooltip, hideTooltip } from "./tooltip";
+import pb from "../backend/proto";
+import { assertUnreachable } from "../typing";
+
+export interface IntervalGraphData {
+    intervals: number[];
+}
+
+export enum IntervalRange {
+    Month = 0,
+    Percentile50 = 1,
+    Percentile95 = 2,
+    Percentile999 = 3,
+    All = 4,
+}
+
+export function gatherIntervalData(cards: pb.BackendProto.Card[]): IntervalGraphData {
+    const intervals = cards
+        .filter((c) => c.queue == CardQueue.Review)
+        .map((c) => c.ivl);
+    return { intervals };
+}
+
+export type IntervalUpdateFn = (
+    arg0: IntervalGraphData,
+    maxDays: IntervalRange
+) => void;
+
+/// Creates an interval graph, returning a function used to update it.
+export function intervalGraph(svgElem: SVGElement): IntervalUpdateFn {
+    const margin = { top: 20, right: 20, bottom: 40, left: 100 };
+    const height = 250;
+    const width = 600;
+    const xTicks = 6;
+
+    // svg elements
+    const svg = select(svgElem).attr("viewBox", [0, 0, width, height].join(" "));
+    const barGroup = svg.append("g");
+    const hoverGroup = svg.append("g");
+    const areaPath = svg.select("path.area");
+    const xAxisGroup = svg.append("g").classed("no-domain-line", true);
+    const yAxisGroup = svg.append("g").classed("no-domain-line", true);
+
+    // x axis
+    const xScale = scaleLinear()
+        .range([margin.left, width - margin.right])
+        .domain([0, 0]);
+    svg.append("text")
+        .attr("transform", `translate(${width / 2}, ${height - 5})`)
+        .style("text-anchor", "middle")
+        .style("font-size", 10)
+        .text("Interval (days)");
+
+    // y axis
+    const yScale = scaleLinear()
+        .domain([0, 0])
+        .range([height - margin.bottom, margin.top]);
+    svg.append("text")
+        .attr(
+            "transform",
+            `translate(${margin.left / 3}, ${
+                (height - margin.bottom) / 2 + margin.top
+            }) rotate(-180)`
+        )
+        .style("text-anchor", "middle")
+        .style("writing-mode", "vertical-rl")
+        .style("rotate", "180")
+        .style("font-size", 10)
+        .text("Number of cards");
+
+    function update(graphData: IntervalGraphData, maxDays: IntervalRange) {
+        const allIntervals = graphData.intervals;
+
+        const [xMin, origXMax] = extent(allIntervals);
+
+        let desiredBars = 70;
+        let xMax = origXMax;
+        switch (maxDays) {
+            case IntervalRange.Month:
+                xMax = Math.min(xMax!, 31);
+                desiredBars = 31;
+                break;
+            case IntervalRange.Percentile50:
+                xMax = quantile(allIntervals, 0.5);
+                break;
+            case IntervalRange.Percentile95:
+                xMax = quantile(allIntervals, 0.95);
+                break;
+            case IntervalRange.Percentile999:
+                xMax = quantile(allIntervals, 0.999);
+                break;
+            case IntervalRange.All:
+                break;
+            default:
+                assertUnreachable(maxDays);
+        }
+
+        const x = xScale.copy().domain([xMin!, xMax!]);
+        // .nice();
+
+        const data = histogram()
+            .domain(x.domain() as any)
+            .thresholds(x.ticks(desiredBars))(allIntervals);
+
+        const yMax = max(data, (d) => d.length);
+
+        const colourScale = scaleSequential(interpolateBlues).domain([
+            -20,
+            data.length,
+        ]);
+
+        const y = yScale.copy().domain([0, yMax!]);
+
+        const t = svg.transition().duration(600);
+
+        const updateXAxis = (
+            g: Selection<SVGGElement, unknown, null, undefined>,
+            scale: any
+        ) =>
+            g
+                .attr("transform", `translate(0,${height - margin.bottom})`)
+                .call(axisBottom(scale).ticks(xTicks).tickSizeOuter(0));
+
+        xAxisGroup.transition(t as any).call(updateXAxis as any, x);
+
+        const updateYAxis = (
+            g: Selection<SVGGElement, unknown, null, undefined>,
+            scale: any
+        ) =>
+            g.attr("transform", `translate(${margin.left}, 0)`).call(
+                axisLeft(scale)
+                    .ticks(height / 80)
+                    .tickSizeOuter(0)
+            );
+
+        yAxisGroup.transition(t as any).call(updateYAxis as any, y);
+
+        const updateBar = (sel: any) => {
+            return sel.call((sel) =>
+                sel
+                    .transition(t as any)
+                    .attr("width", (d: any) => Math.max(0, x(d.x1) - x(d.x0) - 1))
+                    .attr("x", (d: any) => x(d.x0))
+                    .attr("y", (d: any) => y(d.length)!)
+                    .attr("height", (d: any) => y(0) - y(d.length))
+                    .attr("fill", (d, idx) => colourScale(idx))
+            );
+        };
+
+        barGroup
+            .selectAll("rect")
+            .data(data)
+            .join(
+                (enter) =>
+                    updateBar(
+                        enter
+                            .append("rect")
+                            .attr("rx", 1)
+                            .attr("x", (d: any) => x(d.x0))
+                            .attr("y", y(0))
+                            .attr("height", 0)
+                    ),
+                (update) => updateBar(update),
+                (remove) =>
+                    remove.call((remove) =>
+                        remove
+                            .transition(t as any)
+                            .attr("height", 0)
+                            .attr("y", y(0))
+                    )
+            );
+
+        const areaData = cumsum(data.map((d) => d.length));
+        const xAreaScale = x.copy().domain([0, areaData.length]);
+        const yAreaScale = y.copy().domain([0, allIntervals.length]);
+
+        areaPath
+            .datum(areaData as any)
+            .attr("fill", "grey")
+            .attr(
+                "d",
+                area()
+                    .x((d: any, idx) => {
+                        return xAreaScale(idx);
+                    })
+                    .y0(height - margin.bottom)
+                    .y1((d: any) => yAreaScale(d)) as any
+            );
+
+        hoverGroup
+            .selectAll("rect")
+            .data(data)
+            .join("rect")
+            .attr("x", (d: any) => x(d.x0))
+            .attr("y", () => y(yMax!))
+            .attr("width", (d: any) => Math.max(0, x(d.x1) - x(d.x0) - 1))
+            .attr("height", () => y(0) - y(yMax!))
+            .attr("fill", "none")
+            .attr("pointer-events", "all")
+            .on("mousemove", function (this: any, d: any, idx) {
+                const [x, y] = mouse(document.body);
+                const pct = ((areaData[idx] / allIntervals.length) * 100).toFixed(2);
+                showTooltip(
+                    `${d.length} cards with interval ${d.x0}~${d.x1} days. ` +
+                        `<br>${pct}% cards below this point.`,
+                    x,
+                    y
+                );
+            })
+            .on("mouseout", hideTooltip);
+    }
+
+    return update;
+}

ts/src/stats/tooltip.ts

@@ -0,0 +1,28 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+import throttle from "lodash.throttle";
+
+let tooltipDiv: HTMLDivElement | null = null;
+
+function showTooltipInner(msg: string, x: number, y: number): void {
+    if (!tooltipDiv) {
+        tooltipDiv = document.createElement("div");
+        tooltipDiv.className = "graph-tooltip";
+        document.body.appendChild(tooltipDiv);
+    }
+    tooltipDiv.innerHTML = msg;
+    tooltipDiv.style.left = `${x - 50}px`;
+    tooltipDiv.style.top = `${y - 50}px`;
+
+    tooltipDiv.style.opacity = "1";
+}
+
+export const showTooltip = throttle(showTooltipInner, 16);
+
+export function hideTooltip(): void {
+    showTooltip.cancel();
+    if (tooltipDiv) {
+        tooltipDiv.style.opacity = "0";
+    }
+}

ts/src/typing.ts

@@ -0,0 +1,3 @@
+export function assertUnreachable(x: never): never {
+    throw new Error(`unreachable: ${x}`);
+}

ts/svelte.config.js

@@ -0,0 +1,7 @@
+const sveltePreprocess = require("svelte-preprocess");
+
+module.exports = {
+    preprocess: sveltePreprocess({
+        typescript: { compilerOptions: { declaration: false, outDir: null } },
+    }),
+};

ts/tsconfig.json

@@ -0,0 +1,26 @@
+{
+    "exclude": ["node_modules", "dist"],
+    "compilerOptions": {
+        /* Basic Options */
+        "target": "es6",
+        "module": "es6",
+        "declaration": false /* Generates corresponding '.d.ts' file. */,
+        "rootDir": "src",
+        "outDir": "dist",
+        /* Strict Type-Checking Options */
+        "strict": true /* Enable all strict type-checking options. */,
+        "noImplicitAny": false /* Raise error on expressions and declarations with an implied 'any' type. */,
+        "strictNullChecks": true /* Enable strict null checks. */,
+        "strictFunctionTypes": true /* Enable strict checking of function types. */,
+        "strictBindCallApply": true /* Enable strict 'bind', 'call', and 'apply' methods on functions. */,
+        "strictPropertyInitialization": true /* Enable strict checking of property initialization in classes. */,
+        "noImplicitThis": true /* Raise error on 'this' expressions with an implied 'any' type. */,
+        "alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */,
+        /* Module Resolution Options */
+        "moduleResolution": "node" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */,
+        "allowSyntheticDefaultImports": true /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */,
+        "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */,
+        "jsx": "react",
+        "types": ["svelte", "long"]
+    }
+}

ts/webpack.config.js

@@ -0,0 +1,89 @@
+const HtmlWebpackPlugin = require("html-webpack-plugin");
+const mode = process.env.NODE_ENV || "development";
+const prod = mode === "production";
+var path = require("path");
+
+module.exports = {
+    entry: {
+        graphs: ["./src/stats/graphs.ts"],
+    },
+    output: {
+        library: "anki",
+    },
+    plugins: [
+        new HtmlWebpackPlugin({
+            filename: "graphs.html",
+            chunks: ["graphs"],
+            template: "src/html/graphs.html",
+        }),
+    ],
+    externals: {
+        moment: "moment",
+    },
+    devServer: {
+        contentBase: "./dist",
+        port: 9000,
+        // host: "0.0.0.0",
+        // disableHostCheck: true,
+        proxy: {
+            "/_anki": {
+                target: "http://localhost:9001",
+            },
+        },
+    },
+    resolve: {
+        alias: {
+            svelte: path.resolve("node_modules", "svelte"),
+        },
+        extensions: [".mjs", ".js", ".svelte", ".ts", ".tsx"],
+        mainFields: ["svelte", "browser", "module", "main"],
+    },
+    module: {
+        rules: [
+            {
+                test: /\.css$/i,
+                use: ["style-loader", "css-loader"],
+            },
+            {
+                test: /\.(svelte)$/,
+                exclude: /node_modules/,
+                use: [
+                    {
+                        loader: "svelte-loader",
+                        options: {
+                            emitCss: true,
+                            preprocess: require("svelte-preprocess")({
+                                typescript: {
+                                    transpileOnly: true,
+                                    compilerOptions: {
+                                        declaration: false,
+                                    },
+                                },
+                            }),
+                        },
+                    },
+                ],
+            },
+            {
+                test: /\.tsx?$/,
+                use: ["ts-loader"],
+                exclude: /node_modules/,
+            },
+            {
+                test: /\.(tsx?|js)$/,
+                loader: "eslint-loader",
+                exclude: /node_modules/,
+                options: {
+                    fix: true,
+                },
+            },
+        ],
+    },
+    mode,
+    devtool: prod ? false : "source-map",
+    optimization: {
+        splitChunks: {
+            // chunks: "all",
+        },
+    },
+};