diff --git a/ts/.eslintignore b/ts/.eslintignore new file mode 100644 index 000000000..89b902fe7 --- /dev/null +++ b/ts/.eslintignore @@ -0,0 +1,2 @@ +src/backend/proto.d.ts +webpack.config.js diff --git a/ts/.eslintrc.js b/ts/.eslintrc.js new file mode 100644 index 000000000..9fde61347 --- /dev/null +++ b/ts/.eslintrc.js @@ -0,0 +1,23 @@ +module.exports = { + extends: [ + "eslint:recommended", + "plugin:@typescript-eslint/eslint-recommended", + "plugin:@typescript-eslint/recommended", + ], + parser: "@typescript-eslint/parser", + plugins: ["@typescript-eslint", "svelte3"], + rules: { + "prefer-const": "warn", + "@typescript-eslint/ban-ts-ignore": "warn", + }, + overrides: [ + { + files: ["*.svelte"], + processor: "svelte3/svelte3", + rules: { + "@typescript-eslint/explicit-function-return-type": "off", + }, + }, + ], + env: { browser: true }, +}; diff --git a/ts/.gitignore b/ts/.gitignore new file mode 100644 index 000000000..23f1564cb --- /dev/null +++ b/ts/.gitignore @@ -0,0 +1,5 @@ +node_modules +.build +dist +package-lock.json + diff --git a/ts/.prettierignore b/ts/.prettierignore new file mode 100644 index 000000000..1facfe957 --- /dev/null +++ b/ts/.prettierignore @@ -0,0 +1 @@ +src/backend/proto.d.ts diff --git a/ts/.prettierrc b/ts/.prettierrc new file mode 100644 index 000000000..ef6f4ce45 --- /dev/null +++ b/ts/.prettierrc @@ -0,0 +1,6 @@ +{ + "trailingComma": "es5", + "printWidth": 88, + "tabWidth": 4, + "semi": true +} diff --git a/ts/Makefile b/ts/Makefile new file mode 100644 index 000000000..69e812d1c --- /dev/null +++ b/ts/Makefile @@ -0,0 +1,51 @@ +SHELL := /bin/bash + +ifndef SHELLFLAGS + SHELLFLAGS := +endif + +.SHELLFLAGS := -eu -o pipefail ${SHELLFLAGS} -c +MAKEFLAGS += --warn-undefined-variables +MAKEFLAGS += --no-builtin-rules + +ifndef OS + OS := unknown +endif + +.DELETE_ON_ERROR: +.SUFFIXES: + +$(shell mkdir -p .build) + +PHONY: all +all: check + +.build/npm: package.json + npm i + @touch $@ + +PROTODEPS := ../proto/backend.proto ../proto/fluent.proto +BUILDDEPS := .build/npm webpack.config.js + +.build/proto: $(BUILDDEPS) $(PROTODEPS) + npm run proto + @touch $@ + +PHONY: dev +dev: .build/proto + npm run dev + +PHONY: build +build: .build/build + +.build/build: .build/proto $(BUILDDEPS) $(wildcard src/*/*.svelte src/*/*.ts) + npm run build + @touch $@ + +.PHONY: check +check: + npm run check + +.PHONY: fix +fix: + npm run fix diff --git a/ts/css.d.ts b/ts/css.d.ts new file mode 100644 index 000000000..cbe652dbe --- /dev/null +++ b/ts/css.d.ts @@ -0,0 +1 @@ +declare module "*.css"; diff --git a/ts/d3_missing.d.ts b/ts/d3_missing.d.ts new file mode 100644 index 000000000..7d3d27f85 --- /dev/null +++ b/ts/d3_missing.d.ts @@ -0,0 +1,4 @@ +import "d3-array"; +declare module "d3-array" { + export function cumsum(arg0: number[]): Float64Array; +} diff --git a/ts/package.json b/ts/package.json new file mode 100644 index 000000000..5c1fb531f --- /dev/null +++ b/ts/package.json @@ -0,0 +1,56 @@ +{ + "name": "anki", + "version": "0.1.0", + "devDependencies": { + "@pyoner/svelte-types": "^3.4.4-2", + "@types/d3-array": "^2.0.0", + "@types/d3-axis": "^1.0.12", + "@types/d3-scale": "^2.2.0", + "@types/d3-scale-chromatic": "^1.5.0", + "@types/d3-selection": "^1.4.1", + "@types/d3-shape": "^1.3.2", + "@types/d3-transition": "^1.1.6", + "@types/lodash.throttle": "^4.1.6", + "@types/long": "^4.0.1", + "@typescript-eslint/eslint-plugin": "^2.11.0", + "@typescript-eslint/parser": "^2.11.0", + "cross-env": "^7.0.2", + "css-loader": "^3.6.0", + "eslint": "^6.7.2", + "eslint-loader": "^4.0.2", + "eslint-plugin-svelte3": "^2.7.3", + "html-webpack-plugin": "^4.3.0", + "prettier": "^2.0.0", + "prettier-plugin-svelte": "^1.1.0", + "style-loader": "^1.2.1", + "svelte": "^3.23.2", + "svelte-loader": "^2.13.6", + "svelte-preprocess": "^3.9.9", + "ts-loader": "^7.0.5", + "typescript": "^3.7.3", + "webpack": "^4.43.0", + "webpack-cli": "^3.3.11", + "webpack-dev-server": "^3.11.0" + }, + "scripts": { + "proto": "pbjs -t json-module -w es6 ../proto/backend.proto -o src/backend/proto.js; pbjs -t static-module ../proto/backend.proto | pbts -o src/backend/proto.d.ts -", + "fix": "prettier --write src/*/*.ts", + "check": "prettier --check src/*/*.ts && eslint --max-warnings=0 --ext .ts src", + "build": "cross-env NODE_ENV=production webpack", + "dev": "webpack-dev-server" + }, + "dependencies": { + "d3-array": "^2.4.0", + "d3-axis": "^1.0.12", + "d3-scale": "^3.2.1", + "d3-scale-chromatic": "^1.5.0", + "d3-selection": "^1.4.1", + "d3-shape": "^1.3.7", + "d3-transition": "^1.3.2", + "lodash.throttle": "^4.1.1", + "protobufjs": "^6.9.0" + }, + "files": [ + "dist/*" + ] +} diff --git a/ts/src/backend/.gitignore b/ts/src/backend/.gitignore new file mode 100644 index 000000000..684374f9b --- /dev/null +++ b/ts/src/backend/.gitignore @@ -0,0 +1,2 @@ +proto.js +proto.d.ts diff --git a/ts/src/cards.ts b/ts/src/cards.ts new file mode 100644 index 000000000..b97e63e0d --- /dev/null +++ b/ts/src/cards.ts @@ -0,0 +1,19 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +export enum CardQueue { + /// due is the order cards are shown in + New = 0, + /// due is a unix timestamp + Learn = 1, + /// due is days since creation date + Review = 2, + DayLearn = 3, + /// due is a unix timestamp. + /// preview cards only placed here when failed. + PreviewRepeat = 4, + /// cards are not due in these states + Suspended = -1, + UserBuried = -2, + SchedBuried = -3, +} diff --git a/ts/src/html/graphs.html b/ts/src/html/graphs.html new file mode 100644 index 000000000..33a634c71 --- /dev/null +++ b/ts/src/html/graphs.html @@ -0,0 +1,12 @@ + + + + + + +
+ + + diff --git a/ts/src/stats/IntervalsGraph.svelte b/ts/src/stats/IntervalsGraph.svelte new file mode 100644 index 000000000..ef656d8bd --- /dev/null +++ b/ts/src/stats/IntervalsGraph.svelte @@ -0,0 +1,62 @@ + + +
+

Review Intervals

+ +
+ + + + + +
+ + + + +
diff --git a/ts/src/stats/graphs.css b/ts/src/stats/graphs.css new file mode 100644 index 000000000..c8e10b34f --- /dev/null +++ b/ts/src/stats/graphs.css @@ -0,0 +1,58 @@ +* { + box-sizing: border-box; +} + +.graph-tooltip { + position: absolute; + background-color: white; + padding: 15px; + border-radius: 5px; + font-size: 20px; + opacity: 0; + pointer-events: none; + transition: opacity 0.3s; + background: #fff; +} + +.graph { + margin-left: auto; + margin-right: auto; +} + +.graph h1 { + text-align: center; +} + +/* .cards-graph-grid { + display: flex; + flex-wrap: wrap; + padding-left: 0; + padding-right: 1em; + align-items: center; +} + +.cards-graph-grid svg { + flex: 3 0 0; + flex-direction: column; + min-width: 500px; +} + +.cards-graph-grid div.graph-description { + flex: 0 0 10em; + font-family: "Arial"; +} */ + +.no-domain-line .domain { + display: none; +} + +.range-box { + display: flex; + justify-content: center; +} + +.intervals .area { + opacity: 0.05; + pointer-events: none; + stroke: black; +} diff --git a/ts/src/stats/graphs.ts b/ts/src/stats/graphs.ts new file mode 100644 index 000000000..4b193642b --- /dev/null +++ b/ts/src/stats/graphs.ts @@ -0,0 +1,46 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +/* eslint +@typescript-eslint/no-non-null-assertion: "off", +@typescript-eslint/no-explicit-any: "off", +@typescript-eslint/ban-ts-ignore: "off" */ + +import pb from "../backend/proto"; +import style from "./graphs.css"; + +async function fetchData(): Promise { + let t = performance.now(); + const resp = await fetch("/_anki/graphData", { + method: "POST", + }); + if (!resp.ok) { + throw Error(`unexpected reply: ${resp.statusText}`); + } + console.log(`fetch in ${performance.now() - t}ms`); + t = performance.now(); + // get returned bytes + const respBlob = await resp.blob(); + const respBuf = await new Response(respBlob).arrayBuffer(); + const bytes = new Uint8Array(respBuf); + console.log(`bytes in ${performance.now() - t}ms`); + t = performance.now(); + return bytes; +} + +import IntervalGraph from "./IntervalsGraph.svelte"; + +export async function renderGraphs(): Promise { + const bytes = await fetchData(); + let t = performance.now(); + const data = pb.BackendProto.GraphsOut.decode(bytes); + console.log(`decode in ${performance.now() - t}ms`); + t = performance.now(); + + document.head.append(style); + + new IntervalGraph({ + target: document.getElementById("svelte")!, + props: { cards: data.cards2 }, + }); +} diff --git a/ts/src/stats/intervals.ts b/ts/src/stats/intervals.ts new file mode 100644 index 000000000..9e8a87c14 --- /dev/null +++ b/ts/src/stats/intervals.ts @@ -0,0 +1,231 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +/* eslint +@typescript-eslint/no-non-null-assertion: "off", +@typescript-eslint/no-explicit-any: "off", +@typescript-eslint/ban-ts-ignore: "off", +@typescript-eslint/explicit-function-return-type: "off" */ + +import { select, mouse, Selection } from "d3-selection"; +import { cumsum, extent, max, histogram, quantile } from "d3-array"; +import { interpolateBlues } from "d3-scale-chromatic"; +import { scaleLinear, scaleSequential } from "d3-scale"; +import { axisBottom, axisLeft } from "d3-axis"; +import { area } from "d3-shape"; +import "d3-transition"; +import { CardQueue } from "../cards"; +import { showTooltip, hideTooltip } from "./tooltip"; +import pb from "../backend/proto"; +import { assertUnreachable } from "../typing"; + +export interface IntervalGraphData { + intervals: number[]; +} + +export enum IntervalRange { + Month = 0, + Percentile50 = 1, + Percentile95 = 2, + Percentile999 = 3, + All = 4, +} + +export function gatherIntervalData(cards: pb.BackendProto.Card[]): IntervalGraphData { + const intervals = cards + .filter((c) => c.queue == CardQueue.Review) + .map((c) => c.ivl); + return { intervals }; +} + +export type IntervalUpdateFn = ( + arg0: IntervalGraphData, + maxDays: IntervalRange +) => void; + +/// Creates an interval graph, returning a function used to update it. +export function intervalGraph(svgElem: SVGElement): IntervalUpdateFn { + const margin = { top: 20, right: 20, bottom: 40, left: 100 }; + const height = 250; + const width = 600; + const xTicks = 6; + + // svg elements + const svg = select(svgElem).attr("viewBox", [0, 0, width, height].join(" ")); + const barGroup = svg.append("g"); + const hoverGroup = svg.append("g"); + const areaPath = svg.select("path.area"); + const xAxisGroup = svg.append("g").classed("no-domain-line", true); + const yAxisGroup = svg.append("g").classed("no-domain-line", true); + + // x axis + const xScale = scaleLinear() + .range([margin.left, width - margin.right]) + .domain([0, 0]); + svg.append("text") + .attr("transform", `translate(${width / 2}, ${height - 5})`) + .style("text-anchor", "middle") + .style("font-size", 10) + .text("Interval (days)"); + + // y axis + const yScale = scaleLinear() + .domain([0, 0]) + .range([height - margin.bottom, margin.top]); + svg.append("text") + .attr( + "transform", + `translate(${margin.left / 3}, ${ + (height - margin.bottom) / 2 + margin.top + }) rotate(-180)` + ) + .style("text-anchor", "middle") + .style("writing-mode", "vertical-rl") + .style("rotate", "180") + .style("font-size", 10) + .text("Number of cards"); + + function update(graphData: IntervalGraphData, maxDays: IntervalRange) { + const allIntervals = graphData.intervals; + + const [xMin, origXMax] = extent(allIntervals); + + let desiredBars = 70; + let xMax = origXMax; + switch (maxDays) { + case IntervalRange.Month: + xMax = Math.min(xMax!, 31); + desiredBars = 31; + break; + case IntervalRange.Percentile50: + xMax = quantile(allIntervals, 0.5); + break; + case IntervalRange.Percentile95: + xMax = quantile(allIntervals, 0.95); + break; + case IntervalRange.Percentile999: + xMax = quantile(allIntervals, 0.999); + break; + case IntervalRange.All: + break; + default: + assertUnreachable(maxDays); + } + + const x = xScale.copy().domain([xMin!, xMax!]); + // .nice(); + + const data = histogram() + .domain(x.domain() as any) + .thresholds(x.ticks(desiredBars))(allIntervals); + + const yMax = max(data, (d) => d.length); + + const colourScale = scaleSequential(interpolateBlues).domain([ + -20, + data.length, + ]); + + const y = yScale.copy().domain([0, yMax!]); + + const t = svg.transition().duration(600); + + const updateXAxis = ( + g: Selection, + scale: any + ) => + g + .attr("transform", `translate(0,${height - margin.bottom})`) + .call(axisBottom(scale).ticks(xTicks).tickSizeOuter(0)); + + xAxisGroup.transition(t as any).call(updateXAxis as any, x); + + const updateYAxis = ( + g: Selection, + scale: any + ) => + g.attr("transform", `translate(${margin.left}, 0)`).call( + axisLeft(scale) + .ticks(height / 80) + .tickSizeOuter(0) + ); + + yAxisGroup.transition(t as any).call(updateYAxis as any, y); + + const updateBar = (sel: any) => { + return sel.call((sel) => + sel + .transition(t as any) + .attr("width", (d: any) => Math.max(0, x(d.x1) - x(d.x0) - 1)) + .attr("x", (d: any) => x(d.x0)) + .attr("y", (d: any) => y(d.length)!) + .attr("height", (d: any) => y(0) - y(d.length)) + .attr("fill", (d, idx) => colourScale(idx)) + ); + }; + + barGroup + .selectAll("rect") + .data(data) + .join( + (enter) => + updateBar( + enter + .append("rect") + .attr("rx", 1) + .attr("x", (d: any) => x(d.x0)) + .attr("y", y(0)) + .attr("height", 0) + ), + (update) => updateBar(update), + (remove) => + remove.call((remove) => + remove + .transition(t as any) + .attr("height", 0) + .attr("y", y(0)) + ) + ); + + const areaData = cumsum(data.map((d) => d.length)); + const xAreaScale = x.copy().domain([0, areaData.length]); + const yAreaScale = y.copy().domain([0, allIntervals.length]); + + areaPath + .datum(areaData as any) + .attr("fill", "grey") + .attr( + "d", + area() + .x((d: any, idx) => { + return xAreaScale(idx); + }) + .y0(height - margin.bottom) + .y1((d: any) => yAreaScale(d)) as any + ); + + hoverGroup + .selectAll("rect") + .data(data) + .join("rect") + .attr("x", (d: any) => x(d.x0)) + .attr("y", () => y(yMax!)) + .attr("width", (d: any) => Math.max(0, x(d.x1) - x(d.x0) - 1)) + .attr("height", () => y(0) - y(yMax!)) + .attr("fill", "none") + .attr("pointer-events", "all") + .on("mousemove", function (this: any, d: any, idx) { + const [x, y] = mouse(document.body); + const pct = ((areaData[idx] / allIntervals.length) * 100).toFixed(2); + showTooltip( + `${d.length} cards with interval ${d.x0}~${d.x1} days. ` + + `
${pct}% cards below this point.`, + x, + y + ); + }) + .on("mouseout", hideTooltip); + } + + return update; +} diff --git a/ts/src/stats/tooltip.ts b/ts/src/stats/tooltip.ts new file mode 100644 index 000000000..be905e19d --- /dev/null +++ b/ts/src/stats/tooltip.ts @@ -0,0 +1,28 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +import throttle from "lodash.throttle"; + +let tooltipDiv: HTMLDivElement | null = null; + +function showTooltipInner(msg: string, x: number, y: number): void { + if (!tooltipDiv) { + tooltipDiv = document.createElement("div"); + tooltipDiv.className = "graph-tooltip"; + document.body.appendChild(tooltipDiv); + } + tooltipDiv.innerHTML = msg; + tooltipDiv.style.left = `${x - 50}px`; + tooltipDiv.style.top = `${y - 50}px`; + + tooltipDiv.style.opacity = "1"; +} + +export const showTooltip = throttle(showTooltipInner, 16); + +export function hideTooltip(): void { + showTooltip.cancel(); + if (tooltipDiv) { + tooltipDiv.style.opacity = "0"; + } +} diff --git a/ts/src/typing.ts b/ts/src/typing.ts new file mode 100644 index 000000000..e9cd02a0b --- /dev/null +++ b/ts/src/typing.ts @@ -0,0 +1,3 @@ +export function assertUnreachable(x: never): never { + throw new Error(`unreachable: ${x}`); +} diff --git a/ts/svelte.config.js b/ts/svelte.config.js new file mode 100644 index 000000000..b404b169a --- /dev/null +++ b/ts/svelte.config.js @@ -0,0 +1,7 @@ +const sveltePreprocess = require("svelte-preprocess"); + +module.exports = { + preprocess: sveltePreprocess({ + typescript: { compilerOptions: { declaration: false, outDir: null } }, + }), +}; diff --git a/ts/tsconfig.json b/ts/tsconfig.json new file mode 100644 index 000000000..45345549c --- /dev/null +++ b/ts/tsconfig.json @@ -0,0 +1,26 @@ +{ + "exclude": ["node_modules", "dist"], + "compilerOptions": { + /* Basic Options */ + "target": "es6", + "module": "es6", + "declaration": false /* Generates corresponding '.d.ts' file. */, + "rootDir": "src", + "outDir": "dist", + /* Strict Type-Checking Options */ + "strict": true /* Enable all strict type-checking options. */, + "noImplicitAny": false /* Raise error on expressions and declarations with an implied 'any' type. */, + "strictNullChecks": true /* Enable strict null checks. */, + "strictFunctionTypes": true /* Enable strict checking of function types. */, + "strictBindCallApply": true /* Enable strict 'bind', 'call', and 'apply' methods on functions. */, + "strictPropertyInitialization": true /* Enable strict checking of property initialization in classes. */, + "noImplicitThis": true /* Raise error on 'this' expressions with an implied 'any' type. */, + "alwaysStrict": true /* Parse in strict mode and emit "use strict" for each source file. */, + /* Module Resolution Options */ + "moduleResolution": "node" /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */, + "allowSyntheticDefaultImports": true /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */, + "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, + "jsx": "react", + "types": ["svelte", "long"] + } +} diff --git a/ts/webpack.config.js b/ts/webpack.config.js new file mode 100644 index 000000000..807669a9e --- /dev/null +++ b/ts/webpack.config.js @@ -0,0 +1,89 @@ +const HtmlWebpackPlugin = require("html-webpack-plugin"); +const mode = process.env.NODE_ENV || "development"; +const prod = mode === "production"; +var path = require("path"); + +module.exports = { + entry: { + graphs: ["./src/stats/graphs.ts"], + }, + output: { + library: "anki", + }, + plugins: [ + new HtmlWebpackPlugin({ + filename: "graphs.html", + chunks: ["graphs"], + template: "src/html/graphs.html", + }), + ], + externals: { + moment: "moment", + }, + devServer: { + contentBase: "./dist", + port: 9000, + // host: "0.0.0.0", + // disableHostCheck: true, + proxy: { + "/_anki": { + target: "http://localhost:9001", + }, + }, + }, + resolve: { + alias: { + svelte: path.resolve("node_modules", "svelte"), + }, + extensions: [".mjs", ".js", ".svelte", ".ts", ".tsx"], + mainFields: ["svelte", "browser", "module", "main"], + }, + module: { + rules: [ + { + test: /\.css$/i, + use: ["style-loader", "css-loader"], + }, + { + test: /\.(svelte)$/, + exclude: /node_modules/, + use: [ + { + loader: "svelte-loader", + options: { + emitCss: true, + preprocess: require("svelte-preprocess")({ + typescript: { + transpileOnly: true, + compilerOptions: { + declaration: false, + }, + }, + }), + }, + }, + ], + }, + { + test: /\.tsx?$/, + use: ["ts-loader"], + exclude: /node_modules/, + }, + { + test: /\.(tsx?|js)$/, + loader: "eslint-loader", + exclude: /node_modules/, + options: { + fix: true, + }, + }, + ], + }, + mode, + devtool: prod ? false : "source-map", + optimization: { + splitChunks: { + // chunks: "all", + }, + }, +};