Fuzzy search in symbol insertion overlay (#2059)

* Add flag for enabling insert symbols feature

* Add symbols overlay directory

* Detect if :xy is inserted into editable

* Allow naive updating of overlay, and special handling of ':'

* First step towards better Virtual Element support

* Update floating to reference range on insert text

* Position SymbolsOverlay always on top or bottom

* Add a data-provider to emulate API

* Show correct suggestions in symbols overlay

* Rename to replacementLength

* Allow replacing via clicking in menu

* Optionally remove inline padding of Popover

* Hide Symbols overlay on blur of content editable

* Add specialKey to inputHandler and generalize how arrow movement is detected

- This way macOS users can use Ctrl-N to mean down, etc.

* Detect special key from within SymbolsOverlay

* Implement full backwards search while typing

* Allow navigating symbol menu and accepting with enter

* Add some entries to data-provider

* Satisfy eslint

* Generate symbolsTable from sources

* Use other github source, allow multiple names

In return, symbol must be unique

* Automatically scroll in symbols dropdown

* Use from npm packages rather than downloading from URL

* Remove console.log

* Remove print

Co-authored-by: Damien Elmes <dae@users.noreply.github.com>

* Add pointerDown event to input-handler

- so that SymbolsOverlay can reset on field click

* Make tab do the same as enter

* Make font a bit smaller but increase relative icon size

* Satisfy type requirement of handlerlist

* Revert changing default size of DropdownItems

* Remove some now unused code for bootstrap dropdowns

* Use fuse to allow fuzzy searching of symbols

* Remove unnecessary async handling in data-provider

I did that because at first I was still expecting to fetch the symbols
from the backend

* Apply field font family in symbol preview

* Remove inline padding from latex popover

* Rename data-provier to symbols-table

* Add some explaining comments to interface

* Allow for auto insertion symbols

* Use deleteData and after instead of replaceData

* Allow using html in symbols

* Show html symbols as html

* Add SymbolsEntry component

* Also include containshtml at low search precedence

* Put character entities and gemoji into their own files

* Factor out prepareInsertion method

* Allow deletion while searching for correct symbol

* Respect insertCompositionText

* Delete data-provider

* Restrict auto insert queries to max 5 characters

* Satisfy svelte check

* Fix the overlay sometimes not showing

This will make sure to always normalize text nodes before searching.
However it adjacent text is partially formatted, this will still not
find the whole query.

For example, currently, entering `<b>:for</b>al` and then inputting `l`,
will not trigger a search for `forall`, because of the <b> formatting

* Add empty line

* Do not trigger overlay, when last character is whitespace or colon

* Add missing fuse license

package.json

@@ -71,6 +71,7 @@
         "eslint-plugin-import": "^2.25.4",
         "eslint-plugin-simple-import-sort": "^7.0.0",
         "eslint-plugin-svelte3": "^3.4.0",
+        "fuse.js": "^6.6.2",
         "gemoji": "^7.1.0",
         "intl-pluralrules": "^1.2.2",
         "jquery": "^3.5.1",

ts/editor/BUILD.bazel

@@ -33,6 +33,7 @@ _ts_deps = [
     "@npm//svelte",
     "@npm//character-entities",
     "@npm//gemoji",
+    "@npm//fuse.js",
 ]
 
 compile_svelte(

ts/editor/editor-toolbar/LatexButton.svelte

@@ -84,7 +84,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
         </IconButton>
     </span>
 
-    <Popover slot="floating">
+    <Popover slot="floating" --popover-padding-inline="0">
         {#each dropdownItems as [callback, keyCombination, label]}
             <DropdownItem on:click={callback}>
                 <span>{label}</span>

ts/editor/symbols-overlay/SymbolsEntry.svelte

@@ -0,0 +1,58 @@
+<!--
+Copyright: Ankitects Pty Ltd and contributors
+License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+-->
+<script lang="ts">
+    export let symbol: string;
+
+    export let containsHTML: boolean;
+    export let fontFamily: string;
+
+    export let names: string[];
+
+    // Emojis can have a string length of up to 5 characters.
+    // This could be done better with Intl.Segmenter once it has wider support:
+    // const segmenter = new Intl.Segmenter();
+    // const displayInTwoRows = [...segmenter.segment(symbol)].length;
+    $: displayInTwoRows = symbol.length > 6;
+</script>
+
+<div class="symbols-entry" style:flex-direction={displayInTwoRows ? "column" : "row"}>
+    <div class="symbol" style:font-family={fontFamily}>
+        {#if containsHTML}
+            {@html symbol}
+        {:else}
+            {symbol}
+        {/if}
+    </div>
+
+    <div class="description">
+        {#each names as name}
+            <span class="name">
+                <slot symbolName={name} />
+            </span>
+        {/each}
+    </div>
+</div>
+
+<style lang="scss">
+    .symbols-entry {
+        display: flex;
+    }
+
+    .symbol {
+        transform: scale(1.1);
+        font-size: 150%;
+        /* The widest emojis I could find were couple_with_heart_ */
+        /* We should make sure it can still be properly displayed */
+        width: 38px;
+    }
+
+    .description {
+        align-self: center;
+    }
+
+    .name {
+        margin-left: 3px;
+    }
+</style>

ts/editor/symbols-overlay/SymbolsOverlay.svelte

@@ -3,205 +3,168 @@ Copyright: Ankitects Pty Ltd and contributors
 License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 -->
 <script lang="ts">
-    import { onMount } from "svelte";
+    import { getContext, onMount } from "svelte";
+    import type { Readable } from "svelte/store";
 
     import DropdownItem from "../../components/DropdownItem.svelte";
     import Popover from "../../components/Popover.svelte";
     import WithFloating from "../../components/WithFloating.svelte";
-    import {
-        getRange,
-        getSelection,
-        isSelectionCollapsed,
-    } from "../../lib/cross-browser";
+    import { fontFamilyKey } from "../../lib/context-keys";
+    import { getRange, getSelection } from "../../lib/cross-browser";
+    import { createDummyDoc } from "../../lib/parsing";
     import type { Callback } from "../../lib/typing";
     import { singleCallback } from "../../lib/typing";
+    import type { SpecialKeyParams } from "../../sveltelib/input-handler";
     import { context } from "../rich-text-input";
-    import type { SymbolsTable } from "./data-provider";
-    import { getSymbolExact, getSymbols } from "./data-provider";
+    import { findSymbols, getAutoInsertSymbol, getExactSymbol } from "./symbols-table";
+    import type {
+        SymbolsEntry as SymbolsEntryType,
+        SymbolsTable,
+    } from "./symbols-types";
+    import SymbolsEntry from "./SymbolsEntry.svelte";
 
-    const SYMBOLS_DELIMITER = ":";
+    const symbolsDelimiter = ":";
+    const queryMinLength = 2;
+    const autoInsertQueryMaxLength = 5;
 
-    const { inputHandler, editable } = context.get();
+    const whitespaceCharacters = [" ", "\u00a0"];
 
-    let referenceRange: Range | undefined = undefined;
-    let cleanup: Callback;
-    let query: string = "";
-    let activeItem = 0;
+    const { editable, inputHandler } = context.get();
+    const fontFamily = getContext<Readable<string>>(fontFamilyKey);
 
     let foundSymbols: SymbolsTable = [];
 
+    let referenceRange: Range | undefined = undefined;
+    let activeItem = 0;
+    let cleanup: Callback;
+
     function unsetReferenceRange() {
         referenceRange = undefined;
         activeItem = 0;
         cleanup?.();
     }
 
-    async function maybeShowOverlay(
-        selection: Selection,
-        event: InputEvent,
-    ): Promise<void> {
-        if (
-            event.inputType !== "insertText" ||
-            event.data === SYMBOLS_DELIMITER ||
-            !isSelectionCollapsed(selection)
-        ) {
-            return unsetReferenceRange();
-        }
+    function replaceText(selection: Selection, text: Text, nodes: Node[]): void {
+        text.deleteData(0, text.length);
+        text.after(...nodes);
 
-        const currentRange = getRange(selection)!;
-        const offset = currentRange.endOffset;
-
-        if (!(currentRange.commonAncestorContainer instanceof Text) || offset < 2) {
-            return unsetReferenceRange();
-        }
-
-        const wholeText = currentRange.commonAncestorContainer.wholeText;
-
-        for (let index = offset - 1; index >= 0; index--) {
-            const currentCharacter = wholeText[index];
-
-            if (currentCharacter === " ") {
-                return unsetReferenceRange();
-            } else if (currentCharacter === SYMBOLS_DELIMITER) {
-                const possibleQuery =
-                    wholeText.substring(index + 1, offset) + event.data;
-
-                if (possibleQuery.length < 2) {
-                    return unsetReferenceRange();
-                }
-
-                query = possibleQuery;
-                referenceRange = currentRange;
-                foundSymbols = await getSymbols(query);
-
-                cleanup = editable.focusHandler.blur.on(
-                    async () => unsetReferenceRange(),
-                    {
-                        once: true,
-                    },
-                );
-                return;
-            }
-        }
-    }
-
-    async function replaceText(
-        selection: Selection,
-        text: Text,
-        symbolCharacter: string,
-    ): Promise<void> {
-        text.replaceData(0, text.length, symbolCharacter);
         unsetReferenceRange();
 
         // Place caret behind it
         const range = new Range();
-        range.selectNode(text);
-
-        selection.removeAllRanges();
-        selection.addRange(range);
-        selection.collapseToEnd();
-    }
-
-    function replaceTextOnDemand(symbolCharacter: string): void {
-        const commonAncestor = referenceRange!.commonAncestorContainer as Text;
-        const selection = getSelection(commonAncestor)!;
-
-        const replacementLength =
-            commonAncestor.data
-                .substring(0, referenceRange!.endOffset)
-                .split("")
-                .reverse()
-                .join("")
-                .indexOf(SYMBOLS_DELIMITER) + 1;
-
-        const newOffset = referenceRange!.endOffset - replacementLength + 1;
-
-        commonAncestor.replaceData(
-            referenceRange!.endOffset - replacementLength,
-            replacementLength + 1,
-            symbolCharacter,
-        );
-
-        // Place caret behind it
-        const range = new Range();
-        range.setEnd(commonAncestor, newOffset);
+        range.setEndAfter(nodes[nodes.length - 1]);
         range.collapse(false);
 
         selection.removeAllRanges();
         selection.addRange(range);
-
-        unsetReferenceRange();
     }
 
-    async function updateOverlay(
-        selection: Selection,
-        event: InputEvent,
-    ): Promise<void> {
-        if (event.inputType !== "insertText") {
-            return unsetReferenceRange();
-        }
+    const parser = new DOMParser();
 
-        const data = event.data;
-        referenceRange = getRange(selection)!;
-
-        if (data === SYMBOLS_DELIMITER && query) {
-            const symbol = await getSymbolExact(query);
-
-            if (!symbol) {
-                return unsetReferenceRange();
-            }
-
-            const currentRange = getRange(selection)!;
-            const offset = currentRange.endOffset;
-
-            if (!(currentRange.commonAncestorContainer instanceof Text) || offset < 2) {
-                return unsetReferenceRange();
-            }
-
-            const commonAncestor = currentRange.commonAncestorContainer;
-
-            const replacementLength =
-                commonAncestor.data
-                    .substring(0, currentRange.endOffset)
-                    .split("")
-                    .reverse()
-                    .join("")
-                    .indexOf(SYMBOLS_DELIMITER) + 1;
-
-            commonAncestor.deleteData(
-                currentRange.endOffset - replacementLength,
-                replacementLength,
+    function symbolsEntryToReplacement(entry: SymbolsEntryType): Node[] {
+        if (entry.containsHTML) {
+            const doc = parser.parseFromString(
+                createDummyDoc(entry.symbol),
+                "text/html",
             );
-
-            inputHandler.insertText.on(
-                async ({ text }) => replaceText(selection, text, symbol),
-                {
-                    once: true,
-                },
-            );
-        } else if (query) {
-            query += data!;
-            foundSymbols = await getSymbols(query);
-        }
-    }
-
-    async function onBeforeInput({ event }): Promise<void> {
-        const selection = getSelection(event.target)!;
-
-        if (referenceRange) {
-            await updateOverlay(selection, event);
+            return [...doc.body.childNodes];
         } else {
-            await maybeShowOverlay(selection, event);
+            return [new Text(entry.symbol)];
         }
     }
 
-    $: showSymbolsOverlay = referenceRange && foundSymbols.length > 0;
+    function tryAutoInsert(selection: Selection, range: Range, query: string): boolean {
+        if (
+            query.length >= queryMinLength &&
+            query.length <= autoInsertQueryMaxLength
+        ) {
+            const symbolEntry = getAutoInsertSymbol(query);
 
-    async function onSpecialKey({ event, action }): Promise<void> {
-        if (!showSymbolsOverlay) {
-            return;
+            if (symbolEntry) {
+                const commonAncestor = range.commonAncestorContainer as Text;
+                const replacementLength = query.length;
+
+                commonAncestor.parentElement?.normalize();
+                commonAncestor.deleteData(
+                    range.endOffset - replacementLength + 1,
+                    replacementLength,
+                );
+
+                inputHandler.insertText.on(
+                    async ({ text }) => {
+                        replaceText(
+                            selection,
+                            text,
+                            symbolsEntryToReplacement(symbolEntry),
+                        );
+                    },
+                    {
+                        once: true,
+                    },
+                );
+
+                return true;
+            }
         }
 
+        return false;
+    }
+
+    function findValidSearchQuery(
+        selection: Selection,
+        range: Range,
+        startQuery = "",
+        shouldFinishEarly: (
+            selection: Selection,
+            range: Range,
+            query: string,
+        ) => boolean = () => false,
+    ): string | null {
+        if (
+            whitespaceCharacters.includes(startQuery) ||
+            startQuery === symbolsDelimiter
+        ) {
+            return null;
+        }
+
+        const offset = range.endOffset;
+
+        if (!(range.commonAncestorContainer instanceof Text)) {
+            return null;
+        }
+
+        if (range.commonAncestorContainer.parentElement) {
+            // This call can change range.commonAncestor
+            range.commonAncestorContainer.parentElement.normalize();
+        }
+
+        const commonAncestorContainer = range.commonAncestorContainer;
+        let query = startQuery;
+
+        for (let index = offset - 1; index >= 0; index--) {
+            const currentCharacter = commonAncestorContainer.wholeText[index];
+
+            if (whitespaceCharacters.includes(currentCharacter)) {
+                return null;
+            } else if (currentCharacter === symbolsDelimiter) {
+                if (query.length < queryMinLength) {
+                    return null;
+                }
+
+                return query;
+            }
+
+            query = currentCharacter + query;
+
+            if (shouldFinishEarly(selection, range, query)) {
+                return null;
+            }
+        }
+
+        return null;
+    }
+
+    function onSpecialKey({ event, action }): void {
         if (["caretLeft", "caretRight"].includes(action)) {
             return unsetReferenceRange();
         }
@@ -221,23 +184,176 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
                 activeItem++;
             }
         } else if (action === "enter" || action === "tab") {
-            replaceTextOnDemand(foundSymbols[activeItem].symbol);
+            replaceTextOnDemand(foundSymbols[activeItem]);
         } else if (action === "escape") {
             unsetReferenceRange();
         }
     }
 
+    function maybeShowOverlay(selection: Selection, event: InputEvent): void {
+        if (!event.data) {
+            return;
+        }
+
+        const currentRange = getRange(selection)!;
+
+        // The input event opening the overlay or triggering the auto-insert
+        // must be an insertion, so event.data must be a string.
+        // If the inputType is insertCompositionText, the event.data will
+        // contain the current composition, but the document will also
+        // contain the whole composition except the last character.
+        // So we only take the last character from event.data and retrieve the
+        // rest from the document
+        const startQuery = event.data[event.data.length - 1];
+        const query = findValidSearchQuery(
+            selection,
+            currentRange,
+            startQuery,
+            tryAutoInsert,
+        );
+
+        if (query) {
+            foundSymbols = findSymbols(query);
+
+            if (foundSymbols.length > 0) {
+                referenceRange = currentRange;
+                cleanup = singleCallback(
+                    editable.focusHandler.blur.on(async () => unsetReferenceRange(), {
+                        once: true,
+                    }),
+                    inputHandler.pointerDown.on(async () => unsetReferenceRange()),
+                    inputHandler.specialKey.on(async (input: SpecialKeyParams) =>
+                        onSpecialKey(input),
+                    ),
+                );
+            }
+        }
+    }
+
+    function replaceTextOnDemand(entry: SymbolsEntryType): void {
+        const commonAncestor = referenceRange!.commonAncestorContainer as Text;
+        const selection = getSelection(commonAncestor)!;
+
+        const replacementLength =
+            commonAncestor.data
+                .substring(0, referenceRange!.endOffset)
+                .split("")
+                .reverse()
+                .join("")
+                .indexOf(symbolsDelimiter) + 1;
+
+        commonAncestor.deleteData(
+            referenceRange!.endOffset - replacementLength,
+            replacementLength + 1,
+        );
+
+        const nodes = symbolsEntryToReplacement(entry);
+        commonAncestor.after(...nodes);
+
+        const range = new Range();
+        range.setEndAfter(nodes[nodes.length - 1]);
+        range.collapse(false);
+
+        selection.removeAllRanges();
+        selection.addRange(range);
+
+        unsetReferenceRange();
+    }
+
+    function prepareInsertion(selection: Selection, query: string): void {
+        const symbolEntry = getExactSymbol(query);
+
+        if (!symbolEntry) {
+            return unsetReferenceRange();
+        }
+
+        const currentRange = getRange(selection)!;
+        const offset = currentRange.endOffset;
+
+        if (
+            !(currentRange.commonAncestorContainer instanceof Text) ||
+            offset < queryMinLength
+        ) {
+            return unsetReferenceRange();
+        }
+
+        const commonAncestor = currentRange.commonAncestorContainer;
+
+        const replacementLength =
+            commonAncestor.data
+                .substring(0, currentRange.endOffset)
+                .split("")
+                .reverse()
+                .join("")
+                .indexOf(symbolsDelimiter) + 1;
+
+        commonAncestor.deleteData(
+            currentRange.endOffset - replacementLength,
+            replacementLength,
+        );
+
+        inputHandler.insertText.on(
+            async ({ text }) =>
+                replaceText(selection, text, symbolsEntryToReplacement(symbolEntry)),
+            {
+                once: true,
+            },
+        );
+    }
+
+    function updateOverlay(selection: Selection, event: InputEvent): void {
+        if (event.data === symbolsDelimiter) {
+            const query = findValidSearchQuery(selection, getRange(selection)!);
+
+            if (query) {
+                prepareInsertion(selection, query);
+            } else {
+                unsetReferenceRange();
+            }
+        }
+
+        // We have to wait for afterInput to update the symbols, because we also
+        // want to update in the case of a deletion
+        inputHandler.afterInput.on(
+            async (): Promise<void> => {
+                const currentRange = getRange(selection)!;
+                const query = findValidSearchQuery(selection, currentRange);
+
+                if (!query) {
+                    return unsetReferenceRange();
+                }
+
+                foundSymbols = findSymbols(query);
+
+                if (foundSymbols.length === 0) {
+                    unsetReferenceRange();
+                } else {
+                    referenceRange = currentRange;
+                }
+            },
+            { once: true },
+        );
+    }
+
+    function onBeforeInput({ event }): void {
+        const selection = getSelection(event.target)!;
+
+        if (referenceRange) {
+            updateOverlay(selection, event);
+        } else {
+            maybeShowOverlay(selection, event);
+        }
+    }
+
     onMount(() =>
-        singleCallback(
-            inputHandler.beforeInput.on(onBeforeInput),
-            inputHandler.specialKey.on(onSpecialKey),
-            inputHandler.pointerDown.on(async () => unsetReferenceRange()),
+        inputHandler.beforeInput.on(
+            async (input: { event: Event }): Promise<void> => onBeforeInput(input),
         ),
     );
 </script>
 
 <div class="symbols-overlay">
-    {#if showSymbolsOverlay}
+    {#if referenceRange}
         <WithFloating
             reference={referenceRange}
             placement={["top", "bottom"]}
@@ -248,16 +364,17 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
                     {#each foundSymbols as found, index (found.symbol)}
                         <DropdownItem
                             active={index === activeItem}
-                            on:click={() => replaceTextOnDemand(found.symbol)}
+                            on:click={() => replaceTextOnDemand(found)}
                         >
-                            <div class="symbol">{found.symbol}</div>
-                            <div class="description">
-                                {#each found.names as name}
-                                    <span class="name">
-                                        {SYMBOLS_DELIMITER}{name}{SYMBOLS_DELIMITER}
-                                    </span>
-                                {/each}
-                            </div>
+                            <SymbolsEntry
+                                let:symbolName
+                                symbol={found.symbol}
+                                names={found.names}
+                                containsHTML={Boolean(found.containsHTML)}
+                                fontFamily={$fontFamily}
+                            >
+                                {symbolsDelimiter}{symbolName}{symbolsDelimiter}
+                            </SymbolsEntry>
                         </DropdownItem>
                     {/each}
                 </div>
@@ -279,19 +396,4 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
         text-overflow: ellipsis;
         overflow-y: auto;
     }
-
-    .symbol {
-        transform: scale(1.1);
-        font-size: 150%;
-        /* The widest emojis I could find were couple_with_heart_ */
-        width: 38px;
-    }
-
-    .description {
-        align-self: center;
-    }
-
-    .name {
-        margin-left: 3px;
-    }
 </style>

ts/editor/symbols-overlay/character-entities.ts

@@ -0,0 +1,33 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+import { characterEntities } from "character-entities";
+
+import type { SymbolsTable } from "./symbols-types";
+
+// Not all characters work well in the editor field
+delete characterEntities["Tab"];
+
+// A single character entity can be present under differnet names
+// So we change the mapping to symbol => name[]
+const characterTable: Record<string, string[]> = {};
+
+for (const [name, character] of Object.entries(characterEntities)) {
+    if (character in characterTable) {
+        characterTable[character].push(name);
+    } else {
+        characterTable[character] = [name];
+    }
+}
+
+const characterSymbolsTable: SymbolsTable = [];
+
+for (const [character, names] of Object.entries(characterTable)) {
+    characterSymbolsTable.push({
+        symbol: character,
+        names,
+        tags: [],
+    });
+}
+
+export default characterSymbolsTable;

ts/editor/symbols-overlay/data-provider.ts

@@ -1,60 +0,0 @@
-// Copyright: Ankitects Pty Ltd and contributors
-// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
-
-import { characterEntities } from "character-entities";
-import { gemoji } from "gemoji";
-
-interface SymbolsEntry {
-    symbol: string;
-    names: string[];
-    tags: string[];
-    containsHTML?: boolean;
-    autoInsert?: boolean;
-}
-
-export type SymbolsTable = SymbolsEntry[];
-
-const symbolsTable: SymbolsTable = [];
-
-const characterTable: Record<string, string[]> = {};
-
-// Not all characters work well in the editor field
-delete characterEntities["Tab"];
-
-for (const [name, character] of Object.entries(characterEntities)) {
-    if (character in characterTable) {
-        characterTable[character].push(name);
-    } else {
-        characterTable[character] = [name];
-    }
-}
-
-for (const [character, names] of Object.entries(characterTable)) {
-    symbolsTable.push({
-        symbol: character,
-        names,
-        tags: [],
-    });
-}
-
-for (const { emoji, names, tags } of gemoji) {
-    symbolsTable.push({
-        symbol: emoji,
-        names,
-        tags,
-    });
-}
-
-export async function getSymbols(query: string): Promise<SymbolsTable> {
-    return symbolsTable.filter(
-        ({ names, tags }) =>
-            names.some((name) => name.includes(query)) ||
-            tags.some((tag) => tag.includes(query)),
-    );
-}
-
-export async function getSymbolExact(query: string): Promise<string | null> {
-    const found = symbolsTable.find(({ names }) => names.includes(query));
-
-    return found ? found.symbol : null;
-}

ts/editor/symbols-overlay/gemoji.ts

@@ -0,0 +1,18 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+import { gemoji } from "gemoji";
+
+import type { SymbolsTable } from "./symbols-types";
+
+const gemojiSymbolsTable: SymbolsTable = [];
+
+for (const { emoji, names, tags } of gemoji) {
+    gemojiSymbolsTable.push({
+        symbol: emoji,
+        names,
+        tags,
+    });
+}
+
+export default gemojiSymbolsTable;

ts/editor/symbols-overlay/symbols-table.ts

@@ -0,0 +1,54 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+import Fuse from "fuse.js";
+
+import characterEntities from "./character-entities";
+import gemoji from "./gemoji";
+import type { SymbolsEntry, SymbolsTable } from "./symbols-types";
+
+const symbolsTable: SymbolsTable = [...characterEntities, ...gemoji];
+
+const symbolsFuse = new Fuse(symbolsTable, {
+    threshold: 0.2,
+    minMatchCharLength: 2,
+    useExtendedSearch: true,
+    isCaseSensitive: true,
+    keys: [
+        {
+            name: "names",
+            weight: 7,
+        },
+        {
+            name: "tags",
+            weight: 3,
+        },
+        {
+            name: "autoInsert",
+            weight: 0.1,
+        },
+        {
+            name: "containsHTML",
+            weight: 0.1,
+        },
+    ],
+});
+
+export function findSymbols(query: string): SymbolsTable {
+    return symbolsFuse.search(query).map(({ item }) => item);
+}
+
+export function getExactSymbol(query: string): SymbolsEntry | null {
+    const [found] = symbolsFuse.search({ names: `="${query}"` }, { limit: 1 });
+
+    return found ? found.item : null;
+}
+
+export function getAutoInsertSymbol(query: string): SymbolsEntry | null {
+    const [found] = symbolsFuse.search({
+        names: `="${query}"`,
+        autoInsert: "=autoInsert",
+    });
+
+    return found ? found.item : null;
+}

ts/editor/symbols-overlay/symbols-types.d.ts

@@ -0,0 +1,27 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+export interface SymbolsEntry {
+    symbol: string;
+    /**
+     * Used for searching and direct insertion
+     */
+    names: string[];
+    /**
+     * Used for searching
+     */
+    tags: string[];
+    /**
+     * If symbols contain HTML, they need to be treated specially.
+     */
+    containsHTML?: "containsHTML";
+    /**
+     * Symbols can be automak inserted, when you enter a full name within delimiters.
+     * If you enable auto insertion, you can use direction insertion without
+     * using the delimiter and triggering the search dropdown.
+     * To falicitate interacting with fuse.js this is a string value rather than a boolean.
+     */
+    autoInsert?: "autoInsert";
+}
+
+export type SymbolsTable = SymbolsEntry[];

ts/licenses.json

@@ -1005,6 +1005,15 @@
         "path": "node_modules/functional-red-black-tree",
         "licenseFile": "node_modules/functional-red-black-tree/LICENSE"
     },
+    "fuse.js@6.6.2": {
+        "licenses": "Apache-2.0",
+        "repository": "https://github.com/krisk/Fuse",
+        "publisher": "Kiro Risk",
+        "email": "kirollos@gmail.com",
+        "url": "http://kiro.me",
+        "path": "node_modules/fuse.js",
+        "licenseFile": "node_modules/fuse.js/LICENSE"
+    },
     "gemoji@7.1.0": {
         "licenses": "MIT",
         "repository": "https://github.com/wooorm/gemoji",

ts/sveltelib/input-handler.ts

@@ -15,6 +15,10 @@ export interface InputEventParams {
     event: InputEvent;
 }
 
+export interface EventParams {
+    event: Event;
+}
+
 export interface InsertTextParams {
     event: InputEvent;
     text: Text;
@@ -36,6 +40,7 @@ export interface SpecialKeyParams {
 export interface InputHandlerAPI {
     readonly beforeInput: HandlerList<InputEventParams>;
     readonly insertText: HandlerList<InsertTextParams>;
+    readonly afterInput: HandlerList<EventParams>;
     readonly pointerDown: HandlerList<{ event: PointerEvent }>;
     readonly specialKey: HandlerList<SpecialKeyParams>;
 }
@@ -49,6 +54,7 @@ export interface InputHandlerAPI {
 function useInputHandler(): [InputHandlerAPI, SetupInputHandlerAction] {
     const beforeInput = new HandlerList<InputEventParams>();
     const insertText = new HandlerList<InsertTextParams>();
+    const afterInput = new HandlerList<EventParams>();
 
     async function onBeforeInput(this: Element, event: InputEvent): Promise<void> {
         const selection = getSelection(this)!;
@@ -56,7 +62,11 @@ function useInputHandler(): [InputHandlerAPI, SetupInputHandlerAction] {
 
         await beforeInput.dispatch({ event });
 
-        if (!range || event.inputType !== "insertText" || insertText.length === 0) {
+        if (
+            !range ||
+            !event.inputType.startsWith("insert") ||
+            insertText.length === 0
+        ) {
             return;
         }
 
@@ -73,6 +83,14 @@ function useInputHandler(): [InputHandlerAPI, SetupInputHandlerAction] {
         await insertText.dispatch({ event, text });
 
         range.commonAncestorContainer.normalize();
+
+        // We emulate the after input event here, because we prevent
+        // the default behavior earlier
+        await afterInput.dispatch({ event });
+    }
+
+    async function onInput(this: Element, event: Event): Promise<void> {
+        await afterInput.dispatch({ event });
     }
 
     const pointerDown = new HandlerList<{ event: PointerEvent }>();
@@ -107,6 +125,7 @@ function useInputHandler(): [InputHandlerAPI, SetupInputHandlerAction] {
     function setupHandler(element: HTMLElement): { destroy(): void } {
         const destroy = singleCallback(
             on(element, "beforeinput", onBeforeInput),
+            on(element, "input", onInput),
             on(element, "blur", clearInsertText),
             on(element, "pointerdown", onPointerDown),
             on(element, "keydown", onKeyDown),
@@ -120,6 +139,7 @@ function useInputHandler(): [InputHandlerAPI, SetupInputHandlerAction] {
         {
             beforeInput,
             insertText,
+            afterInput,
             specialKey,
             pointerDown,
         },

yarn.lock

@@ -2619,6 +2619,11 @@ functional-red-black-tree@^1.0.1:
   resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327"
   integrity sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=
 
+fuse.js@^6.6.2:
+  version "6.6.2"
+  resolved "https://registry.yarnpkg.com/fuse.js/-/fuse.js-6.6.2.tgz#fe463fed4b98c0226ac3da2856a415576dc9a111"
+  integrity sha512-cJaJkxCCxC8qIIcPBF9yGxY0W/tVZS3uEISDxhYIdtk8OL93pe+6Zj7LjCqVV4dzbqcriOZ+kQ/NE4RXZHsIGA==
+
 gemoji@^7.1.0:
   version "7.1.0"
   resolved "https://registry.yarnpkg.com/gemoji/-/gemoji-7.1.0.tgz#165403777681a9690d649aabd104da037bdd7739"