Start work on copy & paste handling

proto/anki/frontend.proto

@@ -31,6 +31,9 @@ service FrontendService {
 
   // Editor
   rpc editorUpdateNote(notes.UpdateNotesRequest) returns (generic.Empty);
+  rpc convertPastedImage(ConvertPastedImageRequest)
+      returns (ConvertPastedImageResponse);
+  rpc retrieveUrl(generic.String) returns (RetrieveUrlResponse);
 
   // Profile config
   rpc GetProfileConfigJson(generic.String) returns (generic.Json);
@@ -53,6 +56,20 @@ message SetSchedulingStatesRequest {
   scheduler.SchedulingStates states = 2;
 }
 
+message ConvertPastedImageRequest {
+  bytes data = 1;
+  string ext = 2;
+}
+
+message ConvertPastedImageResponse {
+  bytes data = 1;
+}
+
+message RetrieveUrlResponse {
+  string filename = 1;
+  string error = 2;
+}
+
 message SetSettingJsonRequest {
   string key = 1;
   bytes value_json = 2;

qt/aqt/editor.py

@@ -31,7 +31,6 @@ import aqt.sound
 from anki._legacy import deprecated
 from anki.cards import Card
 from anki.collection import Config
-from anki.consts import MODEL_CLOZE
 from anki.hooks import runFilter
 from anki.httpclient import HttpClient
 from anki.models import NotetypeDict, NotetypeId, StockNotetype
@@ -173,6 +172,7 @@ class Editor:
     def setupWeb(self) -> None:
         editor_key = self.mw.pm.editor_key(self.editorMode)
         self.web.load_sveltekit_page(f"editor/?mode={editor_key}")
+        self.web.allow_drops = True
 
     def _set_ready(self) -> None:
         lefttopbtns: list[str] = []
@@ -1124,37 +1124,37 @@ class EditorWebView(AnkiWebView):
     def onMiddleClickPaste(self) -> None:
         self._onPaste(QClipboard.Mode.Selection)
 
-    def dragEnterEvent(self, evt: QDragEnterEvent | None) -> None:
+    # def dragEnterEvent(self, evt: QDragEnterEvent | None) -> None:
-        assert evt is not None
+    #     assert evt is not None
-        evt.accept()
+    #     evt.accept()
 
-    def dropEvent(self, evt: QDropEvent | None) -> None:
+    # def dropEvent(self, evt: QDropEvent | None) -> None:
-        assert evt is not None
+    #     assert evt is not None
-        extended = self._wantsExtendedPaste()
+    #     extended = self._wantsExtendedPaste()
-        mime = evt.mimeData()
+    #     mime = evt.mimeData()
-        assert mime is not None
+    #     assert mime is not None
 
-        if (
+    #     if (
-            self.editor.state is EditorState.IO_PICKER
+    #         self.editor.state is EditorState.IO_PICKER
-            and (html := self._processUrls(mime, allowed_suffixes=pics))
+    #         and (html := self._processUrls(mime, allowed_suffixes=pics))
-            and (path := self.editor.extract_img_path_from_html(html))
+    #         and (path := self.editor.extract_img_path_from_html(html))
-        ):
+    #     ):
-            self.editor.setup_mask_editor(path)
+    #         self.editor.setup_mask_editor(path)
-            return
+    #         return
 
-        evt_pos = evt.position()
+    #     evt_pos = evt.position()
-        cursor_pos = QPoint(int(evt_pos.x()), int(evt_pos.y()))
+    #     cursor_pos = QPoint(int(evt_pos.x()), int(evt_pos.y()))
 
-        if evt.source() and mime.hasHtml():
+    #     if evt.source() and mime.hasHtml():
-            # don't filter html from other fields
+    #         # don't filter html from other fields
-            html, internal = mime.html(), True
+    #         html, internal = mime.html(), True
-        else:
+    #     else:
-            html, internal = self._processMime(mime, extended, drop_event=True)
+    #         html, internal = self._processMime(mime, extended, drop_event=True)
 
-        if not html:
+    #     if not html:
-            return
+    #         return
 
-        self.editor.doDrop(html, internal, extended, cursor_pos)
+    #     self.editor.doDrop(html, internal, extended, cursor_pos)
 
     # returns (html, isInternal)
     def _processMime(

qt/aqt/mediasrv.py

@@ -16,7 +16,7 @@ from collections.abc import Callable
 from dataclasses import dataclass
 from errno import EPROTOTYPE
 from http import HTTPStatus
-from typing import Any
+from typing import Any, cast
 
 import flask
 import flask_cors
@@ -665,6 +665,34 @@ def set_config_json() -> bytes:
     return set_setting_json(aqt.mw.col.set_config)
 
 
+def convert_pasted_image() -> bytes:
+    req = frontend_pb2.ConvertPastedImageRequest()
+    req.ParseFromString(request.data)
+    image = QImage.fromData(req.data)
+    buffer = QBuffer()
+    buffer.open(QBuffer.OpenModeFlag.ReadWrite)
+    if req.ext == "png":
+        quality = 50
+    else:
+        quality = 80
+    image.save(buffer, req.ext, quality)
+    buffer.reset()
+    data = bytes(cast(bytes, buffer.readAll()))
+    return frontend_pb2.ConvertPastedImageResponse(data=data).SerializeToString()
+
+
+def retrieve_url() -> bytes:
+    from aqt.utils import retrieve_url
+
+    req = generic_pb2.String()
+    req.ParseFromString(request.data)
+    url = req.val
+    filename, error = retrieve_url(url)
+    return frontend_pb2.RetrieveUrlResponse(
+        filename=filename, error=error
+    ).SerializeToString()
+
+
 post_handler_list = [
     congrats_info,
     get_deck_configs_for_update,
@@ -686,6 +714,8 @@ post_handler_list = [
     get_meta_json,
     set_meta_json,
     get_config_json,
+    convert_pasted_image,
+    retrieve_url,
 ]
 
 
@@ -739,6 +769,9 @@ exposed_backend_list = [
     "decode_iri_paths",
     # ConfigService
     "set_config_json",
+    "get_config_bool",
+    # MediaService
+    "add_media_file",
 ]
 
 

qt/aqt/utils.py

@@ -9,16 +9,19 @@ import re
 import shutil
 import subprocess
 import sys
+import urllib
 from collections.abc import Callable, Sequence
 from functools import partial, wraps
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Literal, Union
 
+import requests
 from send2trash import send2trash
 
 import aqt
 from anki._legacy import DeprecatedNamesMixinForModule
 from anki.collection import Collection, HelpPage
+from anki.httpclient import HttpClient
 from anki.lang import TR, tr_legacyglobal  # pylint: disable=unused-import
 from anki.utils import (
     call,
@@ -134,6 +137,49 @@ def openLink(link: str | QUrl) -> None:
         QDesktopServices.openUrl(QUrl(link))
 
 
+def retrieve_url(url: str) -> tuple[str, str]:
+    "Download file into media folder and return local filename or None."
+
+    local = url.lower().startswith("file://")
+    content_type = None
+    error_msg: str | None = None
+    try:
+        if local:
+            # urllib doesn't understand percent-escaped utf8, but requires things like
+            # '#' to be escaped.
+            url = urllib.parse.unquote(url)
+            url = url.replace("%", "%25")
+            url = url.replace("#", "%23")
+            req = urllib.request.Request(
+                url, None, {"User-Agent": "Mozilla/5.0 (compatible; Anki)"}
+            )
+            with urllib.request.urlopen(req) as response:
+                filecontents = response.read()
+        else:
+            with HttpClient() as client:
+                client.timeout = 30
+                with client.get(url) as response:
+                    if response.status_code != 200:
+                        error_msg = tr.qt_misc_unexpected_response_code(
+                            val=response.status_code,
+                        )
+                        return "", error_msg
+                    filecontents = response.content
+                    content_type = response.headers.get("content-type")
+    except (urllib.error.URLError, requests.exceptions.RequestException) as e:
+        error_msg = tr.editing_an_error_occurred_while_opening(val=str(e))
+        return "", error_msg
+    # strip off any query string
+    url = re.sub(r"\?.*?$", "", url)
+    fname = os.path.basename(urllib.parse.unquote(url))
+    if not fname.strip():
+        fname = "paste"
+    if content_type:
+        fname = aqt.mw.col.media.add_extension_based_on_mime(fname, content_type)
+
+    return aqt.mw.col.media.write_data(fname, filecontents), ""
+
+
 class MessageBox(QMessageBox):
     def __init__(
         self,

ts/lib/html-filter/element.ts

@@ -34,6 +34,12 @@ const allow = (attrs: string[]): FilterMethod => (element: Element): void =>
         element,
     );
 
+function convertToDiv(element: Element): void {
+    const div = document.createElement("div");
+    div.innerHTML = element.innerHTML;
+    element.replaceWith(div);
+}
+
 function unwrapElement(element: Element): void {
     element.replaceWith(...element.childNodes);
 }
@@ -50,7 +56,7 @@ const tagsAllowedBasic: TagsAllowed = {
     BR: allowNone,
     IMG: allow(["SRC", "ALT"]),
     DIV: allowNone,
-    P: allowNone,
+    P: convertToDiv,
     SUB: allowNone,
     SUP: allowNone,
     TITLE: removeElement,

ts/lib/html-filter/index.ts

@@ -33,6 +33,11 @@ const outputHTMLProcessors: Record<FilterMode, (outputHTML: string) => string> =
 };
 
 export function filterHTML(html: string, internal: boolean, extended: boolean): string {
+    // https://anki.tenderapp.com/discussions/ankidesktop/39543-anki-is-replacing-the-character-by-when-i-exit-the-html-edit-mode-ctrlshiftx
+    if (html.indexOf(">") < 0) {
+        return html;
+    }
+
     const template = document.createElement("template");
     template.innerHTML = html;
 

ts/routes/editor/NoteEditor.svelte

@@ -861,6 +861,13 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
     }
 </script>
 
+<!-- Block Qt's default drag & drop behavior -->
+<svelte:body
+    on:dragenter|preventDefault
+    on:dragover|preventDefault
+    on:drop|preventDefault
+/>
+
 <!--
 @component
 Serves as a pre-slotted convenience component which combines all the common

ts/routes/editor/rich-text-input/data-transfer.ts

@@ -0,0 +1,346 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+import { ConfigKey_Bool } from "@generated/anki/config_pb";
+import { addMediaFile, convertPastedImage, getConfigBool, retrieveUrl as retrieveUrlBackend } from "@generated/backend";
+import { bridgeCommand } from "@tslib/bridgecommand";
+import { shiftPressed } from "@tslib/keys";
+import { pasteHTML } from "../old-editor-adapter";
+
+type ImageData = string | Uint8Array;
+const imageSuffixes = ["jpg", "jpeg", "png", "gif", "svg", "webp", "ico", "avif"];
+const audioSuffixes = [
+    "3gp",
+    "aac",
+    "avi",
+    "flac",
+    "flv",
+    "m4a",
+    "mkv",
+    "mov",
+    "mp3",
+    "mp4",
+    "mpeg",
+    "mpg",
+    "oga",
+    "ogg",
+    "ogv",
+    "ogx",
+    "opus",
+    "spx",
+    "swf",
+    "wav",
+    "webm",
+];
+const mediaSuffixes = [...imageSuffixes, ...audioSuffixes];
+
+function imageDataToUint8Array(data: ImageData): Uint8Array {
+    return typeof data === "string" ? new TextEncoder().encode(data) : data;
+}
+
+// TODO
+async function _wantsExtendedPaste(event: MouseEvent | KeyboardEvent): Promise<boolean> {
+    let stripHtml = (await getConfigBool({
+        key: ConfigKey_Bool.PASTE_STRIPS_FORMATTING,
+    })).val;
+    if (shiftPressed(event)) {
+        stripHtml = !stripHtml;
+    }
+    return !stripHtml;
+}
+
+function escapeHtml(text: string, quote = true): string {
+    text = text
+        .replaceAll("&", "&amp;")
+        .replaceAll("<", "&lt;")
+        .replaceAll(">", "&gt;");
+    if (quote) {
+        text = text.replaceAll("\"", "&quot;")
+            .replaceAll("'", "&#039;");
+    }
+    return text;
+}
+function getUrls(data: DataTransfer): string[] {
+    const urls = data.getData("text/uri-list").split("\n");
+    return urls[0] ? urls : [];
+}
+
+function getText(data: DataTransfer): string {
+    return data.getData("text/plain") ?? "";
+}
+
+const QIMAGE_FORMATS = [
+    "image/jpeg",
+    "image/png",
+    "image/gif",
+    "image/svg+xml",
+    "image/bmp",
+    "image/x-portable-bitmap",
+    "image/x-portable-graymap",
+    "image/x-portable-pixmap",
+    "image/x-xbitmap",
+    "image/x-xpixmap",
+];
+
+async function getImageData(data: DataTransfer): Promise<ImageData | null> {
+    for (const type of QIMAGE_FORMATS) {
+        const image = data.getData(type);
+        if (image) {
+            return image;
+        } else {
+            for (const file of data.files ?? []) {
+                if (file.type === type) {
+                    return new Uint8Array(await file.arrayBuffer());
+                }
+            }
+        }
+    }
+    return null;
+}
+
+async function retrieveUrl(url: string): Promise<string | null> {
+    const response = await retrieveUrlBackend({ val: url });
+    if (response.error) {
+        alert(response.error);
+        return null;
+    }
+
+    return response.filename;
+}
+
+async function urlToFile(url: string): Promise<string | null> {
+    const lowerUrl = url.toLowerCase();
+    for (const suffix of mediaSuffixes) {
+        if (lowerUrl.endsWith(`.${suffix}`)) {
+            return await retrieveUrl(url);
+        }
+    }
+    // Not a supported type
+    return null;
+}
+
+function filenameToLink(filename: string): string {
+    const filenameParts = filename.split(".");
+    const ext = filenameParts[filenameParts.length - 1].toLowerCase();
+    if (imageSuffixes.includes(ext)) {
+        return `<img src="${encodeURI(filename)}">`;
+    } else {
+        return `[sound:${escapeHtml(filename, false)}]`;
+    }
+}
+
+async function urlToLink(url: string): Promise<string> {
+    const filename = await urlToFile(url);
+    if (!filename) {
+        const escapedTitle = escapeHtml(decodeURI(url));
+        return `<a href="${url}">${escapedTitle}</a>`;
+    }
+    return filenameToLink(filename);
+}
+
+async function checksum(data: string | Uint8Array): Promise<string> {
+    const bytes = imageDataToUint8Array(data);
+    const hashBuffer = await crypto.subtle.digest("SHA-1", bytes);
+    const hashArray = Array.from(new Uint8Array(hashBuffer));
+    const hashHex = hashArray.map(b => b.toString(16).padStart(2, "0")).join("");
+
+    return hashHex;
+}
+
+async function addMediaFromData(filename: string, data: ImageData): Promise<string> {
+    filename = (await addMediaFile({
+        desiredName: filename,
+        data: imageDataToUint8Array(data),
+    })).val;
+    return filenameToLink(filename);
+}
+
+async function pastedImageFilename(data: ImageData, ext: string): Promise<string> {
+    const csum = await checksum(data);
+    return `paste-${csum}.${ext}`;
+}
+
+async function addPastedImage(data: ImageData, ext: string, convert = false): Promise<string> {
+    const filename = await pastedImageFilename(data, ext);
+    if (convert) {
+        data = (await convertPastedImage({ data: imageDataToUint8Array(data), ext })).data;
+    }
+    return await addMediaFromData(filename, data);
+}
+
+async function inlinedImageToFilename(src: string): Promise<string> {
+    const prefix = "data:image/";
+    const suffix = ";base64,";
+    for (let ext of ["jpg", "jpeg", "png", "gif"]) {
+        const fullPrefix = prefix + ext + suffix;
+        if (src.startsWith(fullPrefix)) {
+            const b64data = src.slice(fullPrefix.length).trim();
+            const data = atob(b64data);
+            if (ext === "jpeg") {
+                ext = "jpg";
+            }
+            return await addPastedImage(data, ext);
+        }
+    }
+    return "";
+}
+
+async function inlinedImageToLink(src: string): Promise<string> {
+    const filename = await inlinedImageToFilename(src);
+    if (filename) {
+        return filenameToLink(filename);
+    }
+
+    return "";
+}
+
+function isURL(s: string): boolean {
+    s = s.toLowerCase();
+    const prefixes = ["http://", "https://", "ftp://", "file://"];
+    return prefixes.some(prefix => s.startsWith(prefix));
+}
+
+async function processUrls(data: DataTransfer, _extended: boolean): Promise<string | null> {
+    const urls = getUrls(data);
+    if (urls.length === 0) {
+        return null;
+    }
+    let text = "";
+    for (let url of urls) {
+        // Chrome likes to give us the URL twice with a \n
+        const lines = url.split("\n");
+        url = lines[0];
+        text += await urlToLink(url);
+    }
+
+    return text;
+}
+
+async function processImages(data: DataTransfer, _extended: boolean): Promise<string | null> {
+    const image = await getImageData(data);
+    if (!image) {
+        return null;
+    }
+    let ext: string;
+    if (await getConfigBool({ key: ConfigKey_Bool.PASTE_IMAGES_AS_PNG })) {
+        ext = "png";
+    } else {
+        ext = "jpg";
+    }
+    return await addPastedImage(image, ext, true);
+}
+
+async function processText(data: DataTransfer, extended: boolean): Promise<string | null> {
+    function replaceSpaces(match: string, p1: string): string {
+        return `${p1.replaceAll(" ", "&nbsp;")} `;
+    }
+
+    const text = getText(data);
+    if (text.length === 0) {
+        return null;
+    }
+    const processed: string[] = [];
+    for (const line of text.split("\n")) {
+        for (let token of line.split(/(\S+)/g)) {
+            // Inlined data in base64?
+            if (extended && token.startsWith("data:image/")) {
+                processed.push(await inlinedImageToLink(token));
+            } else if (extended && isURL(token)) {
+                // If the user is pasting an image or sound link, convert it to local, otherwise paste as a hyperlink
+                processed.push(await urlToLink(token));
+            } else {
+                token = escapeHtml(token).replaceAll("\t", " ".repeat(4));
+
+                // If there's more than one consecutive space,
+                // use non-breaking spaces for the second one on
+                token = token.replace(/ ( +)/g, replaceSpaces);
+                processed.push(token);
+            }
+        }
+        processed.push("<br>");
+    }
+    processed.pop();
+    return processed.join("");
+}
+
+async function processClipboardData(data: DataTransfer, extended = false): Promise<string | null> {
+    const html = data.getData("text/html");
+    if (html) {
+        return html;
+    }
+    const urls = getUrls(data);
+    let handlers: ((data: DataTransfer, extended: boolean) => Promise<string | null>)[];
+    if (urls.length > 0 && urls[0].startsWith("file://")) {
+        handlers = [processUrls, processImages, processText];
+    } else {
+        handlers = [processImages, processUrls, processText];
+    }
+
+    for (const handler of handlers) {
+        const html = await handler(data, extended);
+        if (html) {
+            return html;
+        }
+    }
+
+    return null;
+}
+
+async function runPreFilter(html: string, internal = false): Promise<string> {
+    const template = document.createElement("template");
+    template.innerHTML = html;
+    const content = template.content;
+    for (const img of content.querySelectorAll("img")) {
+        let src = img.getAttribute("src");
+        if (!src) {
+            continue;
+        }
+        // In internal pastes, rewrite mediasrv references to relative
+        if (internal) {
+            const match = /http:\/\/127\.0\.0\.1:\d+\/(.*)$/.exec(src);
+            if (match) {
+                src = match[1];
+            }
+        } else {
+            if (isURL(src)) {
+                const filename = await retrieveUrl(src);
+                if (filename) {
+                    src = filename;
+                }
+            } else if (src.startsWith("data:image/")) {
+                src = await inlinedImageToFilename(src);
+            }
+        }
+        img.src = src;
+    }
+    return template.innerHTML;
+}
+
+export async function handlePaste(event: ClipboardEvent) {
+    // bridgeCommand("paste");
+    event.preventDefault();
+    const data = event.clipboardData!;
+    let html = await processClipboardData(data, true);
+    if (html) {
+        html = await runPreFilter(html);
+        pasteHTML(html, false, false);
+    }
+}
+
+export async function handleDrop(event: DragEvent) {
+    event.preventDefault();
+    const data = event.dataTransfer!;
+    let html = await processClipboardData(data, true);
+    if (html) {
+        html = await runPreFilter(html);
+        pasteHTML(html, false, false);
+    }
+}
+
+export async function handleDragover(event: DragEvent) {
+    event.preventDefault();
+}
+
+export function handleCutOrCopy() {
+    bridgeCommand("cutOrCopy");
+}

ts/routes/editor/rich-text-input/rich-text-resolve.ts

@@ -1,29 +1,23 @@
 // Copyright: Ankitects Pty Ltd and contributors
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
-import { bridgeCommand } from "@tslib/bridgecommand";
 import { on } from "@tslib/events";
 import { promiseWithResolver } from "@tslib/promise";
+import { handleCutOrCopy, handleDragover, handleDrop, handlePaste } from "./data-transfer";
 
 function bridgeCopyPasteCommands(input: HTMLElement): { destroy(): void } {
-    function onPaste(event: Event): void {
+    const removePaste = on(input, "paste", handlePaste);
-        event.preventDefault();
+    const removeCopy = on(input, "copy", handleCutOrCopy);
-        bridgeCommand("paste");
+    const removeCut = on(input, "cut", handleCutOrCopy);
-    }
+    const removeDragover = on(input, "dragover", handleDragover);
-
+    const removeDrop = on(input, "drop", handleDrop);
-    function onCutOrCopy(): void {
-        bridgeCommand("cutOrCopy");
-    }
-
-    const removePaste = on(input, "paste", onPaste);
-    const removeCopy = on(input, "copy", onCutOrCopy);
-    const removeCut = on(input, "cut", onCutOrCopy);
-
     return {
         destroy() {
             removePaste();
             removeCopy();
             removeCut();
+            removeDragover();
+            removeDrop();
         },
     };
 }