From ffd392de211d8c107b1927c6755d103b34bbae23 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 7 Sep 2023 12:37:15 +1000 Subject: [PATCH] Change Anki's version scheme; bump to 23.09 (#2640) * Accept iterables as inputs to backend methods * Shift add-on check to backend; use new endpoint The new endpoint will return info on a suitable branch if found, instead of returning all branches. This simplifies the frontend code, and means that you can now drop support for certain versions without it also remotely disabling the add-on for people who are running one of the excluded versions, like in https://forums.ankiweb.net/t/prevent-add-ons-from-being-disabled-remote-stealthily-surreptitiously/33427 * Bump version to 23.09 This changes Anki's version numbering system to year.month.patch, as previously mentioned on https://forums.ankiweb.net/t/use-a-different-versioning-system-semver-perhaps/20046/5 This is shaping up to be a big release, with the introduction of FSRS and image occlusion, and it seems like a good time to be finally updating the version scheme as well. AnkiWeb has been updated to understand the new format, and add-on authors will now specify version compatibility using the full version number, as can be seen here: https://ankiweb.net/shared/info/3918629684 * Shift update check to backend, and tidy up update.py * Use the shared client for sync connections too --- .version | 2 +- proto/anki/ankiweb.proto | 49 +++++++ pylib/anki/collection.py | 3 + pylib/anki/utils.py | 23 ++- qt/aqt/addons.py | 195 ++++++++------------------ qt/aqt/main.py | 28 +--- qt/aqt/profiles.py | 6 +- qt/aqt/update.py | 116 ++++++++------- qt/tests/test_addons.py | 29 +--- rslib/Cargo.toml | 1 + rslib/proto/python.rs | 18 ++- rslib/proto/src/lib.rs | 1 + rslib/src/backend/ankiweb.rs | 72 ++++++++++ rslib/src/backend/mod.rs | 10 ++ rslib/src/backend/sync.rs | 20 ++- rslib/src/media/mod.rs | 4 +- rslib/src/sync/collection/download.rs | 5 +- rslib/src/sync/collection/normal.rs | 9 +- rslib/src/sync/collection/progress.rs | 6 +- rslib/src/sync/collection/tests.rs | 3 +- rslib/src/sync/collection/upload.rs | 5 +- rslib/src/sync/http_client/mod.rs | 4 +- rslib/src/sync/login.rs | 4 +- 23 files changed, 336 insertions(+), 277 deletions(-) create mode 100644 proto/anki/ankiweb.proto create mode 100644 rslib/src/backend/ankiweb.rs diff --git a/.version b/.version index 55c60bfe4..bbf80af94 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.1.66 +23.09 diff --git a/proto/anki/ankiweb.proto b/proto/anki/ankiweb.proto new file mode 100644 index 000000000..d411a833f --- /dev/null +++ b/proto/anki/ankiweb.proto @@ -0,0 +1,49 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +syntax = "proto3"; + +option java_multiple_files = true; + +package anki.ankiweb; + +service AnkiwebService {} + +service BackendAnkiwebService { + // Fetch info on add-ons from AnkiWeb. A maximum of 25 can be queried at one + // time. If an add-on doesn't have a branch compatible with the provided + // version, that add-on will not be included in the returned list. + rpc GetAddonInfo(GetAddonInfoRequest) returns (GetAddonInfoResponse); + rpc CheckForUpdate(CheckForUpdateRequest) returns (CheckForUpdateResponse); +} + +message GetAddonInfoRequest { + uint32 client_version = 1; + repeated uint32 addon_ids = 2; +} + +message GetAddonInfoResponse { + repeated AddonInfo info = 1; +} + +message AddonInfo { + uint32 id = 1; + int64 modified = 2; + uint32 min_version = 3; + uint32 max_version = 4; +} + +message CheckForUpdateRequest { + uint32 version = 1; + string buildhash = 2; + string os = 3; + int64 install_id = 4; + uint32 last_message_id = 5; +} + +message CheckForUpdateResponse { + optional string new_version = 1; + int64 current_time = 2; + optional string message = 3; + uint32 last_message_id = 4; +} diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index adf3cca5a..9f2df734f 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -6,6 +6,7 @@ from __future__ import annotations from typing import Any, Generator, Literal, Sequence, Union, cast from anki import ( + ankiweb_pb2, card_rendering_pb2, collection_pb2, config_pb2, @@ -45,6 +46,8 @@ TtsVoice = card_rendering_pb2.AllTtsVoicesResponse.TtsVoice GetImageForOcclusionResponse = image_occlusion_pb2.GetImageForOcclusionResponse AddImageOcclusionNoteRequest = image_occlusion_pb2.AddImageOcclusionNoteRequest GetImageOcclusionNoteResponse = image_occlusion_pb2.GetImageOcclusionNoteResponse +AddonInfo = ankiweb_pb2.AddonInfo +CheckForUpdateResponse = ankiweb_pb2.CheckForUpdateResponse import copy import os diff --git a/pylib/anki/utils.py b/pylib/anki/utils.py index 5fbc29db9..817d7ade8 100644 --- a/pylib/anki/utils.py +++ b/pylib/anki/utils.py @@ -304,14 +304,29 @@ def version_with_build() -> str: return f"{version} ({buildhash})" -def point_version() -> int: +def int_version() -> int: + """Anki's version as an integer in the form YYMMPP, e.g. 230900. + (year, month, patch). + In 2.1.x releases, this was just the last number.""" from anki.buildinfo import version - return int(version.rsplit(".", maxsplit=1)[-1]) + try: + [year, month, patch] = version.split(".") + except ValueError: + [year, month] = version.split(".") + patch = "0" + + year_num = int(year) + month_num = int(month) + patch_num = int(patch) + + return year_num * 10_000 + month_num * 100 + patch_num -# keep the legacy alias around without a deprecation warning for now -pointVersion = point_version +# these two legacy aliases are provided without deprecation warnings, as add-ons that want to support +# old versions could not use the new name without catching cases where it doesn't exist +point_version = int_version +pointVersion = int_version _deprecated_names = DeprecatedNamesMixinForModule(globals()) _deprecated_names.register_deprecated_aliases( diff --git a/qt/aqt/addons.py b/qt/aqt/addons.py index 8520343a5..e279869db 100644 --- a/qt/aqt/addons.py +++ b/qt/aqt/addons.py @@ -14,7 +14,7 @@ from concurrent.futures import Future from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import IO, Any, Callable, Iterable, Union +from typing import IO, Any, Callable, Iterable, Sequence, Union from urllib.parse import parse_qs, urlparse from zipfile import ZipFile @@ -28,6 +28,7 @@ import anki.utils import aqt import aqt.forms import aqt.main +from anki.collection import AddonInfo from anki.httpclient import HttpClient from anki.lang import without_unicode_isolation from aqt import gui_hooks @@ -92,18 +93,9 @@ class DownloadError: DownloadLogEntry = tuple[int, Union[DownloadError, InstallError, InstallOk]] -@dataclass -class UpdateInfo: - id: int - suitable_branch_last_modified: int - current_branch_last_modified: int - current_branch_min_point_ver: int - current_branch_max_point_ver: int - - ANKIWEB_ID_RE = re.compile(r"^\d+$") -current_point_version = anki.utils.point_version() +_current_version = anki.utils.int_version() @dataclass @@ -113,8 +105,8 @@ class AddonMeta: enabled: bool installed_at: int conflicts: list[str] - min_point_version: int - max_point_version: int + min_version: int + max_version: int branch_index: int human_version: str | None update_enabled: bool @@ -131,11 +123,11 @@ class AddonMeta: return None def compatible(self) -> bool: - min = self.min_point_version - if min is not None and current_point_version < min: + min = self.min_version + if min is not None and _current_version < min: return False - max = self.max_point_version - if max is not None and max < 0 and current_point_version > abs(max): + max = self.max_version + if max is not None and max < 0 and _current_version > abs(max): return False return True @@ -155,8 +147,8 @@ class AddonMeta: enabled=not json_meta.get("disabled"), installed_at=json_meta.get("mod", 0), conflicts=json_meta.get("conflicts", []), - min_point_version=json_meta.get("min_point_version", 0) or 0, - max_point_version=json_meta.get("max_point_version", 0) or 0, + min_version=json_meta.get("min_point_version", 0) or 0, + max_version=json_meta.get("max_point_version", 0) or 0, branch_index=json_meta.get("branch_index", 0) or 0, human_version=json_meta.get("human_version"), update_enabled=json_meta.get("update_enabled", True), @@ -191,9 +183,10 @@ class AddonManager: "mod": {"type": "number", "meta": True}, # a list of other packages that conflict "conflicts": {"type": "array", "items": {"type": "string"}, "meta": True}, - # the minimum 2.1.x version this add-on supports + # x for anki 2.1.x; int_version() for more recent releases "min_point_version": {"type": "number", "meta": True}, - # if negative, abs(n) is the maximum 2.1.x version this add-on supports + # x for anki 2.1.x; int_version() for more recent releases + # if negative, abs(n) is the maximum version this add-on supports # if positive, indicates version tested on, and is ignored "max_point_version": {"type": "number", "meta": True}, # AnkiWeb sends this to indicate which branch the user downloaded. @@ -280,8 +273,8 @@ class AddonManager: json_obj["disabled"] = not addon.enabled json_obj["mod"] = addon.installed_at json_obj["conflicts"] = addon.conflicts - json_obj["max_point_version"] = addon.max_point_version - json_obj["min_point_version"] = addon.min_point_version + json_obj["max_point_version"] = addon.max_version + json_obj["min_point_version"] = addon.min_version json_obj["branch_index"] = addon.branch_index if addon.human_version is not None: json_obj["human_version"] = addon.human_version @@ -552,60 +545,39 @@ class AddonManager: # Updating ###################################################################### - def extract_update_info(self, items: list[dict]) -> list[UpdateInfo]: - def extract_one(item: dict) -> UpdateInfo: - id = item["id"] - meta = self.addon_meta(str(id)) - branch_idx = meta.branch_index - return extract_update_info(current_point_version, branch_idx, item) + def update_supported_versions(self, items: list[AddonInfo]) -> None: + """Adjust the supported version range after an update check. - return list(map(extract_one, items)) + AnkiWeb will not have sent us any add-ons that don't support our + version, so this cannot disable add-ons that users are using. It + does allow the add-on author to mark an add-on as not supporting + a future release, causing the add-on to be disabled when the user + upgrades. + """ - def update_supported_versions(self, items: list[UpdateInfo]) -> None: for item in items: - self.update_supported_version(item) + addon = self.addon_meta(str(item.id)) + updated = False - def update_supported_version(self, item: UpdateInfo) -> None: - addon = self.addon_meta(str(item.id)) - updated = False - is_latest = addon.is_latest(item.current_branch_last_modified) - - # if max different to the stored value - cur_max = item.current_branch_max_point_ver - if addon.max_point_version != cur_max: - if is_latest: - addon.max_point_version = cur_max + if addon.max_version != item.max_version: + addon.max_version = item.max_version updated = True - else: - # user is not up to date; only update if new version is stricter - if cur_max is not None and cur_max < addon.max_point_version: - addon.max_point_version = cur_max - updated = True - - # if min different to the stored value - cur_min = item.current_branch_min_point_ver - if addon.min_point_version != cur_min: - if is_latest: - addon.min_point_version = cur_min + if addon.min_version != item.min_version: + addon.min_version = item.min_version updated = True - else: - # user is not up to date; only update if new version is stricter - if cur_min is not None and cur_min > addon.min_point_version: - addon.min_point_version = cur_min - updated = True - if updated: - self.write_addon_meta(addon) + if updated: + self.write_addon_meta(addon) - def updates_required(self, items: list[UpdateInfo]) -> list[UpdateInfo]: + def get_updated_addons(self, items: list[AddonInfo]) -> list[AddonInfo]: """Return ids of add-ons requiring an update.""" need_update = [] for item in items: addon = self.addon_meta(str(item.id)) # update if server mtime is newer - if not addon.is_latest(item.suitable_branch_last_modified): + if not addon.is_latest(item.modified): need_update.append(item) - elif not addon.compatible() and item.suitable_branch_last_modified > 0: + elif not addon.compatible(): # Addon is currently disabled, and a suitable branch was found on the # server. Ignore our stored mtime (which may have been set incorrectly # in the past) and require an update. @@ -808,11 +780,11 @@ class AddonsDialog(QDialog): return name def compatible_string(self, addon: AddonMeta) -> str: - min = addon.min_point_version - if min is not None and min > current_point_version: + min = addon.min_version + if min is not None and min > _current_version: return f"Anki >= 2.1.{min}" else: - max = abs(addon.max_point_version) + max = abs(addon.max_version) return f"Anki <= 2.1.{max}" def should_grey(self, addon: AddonMeta) -> bool: @@ -1020,9 +992,7 @@ class GetAddons(QDialog): def download_addon(client: HttpClient, id: int) -> DownloadOk | DownloadError: "Fetch a single add-on from AnkiWeb." try: - resp = client.get( - f"{aqt.appShared}download/{id}?v=2.1&p={current_point_version}" - ) + resp = client.get(f"{aqt.appShared}download/{id}?v=2.1&p={_current_version}") if resp.status_code != 200: return DownloadError(status_code=resp.status_code) @@ -1230,13 +1200,11 @@ class ChooseAddonsToUpdateList(QListWidget): self, parent: QWidget, mgr: AddonManager, - updated_addons: list[UpdateInfo], + updated_addons: list[AddonInfo], ) -> None: QListWidget.__init__(self, parent) self.mgr = mgr - self.updated_addons = sorted( - updated_addons, key=lambda addon: addon.suitable_branch_last_modified - ) + self.updated_addons = sorted(updated_addons, key=lambda addon: addon.modified) self.ignore_check_evt = False self.setup() self.setContextMenuPolicy(Qt.ContextMenuPolicy.CustomContextMenu) @@ -1256,7 +1224,7 @@ class ChooseAddonsToUpdateList(QListWidget): addon_meta = self.mgr.addon_meta(str(addon_id)) update_enabled = addon_meta.update_enabled addon_name = addon_meta.human_name() - update_timestamp = update_info.suitable_branch_last_modified + update_timestamp = update_info.modified update_time = datetime.fromtimestamp(update_timestamp) addon_label = f"{update_time:%Y-%m-%d} {addon_name}" @@ -1344,7 +1312,7 @@ class ChooseAddonsToUpdateList(QListWidget): class ChooseAddonsToUpdateDialog(QDialog): def __init__( - self, parent: QWidget, mgr: AddonManager, updated_addons: list[UpdateInfo] + self, parent: QWidget, mgr: AddonManager, updated_addons: list[AddonInfo] ) -> None: QDialog.__init__(self, parent) self.setWindowTitle(tr.addons_choose_update_window_title()) @@ -1386,32 +1354,25 @@ class ChooseAddonsToUpdateDialog(QDialog): return [] -def fetch_update_info(client: HttpClient, ids: list[int]) -> list[dict]: +def fetch_update_info(ids: list[int]) -> list[AddonInfo]: """Fetch update info from AnkiWeb in one or more batches.""" - all_info: list[dict] = [] + all_info: list[AddonInfo] = [] while ids: # get another chunk chunk = ids[:25] del ids[:25] - batch_results = _fetch_update_info_batch(client, map(str, chunk)) + batch_results = _fetch_update_info_batch(chunk) all_info.extend(batch_results) return all_info -def _fetch_update_info_batch( - client: HttpClient, chunk: Iterable[str] -) -> Iterable[dict]: - """Get update info from AnkiWeb. - - Chunk must not contain more than 25 ids.""" - resp = client.get(f"{aqt.appShared}updates/{','.join(chunk)}?v=3") - if resp.status_code == 200: - return resp.json() - else: - raise Exception(f"Unexpected response code from AnkiWeb: {resp.status_code}") +def _fetch_update_info_batch(chunk: Iterable[int]) -> Sequence[AddonInfo]: + return aqt.mw.backend.get_addon_info( + client_version=_current_version, addon_ids=chunk + ) def check_and_prompt_for_updates( @@ -1420,19 +1381,17 @@ def check_and_prompt_for_updates( on_done: Callable[[list[DownloadLogEntry]], None], requested_by_user: bool = True, ) -> None: - def on_updates_received(client: HttpClient, items: list[dict]) -> None: - handle_update_info(parent, mgr, client, items, on_done, requested_by_user) + def on_updates_received(items: list[AddonInfo]) -> None: + handle_update_info(parent, mgr, items, on_done, requested_by_user) check_for_updates(mgr, on_updates_received) def check_for_updates( - mgr: AddonManager, on_done: Callable[[HttpClient, list[dict]], None] + mgr: AddonManager, on_done: Callable[[list[AddonInfo]], None] ) -> None: - client = HttpClient() - - def check() -> list[dict]: - return fetch_update_info(client, mgr.ankiweb_addons()) + def check() -> list[AddonInfo]: + return fetch_update_info(mgr.ankiweb_addons()) def update_info_received(future: Future) -> None: # if syncing/in profile screen, defer message delivery @@ -1451,66 +1410,36 @@ def check_for_updates( else: result = future.result() - on_done(client, result) + on_done(result) mgr.mw.taskman.run_in_background(check, update_info_received) -def extract_update_info( - current_point_version: int, current_branch_idx: int, info_json: dict -) -> UpdateInfo: - "Process branches to determine the updated mod time and min/max versions." - branches = info_json["branches"] - try: - current = branches[current_branch_idx] - except IndexError: - current = branches[0] - - last_mod = 0 - for branch in branches: - if branch["minpt"] > current_point_version: - continue - if branch["maxpt"] < 0 and abs(branch["maxpt"]) < current_point_version: - continue - last_mod = branch["fmod"] - - return UpdateInfo( - id=info_json["id"], - suitable_branch_last_modified=last_mod, - current_branch_last_modified=current["fmod"], - current_branch_min_point_ver=current["minpt"], - current_branch_max_point_ver=current["maxpt"], - ) - - def handle_update_info( parent: QWidget, mgr: AddonManager, - client: HttpClient, - items: list[dict], + items: list[AddonInfo], on_done: Callable[[list[DownloadLogEntry]], None], requested_by_user: bool = True, ) -> None: - update_info = mgr.extract_update_info(items) - mgr.update_supported_versions(update_info) - - updated_addons = mgr.updates_required(update_info) + mgr.update_supported_versions(items) + updated_addons = mgr.get_updated_addons(items) if not updated_addons: on_done([]) return - prompt_to_update(parent, mgr, client, updated_addons, on_done, requested_by_user) + prompt_to_update(parent, mgr, updated_addons, on_done, requested_by_user) def prompt_to_update( parent: QWidget, mgr: AddonManager, - client: HttpClient, - updated_addons: list[UpdateInfo], + updated_addons: list[AddonInfo], on_done: Callable[[list[DownloadLogEntry]], None], requested_by_user: bool = True, ) -> None: + client = HttpClient() if not requested_by_user: prompt_update = False for addon in updated_addons: diff --git a/qt/aqt/main.py b/qt/aqt/main.py index 7047080c6..a268c2589 100644 --- a/qt/aqt/main.py +++ b/qt/aqt/main.py @@ -37,10 +37,10 @@ from anki.utils import ( dev_mode, ids2str, int_time, + int_version, is_lin, is_mac, is_win, - point_version, split_fields, ) from aqt import gui_hooks @@ -954,7 +954,7 @@ title="{}" {}>{}""".format( if on_done: on_done() - if elap > 86_400 or self.pm.last_run_version != point_version(): + if elap > 86_400 or self.pm.last_run_version != int_version(): check_and_prompt_for_updates( self, self.addonManager, @@ -1401,29 +1401,9 @@ title="{}" {}>{}""".format( ########################################################################## def setupAutoUpdate(self) -> None: - import aqt.update + from aqt.update import check_for_update - self.autoUpdate = aqt.update.LatestVersionFinder(self) - qconnect(self.autoUpdate.newVerAvail, self.newVerAvail) - qconnect(self.autoUpdate.newMsg, self.newMsg) - qconnect(self.autoUpdate.clockIsOff, self.clockIsOff) - self.autoUpdate.start() - - def newVerAvail(self, ver: str) -> None: - if self.pm.meta.get("suppressUpdate", None) != ver: - aqt.update.askAndUpdate(self, ver) - - def newMsg(self, data: dict) -> None: - aqt.update.showMessages(self, data) - - def clockIsOff(self, diff: int) -> None: - if dev_mode: - print("clock is off; ignoring") - return - diffText = tr.qt_misc_second(count=diff) - warn = tr.qt_misc_in_order_to_ensure_your_collection(val="%s") % diffText - showWarning(warn) - self.app.closeAllWindows() + check_for_update() # Timers ########################################################################## diff --git a/qt/aqt/profiles.py b/qt/aqt/profiles.py index 7cfc38fce..d1b752ee6 100644 --- a/qt/aqt/profiles.py +++ b/qt/aqt/profiles.py @@ -20,7 +20,7 @@ from anki.collection import Collection from anki.db import DB from anki.lang import without_unicode_isolation from anki.sync import SyncAuth -from anki.utils import int_time, is_mac, is_win, point_version +from anki.utils import int_time, int_version, is_mac, is_win from aqt import appHelpSite, gui_hooks from aqt.qt import * from aqt.theme import Theme, WidgetStyle, theme_manager @@ -81,7 +81,7 @@ metaConf = dict( updates=True, created=int_time(), id=random.randrange(0, 2**63), - lastMsg=-1, + lastMsg=0, suppressUpdate=False, firstRun=True, defaultLang=None, @@ -134,7 +134,7 @@ class ProfileManager: res = self._loadMeta() self.firstRun = res.firstTime self.last_run_version = self.meta.get("last_run_version", self.last_run_version) - self.meta["last_run_version"] = point_version() + self.meta["last_run_version"] = int_version() return res # -p profile provided on command line. diff --git a/qt/aqt/update.py b/qt/aqt/update.py index dbec81122..212ddf93d 100644 --- a/qt/aqt/update.py +++ b/qt/aqt/update.py @@ -1,77 +1,75 @@ # Copyright: Ankitects Pty Ltd and contributors # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -import time -from typing import Any - -import requests - import aqt -from anki.utils import plat_desc, version_with_build -from aqt.main import AnkiQt +from anki.buildinfo import buildhash +from anki.collection import CheckForUpdateResponse, Collection +from anki.utils import dev_mode, int_time, int_version, plat_desc +from aqt.operations import QueryOp from aqt.qt import * -from aqt.utils import openLink, showText, tr +from aqt.utils import openLink, show_warning, showText, tr -class LatestVersionFinder(QThread): - newVerAvail = pyqtSignal(str) - newMsg = pyqtSignal(dict) - clockIsOff = pyqtSignal(float) +def check_for_update() -> None: + from aqt import mw - def __init__(self, main: AnkiQt) -> None: - QThread.__init__(self) - self.main = main - self.config = main.pm.meta + def do_check(_col: Collection) -> CheckForUpdateResponse: + return mw.backend.check_for_update( + version=int_version(), + buildhash=buildhash, + os=plat_desc(), + install_id=mw.pm.meta["id"], + last_message_id=max(0, mw.pm.meta["lastMsg"]), + ) - def _data(self) -> dict[str, Any]: - return { - "ver": version_with_build(), - "os": plat_desc(), - "id": self.config["id"], - "lm": self.config["lastMsg"], - "crt": self.config["created"], - } + def on_done(resp: CheckForUpdateResponse) -> None: + # is clock off? + if not dev_mode: + diff = abs(resp.current_time - int_time()) + if diff > 300: + diff_text = tr.qt_misc_second(count=diff) + warn = ( + tr.qt_misc_in_order_to_ensure_your_collection(val="%s") % diff_text + ) + show_warning(warn) + mw.app.closeAllWindows() + return + # should we show a message? + if msg := resp.message: + showText(msg, parent=mw, type="html") + mw.pm.meta["lastMsg"] = resp.last_message_id + # has Anki been updated? + if ver := resp.new_version: + prompt_to_update(mw, ver) - def run(self) -> None: - if not self.config["updates"]: - return - d = self._data() - d["proto"] = 1 + def on_fail(exc: Exception) -> None: + print(f"update check failed: {exc}") - try: - r = requests.post(aqt.appUpdate, data=d, timeout=60) - r.raise_for_status() - resp = r.json() - except: - # behind proxy, corrupt message, etc - print("update check failed") - return - if resp["msg"]: - self.newMsg.emit(resp) # type: ignore - if resp["ver"]: - self.newVerAvail.emit(resp["ver"]) # type: ignore - diff = resp["time"] - time.time() - if abs(diff) > 300: - self.clockIsOff.emit(diff) # type: ignore + QueryOp(parent=mw, op=do_check, success=on_done).failure( + on_fail + ).run_in_background() -def askAndUpdate(mw: aqt.AnkiQt, ver: str) -> None: - baseStr = tr.qt_misc_anki_updatedanki_has_been_released(val=ver) - msg = QMessageBox(mw) - msg.setStandardButtons(QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No) # type: ignore - msg.setIcon(QMessageBox.Icon.Information) - msg.setText(baseStr + tr.qt_misc_would_you_like_to_download_it()) +def prompt_to_update(mw: aqt.AnkiQt, ver: str) -> None: + msg = ( + tr.qt_misc_anki_updatedanki_has_been_released(val=ver) + + tr.qt_misc_would_you_like_to_download_it() + ) + + msgbox = QMessageBox(mw) + msgbox.setStandardButtons( + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No + ) + msgbox.setIcon(QMessageBox.Icon.Information) + msgbox.setText(msg) + button = QPushButton(tr.qt_misc_ignore_this_update()) - msg.addButton(button, QMessageBox.ButtonRole.RejectRole) - msg.setDefaultButton(QMessageBox.StandardButton.Yes) - ret = msg.exec() - if msg.clickedButton() == button: + msgbox.addButton(button, QMessageBox.ButtonRole.RejectRole) + msgbox.setDefaultButton(QMessageBox.StandardButton.Yes) + ret = msgbox.exec() + + if msgbox.clickedButton() == button: # ignore this update mw.pm.meta["suppressUpdate"] = ver elif ret == QMessageBox.StandardButton.Yes: openLink(aqt.appWebsiteDownloadSection) - - -def showMessages(mw: aqt.AnkiQt, data: dict) -> None: - showText(data["msg"], parent=mw, type="html") - mw.pm.meta["lastMsg"] = data["msgId"] diff --git a/qt/tests/test_addons.py b/qt/tests/test_addons.py index f06955878..b4dcdc6de 100644 --- a/qt/tests/test_addons.py +++ b/qt/tests/test_addons.py @@ -7,7 +7,7 @@ from zipfile import ZipFile from mock import MagicMock -from aqt.addons import AddonManager, extract_update_info, package_name_valid +from aqt.addons import AddonManager, package_name_valid def test_readMinimalManifest(): @@ -69,33 +69,6 @@ def assertReadManifest(contents, expectedManifest, nameInZip="manifest.json"): assert adm.readManifestFile(zfile) == expectedManifest -def test_update_info(): - json_info = dict( - id=999, - branches=[ - {"minpt": 0, "maxpt": -15, "fmod": 222}, - {"minpt": 20, "maxpt": -25, "fmod": 333}, - {"minpt": 30, "maxpt": 35, "fmod": 444}, - ], - ) - - r = extract_update_info(5, 0, json_info) - assert r.current_branch_max_point_ver == -15 - assert r.suitable_branch_last_modified == 222 - - r = extract_update_info(5, 1, json_info) - assert r.current_branch_max_point_ver == -25 - assert r.suitable_branch_last_modified == 222 - - r = extract_update_info(19, 1, json_info) - assert r.current_branch_max_point_ver == -25 - assert r.suitable_branch_last_modified == 0 - - r = extract_update_info(20, 1, json_info) - assert r.current_branch_max_point_ver == -25 - assert r.suitable_branch_last_modified == 333 - - def test_package_name_validation(): assert not package_name_valid("") assert not package_name_valid("/") diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index 8e90fd980..9995ebdfc 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -33,6 +33,7 @@ syn.workspace = true [dev-dependencies] async-stream.workspace = true +reqwest = { workspace = true, features = ["native-tls"] } wiremock.workspace = true [dependencies] diff --git a/rslib/proto/python.rs b/rslib/proto/python.rs index 67788568b..bf9d30944 100644 --- a/rslib/proto/python.rs +++ b/rslib/proto/python.rs @@ -124,7 +124,7 @@ fn build_method_arguments(input: &MessageDescriptor) -> String { args.push("*".to_string()); } for field in fields { - let arg = format!("{}: {}", field.name(), python_type(&field)); + let arg = format!("{}: {}", field.name(), python_type(&field, false)); args.push(arg); } args.join(", ") @@ -150,14 +150,17 @@ fn maybe_destructured_output(output: &MessageDescriptor) -> (String, String) { if output.fields().len() == 1 && !matches!(first_field.as_ref().unwrap().kind(), Kind::Enum(_)) { let field = first_field.unwrap(); - (format!("output.{}", field.name()), python_type(&field)) + ( + format!("output.{}", field.name()), + python_type(&field, true), + ) } else { ("output".into(), full_name_to_python(output.full_name())) } } /// e.g. uint32 -> int; repeated bool -> Sequence[bool] -fn python_type(field: &FieldDescriptor) -> String { +fn python_type(field: &FieldDescriptor, output: bool) -> String { let kind = match field.kind() { Kind::Int32 | Kind::Int64 @@ -177,11 +180,15 @@ fn python_type(field: &FieldDescriptor) -> String { Kind::Enum(en) => format!("{}.V", full_name_to_python(en.full_name())), }; if field.is_list() { - format!("Sequence[{}]", kind) + if output { + format!("Sequence[{}]", kind) + } else { + format!("Iterable[{}]", kind) + } } else if field.is_map() { let map_kind = field.kind(); let map_kind = map_kind.as_message().unwrap(); - let map_kv: Vec<_> = map_kind.fields().map(|f| python_type(&f)).collect(); + let map_kv: Vec<_> = map_kind.fields().map(|f| python_type(&f, output)).collect(); format!("Mapping[{}, {}]", map_kv[0], map_kv[1]) } else { kind @@ -220,6 +227,7 @@ col.decks.all_config() from typing import * import anki +import anki.ankiweb_pb2 import anki.backend_pb2 import anki.card_rendering_pb2 import anki.cards_pb2 diff --git a/rslib/proto/src/lib.rs b/rslib/proto/src/lib.rs index e7d8ad6b3..85cd528cf 100644 --- a/rslib/proto/src/lib.rs +++ b/rslib/proto/src/lib.rs @@ -15,6 +15,7 @@ macro_rules! protobuf { } protobuf!(ankidroid, "ankidroid"); +protobuf!(ankiweb, "ankiweb"); protobuf!(backend, "backend"); protobuf!(card_rendering, "card_rendering"); protobuf!(cards, "cards"); diff --git a/rslib/src/backend/ankiweb.rs b/rslib/src/backend/ankiweb.rs new file mode 100644 index 000000000..e6dcd4e5e --- /dev/null +++ b/rslib/src/backend/ankiweb.rs @@ -0,0 +1,72 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::time::Duration; + +use anki_proto::ankiweb::CheckForUpdateRequest; +use anki_proto::ankiweb::CheckForUpdateResponse; +use anki_proto::ankiweb::GetAddonInfoRequest; +use anki_proto::ankiweb::GetAddonInfoResponse; +use prost::Message; + +use super::Backend; +use crate::prelude::*; +use crate::services::BackendAnkiwebService; + +fn service_url(service: &str) -> String { + format!("https://ankiweb.net/svc/{service}") +} + +impl Backend { + fn post(&self, service: &str, input: I) -> Result + where + I: Message, + O: Message + Default, + { + self.runtime_handle().block_on(async move { + let out = self + .web_client() + .post(service_url(service)) + .body(input.encode_to_vec()) + .timeout(Duration::from_secs(60)) + .send() + .await? + .error_for_status()? + .bytes() + .await?; + let out: O = O::decode(&out[..])?; + Ok(out) + }) + } +} + +impl BackendAnkiwebService for Backend { + fn get_addon_info(&self, input: GetAddonInfoRequest) -> Result { + self.post("desktop/addon-info", input) + } + + fn check_for_update(&self, input: CheckForUpdateRequest) -> Result { + self.post("desktop/check-for-update", input) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn addon_info() -> Result<()> { + if std::env::var("ONLINE_TESTS").is_err() { + println!("test disabled; ONLINE_TESTS not set"); + return Ok(()); + } + let backend = Backend::new(I18n::template_only(), false); + let info = backend.get_addon_info(GetAddonInfoRequest { + client_version: 30, + addon_ids: vec![3918629684], + })?; + assert_eq!(info.info[0].min_version, 0); + assert_eq!(info.info[0].max_version, 49); + Ok(()) + } +} diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index f47672550..441d35f72 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -3,6 +3,7 @@ mod adding; mod ankidroid; +mod ankiweb; mod card_rendering; mod collection; mod config; @@ -20,6 +21,7 @@ use std::thread::JoinHandle; use once_cell::sync::OnceCell; use prost::Message; +use reqwest::Client; use tokio::runtime; use tokio::runtime::Runtime; @@ -40,6 +42,7 @@ pub struct Backend { runtime: OnceCell, state: Arc>, backup_task: Arc>>>>, + web_client: OnceCell, } #[derive(Default)] @@ -73,6 +76,7 @@ impl Backend { runtime: OnceCell::new(), state: Arc::new(Mutex::new(BackendState::default())), backup_task: Arc::new(Mutex::new(None)), + web_client: OnceCell::new(), } } @@ -118,6 +122,12 @@ impl Backend { .clone() } + fn web_client(&self) -> &Client { + // currently limited to http1, as nginx doesn't support http2 proxies + self.web_client + .get_or_init(|| Client::builder().http1_only().build().unwrap()) + } + fn db_command(&self, input: &[u8]) -> Result> { self.with_col(|col| db_command_bytes(col, input)) } diff --git a/rslib/src/backend/sync.rs b/rslib/src/backend/sync.rs index 3e781a855..234c66a84 100644 --- a/rslib/src/backend/sync.rs +++ b/rslib/src/backend/sync.rs @@ -198,7 +198,7 @@ impl Backend { (col.media()?, col.new_progress_handler()) }; let rt = self.runtime_handle(); - let sync_fut = mgr.sync_media(progress, auth); + let sync_fut = mgr.sync_media(progress, auth, self.web_client().clone()); let abortable_sync = Abortable::new(sync_fut, abort_reg); let result = rt.block_on(abortable_sync); @@ -238,7 +238,12 @@ impl Backend { let (_guard, abort_reg) = self.sync_abort_handle()?; let rt = self.runtime_handle(); - let sync_fut = sync_login(input.username, input.password, input.endpoint); + let sync_fut = sync_login( + input.username, + input.password, + input.endpoint, + self.web_client().clone(), + ); let abortable_sync = Abortable::new(sync_fut, abort_reg); let ret = match rt.block_on(abortable_sync) { Ok(sync_result) => sync_result, @@ -276,7 +281,7 @@ impl Backend { let rt = self.runtime_handle(); let time_at_check_begin = TimestampSecs::now(); let local = self.with_col(|col| col.sync_meta())?; - let mut client = HttpSyncClient::new(auth); + let mut client = HttpSyncClient::new(auth, self.web_client().clone()); let state = rt.block_on(online_sync_status_check(local, &mut client))?; { let mut guard = self.state.lock().unwrap(); @@ -301,9 +306,10 @@ impl Backend { let (_guard, abort_reg) = self.sync_abort_handle()?; let rt = self.runtime_handle(); + let client = self.web_client().clone(); let ret = self.with_col(|col| { - let sync_fut = col.normal_sync(auth.clone()); + let sync_fut = col.normal_sync(auth.clone(), client.clone()); let abortable_sync = Abortable::new(sync_fut, abort_reg); match rt.block_on(abortable_sync) { @@ -313,7 +319,7 @@ impl Backend { col.storage.rollback_trx()?; // and tell AnkiWeb to clean up let _handle = std::thread::spawn(move || { - let _ = rt.block_on(sync_abort(auth)); + let _ = rt.block_on(sync_abort(auth, client)); }); Err(AnkiError::Interrupted) @@ -353,11 +359,11 @@ impl Backend { let mut builder = col_inner.as_builder(); let result = if upload { - let sync_fut = col_inner.full_upload(auth); + let sync_fut = col_inner.full_upload(auth, self.web_client().clone()); let abortable_sync = Abortable::new(sync_fut, abort_reg); rt.block_on(abortable_sync) } else { - let sync_fut = col_inner.full_download(auth); + let sync_fut = col_inner.full_download(auth, self.web_client().clone()); let abortable_sync = Abortable::new(sync_fut, abort_reg); rt.block_on(abortable_sync) }; diff --git a/rslib/src/media/mod.rs b/rslib/src/media/mod.rs index dba9989ff..f216706ca 100644 --- a/rslib/src/media/mod.rs +++ b/rslib/src/media/mod.rs @@ -11,6 +11,7 @@ use std::path::Path; use std::path::PathBuf; use anki_io::create_dir_all; +use reqwest::Client; use crate::media::files::add_data_to_folder_uniquely; use crate::media::files::mtime_as_i64; @@ -145,8 +146,9 @@ impl MediaManager { self, progress: ThrottlingProgressHandler, auth: SyncAuth, + client: Client, ) -> Result<()> { - let client = HttpSyncClient::new(auth); + let client = HttpSyncClient::new(auth, client); let mut syncer = MediaSyncer::new(self, progress, client)?; syncer.sync().await } diff --git a/rslib/src/sync/collection/download.rs b/rslib/src/sync/collection/download.rs index 66c5400e4..72d9f33e6 100644 --- a/rslib/src/sync/collection/download.rs +++ b/rslib/src/sync/collection/download.rs @@ -5,6 +5,7 @@ use anki_io::atomic_rename; use anki_io::new_tempfile_in_parent_of; use anki_io::read_file; use anki_io::write_file; +use reqwest::Client; use crate::collection::CollectionBuilder; use crate::prelude::*; @@ -17,8 +18,8 @@ use crate::sync::login::SyncAuth; impl Collection { /// Download collection from AnkiWeb. Caller must re-open afterwards. - pub async fn full_download(self, auth: SyncAuth) -> Result<()> { - self.full_download_with_server(HttpSyncClient::new(auth)) + pub async fn full_download(self, auth: SyncAuth, client: Client) -> Result<()> { + self.full_download_with_server(HttpSyncClient::new(auth, client)) .await } diff --git a/rslib/src/sync/collection/normal.rs b/rslib/src/sync/collection/normal.rs index 43b1d9b0d..981f9a76d 100644 --- a/rslib/src/sync/collection/normal.rs +++ b/rslib/src/sync/collection/normal.rs @@ -1,6 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use reqwest::Client; use tracing::debug; use crate::collection::Collection; @@ -152,8 +153,12 @@ impl From for SyncOutput { } impl Collection { - pub async fn normal_sync(&mut self, auth: SyncAuth) -> error::Result { - NormalSyncer::new(self, HttpSyncClient::new(auth)) + pub async fn normal_sync( + &mut self, + auth: SyncAuth, + client: Client, + ) -> error::Result { + NormalSyncer::new(self, HttpSyncClient::new(auth, client)) .sync() .await } diff --git a/rslib/src/sync/collection/progress.rs b/rslib/src/sync/collection/progress.rs index 419d80fcf..619d06512 100644 --- a/rslib/src/sync/collection/progress.rs +++ b/rslib/src/sync/collection/progress.rs @@ -1,6 +1,8 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use reqwest::Client; + use crate::error; use crate::sync::collection::protocol::EmptyInput; use crate::sync::collection::protocol::SyncProtocol; @@ -21,8 +23,8 @@ pub struct FullSyncProgress { pub total_bytes: usize, } -pub async fn sync_abort(auth: SyncAuth) -> error::Result<()> { - HttpSyncClient::new(auth) +pub async fn sync_abort(auth: SyncAuth, client: Client) -> error::Result<()> { + HttpSyncClient::new(auth, client) .abort(EmptyInput::request()) .await? .json() diff --git a/rslib/src/sync/collection/tests.rs b/rslib/src/sync/collection/tests.rs index bf8ea38e4..998ed57e2 100644 --- a/rslib/src/sync/collection/tests.rs +++ b/rslib/src/sync/collection/tests.rs @@ -7,6 +7,7 @@ use std::future::Future; use axum::http::StatusCode; use once_cell::sync::Lazy; +use reqwest::Client; use reqwest::Url; use serde_json::json; use tempfile::tempdir; @@ -106,7 +107,7 @@ where endpoint: Some(endpoint), io_timeout_secs: None, }; - let client = HttpSyncClient::new(auth); + let client = HttpSyncClient::new(auth, Client::new()); op(client).await } diff --git a/rslib/src/sync/collection/upload.rs b/rslib/src/sync/collection/upload.rs index 15dcafa02..85800a329 100644 --- a/rslib/src/sync/collection/upload.rs +++ b/rslib/src/sync/collection/upload.rs @@ -12,6 +12,7 @@ use axum::response::Response; use flate2::write::GzEncoder; use flate2::Compression; use futures::StreamExt; +use reqwest::Client; use tokio_util::io::ReaderStream; use crate::collection::CollectionBuilder; @@ -32,8 +33,8 @@ pub const CORRUPT_MESSAGE: &str = impl Collection { /// Upload collection to AnkiWeb. Caller must re-open afterwards. - pub async fn full_upload(self, auth: SyncAuth) -> Result<()> { - self.full_upload_with_server(HttpSyncClient::new(auth)) + pub async fn full_upload(self, auth: SyncAuth, client: Client) -> Result<()> { + self.full_upload_with_server(HttpSyncClient::new(auth, client)) .await } diff --git a/rslib/src/sync/http_client/mod.rs b/rslib/src/sync/http_client/mod.rs index 0131fd158..7366cfbbe 100644 --- a/rslib/src/sync/http_client/mod.rs +++ b/rslib/src/sync/http_client/mod.rs @@ -34,12 +34,12 @@ pub struct HttpSyncClient { } impl HttpSyncClient { - pub fn new(auth: SyncAuth) -> HttpSyncClient { + pub fn new(auth: SyncAuth, client: Client) -> HttpSyncClient { let io_timeout = Duration::from_secs(auth.io_timeout_secs.unwrap_or(30) as u64); HttpSyncClient { sync_key: auth.hkey, session_key: simple_session_id(), - client: Client::builder().http1_only().build().unwrap(), + client, endpoint: auth .endpoint .unwrap_or_else(|| Url::try_from("https://sync.ankiweb.net/").unwrap()), diff --git a/rslib/src/sync/login.rs b/rslib/src/sync/login.rs index a8805bc1b..0e61312cc 100644 --- a/rslib/src/sync/login.rs +++ b/rslib/src/sync/login.rs @@ -1,6 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use reqwest::Client; use reqwest::Url; use serde::Deserialize; use serde::Serialize; @@ -34,13 +35,14 @@ pub async fn sync_login>( username: S, password: S, endpoint: Option, + client: Client, ) -> Result { let auth = anki_proto::sync::SyncAuth { endpoint, ..Default::default() } .try_into()?; - let client = HttpSyncClient::new(auth); + let client = HttpSyncClient::new(auth, client); let resp = client .host_key( HostKeyRequest {