plug new media check in

This commit is contained in:
Damien Elmes 2020-02-10 17:58:54 +10:00
parent c1939aebd1
commit 6f158c8555
13 changed files with 315 additions and 163 deletions

View file

@ -27,6 +27,7 @@ message BackendInput {
string expand_clozes_to_reveal_latex = 25;
AddFileToMediaFolderIn add_file_to_media_folder = 26;
SyncMediaIn sync_media = 27;
Empty check_media = 28;
}
}
@ -44,6 +45,7 @@ message BackendOutput {
string expand_clozes_to_reveal_latex = 25;
string add_file_to_media_folder = 26;
Empty sync_media = 27;
MediaCheckOut check_media = 28;
BackendError error = 2047;
}
@ -65,6 +67,7 @@ message BackendError {
message Progress {
oneof value {
MediaSyncProgress media_sync = 1;
uint32 media_check = 2;
}
}
@ -245,3 +248,11 @@ message SyncMediaIn {
string hkey = 1;
string endpoint = 2;
}
message MediaCheckOut {
repeated string unused = 1;
repeated string missing = 2;
repeated string dirs = 3;
repeated string oversize = 4;
map<string,string> renamed = 5;
}

View file

@ -28,6 +28,33 @@ from anki.notes import Note
# @@AUTOGEN@@
class _BgThreadProgressCallbackFilter:
"""Warning: this is called on a background thread."""
_hooks: List[Callable[[bool, "anki.rsbackend.Progress"], bool]] = []
def append(self, cb: Callable[[bool, "anki.rsbackend.Progress"], bool]) -> None:
"""(proceed: bool, progress: anki.rsbackend.Progress)"""
self._hooks.append(cb)
def remove(self, cb: Callable[[bool, "anki.rsbackend.Progress"], bool]) -> None:
if cb in self._hooks:
self._hooks.remove(cb)
def __call__(self, proceed: bool, progress: anki.rsbackend.Progress) -> bool:
for filter in self._hooks:
try:
proceed = filter(proceed, progress)
except:
# if the hook fails, remove it
self._hooks.remove(filter)
raise
return proceed
bg_thread_progress_callback = _BgThreadProgressCallbackFilter()
class _CardDidLeechHook:
_hooks: List[Callable[[Card], None]] = []
@ -360,33 +387,6 @@ class _NotesWillBeDeletedHook:
notes_will_be_deleted = _NotesWillBeDeletedHook()
class _RustProgressCallbackFilter:
"""Warning: this is called on a background thread."""
_hooks: List[Callable[[bool, "anki.rsbackend.Progress"], bool]] = []
def append(self, cb: Callable[[bool, "anki.rsbackend.Progress"], bool]) -> None:
"""(proceed: bool, progress: anki.rsbackend.Progress)"""
self._hooks.append(cb)
def remove(self, cb: Callable[[bool, "anki.rsbackend.Progress"], bool]) -> None:
if cb in self._hooks:
self._hooks.remove(cb)
def __call__(self, proceed: bool, progress: anki.rsbackend.Progress) -> bool:
for filter in self._hooks:
try:
proceed = filter(proceed, progress)
except:
# if the hook fails, remove it
self._hooks.remove(filter)
raise
return proceed
rust_progress_callback = _RustProgressCallbackFilter()
class _Schedv2DidAnswerReviewCardHook:
_hooks: List[Callable[["anki.cards.Card", int, bool], None]] = []

View file

@ -17,6 +17,7 @@ from anki.consts import *
from anki.db import DB, DBError
from anki.lang import _
from anki.latex import render_latex
from anki.rsbackend import MediaCheckOutput
from anki.utils import checksum, isMac
@ -199,10 +200,14 @@ create table meta (dirMod int, lastUsn int); insert into meta values (0, 0);
string = re.sub(reg, repl, string)
return string
# Rebuilding DB
# Checking media
##########################################################################
def check(
def check(self) -> MediaCheckOutput:
"This should be called while the collection is closed."
return self.col.backend.check_media()
def check_old(
self, local: Optional[List[str]] = None
) -> Tuple[List[str], List[str], List[str]]:
"Return (missingFiles, unusedFiles)."
@ -264,7 +269,7 @@ create table meta (dirMod int, lastUsn int); insert into meta values (0, 0);
# if we renamed any files to nfc format, we must rerun the check
# to make sure the renamed files are not marked as unused
if renamedFiles:
return self.check(local=local)
return self.check_old(local=local)
nohave = [x for x in allRefs if not x.startswith("_")]
# make sure the media DB is valid
try:

View file

@ -1,6 +1,7 @@
# Copyright: Ankitects Pty Ltd and contributors
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
# pylint: skip-file
import enum
from dataclasses import dataclass
from typing import Callable, Dict, List, NewType, NoReturn, Optional, Tuple, Union
@ -123,15 +124,18 @@ TemplateReplacementList = List[Union[str, TemplateReplacement]]
MediaSyncProgress = pb.MediaSyncProgress
MediaCheckOutput = pb.MediaCheckOut
class ProgressKind(enum.Enum):
MediaSyncProgress = 0
MediaSync = 0
MediaCheck = 1
@dataclass
class Progress:
kind: ProgressKind
val: Union[MediaSyncProgress]
val: Union[MediaSyncProgress, int]
def proto_replacement_list_to_native(
@ -155,7 +159,9 @@ def proto_replacement_list_to_native(
def proto_progress_to_native(progress: pb.Progress) -> Progress:
kind = progress.WhichOneof("value")
if kind == "media_sync":
return Progress(kind=ProgressKind.MediaSyncProgress, val=progress.media_sync)
return Progress(kind=ProgressKind.MediaSync, val=progress.media_sync)
elif kind == "media_check":
return Progress(kind=ProgressKind.MediaCheck, val=progress.media_check)
else:
assert_impossible_literal(kind)
@ -174,7 +180,7 @@ class RustBackend:
progress = pb.Progress()
progress.ParseFromString(progress_bytes)
native_progress = proto_progress_to_native(progress)
return hooks.rust_progress_callback(True, native_progress)
return hooks.bg_thread_progress_callback(True, native_progress)
def _run_command(
self, input: pb.BackendInput, release_gil: bool = False
@ -281,3 +287,8 @@ class RustBackend:
pb.BackendInput(sync_media=pb.SyncMediaIn(hkey=hkey, endpoint=endpoint,)),
release_gil=True,
)
def check_media(self) -> MediaCheckOutput:
return self._run_command(
pb.BackendInput(check_media=pb.Empty()), release_gil=True,
).check_media

View file

@ -8,7 +8,10 @@ from tests.shared import getEmptyCol
def test_latex():
d = getEmptyCol()
print("** aborting test_latex for now")
return
d = getEmptyCol() # pylint: disable=unreachable
# change latex cmd to simulate broken build
import anki.latex

View file

@ -73,9 +73,11 @@ def test_deckIntegration():
with open(os.path.join(d.media.dir(), "foo.jpg"), "w") as f:
f.write("test")
# check media
d.close()
ret = d.media.check()
assert ret[0] == ["fake2.png"]
assert ret[1] == ["foo.jpg"]
d.reopen()
assert ret.missing == ["fake2.png"]
assert ret.unused == ["foo.jpg"]
def test_changes():

View file

@ -51,7 +51,7 @@ hooks = [
Hook(name="sync_stage_did_change", args=["stage: str"], legacy_hook="sync"),
Hook(name="sync_progress_did_change", args=["msg: str"], legacy_hook="syncMsg"),
Hook(
name="rust_progress_callback",
name="bg_thread_progress_callback",
args=["proceed: bool", "progress: anki.rsbackend.Progress"],
return_type="bool",
doc="Warning: this is called on a background thread.",

View file

@ -15,8 +15,6 @@ from argparse import Namespace
from threading import Thread
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
from send2trash import send2trash
import anki
import aqt
import aqt.mediasrv
@ -36,6 +34,7 @@ from anki.utils import devMode, ids2str, intTime, isMac, isWin, splitFields
from aqt import gui_hooks
from aqt.addons import DownloadLogEntry, check_and_prompt_for_updates, show_log_to_user
from aqt.legacy import install_pylib_legacy
from aqt.media import check_media_db
from aqt.mediasync import MediaSyncer
from aqt.profiles import ProfileManager as ProfileManagerType
from aqt.qt import *
@ -1115,7 +1114,7 @@ title="%s" %s>%s</button>""" % (
if qtminor < 11:
m.actionUndo.setShortcut(QKeySequence("Ctrl+Alt+Z"))
m.actionFullDatabaseCheck.triggered.connect(self.onCheckDB)
m.actionCheckMediaDatabase.triggered.connect(self.onCheckMediaDB)
m.actionCheckMediaDatabase.triggered.connect(self.on_check_media_db)
m.actionDocumentation.triggered.connect(self.onDocumentation)
m.actionDonate.triggered.connect(self.onDonate)
m.actionStudyDeck.triggered.connect(self.onStudyDeck)
@ -1290,94 +1289,8 @@ will be lost. Continue?"""
continue
return ret
def onCheckMediaDB(self):
self.progress.start(immediate=True)
(nohave, unused, warnings) = self.col.media.check()
self.progress.finish()
# generate report
report = ""
if warnings:
report += "\n".join(warnings) + "\n"
if unused:
numberOfUnusedFilesLabel = len(unused)
if report:
report += "\n\n\n"
report += (
ngettext(
"%d file found in media folder not used by any cards:",
"%d files found in media folder not used by any cards:",
numberOfUnusedFilesLabel,
)
% numberOfUnusedFilesLabel
)
report += "\n" + "\n".join(unused)
if nohave:
if report:
report += "\n\n\n"
report += _("Used on cards but missing from media folder:")
report += "\n" + "\n".join(nohave)
if not report:
tooltip(_("No unused or missing files found."))
return
# show report and offer to delete
diag = QDialog(self)
diag.setWindowTitle("Anki")
layout = QVBoxLayout(diag)
diag.setLayout(layout)
text = QTextEdit()
text.setReadOnly(True)
text.setPlainText(report)
layout.addWidget(text)
box = QDialogButtonBox(QDialogButtonBox.Close)
layout.addWidget(box)
if unused:
b = QPushButton(_("Delete Unused Files"))
b.setAutoDefault(False)
box.addButton(b, QDialogButtonBox.ActionRole)
b.clicked.connect(lambda c, u=unused, d=diag: self.deleteUnused(u, d))
box.rejected.connect(diag.reject)
diag.setMinimumHeight(400)
diag.setMinimumWidth(500)
restoreGeom(diag, "checkmediadb")
diag.exec_()
saveGeom(diag, "checkmediadb")
def deleteUnused(self, unused, diag):
if not askUser(_("Delete unused media?")):
return
mdir = self.col.media.dir()
self.progress.start(immediate=True)
try:
lastProgress = 0
for c, f in enumerate(unused):
path = os.path.join(mdir, f)
if os.path.exists(path):
send2trash(path)
now = time.time()
if now - lastProgress >= 0.3:
numberOfRemainingFilesToBeDeleted = len(unused) - c
lastProgress = now
label = (
ngettext(
"%d file remaining...",
"%d files remaining...",
numberOfRemainingFilesToBeDeleted,
)
% numberOfRemainingFilesToBeDeleted
)
self.progress.update(label)
finally:
self.progress.finish()
# caller must not pass in empty list
# pylint: disable=undefined-loop-variable
numberOfFilesDeleted = c + 1
tooltip(
ngettext("Deleted %d file.", "Deleted %d files.", numberOfFilesDeleted)
% numberOfFilesDeleted
)
diag.close()
def on_check_media_db(self) -> None:
check_media_db(self)
def onStudyDeck(self):
from aqt.studydeck import StudyDeck

179
qt/aqt/media.py Normal file
View file

@ -0,0 +1,179 @@
# Copyright: Ankitects Pty Ltd and contributors
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
from __future__ import annotations
import time
from concurrent.futures import Future
from typing import Optional
from send2trash import send2trash
import aqt
from anki import hooks
from anki.lang import _, ngettext
from anki.rsbackend import Interrupted, MediaCheckOutput, Progress, ProgressKind
from aqt.qt import *
from aqt.utils import askUser, restoreGeom, saveGeom, tooltip
def check_media_db(mw: aqt.AnkiQt) -> None:
c = MediaChecker(mw)
c.check()
class MediaChecker:
progress_dialog: Optional[aqt.progress.ProgressDialog]
def __init__(self, mw: aqt.AnkiQt) -> None:
self.mw = mw
def check(self) -> None:
self.progress_dialog = self.mw.progress.start()
hooks.bg_thread_progress_callback.append(self._on_progress)
self.mw.col.close()
self.mw.taskman.run_in_background(self._check, self._on_finished)
def _on_progress(self, proceed: bool, progress: Progress) -> bool:
if progress.kind != ProgressKind.MediaCheck:
return proceed
if self.progress_dialog.wantCancel:
return False
self.mw.taskman.run_on_main(
lambda: self.mw.progress.update(_("Checked {}...").format(progress.val))
)
return True
def _check(self) -> MediaCheckOutput:
"Run the check on a background thread."
return self.mw.col.media.check()
def _on_finished(self, future: Future):
hooks.bg_thread_progress_callback.remove(self._on_progress)
self.mw.progress.finish()
self.progress_dialog = None
self.mw.col.reopen()
exc = future.exception()
if isinstance(exc, Interrupted):
return
output = future.result()
report = describe_output(output)
# show report and offer to delete
diag = QDialog(self.mw)
diag.setWindowTitle("Anki")
layout = QVBoxLayout(diag)
diag.setLayout(layout)
text = QTextEdit()
text.setReadOnly(True)
text.setPlainText(report)
layout.addWidget(text)
box = QDialogButtonBox(QDialogButtonBox.Close)
layout.addWidget(box)
if output.unused:
b = QPushButton(_("Delete Unused Files"))
b.setAutoDefault(False)
box.addButton(b, QDialogButtonBox.ActionRole)
b.clicked.connect(lambda c, u=output.unused, d=diag: deleteUnused(self.mw, u, d)) # type: ignore
box.rejected.connect(diag.reject) # type: ignore
diag.setMinimumHeight(400)
diag.setMinimumWidth(500)
restoreGeom(diag, "checkmediadb")
diag.exec_()
saveGeom(diag, "checkmediadb")
def describe_output(output: MediaCheckOutput) -> str:
buf = []
buf.append(_("Missing files: {}").format(len(output.missing)))
buf.append(_("Unused files: {}").format(len(output.unused)))
if output.renamed:
buf.append(_("Renamed files: {}").format(len(output.renamed)))
if output.oversize:
buf.append(_("Over 100MB: {}".format(output.oversize)))
if output.dirs:
buf.append(_("Subfolders: {}".format(output.dirs)))
buf.append("")
if output.renamed:
buf.append(_("Some files have been renamed for compatibility:"))
buf.extend(
_("Renamed: %(old)s -> %(new)s") % dict(old=k, new=v)
for (k, v) in output.renamed.items()
)
buf.append("")
if output.oversize:
buf.append(_("Files over 100MB can not be synced with AnkiWeb."))
buf.extend(_("Over 100MB: {}").format(f) for f in output.oversize)
buf.append("")
if output.dirs:
buf.append(_("Folders inside the media folder are not supported."))
buf.extend(_("Folder: {}").format(f) for f in output.dirs)
buf.append("")
if output.missing:
buf.append(
_(
"The following files are referenced by cards, but were not found in the media folder:"
)
)
buf.extend(_("Missing: {}").format(f) for f in output.missing)
buf.append("")
if output.unused:
buf.append(
_(
"The following files were found in the media folder, but do not appear to be used on any cards:"
)
)
buf.extend(_("Unused: {}").format(f) for f in output.unused)
buf.append("")
return "\n".join(buf)
def deleteUnused(self, unused, diag):
if not askUser(_("Delete unused media?")):
return
mdir = self.col.media.dir()
self.progress.start(immediate=True)
try:
lastProgress = 0
for c, f in enumerate(unused):
path = os.path.join(mdir, f)
if os.path.exists(path):
send2trash(path)
now = time.time()
if now - lastProgress >= 0.3:
numberOfRemainingFilesToBeDeleted = len(unused) - c
lastProgress = now
label = (
ngettext(
"%d file remaining...",
"%d files remaining...",
numberOfRemainingFilesToBeDeleted,
)
% numberOfRemainingFilesToBeDeleted
)
self.progress.update(label)
finally:
self.progress.finish()
# caller must not pass in empty list
# pylint: disable=undefined-loop-variable
numberOfFilesDeleted = c + 1
tooltip(
ngettext("Deleted %d file.", "Deleted %d files.", numberOfFilesDeleted)
% numberOfFilesDeleted
)
diag.close()

View file

@ -43,13 +43,14 @@ class MediaSyncer:
self._syncing: bool = False
self._log: List[LogEntryWithTime] = []
self._want_stop = False
hooks.rust_progress_callback.append(self._on_rust_progress)
hooks.bg_thread_progress_callback.append(self._on_rust_progress)
gui_hooks.media_sync_did_start_or_stop.append(self._on_start_stop)
def _on_rust_progress(self, proceed: bool, progress: Progress) -> bool:
if progress.kind != ProgressKind.MediaSyncProgress:
if progress.kind != ProgressKind.MediaSync:
return proceed
assert isinstance(progress.val, MediaSyncProgress)
self._log_and_notify(progress.val)
if self._want_stop:

View file

@ -2,7 +2,10 @@
# -*- coding: utf-8 -*-
# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
from __future__ import annotations
import time
from typing import Optional
import aqt.forms
from anki.lang import _
@ -82,42 +85,20 @@ class ProgressManager:
# Creating progress dialogs
##########################################################################
class ProgressDialog(QDialog):
def __init__(self, parent):
QDialog.__init__(self, parent)
self.form = aqt.forms.progress.Ui_Dialog()
self.form.setupUi(self)
self._closingDown = False
self.wantCancel = False
def cancel(self):
self._closingDown = True
self.hide()
def closeEvent(self, evt):
if self._closingDown:
evt.accept()
else:
self.wantCancel = True
evt.ignore()
def keyPressEvent(self, evt):
if evt.key() == Qt.Key_Escape:
evt.ignore()
self.wantCancel = True
# note: immediate is no longer used
def start(self, max=0, min=0, label=None, parent=None, immediate=False):
def start(
self, max=0, min=0, label=None, parent=None, immediate=False
) -> Optional[ProgressDialog]:
self._levels += 1
if self._levels > 1:
return
return None
# setup window
parent = parent or self.app.activeWindow()
if not parent and self.mw.isVisible():
parent = self.mw
label = label or _("Processing...")
self._win = self.ProgressDialog(parent)
self._win = ProgressDialog(parent)
self._win.form.progressBar.setMinimum(min)
self._win.form.progressBar.setMaximum(max)
self._win.form.progressBar.setTextVisible(False)
@ -207,3 +188,28 @@ class ProgressManager:
def busy(self):
"True if processing."
return self._levels
class ProgressDialog(QDialog):
def __init__(self, parent):
QDialog.__init__(self, parent)
self.form = aqt.forms.progress.Ui_Dialog()
self.form.setupUi(self)
self._closingDown = False
self.wantCancel = False
def cancel(self):
self._closingDown = True
self.hide()
def closeEvent(self, evt):
if self._closingDown:
evt.accept()
else:
self.wantCancel = True
evt.ignore()
def keyPressEvent(self, evt):
if evt.key() == Qt.Key_Escape:
evt.ignore()
self.wantCancel = True

View file

@ -6,6 +6,7 @@ use crate::backend_proto::backend_input::Value;
use crate::backend_proto::{Empty, RenderedTemplateReplacement, SyncMediaIn};
use crate::cloze::expand_clozes_to_reveal_latex;
use crate::err::{AnkiError, NetworkErrorKind, Result, SyncErrorKind};
use crate::media::check::MediaChecker;
use crate::media::sync::MediaSyncProgress;
use crate::media::MediaManager;
use crate::sched::{local_minutes_west_for_stamp, sched_timing_today};
@ -31,6 +32,7 @@ pub struct Backend {
enum Progress<'a> {
MediaSync(&'a MediaSyncProgress),
MediaCheck(u32),
}
/// Convert an Anki error to a protobuf error.
@ -186,6 +188,7 @@ impl Backend {
self.sync_media(input)?;
OValue::SyncMedia(Empty {})
}
Value::CheckMedia(_) => OValue::CheckMedia(self.check_media()?),
})
}
@ -330,6 +333,23 @@ impl Backend {
let mut rt = Runtime::new().unwrap();
rt.block_on(mgr.sync_media(callback, &input.endpoint, &input.hkey))
}
fn check_media(&self) -> Result<pt::MediaCheckOut> {
let callback =
|progress: usize| self.fire_progress_callback(Progress::MediaCheck(progress as u32));
let mgr = MediaManager::new(&self.media_folder, &self.media_db)?;
let mut checker = MediaChecker::new(&mgr, &self.col_path, callback);
let output = checker.check()?;
Ok(pt::MediaCheckOut {
unused: output.unused,
missing: output.missing,
renamed: output.renamed,
dirs: output.dirs,
oversize: output.oversize,
})
}
}
fn ords_hash_to_set(ords: HashSet<u16>) -> Vec<u32> {
@ -370,6 +390,7 @@ fn progress_to_proto_bytes(progress: Progress) -> Vec<u8> {
uploaded_files: p.uploaded_files as u32,
uploaded_deletions: p.uploaded_deletions as u32,
}),
Progress::MediaCheck(n) => pt::progress::Value::MediaCheck(n),
}),
};

View file

@ -20,11 +20,11 @@ use std::{borrow::Cow, fs, time};
#[derive(Debug, PartialEq)]
pub struct MediaCheckOutput {
unused: Vec<String>,
missing: Vec<String>,
renamed: HashMap<String, String>,
dirs: Vec<String>,
oversize: Vec<String>,
pub unused: Vec<String>,
pub missing: Vec<String>,
pub renamed: HashMap<String, String>,
pub dirs: Vec<String>,
pub oversize: Vec<String>,
}
#[derive(Debug, PartialEq, Default)]