Merge pull request #793 from nwwt/object-audio-tags-support

Audio & object tag support

pylib/anki/media.py

@@ -33,14 +33,18 @@ def media_paths_from_col_path(col_path: str) -> Tuple[str, str]:
 
 class MediaManager:
 
-    soundRegexps = [r"(?i)(\[sound:(?P<fname>[^]]+)\])"]
+    sound_regexps = [r"(?i)(\[sound:(?P<fname>[^]]+)\])"]
-    imgRegexps = [
+    html_media_regexps = [
         # src element quoted case
-        r"(?i)(<img[^>]* src=(?P<str>[\"'])(?P<fname>[^>]+?)(?P=str)[^>]*>)",
+        r"(?i)(<[img|audio][^>]* src=(?P<str>[\"'])(?P<fname>[^>]+?)(?P=str)[^>]*>)",
         # unquoted case
-        r"(?i)(<img[^>]* src=(?!['\"])(?P<fname>[^ >]+)[^>]*?>)",
+        r"(?i)(<[img|audio][^>]* src=(?!['\"])(?P<fname>[^ >]+)[^>]*?>)",
+        # src element quoted case
+        r"(?i)(<object[^>]* data=(?P<str>[\"'])(?P<fname>[^>]+?)(?P=str)[^>]*>)",
+        # unquoted case
+        r"(?i)(<object[^>]* data=(?!['\"])(?P<fname>[^ >]+)[^>]*?>)",
     ]
-    regexps = soundRegexps + imgRegexps
+    regexps = sound_regexps + html_media_regexps
 
     def __init__(self, col: anki.collection.Collection, server: bool) -> None:
         self.col = col.weakref()
@@ -159,7 +163,11 @@ class MediaManager:
         return txt
 
     def escapeImages(self, string: str, unescape: bool = False) -> str:
-        "Apply or remove percent encoding to image filenames."
+        "escape_media_filenames alias for compatibility with add-ons."
+        return self.escape_media_filenames(string, unescape)
+
+    def escape_media_filenames(self, string: str, unescape: bool = False) -> str:
+        "Apply or remove percent encoding to filenames in html tags (audio, image, object)."
         fn: Callable
         if unescape:
             fn = urllib.parse.unquote
@@ -173,7 +181,7 @@ class MediaManager:
                 return tag
             return tag.replace(fname, fn(fname))
 
-        for reg in self.imgRegexps:
+        for reg in self.html_media_regexps:
             string = re.sub(reg, repl, string)
         return string
 

pylib/tests/test_media.py

@@ -46,7 +46,7 @@ def test_strings():
     assert sp("aoeu") == "aoeu"
     assert sp("aoeu[sound:foo.mp3]aoeu") == "aoeuaoeu"
     assert sp("a<img src=yo>oeu") == "aoeu"
-    es = col.media.escapeImages
+    es = col.media.escape_media_filenames
     assert es("aoeu") == "aoeu"
     assert es("<img src='http://foo.com'>") == "<img src='http://foo.com'>"
     assert es('<img src="foo bar.jpg">') == '<img src="foo%20bar.jpg">'

qt/aqt/clayout.py

@@ -491,12 +491,14 @@ class CardLayout(QDialog):
             self.have_autoplayed = True
 
             if c.autoplay():
+                AnkiWebView.setPlaybackRequiresGesture(False)
                 if self.pform.preview_front.isChecked():
                     audio = c.question_av_tags()
                 else:
                     audio = c.answer_av_tags()
                 av_player.play_tags(audio)
             else:
+                AnkiWebView.setPlaybackRequiresGesture(True)
                 av_player.clear_queue_and_maybe_interrupt()
 
         self.updateCardNames()

qt/aqt/editor.py

@@ -450,7 +450,8 @@ class Editor:
             return
 
         data = [
-            (fld, self.mw.col.media.escapeImages(val)) for fld, val in self.note.items()
+            (fld, self.mw.col.media.escape_media_filenames(val))
+            for fld, val in self.note.items()
         ]
         self.widget.show()
         self.updateTags()
@@ -547,11 +548,13 @@ class Editor:
         if html.find(">") > -1:
             # filter html through beautifulsoup so we can strip out things like a
             # leading </div>
-            html_escaped = self.mw.col.media.escapeImages(html)
+            html_escaped = self.mw.col.media.escape_media_filenames(html)
             with warnings.catch_warnings():
                 warnings.simplefilter("ignore", UserWarning)
                 html_escaped = str(BeautifulSoup(html_escaped, "html.parser"))
-                html = self.mw.col.media.escapeImages(html_escaped, unescape=True)
+                html = self.mw.col.media.escape_media_filenames(
+                    html_escaped, unescape=True
+                )
         self.note.fields[field] = html
         if not self.addMode:
             self.note.flush()
@@ -1231,7 +1234,7 @@ def remove_null_bytes(txt, editor):
 
 def reverse_url_quoting(txt, editor):
     # reverse the url quoting we added to get images to display
-    return editor.mw.col.media.escapeImages(txt, unescape=True)
+    return editor.mw.col.media.escape_media_filenames(txt, unescape=True)
 
 
 gui_hooks.editor_will_use_font_for_field.append(fontMungeHack)

qt/aqt/main.py

@@ -485,7 +485,7 @@ close the profile or restart Anki."""
             return anki.sound.strip_av_refs(text)
 
     def prepare_card_text_for_display(self, text: str) -> str:
-        text = self.col.media.escapeImages(text)
+        text = self.col.media.escape_media_filenames(text)
         text = self._add_play_buttons(text)
         return text
 

qt/aqt/previewer.py

@@ -184,6 +184,7 @@ class Previewer(QDialog):
             bodyclass = theme_manager.body_classes_for_card_ord(c.ord)
 
             if c.autoplay():
+                AnkiWebView.setPlaybackRequiresGesture(False)
                 if self._show_both_sides:
                     # if we're showing both sides at once, remove any audio
                     # from the answer that's appeared on the question already
@@ -198,6 +199,7 @@ class Previewer(QDialog):
                     audio = c.answer_av_tags()
                 av_player.play_tags(audio)
             else:
+                AnkiWebView.setPlaybackRequiresGesture(True)
                 av_player.clear_queue_and_maybe_interrupt()
 
             txt = self.mw.prepare_card_text_for_display(txt)

qt/aqt/reviewer.py

@@ -23,6 +23,7 @@ from aqt.sound import av_player, getAudio, play_clicked_audio
 from aqt.theme import theme_manager
 from aqt.toolbar import BottomBar
 from aqt.utils import askUserDialog, downArrow, qtMenuShortcutWorkaround, tooltip
+from aqt.webview import AnkiWebView
 
 
 class ReviewerBottomBar:
@@ -184,10 +185,12 @@ class Reviewer:
         q = c.q()
         # play audio?
         if c.autoplay():
+            AnkiWebView.setPlaybackRequiresGesture(False)
             sounds = c.question_av_tags()
             gui_hooks.reviewer_will_play_question_sounds(c, sounds)
             av_player.play_tags(sounds)
         else:
+            AnkiWebView.setPlaybackRequiresGesture(True)
             av_player.clear_queue_and_maybe_interrupt()
             sounds = []
             gui_hooks.reviewer_will_play_question_sounds(c, sounds)

qt/aqt/utils.py

@@ -554,7 +554,7 @@ def restore_combo_history(comboBox: QComboBox, name: str):
 
 def mungeQA(col, txt):
     print("mungeQA() deprecated; use mw.prepare_card_text_for_display()")
-    txt = col.media.escapeImages(txt)
+    txt = col.media.escape_media_filenames(txt)
     return txt
 
 

qt/aqt/webview.py

@@ -340,6 +340,12 @@ class AnkiWebView(QWebEngineView):
         newFactor = desiredScale / qtIntScale
         return max(1, newFactor)
 
+    @staticmethod
+    def setPlaybackRequiresGesture(value: bool) -> None:
+        QWebEngineSettings.globalSettings().setAttribute(
+            QWebEngineSettings.PlaybackRequiresUserGesture, value
+        )
+
     def _getQtIntScale(self, screen) -> int:
         # try to detect if Qt has scaled the screen
         # - qt will round the scale factor to a whole number, so a dpi of 125% = 1x,

rslib/src/notes.rs

@@ -10,7 +10,7 @@ use crate::{
     err::{AnkiError, Result},
     notetype::{CardGenContext, NoteField, NoteType, NoteTypeID},
     template::field_is_empty,
-    text::{ensure_string_in_nfc, normalize_to_nfc, strip_html_preserving_image_filenames},
+    text::{ensure_string_in_nfc, normalize_to_nfc, strip_html_preserving_media_filenames},
     timestamp::TimestampSecs,
     types::Usn,
 };
@@ -100,12 +100,12 @@ impl Note {
             }
         }
 
-        let field1_nohtml = strip_html_preserving_image_filenames(&self.fields()[0]);
+        let field1_nohtml = strip_html_preserving_media_filenames(&self.fields()[0]);
         let checksum = field_checksum(field1_nohtml.as_ref());
         let sort_field = if nt.config.sort_field_idx == 0 {
             field1_nohtml
         } else {
-            strip_html_preserving_image_filenames(
+            strip_html_preserving_media_filenames(
                 self.fields
                     .get(nt.config.sort_field_idx as usize)
                     .map(AsRef::as_ref)
@@ -208,7 +208,7 @@ impl From<pb::Note> for Note {
     }
 }
 
-/// Text must be passed to strip_html_preserving_image_filenames() by
+/// Text must be passed to strip_html_preserving_media_filenames() by
 /// caller prior to passing in here.
 pub(crate) fn field_checksum(text: &str) -> u32 {
     let digest = sha1::Sha1::from(text).digest().bytes();
@@ -429,7 +429,7 @@ impl Collection {
             } else {
                 field1.into()
             };
-            let stripped = strip_html_preserving_image_filenames(&field1);
+            let stripped = strip_html_preserving_media_filenames(&field1);
             if stripped.trim().is_empty() {
                 Ok(DuplicateState::Empty)
             } else {
@@ -438,7 +438,7 @@ impl Collection {
                     self.storage
                         .note_fields_by_checksum(note.id, note.notetype_id, csum)?
                 {
-                    if strip_html_preserving_image_filenames(&field) == stripped {
+                    if strip_html_preserving_media_filenames(&field) == stripped {
                         return Ok(DuplicateState::Duplicate);
                     }
                 }

rslib/src/search/sqlwriter.rs

@@ -10,7 +10,7 @@ use crate::{
     notes::field_checksum,
     notetype::NoteTypeID,
     text::{matches_wildcard, text_to_re},
-    text::{normalize_to_nfc, strip_html_preserving_image_filenames, without_combining},
+    text::{normalize_to_nfc, strip_html_preserving_media_filenames, without_combining},
     timestamp::TimestampSecs,
 };
 use lazy_static::lazy_static;
@@ -424,7 +424,7 @@ impl SqlWriter<'_> {
     }
 
     fn write_dupes(&mut self, ntid: NoteTypeID, text: &str) {
-        let text_nohtml = strip_html_preserving_image_filenames(text);
+        let text_nohtml = strip_html_preserving_media_filenames(text);
         let csum = field_checksum(text_nohtml.as_ref());
         write!(
             self.sql,

rslib/src/text.rs

@@ -32,10 +32,10 @@ lazy_static! {
     ))
     .unwrap();
 
-    static ref IMG_TAG: Regex = Regex::new(
+    static ref HTML_MEDIA_TAGS: Regex = Regex::new(
         r#"(?xsi)
-            # the start of the image tag
+            # the start of the image, audio, or object tag
-            <img[^>]+src=
+            <\b(?:img|audio|object)\b[^>]+\b(?:src|data)\b=
             (?:
                     # 1: double-quoted filename
                     "
@@ -149,7 +149,7 @@ pub(crate) struct MediaRef<'a> {
 pub(crate) fn extract_media_refs(text: &str) -> Vec<MediaRef> {
     let mut out = vec![];
 
-    for caps in IMG_TAG.captures_iter(text) {
+    for caps in HTML_MEDIA_TAGS.captures_iter(text) {
         let fname = caps
             .get(1)
             .or_else(|| caps.get(2))
@@ -213,8 +213,8 @@ fn tts_tag_from_string<'a>(field_text: &'a str, args: &'a str) -> AVTag {
     }
 }
 
-pub fn strip_html_preserving_image_filenames(html: &str) -> Cow<str> {
+pub fn strip_html_preserving_media_filenames(html: &str) -> Cow<str> {
-    let without_fnames = IMG_TAG.replace_all(html, r" ${1}${2}${3} ");
+    let without_fnames = HTML_MEDIA_TAGS.replace_all(html, r" ${1}${2}${3} ");
     let without_html = HTML.replace_all(&without_fnames, "");
     // no changes?
     if let Cow::Borrowed(b) = without_html {
@@ -306,7 +306,7 @@ mod test {
     use super::matches_wildcard;
     use crate::text::without_combining;
     use crate::text::{
-        extract_av_tags, strip_av_tags, strip_html, strip_html_preserving_image_filenames, AVTag,
+        extract_av_tags, strip_av_tags, strip_html, strip_html_preserving_media_filenames, AVTag,
     };
     use std::borrow::Cow;
 
@@ -317,14 +317,14 @@ mod test {
         assert_eq!(strip_html("so<SCRIPT>t<b>e</b>st</script>me"), "some");
 
         assert_eq!(
-            strip_html_preserving_image_filenames("<img src=foo.jpg>"),
+            strip_html_preserving_media_filenames("<img src=foo.jpg>"),
             " foo.jpg "
         );
         assert_eq!(
-            strip_html_preserving_image_filenames("<img src='foo.jpg'><html>"),
+            strip_html_preserving_media_filenames("<img src='foo.jpg'><html>"),
             " foo.jpg "
         );
-        assert_eq!(strip_html_preserving_image_filenames("<html>"), "");
+        assert_eq!(strip_html_preserving_media_filenames("<html>"), "");
     }
 
     #[test]