,
}
-pub struct MediaChecker<'a, P>
+pub struct MediaChecker<'a, 'b, P>
where
P: FnMut(usize) -> bool,
{
+ ctx: &'a mut RequestContext<'b>,
mgr: &'a MediaManager,
- col_path: &'a Path,
progress_cb: P,
checked: usize,
progress_updated: Instant,
- i18n: &'a I18n,
- log: &'a Logger,
}
-impl MediaChecker<'_, P>
+impl
MediaChecker<'_, '_, P>
where
P: FnMut(usize) -> bool,
{
- pub fn new<'a>(
+ pub(crate) fn new<'a, 'b>(
+ ctx: &'a mut RequestContext<'b>,
mgr: &'a MediaManager,
- col_path: &'a Path,
progress_cb: P,
- i18n: &'a I18n,
- log: &'a Logger,
- ) -> MediaChecker<'a, P> {
+ ) -> MediaChecker<'a, 'b, P> {
MediaChecker {
+ ctx,
mgr,
- col_path,
progress_cb,
checked: 0,
progress_updated: Instant::now(),
- i18n,
- log,
}
}
@@ -100,7 +93,7 @@ where
pub fn summarize_output(&self, output: &mut MediaCheckOutput) -> String {
let mut buf = String::new();
- let i = &self.i18n;
+ let i = &self.ctx.i18n;
// top summary area
if output.trash_count > 0 {
@@ -279,7 +272,7 @@ where
}
})?;
let fname = self.mgr.add_file(ctx, disk_fname, &data)?;
- debug!(self.log, "renamed"; "from"=>disk_fname, "to"=>&fname.as_ref());
+ debug!(self.ctx.log, "renamed"; "from"=>disk_fname, "to"=>&fname.as_ref());
assert_ne!(fname.as_ref(), disk_fname);
// remove the original file
@@ -373,7 +366,7 @@ where
self.mgr
.add_file(&mut self.mgr.dbctx(), fname.as_ref(), &data)?;
} else {
- debug!(self.log, "file disappeared while restoring trash"; "fname"=>fname.as_ref());
+ debug!(self.ctx.log, "file disappeared while restoring trash"; "fname"=>fname.as_ref());
}
fs::remove_file(dentry.path())?;
}
@@ -387,14 +380,11 @@ where
&mut self,
renamed: &HashMap,
) -> Result> {
- let mut db = open_or_create_collection_db(self.col_path)?;
- let trx = db.transaction()?;
-
let mut referenced_files = HashSet::new();
- let note_types = get_note_types(&trx)?;
+ let note_types = self.ctx.storage.all_note_types()?;
let mut collection_modified = false;
- for_every_note(&trx, |note| {
+ for_every_note(&self.ctx.storage.db, |note| {
self.checked += 1;
if self.checked % 10 == 0 {
self.maybe_fire_progress_cb()?;
@@ -403,10 +393,16 @@ where
.get(¬e.mid)
.ok_or_else(|| AnkiError::DBError {
info: "missing note type".to_string(),
+ kind: DBErrorKind::MissingEntity,
})?;
- if fix_and_extract_media_refs(note, &mut referenced_files, renamed)? {
+ if fix_and_extract_media_refs(
+ note,
+ &mut referenced_files,
+ renamed,
+ &self.mgr.media_folder,
+ )? {
// note was modified, needs saving
- set_note(&trx, note, nt)?;
+ set_note(&self.ctx.storage.db, note, nt)?;
collection_modified = true;
}
@@ -415,9 +411,8 @@ where
Ok(())
})?;
- if collection_modified {
- mark_collection_modified(&trx)?;
- trx.commit()?;
+ if !collection_modified {
+ self.ctx.should_commit = false;
}
Ok(referenced_files)
@@ -429,11 +424,17 @@ fn fix_and_extract_media_refs(
note: &mut Note,
seen_files: &mut HashSet,
renamed: &HashMap,
+ media_folder: &Path,
) -> Result {
let mut updated = false;
for idx in 0..note.fields().len() {
- let field = normalize_and_maybe_rename_files(¬e.fields()[idx], renamed, seen_files);
+ let field = normalize_and_maybe_rename_files(
+ ¬e.fields()[idx],
+ renamed,
+ seen_files,
+ media_folder,
+ );
if let Cow::Owned(field) = field {
// field was modified, need to save
note.set_field(idx, field)?;
@@ -450,6 +451,7 @@ fn normalize_and_maybe_rename_files<'a>(
field: &'a str,
renamed: &HashMap,
seen_files: &mut HashSet,
+ media_folder: &Path,
) -> Cow<'a, str> {
let refs = extract_media_refs(field);
let mut field: Cow = field.into();
@@ -466,7 +468,21 @@ fn normalize_and_maybe_rename_files<'a>(
if let Some(new_name) = renamed.get(fname.as_ref()) {
fname = new_name.to_owned().into();
}
- // if it was not in NFC or was renamed, update the field
+ // if the filename was in NFC and was not renamed as part of the
+ // media check, it may have already been renamed during a previous
+ // sync. If that's the case and the renamed version exists on disk,
+ // we'll need to update the field to match it. It may be possible
+ // to remove this check in the future once we can be sure all media
+ // files stored on AnkiWeb are in normalized form.
+ if matches!(fname, Cow::Borrowed(_)) {
+ if let Cow::Owned(normname) = normalize_nfc_filename(fname.as_ref().into()) {
+ let path = media_folder.join(&normname);
+ if path.exists() {
+ fname = normname.into();
+ }
+ }
+ }
+ // update the field if the filename was modified
if let Cow::Owned(ref new_name) = fname {
field = rename_media_ref_in_field(field.as_ref(), &media_ref, new_name).into();
}
@@ -510,41 +526,42 @@ fn extract_latex_refs(note: &Note, seen_files: &mut HashSet, svg: bool)
}
#[cfg(test)]
-mod test {
+pub(crate) mod test {
+ pub(crate) const MEDIACHECK_ANKI2: &'static [u8] =
+ include_bytes!("../../tests/support/mediacheck.anki2");
+
+ use crate::collection::{open_collection, Collection};
use crate::err::Result;
use crate::i18n::I18n;
use crate::log;
- use crate::log::Logger;
use crate::media::check::{MediaCheckOutput, MediaChecker};
use crate::media::files::trash_folder;
use crate::media::MediaManager;
- use std::path::{Path, PathBuf};
+ use std::path::Path;
use std::{fs, io};
use tempfile::{tempdir, TempDir};
- fn common_setup() -> Result<(TempDir, MediaManager, PathBuf, Logger, I18n)> {
+ fn common_setup() -> Result<(TempDir, MediaManager, Collection)> {
let dir = tempdir()?;
let media_dir = dir.path().join("media");
fs::create_dir(&media_dir)?;
let media_db = dir.path().join("media.db");
let col_path = dir.path().join("col.anki2");
- fs::write(
- &col_path,
- &include_bytes!("../../tests/support/mediacheck.anki2")[..],
- )?;
+ fs::write(&col_path, MEDIACHECK_ANKI2)?;
- let mgr = MediaManager::new(&media_dir, media_db)?;
+ let mgr = MediaManager::new(&media_dir, media_db.clone())?;
let log = log::terminal();
-
let i18n = I18n::new(&["zz"], "dummy", log.clone());
- Ok((dir, mgr, col_path, log, i18n))
+ let col = open_collection(col_path, media_dir, media_db, false, i18n, log)?;
+
+ Ok((dir, mgr, col))
}
#[test]
fn media_check() -> Result<()> {
- let (_dir, mgr, col_path, log, i18n) = common_setup()?;
+ let (_dir, mgr, col) = common_setup()?;
// add some test files
fs::write(&mgr.media_folder.join("zerobytes"), "")?;
@@ -555,8 +572,13 @@ mod test {
fs::write(&mgr.media_folder.join("unused.jpg"), "foo")?;
let progress = |_n| true;
- let mut checker = MediaChecker::new(&mgr, &col_path, progress, &i18n, &log);
- let mut output = checker.check()?;
+
+ let (output, report) = col.transact(None, |ctx| {
+ let mut checker = MediaChecker::new(ctx, &mgr, progress);
+ let output = checker.check()?;
+ let summary = checker.summarize_output(&mut output.clone());
+ Ok((output, summary))
+ })?;
assert_eq!(
output,
@@ -576,7 +598,6 @@ mod test {
assert!(fs::metadata(&mgr.media_folder.join("foo[.jpg")).is_err());
assert!(fs::metadata(&mgr.media_folder.join("foo.jpg")).is_ok());
- let report = checker.summarize_output(&mut output);
assert_eq!(
report,
"Missing files: 1
@@ -616,14 +637,16 @@ Unused: unused.jpg
#[test]
fn trash_handling() -> Result<()> {
- let (_dir, mgr, col_path, log, i18n) = common_setup()?;
+ let (_dir, mgr, col) = common_setup()?;
let trash_folder = trash_folder(&mgr.media_folder)?;
fs::write(trash_folder.join("test.jpg"), "test")?;
let progress = |_n| true;
- let mut checker = MediaChecker::new(&mgr, &col_path, progress, &i18n, &log);
- checker.restore_trash()?;
+ col.transact(None, |ctx| {
+ let mut checker = MediaChecker::new(ctx, &mgr, progress);
+ checker.restore_trash()
+ })?;
// file should have been moved to media folder
assert_eq!(files_in_dir(&trash_folder), Vec::::new());
@@ -634,7 +657,10 @@ Unused: unused.jpg
// if we repeat the process, restoring should do the same thing if the contents are equal
fs::write(trash_folder.join("test.jpg"), "test")?;
- checker.restore_trash()?;
+ col.transact(None, |ctx| {
+ let mut checker = MediaChecker::new(ctx, &mgr, progress);
+ checker.restore_trash()
+ })?;
assert_eq!(files_in_dir(&trash_folder), Vec::::new());
assert_eq!(
files_in_dir(&mgr.media_folder),
@@ -643,7 +669,10 @@ Unused: unused.jpg
// but rename if required
fs::write(trash_folder.join("test.jpg"), "test2")?;
- checker.restore_trash()?;
+ col.transact(None, |ctx| {
+ let mut checker = MediaChecker::new(ctx, &mgr, progress);
+ checker.restore_trash()
+ })?;
assert_eq!(files_in_dir(&trash_folder), Vec::::new());
assert_eq!(
files_in_dir(&mgr.media_folder),
@@ -658,13 +687,17 @@ Unused: unused.jpg
#[test]
fn unicode_normalization() -> Result<()> {
- let (_dir, mgr, col_path, log, i18n) = common_setup()?;
+ let (_dir, mgr, col) = common_setup()?;
fs::write(&mgr.media_folder.join("ぱぱ.jpg"), "nfd encoding")?;
let progress = |_n| true;
- let mut checker = MediaChecker::new(&mgr, &col_path, progress, &i18n, &log);
- let mut output = checker.check()?;
+
+ let mut output = col.transact(None, |ctx| {
+ let mut checker = MediaChecker::new(ctx, &mgr, progress);
+ checker.check()
+ })?;
+
output.missing.sort();
if cfg!(target_vendor = "apple") {
diff --git a/rslib/src/media/files.rs b/rslib/src/media/files.rs
index bd8dfdcb4..791f0ad5e 100644
--- a/rslib/src/media/files.rs
+++ b/rslib/src/media/files.rs
@@ -84,7 +84,7 @@ pub(crate) fn normalize_filename(fname: &str) -> Cow {
}
/// See normalize_filename(). This function expects NFC-normalized input.
-fn normalize_nfc_filename(mut fname: Cow) -> Cow {
+pub(crate) fn normalize_nfc_filename(mut fname: Cow) -> Cow {
if fname.chars().any(disallowed_char) {
fname = fname.replace(disallowed_char, "").into()
}
diff --git a/rslib/src/media/mod.rs b/rslib/src/media/mod.rs
index c0dcfcca8..a3351b95c 100644
--- a/rslib/src/media/mod.rs
+++ b/rslib/src/media/mod.rs
@@ -12,7 +12,6 @@ use std::path::{Path, PathBuf};
pub mod changetracker;
pub mod check;
-pub mod col;
pub mod database;
pub mod files;
pub mod sync;
diff --git a/rslib/src/media/sync.rs b/rslib/src/media/sync.rs
index 5bf7dbcc6..7717d8b4e 100644
--- a/rslib/src/media/sync.rs
+++ b/rslib/src/media/sync.rs
@@ -717,6 +717,17 @@ fn zip_files<'a>(
break;
}
+ #[cfg(target_vendor = "apple")]
+ {
+ use unicode_normalization::is_nfc;
+ if !is_nfc(&file.fname) {
+ // older Anki versions stored non-normalized filenames in the DB; clean them up
+ debug!(log, "clean up non-nfc entry"; "fname"=>&file.fname);
+ invalid_entries.push(&file.fname);
+ continue;
+ }
+ }
+
let file_data = if file.sha1.is_some() {
match data_for_file(media_folder, &file.fname) {
Ok(data) => data,
diff --git a/rslib/src/media/col.rs b/rslib/src/notes.rs
similarity index 52%
rename from rslib/src/media/col.rs
rename to rslib/src/notes.rs
index a563bb93e..cad1f614c 100644
--- a/rslib/src/media/col.rs
+++ b/rslib/src/notes.rs
@@ -1,17 +1,17 @@
// Copyright: Ankitects Pty Ltd and contributors
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
-/// Basic note reading/updating functionality for the media DB check.
-use crate::err::{AnkiError, Result};
+/// At the moment, this is just basic note reading/updating functionality for
+/// the media DB check.
+use crate::err::{AnkiError, DBErrorKind, Result};
use crate::text::strip_html_preserving_image_filenames;
-use crate::time::{i64_unix_millis, i64_unix_secs};
-use crate::types::{ObjID, Timestamp, Usn};
+use crate::time::i64_unix_secs;
+use crate::{
+ notetypes::NoteType,
+ types::{ObjID, Timestamp, Usn},
+};
use rusqlite::{params, Connection, Row, NO_PARAMS};
-use serde_aux::field_attributes::deserialize_number_from_string;
-use serde_derive::Deserialize;
-use std::collections::HashMap;
use std::convert::TryInto;
-use std::path::Path;
#[derive(Debug)]
pub(super) struct Note {
@@ -40,55 +40,13 @@ impl Note {
}
}
-fn field_checksum(text: &str) -> u32 {
+/// Text must be passed to strip_html_preserving_image_filenames() by
+/// caller prior to passing in here.
+pub(crate) fn field_checksum(text: &str) -> u32 {
let digest = sha1::Sha1::from(text).digest().bytes();
u32::from_be_bytes(digest[..4].try_into().unwrap())
}
-pub(super) fn open_or_create_collection_db(path: &Path) -> Result {
- let db = Connection::open(path)?;
-
- db.pragma_update(None, "locking_mode", &"exclusive")?;
- db.pragma_update(None, "page_size", &4096)?;
- db.pragma_update(None, "cache_size", &(-40 * 1024))?;
- db.pragma_update(None, "legacy_file_format", &false)?;
- db.pragma_update(None, "journal", &"wal")?;
- db.set_prepared_statement_cache_capacity(5);
-
- Ok(db)
-}
-
-#[derive(Deserialize, Debug)]
-pub(super) struct NoteType {
- #[serde(deserialize_with = "deserialize_number_from_string")]
- id: ObjID,
- #[serde(rename = "sortf")]
- sort_field_idx: u16,
-
- #[serde(rename = "latexsvg", default)]
- latex_svg: bool,
-}
-
-impl NoteType {
- pub fn latex_uses_svg(&self) -> bool {
- self.latex_svg
- }
-}
-
-pub(super) fn get_note_types(db: &Connection) -> Result> {
- let mut stmt = db.prepare("select models from col")?;
- let note_types = stmt
- .query_and_then(NO_PARAMS, |row| -> Result> {
- let v: HashMap = serde_json::from_str(row.get_raw(0).as_str()?)?;
- Ok(v)
- })?
- .next()
- .ok_or_else(|| AnkiError::DBError {
- info: "col table empty".to_string(),
- })??;
- Ok(note_types)
-}
-
#[allow(dead_code)]
fn get_note(db: &Connection, nid: ObjID) -> Result> {
let mut stmt = db.prepare_cached("select id, mid, mod, usn, flds from notes where id=?")?;
@@ -130,14 +88,20 @@ pub(super) fn set_note(db: &Connection, note: &mut Note, note_type: &NoteType) -
note.mtime_secs = i64_unix_secs();
// hard-coded for now
note.usn = -1;
- let csum = field_checksum(¬e.fields()[0]);
- let sort_field = strip_html_preserving_image_filenames(
- note.fields()
- .get(note_type.sort_field_idx as usize)
- .ok_or_else(|| AnkiError::DBError {
- info: "sort field out of range".to_string(),
- })?,
- );
+ let field1_nohtml = strip_html_preserving_image_filenames(¬e.fields()[0]);
+ let csum = field_checksum(field1_nohtml.as_ref());
+ let sort_field = if note_type.sort_field_idx == 0 {
+ field1_nohtml
+ } else {
+ strip_html_preserving_image_filenames(
+ note.fields()
+ .get(note_type.sort_field_idx as usize)
+ .ok_or_else(|| AnkiError::DBError {
+ info: "sort field out of range".to_string(),
+ kind: DBErrorKind::MissingEntity,
+ })?,
+ )
+ };
let mut stmt =
db.prepare_cached("update notes set mod=?,usn=?,flds=?,sfld=?,csum=? where id=?")?;
@@ -152,8 +116,3 @@ pub(super) fn set_note(db: &Connection, note: &mut Note, note_type: &NoteType) -
Ok(())
}
-
-pub(super) fn mark_collection_modified(db: &Connection) -> Result<()> {
- db.execute("update col set mod=?", params![i64_unix_millis()])?;
- Ok(())
-}
diff --git a/rslib/src/notetypes.rs b/rslib/src/notetypes.rs
new file mode 100644
index 000000000..4b9295939
--- /dev/null
+++ b/rslib/src/notetypes.rs
@@ -0,0 +1,39 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use crate::types::ObjID;
+use serde_aux::field_attributes::deserialize_number_from_string;
+use serde_derive::Deserialize;
+
+#[derive(Deserialize, Debug)]
+pub(crate) struct NoteType {
+ #[serde(deserialize_with = "deserialize_number_from_string")]
+ pub id: ObjID,
+ pub name: String,
+ #[serde(rename = "sortf")]
+ pub sort_field_idx: u16,
+ #[serde(rename = "latexsvg", default)]
+ pub latex_svg: bool,
+ #[serde(rename = "tmpls")]
+ pub templates: Vec,
+ #[serde(rename = "flds")]
+ pub fields: Vec,
+}
+
+#[derive(Deserialize, Debug)]
+pub(crate) struct CardTemplate {
+ pub name: String,
+ pub ord: u16,
+}
+
+#[derive(Deserialize, Debug)]
+pub(crate) struct NoteField {
+ pub name: String,
+ pub ord: u16,
+}
+
+impl NoteType {
+ pub fn latex_uses_svg(&self) -> bool {
+ self.latex_svg
+ }
+}
diff --git a/rslib/src/sched/cutoff.rs b/rslib/src/sched/cutoff.rs
index 5c99b57a5..ba3ba484e 100644
--- a/rslib/src/sched/cutoff.rs
+++ b/rslib/src/sched/cutoff.rs
@@ -3,6 +3,7 @@
use chrono::{Date, Duration, FixedOffset, Local, TimeZone};
+#[derive(Debug, PartialEq, Clone, Copy)]
pub struct SchedTimingToday {
/// The number of days that have passed since the collection was created.
pub days_elapsed: u32,
@@ -17,7 +18,7 @@ pub struct SchedTimingToday {
/// - now_secs is a timestamp of the current time
/// - now_mins_west is the current offset west of UTC
/// - rollover_hour is the hour of the day the rollover happens (eg 4 for 4am)
-pub fn sched_timing_today(
+pub fn sched_timing_today_v2_new(
created_secs: i64,
created_mins_west: i32,
now_secs: i64,
@@ -90,11 +91,84 @@ pub fn local_minutes_west_for_stamp(stamp: i64) -> i32 {
Local.timestamp(stamp, 0).offset().utc_minus_local() / 60
}
+// Legacy code
+// ----------------------------------
+
+fn sched_timing_today_v1(crt: i64, now: i64) -> SchedTimingToday {
+ let days_elapsed = (now - crt) / 86_400;
+ let next_day_at = crt + (days_elapsed + 1) * 86_400;
+ SchedTimingToday {
+ days_elapsed: days_elapsed as u32,
+ next_day_at,
+ }
+}
+
+fn sched_timing_today_v2_legacy(
+ crt: i64,
+ rollover: i8,
+ now: i64,
+ mins_west: i32,
+) -> SchedTimingToday {
+ let normalized_rollover = normalized_rollover_hour(rollover);
+ let offset = fixed_offset_from_minutes(mins_west);
+
+ let crt_at_rollover = offset
+ .timestamp(crt, 0)
+ .date()
+ .and_hms(normalized_rollover as u32, 0, 0)
+ .timestamp();
+ let days_elapsed = (now - crt_at_rollover) / 86_400;
+
+ let mut next_day_at = offset
+ .timestamp(now, 0)
+ .date()
+ .and_hms(normalized_rollover as u32, 0, 0)
+ .timestamp();
+ if next_day_at < now {
+ next_day_at += 86_400;
+ }
+
+ SchedTimingToday {
+ days_elapsed: days_elapsed as u32,
+ next_day_at,
+ }
+}
+
+// ----------------------------------
+
+/// Based on provided input, get timing info from the relevant function.
+pub(crate) fn sched_timing_today(
+ created_secs: i64,
+ now_secs: i64,
+ created_mins_west: Option,
+ now_mins_west: Option,
+ rollover_hour: Option,
+) -> SchedTimingToday {
+ let now_west = now_mins_west.unwrap_or_else(|| local_minutes_west_for_stamp(now_secs));
+ match (rollover_hour, created_mins_west) {
+ (None, _) => {
+ // if rollover unset, v1 scheduler
+ sched_timing_today_v1(created_secs, now_secs)
+ }
+ (Some(roll), None) => {
+ // if creationOffset unset, v2 scheduler with legacy cutoff handling
+ sched_timing_today_v2_legacy(created_secs, roll, now_secs, now_west)
+ }
+ (Some(roll), Some(crt_west)) => {
+ // v2 scheduler, new cutoff handling
+ sched_timing_today_v2_new(created_secs, crt_west, now_secs, now_west, roll)
+ }
+ }
+}
+
#[cfg(test)]
mod test {
+ use super::SchedTimingToday;
+ use crate::sched::cutoff::sched_timing_today_v1;
+ use crate::sched::cutoff::sched_timing_today_v2_legacy;
use crate::sched::cutoff::{
fixed_offset_from_minutes, local_minutes_west_for_stamp, normalized_rollover_hour,
- sched_timing_today,
+ sched_timing_today_v2_new,
};
use chrono::{FixedOffset, Local, TimeZone, Utc};
@@ -117,7 +191,7 @@ mod test {
// helper
fn elap(start: i64, end: i64, start_west: i32, end_west: i32, rollhour: i8) -> u32 {
- let today = sched_timing_today(start, start_west, end, end_west, rollhour);
+ let today = sched_timing_today_v2_new(start, start_west, end, end_west, rollhour);
today.days_elapsed
}
@@ -228,7 +302,7 @@ mod test {
// before the rollover, the next day should be later on the same day
let now = Local.ymd(2019, 1, 3).and_hms(2, 0, 0);
let next_day_at = Local.ymd(2019, 1, 3).and_hms(rollhour, 0, 0);
- let today = sched_timing_today(
+ let today = sched_timing_today_v2_new(
crt.timestamp(),
crt.offset().utc_minus_local() / 60,
now.timestamp(),
@@ -240,7 +314,7 @@ mod test {
// after the rollover, the next day should be the next day
let now = Local.ymd(2019, 1, 3).and_hms(rollhour, 0, 0);
let next_day_at = Local.ymd(2019, 1, 4).and_hms(rollhour, 0, 0);
- let today = sched_timing_today(
+ let today = sched_timing_today_v2_new(
crt.timestamp(),
crt.offset().utc_minus_local() / 60,
now.timestamp(),
@@ -252,7 +326,7 @@ mod test {
// after the rollover, the next day should be the next day
let now = Local.ymd(2019, 1, 3).and_hms(rollhour + 3, 0, 0);
let next_day_at = Local.ymd(2019, 1, 4).and_hms(rollhour, 0, 0);
- let today = sched_timing_today(
+ let today = sched_timing_today_v2_new(
crt.timestamp(),
crt.offset().utc_minus_local() / 60,
now.timestamp(),
@@ -261,4 +335,34 @@ mod test {
);
assert_eq!(today.next_day_at, next_day_at.timestamp());
}
+
+ #[test]
+ fn legacy_timing() {
+ let now = 1584491078;
+ let mins_west = -600;
+
+ assert_eq!(
+ sched_timing_today_v1(1575226800, now),
+ SchedTimingToday {
+ days_elapsed: 107,
+ next_day_at: 1584558000
+ }
+ );
+
+ assert_eq!(
+ sched_timing_today_v2_legacy(1533564000, 0, now, mins_west),
+ SchedTimingToday {
+ days_elapsed: 589,
+ next_day_at: 1584540000
+ }
+ );
+
+ assert_eq!(
+ sched_timing_today_v2_legacy(1524038400, 4, now, mins_west),
+ SchedTimingToday {
+ days_elapsed: 700,
+ next_day_at: 1584554400
+ }
+ );
+ }
}
diff --git a/rslib/src/search/cards.rs b/rslib/src/search/cards.rs
new file mode 100644
index 000000000..8ab045b7b
--- /dev/null
+++ b/rslib/src/search/cards.rs
@@ -0,0 +1,154 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use super::{parser::Node, sqlwriter::node_to_sql};
+use crate::card::CardType;
+use crate::collection::RequestContext;
+use crate::config::SortKind;
+use crate::err::Result;
+use crate::search::parser::parse;
+use crate::types::ObjID;
+use rusqlite::params;
+
+pub(crate) enum SortMode {
+ NoOrder,
+ FromConfig,
+ Builtin { kind: SortKind, reverse: bool },
+ Custom(String),
+}
+
+pub(crate) fn search_cards<'a, 'b>(
+ req: &'a mut RequestContext<'b>,
+ search: &'a str,
+ order: SortMode,
+) -> Result> {
+ let top_node = Node::Group(parse(search)?);
+ let (sql, args) = node_to_sql(req, &top_node)?;
+
+ let mut sql = format!(
+ "select c.id from cards c, notes n where c.nid=n.id and {}",
+ sql
+ );
+
+ match order {
+ SortMode::NoOrder => (),
+ SortMode::FromConfig => {
+ let conf = req.storage.all_config()?;
+ prepare_sort(req, &conf.browser_sort_kind)?;
+ sql.push_str(" order by ");
+ write_order(&mut sql, &conf.browser_sort_kind, conf.browser_sort_reverse)?;
+ }
+ SortMode::Builtin { kind, reverse } => {
+ prepare_sort(req, &kind)?;
+ sql.push_str(" order by ");
+ write_order(&mut sql, &kind, reverse)?;
+ }
+ SortMode::Custom(order_clause) => {
+ sql.push_str(" order by ");
+ sql.push_str(&order_clause);
+ }
+ }
+
+ let mut stmt = req.storage.db.prepare(&sql)?;
+ let ids: Vec = stmt
+ .query_map(&args, |row| row.get(0))?
+ .collect::>()?;
+
+ Ok(ids)
+}
+
+/// Add the order clause to the sql.
+fn write_order(sql: &mut String, kind: &SortKind, reverse: bool) -> Result<()> {
+ let tmp_str;
+ let order = match kind {
+ SortKind::NoteCreation => "n.id asc, c.ord asc",
+ SortKind::NoteMod => "n.mod asc, c.ord asc",
+ SortKind::NoteField => "n.sfld collate nocase asc, c.ord asc",
+ SortKind::CardMod => "c.mod asc",
+ SortKind::CardReps => "c.reps asc",
+ SortKind::CardDue => "c.type asc, c.due asc",
+ SortKind::CardEase => {
+ tmp_str = format!("c.type = {} asc, c.factor asc", CardType::New as i8);
+ &tmp_str
+ }
+ SortKind::CardLapses => "c.lapses asc",
+ SortKind::CardInterval => "c.ivl asc",
+ SortKind::NoteTags => "n.tags asc",
+ SortKind::CardDeck => "(select v from sort_order where k = c.did) asc",
+ SortKind::NoteType => "(select v from sort_order where k = n.mid) asc",
+ SortKind::CardTemplate => "(select v from sort_order where k1 = n.mid and k2 = c.ord) asc",
+ };
+ if order.is_empty() {
+ return Ok(());
+ }
+ if reverse {
+ sql.push_str(
+ &order
+ .to_ascii_lowercase()
+ .replace(" desc", "")
+ .replace(" asc", " desc"),
+ )
+ } else {
+ sql.push_str(order);
+ }
+ Ok(())
+}
+
+// In the future these items should be moved from JSON into separate SQL tables,
+// - for now we use a temporary deck to sort them.
+fn prepare_sort(req: &mut RequestContext, kind: &SortKind) -> Result<()> {
+ use SortKind::*;
+ match kind {
+ CardDeck | NoteType => {
+ prepare_sort_order_table(req)?;
+ let mut stmt = req
+ .storage
+ .db
+ .prepare("insert into sort_order (k,v) values (?,?)")?;
+
+ match kind {
+ CardDeck => {
+ for (k, v) in req.storage.all_decks()? {
+ stmt.execute(params![k, v.name])?;
+ }
+ }
+ NoteType => {
+ for (k, v) in req.storage.all_note_types()? {
+ stmt.execute(params![k, v.name])?;
+ }
+ }
+ _ => unreachable!(),
+ }
+ }
+ CardTemplate => {
+ prepare_sort_order_table2(req)?;
+ let mut stmt = req
+ .storage
+ .db
+ .prepare("insert into sort_order (k1,k2,v) values (?,?,?)")?;
+
+ for (ntid, nt) in req.storage.all_note_types()? {
+ for tmpl in nt.templates {
+ stmt.execute(params![ntid, tmpl.ord, tmpl.name])?;
+ }
+ }
+ }
+ _ => (),
+ }
+
+ Ok(())
+}
+
+fn prepare_sort_order_table(req: &mut RequestContext) -> Result<()> {
+ req.storage
+ .db
+ .execute_batch(include_str!("sort_order.sql"))?;
+ Ok(())
+}
+
+fn prepare_sort_order_table2(req: &mut RequestContext) -> Result<()> {
+ req.storage
+ .db
+ .execute_batch(include_str!("sort_order2.sql"))?;
+ Ok(())
+}
diff --git a/rslib/src/search/mod.rs b/rslib/src/search/mod.rs
new file mode 100644
index 000000000..14cdb1181
--- /dev/null
+++ b/rslib/src/search/mod.rs
@@ -0,0 +1,7 @@
+mod cards;
+mod notes;
+mod parser;
+mod sqlwriter;
+
+pub(crate) use cards::{search_cards, SortMode};
+pub(crate) use notes::search_notes;
diff --git a/rslib/src/search/notes.rs b/rslib/src/search/notes.rs
new file mode 100644
index 000000000..50021a92e
--- /dev/null
+++ b/rslib/src/search/notes.rs
@@ -0,0 +1,28 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use super::{parser::Node, sqlwriter::node_to_sql};
+use crate::collection::RequestContext;
+use crate::err::Result;
+use crate::search::parser::parse;
+use crate::types::ObjID;
+
+pub(crate) fn search_notes<'a, 'b>(
+ req: &'a mut RequestContext<'b>,
+ search: &'a str,
+) -> Result> {
+ let top_node = Node::Group(parse(search)?);
+ let (sql, args) = node_to_sql(req, &top_node)?;
+
+ let sql = format!(
+ "select n.id from cards c, notes n where c.nid=n.id and {}",
+ sql
+ );
+
+ let mut stmt = req.storage.db.prepare(&sql)?;
+ let ids: Vec = stmt
+ .query_map(&args, |row| row.get(0))?
+ .collect::>()?;
+
+ Ok(ids)
+}
diff --git a/rslib/src/search/parser.rs b/rslib/src/search/parser.rs
new file mode 100644
index 000000000..4018cc56c
--- /dev/null
+++ b/rslib/src/search/parser.rs
@@ -0,0 +1,528 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use crate::err::{AnkiError, Result};
+use crate::types::ObjID;
+use nom::branch::alt;
+use nom::bytes::complete::{escaped, is_not, tag, take_while1};
+use nom::character::complete::{anychar, char, one_of};
+use nom::character::is_digit;
+use nom::combinator::{all_consuming, map, map_res};
+use nom::sequence::{delimited, preceded, tuple};
+use nom::{multi::many0, IResult};
+use std::{borrow::Cow, num};
+
+// fixme: need to preserve \ when used twice in string
+
+struct ParseError {}
+
+impl From for ParseError {
+ fn from(_: num::ParseIntError) -> Self {
+ ParseError {}
+ }
+}
+
+impl From for ParseError {
+ fn from(_: num::ParseFloatError) -> Self {
+ ParseError {}
+ }
+}
+
+impl From> for ParseError {
+ fn from(_: nom::Err<(I, nom::error::ErrorKind)>) -> Self {
+ ParseError {}
+ }
+}
+
+type ParseResult = std::result::Result;
+
+#[derive(Debug, PartialEq)]
+pub(super) enum Node<'a> {
+ And,
+ Or,
+ Not(Box>),
+ Group(Vec>),
+ Search(SearchNode<'a>),
+}
+
+#[derive(Debug, PartialEq)]
+pub(super) enum SearchNode<'a> {
+ // text without a colon
+ UnqualifiedText(Cow<'a, str>),
+ // foo:bar, where foo doesn't match a term below
+ SingleField {
+ field: Cow<'a, str>,
+ text: Cow<'a, str>,
+ is_re: bool,
+ },
+ AddedInDays(u32),
+ CardTemplate(TemplateKind),
+ Deck(Cow<'a, str>),
+ NoteTypeID(ObjID),
+ NoteType(Cow<'a, str>),
+ Rated {
+ days: u32,
+ ease: Option,
+ },
+ Tag(Cow<'a, str>),
+ Duplicates {
+ note_type_id: ObjID,
+ text: String,
+ },
+ State(StateKind),
+ Flag(u8),
+ NoteIDs(Cow<'a, str>),
+ CardIDs(Cow<'a, str>),
+ Property {
+ operator: String,
+ kind: PropertyKind,
+ },
+ WholeCollection,
+ Regex(Cow<'a, str>),
+ NoCombining(Cow<'a, str>),
+}
+
+#[derive(Debug, PartialEq)]
+pub(super) enum PropertyKind {
+ Due(i32),
+ Interval(u32),
+ Reps(u32),
+ Lapses(u32),
+ Ease(f32),
+}
+
+#[derive(Debug, PartialEq)]
+pub(super) enum StateKind {
+ New,
+ Review,
+ Learning,
+ Due,
+ Buried,
+ Suspended,
+}
+
+#[derive(Debug, PartialEq)]
+pub(super) enum TemplateKind {
+ Ordinal(u16),
+ Name(String),
+}
+
+/// Parse the input string into a list of nodes.
+#[allow(dead_code)]
+pub(super) fn parse(input: &str) -> Result> {
+ let input = input.trim();
+ if input.is_empty() {
+ return Ok(vec![Node::Search(SearchNode::WholeCollection)]);
+ }
+
+ let (_, nodes) = all_consuming(group_inner)(input)
+ .map_err(|_e| AnkiError::invalid_input("unable to parse search"))?;
+ Ok(nodes)
+}
+
+/// One or more nodes surrounded by brackets, eg (one OR two)
+fn group(s: &str) -> IResult<&str, Node> {
+ map(delimited(char('('), group_inner, char(')')), |nodes| {
+ Node::Group(nodes)
+ })(s)
+}
+
+/// One or more nodes inside brackets, er 'one OR two -three'
+fn group_inner(input: &str) -> IResult<&str, Vec> {
+ let mut remaining = input;
+ let mut nodes = vec![];
+
+ loop {
+ match node(remaining) {
+ Ok((rem, node)) => {
+ remaining = rem;
+
+ if nodes.len() % 2 == 0 {
+ // before adding the node, if the length is even then the node
+ // must not be a boolean
+ if matches!(node, Node::And | Node::Or) {
+ return Err(nom::Err::Failure(("", nom::error::ErrorKind::NoneOf)));
+ }
+ } else {
+ // if the length is odd, the next item must be a boolean. if it's
+ // not, add an implicit and
+ if !matches!(node, Node::And | Node::Or) {
+ nodes.push(Node::And);
+ }
+ }
+ nodes.push(node);
+ }
+ Err(e) => match e {
+ nom::Err::Error(_) => break,
+ _ => return Err(e),
+ },
+ };
+ }
+
+ if nodes.is_empty() {
+ Err(nom::Err::Error((remaining, nom::error::ErrorKind::Many1)))
+ } else {
+ // chomp any trailing whitespace
+ let (remaining, _) = whitespace0(remaining)?;
+
+ Ok((remaining, nodes))
+ }
+}
+
+fn whitespace0(s: &str) -> IResult<&str, Vec> {
+ many0(one_of(" \u{3000}"))(s)
+}
+
+/// Optional leading space, then a (negated) group or text
+fn node(s: &str) -> IResult<&str, Node> {
+ preceded(whitespace0, alt((negated_node, group, text)))(s)
+}
+
+fn negated_node(s: &str) -> IResult<&str, Node> {
+ map(preceded(char('-'), alt((group, text))), |node| {
+ Node::Not(Box::new(node))
+ })(s)
+}
+
+/// Either quoted or unquoted text
+fn text(s: &str) -> IResult<&str, Node> {
+ alt((quoted_term, partially_quoted_term, unquoted_term))(s)
+}
+
+/// Determine if text is a qualified search, and handle escaped chars.
+fn search_node_for_text(s: &str) -> ParseResult {
+ let mut it = s.splitn(2, ':');
+ let (head, tail) = (
+ unescape_quotes(it.next().unwrap()),
+ it.next().map(unescape_quotes),
+ );
+
+ if let Some(tail) = tail {
+ search_node_for_text_with_argument(head, tail)
+ } else {
+ Ok(SearchNode::UnqualifiedText(head))
+ }
+}
+
+/// \" -> "
+fn unescape_quotes(s: &str) -> Cow {
+ if s.find(r#"\""#).is_some() {
+ s.replace(r#"\""#, "\"").into()
+ } else {
+ s.into()
+ }
+}
+
+/// Unquoted text, terminated by a space or )
+fn unquoted_term(s: &str) -> IResult<&str, Node> {
+ map_res(
+ take_while1(|c| c != ' ' && c != ')' && c != '"'),
+ |text: &str| -> ParseResult {
+ Ok(if text.eq_ignore_ascii_case("or") {
+ Node::Or
+ } else if text.eq_ignore_ascii_case("and") {
+ Node::And
+ } else {
+ Node::Search(search_node_for_text(text)?)
+ })
+ },
+ )(s)
+}
+
+/// Quoted text, including the outer double quotes.
+fn quoted_term(s: &str) -> IResult<&str, Node> {
+ map_res(quoted_term_str, |o| -> ParseResult {
+ Ok(Node::Search(search_node_for_text(o)?))
+ })(s)
+}
+
+fn quoted_term_str(s: &str) -> IResult<&str, &str> {
+ delimited(char('"'), quoted_term_inner, char('"'))(s)
+}
+
+/// Quoted text, terminated by a non-escaped double quote
+fn quoted_term_inner(s: &str) -> IResult<&str, &str> {
+ escaped(is_not(r#""\"#), '\\', anychar)(s)
+}
+
+/// eg deck:"foo bar" - quotes must come after the :
+fn partially_quoted_term(s: &str) -> IResult<&str, Node> {
+ let term = take_while1(|c| c != ' ' && c != ')' && c != ':');
+ let (s, (term, _, quoted_val)) = tuple((term, char(':'), quoted_term_str))(s)?;
+
+ match search_node_for_text_with_argument(term.into(), quoted_val.into()) {
+ Ok(search) => Ok((s, Node::Search(search))),
+ Err(_) => Err(nom::Err::Failure((s, nom::error::ErrorKind::NoneOf))),
+ }
+}
+
+/// Convert a colon-separated key/val pair into the relevant search type.
+fn search_node_for_text_with_argument<'a>(
+ key: Cow<'a, str>,
+ val: Cow<'a, str>,
+) -> ParseResult> {
+ Ok(match key.to_ascii_lowercase().as_str() {
+ "added" => SearchNode::AddedInDays(val.parse()?),
+ "deck" => SearchNode::Deck(val),
+ "note" => SearchNode::NoteType(val),
+ "tag" => SearchNode::Tag(val),
+ "mid" => SearchNode::NoteTypeID(val.parse()?),
+ "nid" => SearchNode::NoteIDs(check_id_list(val)?),
+ "cid" => SearchNode::CardIDs(check_id_list(val)?),
+ "card" => parse_template(val.as_ref()),
+ "is" => parse_state(val.as_ref())?,
+ "flag" => parse_flag(val.as_ref())?,
+ "rated" => parse_rated(val.as_ref())?,
+ "dupes" => parse_dupes(val.as_ref())?,
+ "prop" => parse_prop(val.as_ref())?,
+ "re" => SearchNode::Regex(val),
+ "nc" => SearchNode::NoCombining(val),
+ // anything else is a field search
+ _ => parse_single_field(key.as_ref(), val.as_ref()),
+ })
+}
+
+/// ensure a list of ids contains only numbers and commas, returning unchanged if true
+/// used by nid: and cid:
+fn check_id_list(s: Cow) -> ParseResult> {
+ if s.is_empty() || s.as_bytes().iter().any(|&c| !is_digit(c) && c != b',') {
+ Err(ParseError {})
+ } else {
+ Ok(s)
+ }
+}
+
+/// eg is:due
+fn parse_state(s: &str) -> ParseResult> {
+ use StateKind::*;
+ Ok(SearchNode::State(match s {
+ "new" => New,
+ "review" => Review,
+ "learn" => Learning,
+ "due" => Due,
+ "buried" => Buried,
+ "suspended" => Suspended,
+ _ => return Err(ParseError {}),
+ }))
+}
+
+/// flag:0-4
+fn parse_flag(s: &str) -> ParseResult> {
+ let n: u8 = s.parse()?;
+ if n > 4 {
+ Err(ParseError {})
+ } else {
+ Ok(SearchNode::Flag(n))
+ }
+}
+
+/// eg rated:3 or rated:10:2
+/// second arg must be between 0-4
+fn parse_rated(val: &str) -> ParseResult> {
+ let mut it = val.splitn(2, ':');
+ let days = it.next().unwrap().parse()?;
+ let ease = match it.next() {
+ Some(v) => {
+ let n: u8 = v.parse()?;
+ if n < 5 {
+ Some(n)
+ } else {
+ return Err(ParseError {});
+ }
+ }
+ None => None,
+ };
+
+ Ok(SearchNode::Rated { days, ease })
+}
+
+/// eg dupes:1231,hello
+fn parse_dupes(val: &str) -> ParseResult> {
+ let mut it = val.splitn(2, ',');
+ let mid: ObjID = it.next().unwrap().parse()?;
+ let text = it.next().ok_or(ParseError {})?;
+ Ok(SearchNode::Duplicates {
+ note_type_id: mid,
+ text: text.into(),
+ })
+}
+
+/// eg prop:ivl>3, prop:ease!=2.5
+fn parse_prop(val: &str) -> ParseResult> {
+ let (val, key) = alt((
+ tag("ivl"),
+ tag("due"),
+ tag("reps"),
+ tag("lapses"),
+ tag("ease"),
+ ))(val)?;
+
+ let (val, operator) = alt((
+ tag("<="),
+ tag(">="),
+ tag("!="),
+ tag("="),
+ tag("<"),
+ tag(">"),
+ ))(val)?;
+
+ let kind = if key == "ease" {
+ let num: f32 = val.parse()?;
+ PropertyKind::Ease(num)
+ } else if key == "due" {
+ let num: i32 = val.parse()?;
+ PropertyKind::Due(num)
+ } else {
+ let num: u32 = val.parse()?;
+ match key {
+ "ivl" => PropertyKind::Interval(num),
+ "reps" => PropertyKind::Reps(num),
+ "lapses" => PropertyKind::Lapses(num),
+ _ => unreachable!(),
+ }
+ };
+
+ Ok(SearchNode::Property {
+ operator: operator.to_string(),
+ kind,
+ })
+}
+
+fn parse_template(val: &str) -> SearchNode<'static> {
+ SearchNode::CardTemplate(match val.parse::() {
+ Ok(n) => TemplateKind::Ordinal(n.max(1) - 1),
+ Err(_) => TemplateKind::Name(val.into()),
+ })
+}
+
+fn parse_single_field(key: &str, mut val: &str) -> SearchNode<'static> {
+ let is_re = if val.starts_with("re:") {
+ val = val.trim_start_matches("re:");
+ true
+ } else {
+ false
+ };
+ SearchNode::SingleField {
+ field: key.to_string().into(),
+ text: val.to_string().into(),
+ is_re,
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+
+ #[test]
+ fn parsing() -> Result<()> {
+ use Node::*;
+ use SearchNode::*;
+
+ assert_eq!(parse("")?, vec![Search(SearchNode::WholeCollection)]);
+ assert_eq!(parse(" ")?, vec![Search(SearchNode::WholeCollection)]);
+
+ // leading/trailing/interspersed whitespace
+ assert_eq!(
+ parse(" t t2 ")?,
+ vec![
+ Search(UnqualifiedText("t".into())),
+ And,
+ Search(UnqualifiedText("t2".into()))
+ ]
+ );
+
+ // including in groups
+ assert_eq!(
+ parse("( t t2 )")?,
+ vec![Group(vec![
+ Search(UnqualifiedText("t".into())),
+ And,
+ Search(UnqualifiedText("t2".into()))
+ ])]
+ );
+
+ assert_eq!(
+ parse(r#"hello -(world and "foo:bar baz") OR test"#)?,
+ vec![
+ Search(UnqualifiedText("hello".into())),
+ And,
+ Not(Box::new(Group(vec![
+ Search(UnqualifiedText("world".into())),
+ And,
+ Search(SingleField {
+ field: "foo".into(),
+ text: "bar baz".into(),
+ is_re: false,
+ })
+ ]))),
+ Or,
+ Search(UnqualifiedText("test".into()))
+ ]
+ );
+
+ assert_eq!(
+ parse("foo:re:bar")?,
+ vec![Search(SingleField {
+ field: "foo".into(),
+ text: "bar".into(),
+ is_re: true
+ })]
+ );
+
+ // any character should be escapable in quotes
+ assert_eq!(
+ parse(r#""re:\btest""#)?,
+ vec![Search(Regex(r"\btest".into()))]
+ );
+
+ assert_eq!(parse("added:3")?, vec![Search(AddedInDays(3))]);
+ assert_eq!(
+ parse("card:front")?,
+ vec![Search(CardTemplate(TemplateKind::Name("front".into())))]
+ );
+ assert_eq!(
+ parse("card:3")?,
+ vec![Search(CardTemplate(TemplateKind::Ordinal(2)))]
+ );
+ // 0 must not cause a crash due to underflow
+ assert_eq!(
+ parse("card:0")?,
+ vec![Search(CardTemplate(TemplateKind::Ordinal(0)))]
+ );
+ assert_eq!(parse("deck:default")?, vec![Search(Deck("default".into()))]);
+ assert_eq!(
+ parse("deck:\"default one\"")?,
+ vec![Search(Deck("default one".into()))]
+ );
+
+ assert_eq!(parse("note:basic")?, vec![Search(NoteType("basic".into()))]);
+ assert_eq!(parse("tag:hard")?, vec![Search(Tag("hard".into()))]);
+ assert_eq!(
+ parse("nid:1237123712,2,3")?,
+ vec![Search(NoteIDs("1237123712,2,3".into()))]
+ );
+ assert!(parse("nid:1237123712_2,3").is_err());
+ assert_eq!(parse("is:due")?, vec![Search(State(StateKind::Due))]);
+ assert_eq!(parse("flag:3")?, vec![Search(Flag(3))]);
+ assert!(parse("flag:-1").is_err());
+ assert!(parse("flag:5").is_err());
+
+ assert_eq!(
+ parse("prop:ivl>3")?,
+ vec![Search(Property {
+ operator: ">".into(),
+ kind: PropertyKind::Interval(3)
+ })]
+ );
+ assert!(parse("prop:ivl>3.3").is_err());
+ assert_eq!(
+ parse("prop:ease<=3.3")?,
+ vec![Search(Property {
+ operator: "<=".into(),
+ kind: PropertyKind::Ease(3.3)
+ })]
+ );
+
+ Ok(())
+ }
+}
diff --git a/rslib/src/search/sort_order.sql b/rslib/src/search/sort_order.sql
new file mode 100644
index 000000000..3c77a5a7d
--- /dev/null
+++ b/rslib/src/search/sort_order.sql
@@ -0,0 +1,2 @@
+drop table if exists sort_order;
+create temporary table sort_order (k int primary key, v text);
diff --git a/rslib/src/search/sort_order2.sql b/rslib/src/search/sort_order2.sql
new file mode 100644
index 000000000..62db35752
--- /dev/null
+++ b/rslib/src/search/sort_order2.sql
@@ -0,0 +1,2 @@
+drop table if exists sort_order;
+create temporary table sort_order (k1 int, k2 int, v text, primary key (k1, k2)) without rowid;
\ No newline at end of file
diff --git a/rslib/src/search/sqlwriter.rs b/rslib/src/search/sqlwriter.rs
new file mode 100644
index 000000000..1f5e7ef0c
--- /dev/null
+++ b/rslib/src/search/sqlwriter.rs
@@ -0,0 +1,583 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use super::parser::{Node, PropertyKind, SearchNode, StateKind, TemplateKind};
+use crate::card::CardQueue;
+use crate::decks::child_ids;
+use crate::decks::get_deck;
+use crate::err::{AnkiError, Result};
+use crate::notes::field_checksum;
+use crate::text::matches_wildcard;
+use crate::text::without_combining;
+use crate::{
+ collection::RequestContext, text::strip_html_preserving_image_filenames, types::ObjID,
+};
+use std::fmt::Write;
+
+struct SqlWriter<'a, 'b> {
+ req: &'a mut RequestContext<'b>,
+ sql: String,
+ args: Vec,
+}
+
+pub(super) fn node_to_sql(req: &mut RequestContext, node: &Node) -> Result<(String, Vec)> {
+ let mut sctx = SqlWriter::new(req);
+ sctx.write_node_to_sql(&node)?;
+ Ok((sctx.sql, sctx.args))
+}
+
+impl SqlWriter<'_, '_> {
+ fn new<'a, 'b>(req: &'a mut RequestContext<'b>) -> SqlWriter<'a, 'b> {
+ let sql = String::new();
+ let args = vec![];
+ SqlWriter { req, sql, args }
+ }
+
+ fn write_node_to_sql(&mut self, node: &Node) -> Result<()> {
+ match node {
+ Node::And => write!(self.sql, " and ").unwrap(),
+ Node::Or => write!(self.sql, " or ").unwrap(),
+ Node::Not(node) => {
+ write!(self.sql, "not ").unwrap();
+ self.write_node_to_sql(node)?;
+ }
+ Node::Group(nodes) => {
+ write!(self.sql, "(").unwrap();
+ for node in nodes {
+ self.write_node_to_sql(node)?;
+ }
+ write!(self.sql, ")").unwrap();
+ }
+ Node::Search(search) => self.write_search_node_to_sql(search)?,
+ };
+ Ok(())
+ }
+
+ fn write_search_node_to_sql(&mut self, node: &SearchNode) -> Result<()> {
+ match node {
+ SearchNode::UnqualifiedText(text) => self.write_unqualified(text),
+ SearchNode::SingleField { field, text, is_re } => {
+ self.write_single_field(field.as_ref(), text.as_ref(), *is_re)?
+ }
+ SearchNode::AddedInDays(days) => self.write_added(*days)?,
+ SearchNode::CardTemplate(template) => self.write_template(template)?,
+ SearchNode::Deck(deck) => self.write_deck(deck.as_ref())?,
+ SearchNode::NoteTypeID(ntid) => {
+ write!(self.sql, "n.mid = {}", ntid).unwrap();
+ }
+ SearchNode::NoteType(notetype) => self.write_note_type(notetype.as_ref())?,
+ SearchNode::Rated { days, ease } => self.write_rated(*days, *ease)?,
+ SearchNode::Tag(tag) => self.write_tag(tag),
+ SearchNode::Duplicates { note_type_id, text } => self.write_dupes(*note_type_id, text),
+ SearchNode::State(state) => self.write_state(state)?,
+ SearchNode::Flag(flag) => {
+ write!(self.sql, "(c.flags & 7) == {}", flag).unwrap();
+ }
+ SearchNode::NoteIDs(nids) => {
+ write!(self.sql, "n.id in ({})", nids).unwrap();
+ }
+ SearchNode::CardIDs(cids) => {
+ write!(self.sql, "c.id in ({})", cids).unwrap();
+ }
+ SearchNode::Property { operator, kind } => self.write_prop(operator, kind)?,
+ SearchNode::WholeCollection => write!(self.sql, "true").unwrap(),
+ SearchNode::Regex(re) => self.write_regex(re.as_ref()),
+ SearchNode::NoCombining(text) => self.write_no_combining(text.as_ref()),
+ };
+ Ok(())
+ }
+
+ fn write_unqualified(&mut self, text: &str) {
+ // implicitly wrap in %
+ let text = format!("%{}%", text);
+ self.args.push(text);
+ write!(
+ self.sql,
+ "(n.sfld like ?{n} escape '\\' or n.flds like ?{n} escape '\\')",
+ n = self.args.len(),
+ )
+ .unwrap();
+ }
+
+ fn write_no_combining(&mut self, text: &str) {
+ let text = format!("%{}%", without_combining(text));
+ self.args.push(text);
+ write!(
+ self.sql,
+ concat!(
+ "(coalesce(without_combining(cast(n.sfld as text)), n.sfld) like ?{n} escape '\\' ",
+ "or coalesce(without_combining(n.flds), n.flds) like ?{n} escape '\\')"
+ ),
+ n = self.args.len(),
+ )
+ .unwrap();
+ }
+
+ fn write_tag(&mut self, text: &str) {
+ match text {
+ "none" => {
+ write!(self.sql, "n.tags = ''").unwrap();
+ }
+ "*" | "%" => {
+ write!(self.sql, "true").unwrap();
+ }
+ text => {
+ let tag = format!("% {} %", text.replace('*', "%"));
+ write!(self.sql, "n.tags like ? escape '\\'").unwrap();
+ self.args.push(tag);
+ }
+ }
+ }
+
+ fn write_rated(&mut self, days: u32, ease: Option) -> Result<()> {
+ let today_cutoff = self.req.storage.timing_today()?.next_day_at;
+ let days = days.min(365) as i64;
+ let target_cutoff_ms = (today_cutoff - 86_400 * days) * 1_000;
+ write!(
+ self.sql,
+ "c.id in (select cid from revlog where id>{}",
+ target_cutoff_ms
+ )
+ .unwrap();
+ if let Some(ease) = ease {
+ write!(self.sql, " and ease={})", ease).unwrap();
+ } else {
+ write!(self.sql, ")").unwrap();
+ }
+
+ Ok(())
+ }
+
+ fn write_prop(&mut self, op: &str, kind: &PropertyKind) -> Result<()> {
+ let timing = self.req.storage.timing_today()?;
+ match kind {
+ PropertyKind::Due(days) => {
+ let day = days + (timing.days_elapsed as i32);
+ write!(
+ self.sql,
+ "(c.queue in ({rev},{daylrn}) and due {op} {day})",
+ rev = CardQueue::Review as u8,
+ daylrn = CardQueue::DayLearn as u8,
+ op = op,
+ day = day
+ )
+ }
+ PropertyKind::Interval(ivl) => write!(self.sql, "ivl {} {}", op, ivl),
+ PropertyKind::Reps(reps) => write!(self.sql, "reps {} {}", op, reps),
+ PropertyKind::Lapses(days) => write!(self.sql, "lapses {} {}", op, days),
+ PropertyKind::Ease(ease) => {
+ write!(self.sql, "factor {} {}", op, (ease * 1000.0) as u32)
+ }
+ }
+ .unwrap();
+ Ok(())
+ }
+
+ fn write_state(&mut self, state: &StateKind) -> Result<()> {
+ let timing = self.req.storage.timing_today()?;
+ match state {
+ StateKind::New => write!(self.sql, "c.type = {}", CardQueue::New as i8),
+ StateKind::Review => write!(self.sql, "c.type = {}", CardQueue::Review as i8),
+ StateKind::Learning => write!(
+ self.sql,
+ "c.queue in ({},{})",
+ CardQueue::Learn as i8,
+ CardQueue::DayLearn as i8
+ ),
+ StateKind::Buried => write!(
+ self.sql,
+ "c.queue in ({},{})",
+ CardQueue::SchedBuried as i8,
+ CardQueue::UserBuried as i8
+ ),
+ StateKind::Suspended => write!(self.sql, "c.queue = {}", CardQueue::Suspended as i8),
+ StateKind::Due => write!(
+ self.sql,
+ "
+ (c.queue in ({rev},{daylrn}) and c.due <= {today}) or
+ (c.queue = {lrn} and c.due <= {daycutoff})",
+ rev = CardQueue::Review as i8,
+ daylrn = CardQueue::DayLearn as i8,
+ today = timing.days_elapsed,
+ lrn = CardQueue::Learn as i8,
+ daycutoff = timing.next_day_at,
+ ),
+ }
+ .unwrap();
+ Ok(())
+ }
+
+ fn write_deck(&mut self, deck: &str) -> Result<()> {
+ match deck {
+ "*" => write!(self.sql, "true").unwrap(),
+ "filtered" => write!(self.sql, "c.odid > 0").unwrap(),
+ deck => {
+ let all_decks: Vec<_> = self
+ .req
+ .storage
+ .all_decks()?
+ .into_iter()
+ .map(|(_, v)| v)
+ .collect();
+ let dids_with_children = if deck == "current" {
+ let config = self.req.storage.all_config()?;
+ let mut dids_with_children = vec![config.current_deck_id];
+ let current = get_deck(&all_decks, config.current_deck_id)
+ .ok_or_else(|| AnkiError::invalid_input("invalid current deck"))?;
+ for child_did in child_ids(&all_decks, ¤t.name) {
+ dids_with_children.push(child_did);
+ }
+ dids_with_children
+ } else {
+ let mut dids_with_children = vec![];
+ for deck in all_decks.iter().filter(|d| matches_wildcard(&d.name, deck)) {
+ dids_with_children.push(deck.id);
+ for child_id in child_ids(&all_decks, &deck.name) {
+ dids_with_children.push(child_id);
+ }
+ }
+ dids_with_children
+ };
+
+ self.sql.push_str("c.did in ");
+ ids_to_string(&mut self.sql, &dids_with_children);
+ }
+ };
+ Ok(())
+ }
+
+ fn write_template(&mut self, template: &TemplateKind) -> Result<()> {
+ match template {
+ TemplateKind::Ordinal(n) => {
+ write!(self.sql, "c.ord = {}", n).unwrap();
+ }
+ TemplateKind::Name(name) => {
+ let note_types = self.req.storage.all_note_types()?;
+ let mut id_ords = vec![];
+ for nt in note_types.values() {
+ for tmpl in &nt.templates {
+ if matches_wildcard(&tmpl.name, name) {
+ id_ords.push((nt.id, tmpl.ord));
+ }
+ }
+ }
+
+ // sort for the benefit of unit tests
+ id_ords.sort();
+
+ if id_ords.is_empty() {
+ self.sql.push_str("false");
+ } else {
+ let v: Vec<_> = id_ords
+ .iter()
+ .map(|(ntid, ord)| format!("(n.mid = {} and c.ord = {})", ntid, ord))
+ .collect();
+ write!(self.sql, "({})", v.join(" or ")).unwrap();
+ }
+ }
+ };
+ Ok(())
+ }
+
+ fn write_note_type(&mut self, nt_name: &str) -> Result<()> {
+ let mut ntids: Vec<_> = self
+ .req
+ .storage
+ .all_note_types()?
+ .values()
+ .filter(|nt| matches_wildcard(&nt.name, nt_name))
+ .map(|nt| nt.id)
+ .collect();
+ self.sql.push_str("n.mid in ");
+ // sort for the benefit of unit tests
+ ntids.sort();
+ ids_to_string(&mut self.sql, &ntids);
+ Ok(())
+ }
+
+ fn write_single_field(&mut self, field_name: &str, val: &str, is_re: bool) -> Result<()> {
+ let note_types = self.req.storage.all_note_types()?;
+
+ let mut field_map = vec![];
+ for nt in note_types.values() {
+ for field in &nt.fields {
+ if matches_wildcard(&field.name, field_name) {
+ field_map.push((nt.id, field.ord));
+ }
+ }
+ }
+
+ // for now, sort the map for the benefit of unit tests
+ field_map.sort();
+
+ if field_map.is_empty() {
+ write!(self.sql, "false").unwrap();
+ return Ok(());
+ }
+
+ let cmp;
+ if is_re {
+ cmp = "regexp";
+ self.args.push(format!("(?i){}", val));
+ } else {
+ cmp = "like";
+ self.args.push(val.replace('*', "%"));
+ }
+
+ let arg_idx = self.args.len();
+ let searches: Vec<_> = field_map
+ .iter()
+ .map(|(ntid, ord)| {
+ format!(
+ "(n.mid = {mid} and field_at_index(n.flds, {ord}) {cmp} ?{n})",
+ mid = ntid,
+ ord = ord,
+ cmp = cmp,
+ n = arg_idx
+ )
+ })
+ .collect();
+ write!(self.sql, "({})", searches.join(" or ")).unwrap();
+
+ Ok(())
+ }
+
+ fn write_dupes(&mut self, ntid: ObjID, text: &str) {
+ let text_nohtml = strip_html_preserving_image_filenames(text);
+ let csum = field_checksum(text_nohtml.as_ref());
+ write!(
+ self.sql,
+ "(n.mid = {} and n.csum = {} and field_at_index(n.flds, 0) = ?",
+ ntid, csum
+ )
+ .unwrap();
+ self.args.push(text.to_string());
+ }
+
+ fn write_added(&mut self, days: u32) -> Result<()> {
+ let timing = self.req.storage.timing_today()?;
+ let cutoff = (timing.next_day_at - (86_400 * (days as i64))) * 1_000;
+ write!(self.sql, "c.id > {}", cutoff).unwrap();
+ Ok(())
+ }
+
+ fn write_regex(&mut self, word: &str) {
+ self.sql.push_str("n.flds regexp ?");
+ self.args.push(format!(r"(?i){}", word));
+ }
+}
+
+// Write a list of IDs as '(x,y,...)' into the provided string.
+fn ids_to_string(buf: &mut String, ids: &[T])
+where
+ T: std::fmt::Display,
+{
+ buf.push('(');
+ if !ids.is_empty() {
+ for id in ids.iter().skip(1) {
+ write!(buf, "{},", id).unwrap();
+ }
+ write!(buf, "{}", ids[0]).unwrap();
+ }
+ buf.push(')');
+}
+
+#[cfg(test)]
+mod test {
+ use super::ids_to_string;
+ use crate::{collection::open_collection, i18n::I18n, log};
+ use std::{fs, path::PathBuf};
+ use tempfile::tempdir;
+
+ #[test]
+ fn ids_string() {
+ let mut s = String::new();
+ ids_to_string::(&mut s, &[]);
+ assert_eq!(s, "()");
+ s.clear();
+ ids_to_string(&mut s, &[7]);
+ assert_eq!(s, "(7)");
+ s.clear();
+ ids_to_string(&mut s, &[7, 6]);
+ assert_eq!(s, "(6,7)");
+ s.clear();
+ ids_to_string(&mut s, &[7, 6, 5]);
+ assert_eq!(s, "(6,5,7)");
+ s.clear();
+ }
+
+ use super::super::parser::parse;
+ use super::*;
+
+ // shortcut
+ fn s(req: &mut RequestContext, search: &str) -> (String, Vec) {
+ let node = Node::Group(parse(search).unwrap());
+ node_to_sql(req, &node).unwrap()
+ }
+
+ #[test]
+ fn sql() -> Result<()> {
+ // re-use the mediacheck .anki2 file for now
+ use crate::media::check::test::MEDIACHECK_ANKI2;
+ let dir = tempdir().unwrap();
+ let col_path = dir.path().join("col.anki2");
+ fs::write(&col_path, MEDIACHECK_ANKI2).unwrap();
+
+ let i18n = I18n::new(&[""], "", log::terminal());
+ let col = open_collection(
+ &col_path,
+ &PathBuf::new(),
+ &PathBuf::new(),
+ false,
+ i18n,
+ log::terminal(),
+ )
+ .unwrap();
+
+ col.with_ctx(|ctx| {
+ // unqualified search
+ assert_eq!(
+ s(ctx, "test"),
+ (
+ "((n.sfld like ?1 escape '\\' or n.flds like ?1 escape '\\'))".into(),
+ vec!["%test%".into()]
+ )
+ );
+ assert_eq!(s(ctx, "te%st").1, vec!["%te%st%".to_string()]);
+ // user should be able to escape sql wildcards
+ assert_eq!(s(ctx, r#"te\%s\_t"#).1, vec!["%te\\%s\\_t%".to_string()]);
+
+ // qualified search
+ assert_eq!(
+ s(ctx, "front:te*st"),
+ (
+ concat!(
+ "(((n.mid = 1581236385344 and field_at_index(n.flds, 0) like ?1) or ",
+ "(n.mid = 1581236385345 and field_at_index(n.flds, 0) like ?1) or ",
+ "(n.mid = 1581236385346 and field_at_index(n.flds, 0) like ?1) or ",
+ "(n.mid = 1581236385347 and field_at_index(n.flds, 0) like ?1)))"
+ )
+ .into(),
+ vec!["te%st".into()]
+ )
+ );
+
+ // added
+ let timing = ctx.storage.timing_today().unwrap();
+ assert_eq!(
+ s(ctx, "added:3").0,
+ format!("(c.id > {})", (timing.next_day_at - (86_400 * 3)) * 1_000)
+ );
+
+ // deck
+ assert_eq!(s(ctx, "deck:default"), ("(c.did in (1))".into(), vec![],));
+ assert_eq!(s(ctx, "deck:current"), ("(c.did in (1))".into(), vec![],));
+ assert_eq!(s(ctx, "deck:missing"), ("(c.did in ())".into(), vec![],));
+ assert_eq!(s(ctx, "deck:d*"), ("(c.did in (1))".into(), vec![],));
+ assert_eq!(s(ctx, "deck:filtered"), ("(c.odid > 0)".into(), vec![],));
+
+ // card
+ assert_eq!(s(ctx, "card:front"), ("(false)".into(), vec![],));
+ assert_eq!(
+ s(ctx, r#""card:card 1""#),
+ (
+ concat!(
+ "(((n.mid = 1581236385344 and c.ord = 0) or ",
+ "(n.mid = 1581236385345 and c.ord = 0) or ",
+ "(n.mid = 1581236385346 and c.ord = 0) or ",
+ "(n.mid = 1581236385347 and c.ord = 0)))"
+ )
+ .into(),
+ vec![],
+ )
+ );
+
+ // IDs
+ assert_eq!(s(ctx, "mid:3"), ("(n.mid = 3)".into(), vec![]));
+ assert_eq!(s(ctx, "nid:3"), ("(n.id in (3))".into(), vec![]));
+ assert_eq!(s(ctx, "nid:3,4"), ("(n.id in (3,4))".into(), vec![]));
+ assert_eq!(s(ctx, "cid:3,4"), ("(c.id in (3,4))".into(), vec![]));
+
+ // flags
+ assert_eq!(s(ctx, "flag:2"), ("((c.flags & 7) == 2)".into(), vec![]));
+ assert_eq!(s(ctx, "flag:0"), ("((c.flags & 7) == 0)".into(), vec![]));
+
+ // dupes
+ assert_eq!(
+ s(ctx, "dupes:123,test"),
+ (
+ "((n.mid = 123 and n.csum = 2840236005 and field_at_index(n.flds, 0) = ?)"
+ .into(),
+ vec!["test".into()]
+ )
+ );
+
+ // tags
+ assert_eq!(
+ s(ctx, "tag:one"),
+ ("(n.tags like ? escape '\\')".into(), vec!["% one %".into()])
+ );
+ assert_eq!(
+ s(ctx, "tag:o*e"),
+ ("(n.tags like ? escape '\\')".into(), vec!["% o%e %".into()])
+ );
+ assert_eq!(s(ctx, "tag:none"), ("(n.tags = '')".into(), vec![]));
+ assert_eq!(s(ctx, "tag:*"), ("(true)".into(), vec![]));
+
+ // state
+ assert_eq!(
+ s(ctx, "is:suspended").0,
+ format!("(c.queue = {})", CardQueue::Suspended as i8)
+ );
+ assert_eq!(
+ s(ctx, "is:new").0,
+ format!("(c.type = {})", CardQueue::New as i8)
+ );
+
+ // rated
+ assert_eq!(
+ s(ctx, "rated:2").0,
+ format!(
+ "(c.id in (select cid from revlog where id>{}))",
+ (timing.next_day_at - (86_400 * 2)) * 1_000
+ )
+ );
+ assert_eq!(
+ s(ctx, "rated:400:1").0,
+ format!(
+ "(c.id in (select cid from revlog where id>{} and ease=1))",
+ (timing.next_day_at - (86_400 * 365)) * 1_000
+ )
+ );
+
+ // props
+ assert_eq!(s(ctx, "prop:lapses=3").0, "(lapses = 3)".to_string());
+ assert_eq!(s(ctx, "prop:ease>=2.5").0, "(factor >= 2500)".to_string());
+ assert_eq!(
+ s(ctx, "prop:due!=-1").0,
+ format!(
+ "((c.queue in (2,3) and due != {}))",
+ timing.days_elapsed - 1
+ )
+ );
+
+ // note types by name
+ assert_eq!(&s(ctx, "note:basic").0, "(n.mid in (1581236385347))");
+ assert_eq!(
+ &s(ctx, "note:basic*").0,
+ "(n.mid in (1581236385345,1581236385346,1581236385347,1581236385344))"
+ );
+
+ // regex
+ assert_eq!(
+ s(ctx, r"re:\bone"),
+ ("(n.flds regexp ?)".into(), vec![r"(?i)\bone".into()])
+ );
+
+ Ok(())
+ })
+ .unwrap();
+
+ Ok(())
+ }
+}
diff --git a/rslib/src/storage/mod.rs b/rslib/src/storage/mod.rs
new file mode 100644
index 000000000..2ed04892c
--- /dev/null
+++ b/rslib/src/storage/mod.rs
@@ -0,0 +1,3 @@
+mod sqlite;
+
+pub(crate) use sqlite::{SqliteStorage, StorageContext};
diff --git a/rslib/src/storage/schema11.sql b/rslib/src/storage/schema11.sql
new file mode 100644
index 000000000..e5d1b7f4d
--- /dev/null
+++ b/rslib/src/storage/schema11.sql
@@ -0,0 +1,88 @@
+create table col
+(
+ id integer primary key,
+ crt integer not null,
+ mod integer not null,
+ scm integer not null,
+ ver integer not null,
+ dty integer not null,
+ usn integer not null,
+ ls integer not null,
+ conf text not null,
+ models text not null,
+ decks text not null,
+ dconf text not null,
+ tags text not null
+);
+
+create table notes
+(
+ id integer primary key,
+ guid text not null,
+ mid integer not null,
+ mod integer not null,
+ usn integer not null,
+ tags text not null,
+ flds text not null,
+ sfld integer not null,
+ csum integer not null,
+ flags integer not null,
+ data text not null
+);
+
+create table cards
+(
+ id integer primary key,
+ nid integer not null,
+ did integer not null,
+ ord integer not null,
+ mod integer not null,
+ usn integer not null,
+ type integer not null,
+ queue integer not null,
+ due integer not null,
+ ivl integer not null,
+ factor integer not null,
+ reps integer not null,
+ lapses integer not null,
+ left integer not null,
+ odue integer not null,
+ odid integer not null,
+ flags integer not null,
+ data text not null
+);
+
+create table revlog
+(
+ id integer primary key,
+ cid integer not null,
+ usn integer not null,
+ ease integer not null,
+ ivl integer not null,
+ lastIvl integer not null,
+ factor integer not null,
+ time integer not null,
+ type integer not null
+);
+
+create table graves
+(
+ usn integer not null,
+ oid integer not null,
+ type integer not null
+);
+
+-- syncing
+create index ix_notes_usn on notes (usn);
+create index ix_cards_usn on cards (usn);
+create index ix_revlog_usn on revlog (usn);
+-- card spacing, etc
+create index ix_cards_nid on cards (nid);
+-- scheduling and deck limiting
+create index ix_cards_sched on cards (did, queue, due);
+-- revlog by card
+create index ix_revlog_cid on revlog (cid);
+-- field uniqueness
+create index ix_notes_csum on notes (csum);
+
+insert into col values (1,0,0,0,0,0,0,0,'{}','{}','{}','{}','{}');
diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs
new file mode 100644
index 000000000..7e5b2f87a
--- /dev/null
+++ b/rslib/src/storage/sqlite.rs
@@ -0,0 +1,335 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+use crate::collection::CollectionOp;
+use crate::config::Config;
+use crate::err::Result;
+use crate::err::{AnkiError, DBErrorKind};
+use crate::time::{i64_unix_millis, i64_unix_secs};
+use crate::{
+ decks::Deck,
+ notetypes::NoteType,
+ sched::cutoff::{sched_timing_today, SchedTimingToday},
+ text::without_combining,
+ types::{ObjID, Usn},
+};
+use regex::Regex;
+use rusqlite::{params, Connection, NO_PARAMS};
+use std::cmp::Ordering;
+use std::{
+ borrow::Cow,
+ collections::HashMap,
+ path::{Path, PathBuf},
+};
+use unicase::UniCase;
+
+const SCHEMA_MIN_VERSION: u8 = 11;
+const SCHEMA_MAX_VERSION: u8 = 11;
+
+fn unicase_compare(s1: &str, s2: &str) -> Ordering {
+ UniCase::new(s1).cmp(&UniCase::new(s2))
+}
+
+// currently public for dbproxy
+#[derive(Debug)]
+pub struct SqliteStorage {
+ // currently crate-visible for dbproxy
+ pub(crate) db: Connection,
+
+ // fixme: stored in wrong location?
+ path: PathBuf,
+}
+
+fn open_or_create_collection_db(path: &Path) -> Result {
+ let mut db = Connection::open(path)?;
+
+ if std::env::var("TRACESQL").is_ok() {
+ db.trace(Some(trace));
+ }
+
+ db.busy_timeout(std::time::Duration::from_secs(0))?;
+
+ db.pragma_update(None, "locking_mode", &"exclusive")?;
+ db.pragma_update(None, "page_size", &4096)?;
+ db.pragma_update(None, "cache_size", &(-40 * 1024))?;
+ db.pragma_update(None, "legacy_file_format", &false)?;
+ db.pragma_update(None, "journal_mode", &"wal")?;
+ db.pragma_update(None, "temp_store", &"memory")?;
+
+ db.set_prepared_statement_cache_capacity(50);
+
+ add_field_index_function(&db)?;
+ add_regexp_function(&db)?;
+ add_without_combining_function(&db)?;
+
+ db.create_collation("unicase", unicase_compare)?;
+
+ Ok(db)
+}
+
+/// Adds sql function field_at_index(flds, index)
+/// to split provided fields and return field at zero-based index.
+/// If out of range, returns empty string.
+fn add_field_index_function(db: &Connection) -> rusqlite::Result<()> {
+ db.create_scalar_function("field_at_index", 2, true, |ctx| {
+ let mut fields = ctx.get_raw(0).as_str()?.split('\x1f');
+ let idx: u16 = ctx.get(1)?;
+ Ok(fields.nth(idx as usize).unwrap_or("").to_string())
+ })
+}
+
+fn add_without_combining_function(db: &Connection) -> rusqlite::Result<()> {
+ db.create_scalar_function("without_combining", 1, true, |ctx| {
+ let text = ctx.get_raw(0).as_str()?;
+ Ok(match without_combining(text) {
+ Cow::Borrowed(_) => None,
+ Cow::Owned(o) => Some(o),
+ })
+ })
+}
+
+/// Adds sql function regexp(regex, string) -> is_match
+/// Taken from the rusqlite docs
+fn add_regexp_function(db: &Connection) -> rusqlite::Result<()> {
+ db.create_scalar_function("regexp", 2, true, move |ctx| {
+ assert_eq!(ctx.len(), 2, "called with unexpected number of arguments");
+
+ let saved_re: Option<&Regex> = ctx.get_aux(0)?;
+ let new_re = match saved_re {
+ None => {
+ let s = ctx.get::(0)?;
+ match Regex::new(&s) {
+ Ok(r) => Some(r),
+ Err(err) => return Err(rusqlite::Error::UserFunctionError(Box::new(err))),
+ }
+ }
+ Some(_) => None,
+ };
+
+ let is_match = {
+ let re = saved_re.unwrap_or_else(|| new_re.as_ref().unwrap());
+
+ let text = ctx
+ .get_raw(1)
+ .as_str()
+ .map_err(|e| rusqlite::Error::UserFunctionError(e.into()))?;
+
+ re.is_match(text)
+ };
+
+ if let Some(re) = new_re {
+ ctx.set_aux(0, re);
+ }
+
+ Ok(is_match)
+ })
+}
+
+/// Fetch schema version from database.
+/// Return (must_create, version)
+fn schema_version(db: &Connection) -> Result<(bool, u8)> {
+ if !db
+ .prepare("select null from sqlite_master where type = 'table' and name = 'col'")?
+ .exists(NO_PARAMS)?
+ {
+ return Ok((true, SCHEMA_MAX_VERSION));
+ }
+
+ Ok((
+ false,
+ db.query_row("select ver from col", NO_PARAMS, |r| Ok(r.get(0)?))?,
+ ))
+}
+
+fn trace(s: &str) {
+ println!("sql: {}", s)
+}
+
+impl SqliteStorage {
+ pub(crate) fn open_or_create(path: &Path) -> Result {
+ let db = open_or_create_collection_db(path)?;
+
+ let (create, ver) = schema_version(&db)?;
+ if create {
+ db.prepare_cached("begin exclusive")?.execute(NO_PARAMS)?;
+ db.execute_batch(include_str!("schema11.sql"))?;
+ db.execute("update col set crt=?, ver=?", params![i64_unix_secs(), ver])?;
+ db.prepare_cached("commit")?.execute(NO_PARAMS)?;
+ } else {
+ if ver > SCHEMA_MAX_VERSION {
+ return Err(AnkiError::DBError {
+ info: "".to_string(),
+ kind: DBErrorKind::FileTooNew,
+ });
+ }
+ if ver < SCHEMA_MIN_VERSION {
+ return Err(AnkiError::DBError {
+ info: "".to_string(),
+ kind: DBErrorKind::FileTooOld,
+ });
+ }
+ };
+
+ let storage = Self {
+ db,
+ path: path.to_owned(),
+ };
+
+ Ok(storage)
+ }
+
+ pub(crate) fn context(&self, server: bool) -> StorageContext {
+ StorageContext::new(&self.db, server)
+ }
+}
+
+pub(crate) struct StorageContext<'a> {
+ pub(crate) db: &'a Connection,
+ #[allow(dead_code)]
+ server: bool,
+ #[allow(dead_code)]
+ usn: Option,
+
+ timing_today: Option,
+}
+
+impl StorageContext<'_> {
+ fn new(db: &Connection, server: bool) -> StorageContext {
+ StorageContext {
+ db,
+ server,
+ usn: None,
+ timing_today: None,
+ }
+ }
+
+ // Standard transaction start/stop
+ //////////////////////////////////////
+
+ pub(crate) fn begin_trx(&self) -> Result<()> {
+ self.db
+ .prepare_cached("begin exclusive")?
+ .execute(NO_PARAMS)?;
+ Ok(())
+ }
+
+ pub(crate) fn commit_trx(&self) -> Result<()> {
+ if !self.db.is_autocommit() {
+ self.db.prepare_cached("commit")?.execute(NO_PARAMS)?;
+ }
+ Ok(())
+ }
+
+ pub(crate) fn rollback_trx(&self) -> Result<()> {
+ if !self.db.is_autocommit() {
+ self.db.execute("rollback", NO_PARAMS)?;
+ }
+ Ok(())
+ }
+
+ // Savepoints
+ //////////////////////////////////////////
+ //
+ // This is necessary at the moment because Anki's current architecture uses
+ // long-running transactions as an undo mechanism. Once a proper undo
+ // mechanism has been added to all existing functionality, we could
+ // transition these to standard commits.
+
+ pub(crate) fn begin_rust_trx(&self) -> Result<()> {
+ self.db
+ .prepare_cached("savepoint rust")?
+ .execute(NO_PARAMS)?;
+ Ok(())
+ }
+
+ pub(crate) fn commit_rust_trx(&self) -> Result<()> {
+ self.db.prepare_cached("release rust")?.execute(NO_PARAMS)?;
+ Ok(())
+ }
+
+ pub(crate) fn commit_rust_op(&self, _op: Option) -> Result<()> {
+ self.commit_rust_trx()
+ }
+
+ pub(crate) fn rollback_rust_trx(&self) -> Result<()> {
+ self.db
+ .prepare_cached("rollback to rust")?
+ .execute(NO_PARAMS)?;
+ Ok(())
+ }
+
+ //////////////////////////////////////////
+
+ pub(crate) fn mark_modified(&self) -> Result<()> {
+ self.db
+ .prepare_cached("update col set mod=?")?
+ .execute(params![i64_unix_millis()])?;
+ Ok(())
+ }
+
+ #[allow(dead_code)]
+ pub(crate) fn usn(&mut self) -> Result {
+ if self.server {
+ if self.usn.is_none() {
+ self.usn = Some(
+ self.db
+ .prepare_cached("select usn from col")?
+ .query_row(NO_PARAMS, |row| row.get(0))?,
+ );
+ }
+ Ok(*self.usn.as_ref().unwrap())
+ } else {
+ Ok(-1)
+ }
+ }
+
+ pub(crate) fn all_decks(&self) -> Result> {
+ self.db
+ .query_row_and_then("select decks from col", NO_PARAMS, |row| -> Result<_> {
+ Ok(serde_json::from_str(row.get_raw(0).as_str()?)?)
+ })
+ }
+
+ pub(crate) fn all_config(&self) -> Result {
+ self.db
+ .query_row_and_then("select conf from col", NO_PARAMS, |row| -> Result<_> {
+ Ok(serde_json::from_str(row.get_raw(0).as_str()?)?)
+ })
+ }
+
+ pub(crate) fn all_note_types(&self) -> Result> {
+ let mut stmt = self.db.prepare("select models from col")?;
+ let note_types = stmt
+ .query_and_then(NO_PARAMS, |row| -> Result> {
+ let v: HashMap = serde_json::from_str(row.get_raw(0).as_str()?)?;
+ Ok(v)
+ })?
+ .next()
+ .ok_or_else(|| AnkiError::DBError {
+ info: "col table empty".to_string(),
+ kind: DBErrorKind::MissingEntity,
+ })??;
+ Ok(note_types)
+ }
+
+ #[allow(dead_code)]
+ pub(crate) fn timing_today(&mut self) -> Result {
+ if self.timing_today.is_none() {
+ let crt: i64 = self
+ .db
+ .prepare_cached("select crt from col")?
+ .query_row(NO_PARAMS, |row| row.get(0))?;
+ let conf = self.all_config()?;
+ let now_offset = if self.server { conf.local_offset } else { None };
+
+ self.timing_today = Some(sched_timing_today(
+ crt,
+ i64_unix_secs(),
+ conf.creation_offset,
+ now_offset,
+ conf.rollover,
+ ));
+ }
+ Ok(*self.timing_today.as_ref().unwrap())
+ }
+}
diff --git a/rslib/src/text.rs b/rslib/src/text.rs
index 4d5ff286d..a88bd4a4f 100644
--- a/rslib/src/text.rs
+++ b/rslib/src/text.rs
@@ -5,7 +5,10 @@ use lazy_static::lazy_static;
use regex::{Captures, Regex};
use std::borrow::Cow;
use std::ptr;
-use unicode_normalization::{is_nfc, UnicodeNormalization};
+use unicase::eq as uni_eq;
+use unicode_normalization::{
+ char::is_combining_mark, is_nfc, is_nfkd_quick, IsNormalized, UnicodeNormalization,
+};
#[derive(Debug, PartialEq)]
pub enum AVTag {
@@ -219,11 +222,43 @@ pub(crate) fn normalize_to_nfc(s: &str) -> Cow {
}
}
+/// True if search is equal to text, folding case.
+/// Supports '*' to match 0 or more characters.
+pub(crate) fn matches_wildcard(text: &str, search: &str) -> bool {
+ if search.contains('*') {
+ let search = format!("^(?i){}$", regex::escape(search).replace(r"\*", ".*"));
+ Regex::new(&search).unwrap().is_match(text)
+ } else {
+ uni_eq(text, search)
+ }
+}
+
+/// Convert provided string to NFKD form and strip combining characters.
+pub(crate) fn without_combining(s: &str) -> Cow {
+ // if the string is already normalized
+ if matches!(is_nfkd_quick(s.chars()), IsNormalized::Yes) {
+ // and no combining characters found, return unchanged
+ if !s.chars().any(is_combining_mark) {
+ return s.into();
+ }
+ }
+
+ // we need to create a new string without the combining marks
+ s.chars()
+ .nfkd()
+ .filter(|c| !is_combining_mark(*c))
+ .collect::()
+ .into()
+}
+
#[cfg(test)]
mod test {
+ use super::matches_wildcard;
+ use crate::text::without_combining;
use crate::text::{
extract_av_tags, strip_av_tags, strip_html, strip_html_preserving_image_filenames, AVTag,
};
+ use std::borrow::Cow;
#[test]
fn stripping() {
@@ -265,4 +300,19 @@ mod test {
]
);
}
+
+ #[test]
+ fn wildcard() {
+ assert_eq!(matches_wildcard("foo", "bar"), false);
+ assert_eq!(matches_wildcard("foo", "Foo"), true);
+ assert_eq!(matches_wildcard("foo", "F*"), true);
+ assert_eq!(matches_wildcard("foo", "F*oo"), true);
+ assert_eq!(matches_wildcard("foo", "b*"), false);
+ }
+
+ #[test]
+ fn combining() {
+ assert!(matches!(without_combining("test"), Cow::Borrowed(_)));
+ assert!(matches!(without_combining("Über"), Cow::Owned(_)));
+ }
}
diff --git a/rslib/src/time.rs b/rslib/src/time.rs
index 8d9084c45..e392cbb6e 100644
--- a/rslib/src/time.rs
+++ b/rslib/src/time.rs
@@ -4,15 +4,35 @@
use std::time;
pub(crate) fn i64_unix_secs() -> i64 {
- time::SystemTime::now()
- .duration_since(time::SystemTime::UNIX_EPOCH)
- .unwrap()
- .as_secs() as i64
+ elapsed().as_secs() as i64
}
pub(crate) fn i64_unix_millis() -> i64 {
+ elapsed().as_millis() as i64
+}
+
+#[cfg(not(test))]
+fn elapsed() -> time::Duration {
time::SystemTime::now()
.duration_since(time::SystemTime::UNIX_EPOCH)
.unwrap()
- .as_millis() as i64
+}
+
+// when running in CI, shift the current time away from the cutoff point
+// to accomodate unit tests that depend on the current time
+#[cfg(test)]
+fn elapsed() -> time::Duration {
+ use chrono::{Local, Timelike};
+
+ let now = Local::now();
+
+ let mut elap = time::SystemTime::now()
+ .duration_since(time::SystemTime::UNIX_EPOCH)
+ .unwrap();
+
+ if now.hour() >= 2 && now.hour() < 4 {
+ elap -= time::Duration::from_secs(60 * 60 * 2);
+ }
+
+ elap
}
diff --git a/rspy/Cargo.toml b/rspy/Cargo.toml
index 11cd4f41e..5b593acb2 100644
--- a/rspy/Cargo.toml
+++ b/rspy/Cargo.toml
@@ -1,6 +1,6 @@
[package]
name = "ankirspy"
-version = "2.1.22" # automatically updated
+version = "2.1.24" # automatically updated
edition = "2018"
authors = ["Ankitects Pty Ltd and contributors"]
diff --git a/rspy/src/lib.rs b/rspy/src/lib.rs
index f24fccc76..5ec2375f4 100644
--- a/rspy/src/lib.rs
+++ b/rspy/src/lib.rs
@@ -1,12 +1,11 @@
// Copyright: Ankitects Pty Ltd and contributors
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
-use anki::backend::{
- init_backend, init_i18n_backend, Backend as RustBackend, I18nBackend as RustI18nBackend,
-};
+use anki::backend::{init_backend, Backend as RustBackend};
+use pyo3::exceptions::Exception;
use pyo3::prelude::*;
use pyo3::types::PyBytes;
-use pyo3::{exceptions, wrap_pyfunction};
+use pyo3::{create_exception, exceptions, wrap_pyfunction};
// Regular backend
//////////////////////////////////
@@ -16,6 +15,8 @@ struct Backend {
backend: RustBackend,
}
+create_exception!(ankirspy, DBError, Exception);
+
#[pyfunction]
fn buildhash() -> &'static str {
include_str!("../../meta/buildhash").trim()
@@ -70,29 +71,17 @@ impl Backend {
self.backend.set_progress_callback(Some(Box::new(func)));
}
}
-}
-// I18n backend
-//////////////////////////////////
-
-#[pyclass]
-struct I18nBackend {
- backend: RustI18nBackend,
-}
-
-#[pyfunction]
-fn open_i18n(init_msg: &PyBytes) -> PyResult {
- match init_i18n_backend(init_msg.as_bytes()) {
- Ok(backend) => Ok(I18nBackend { backend }),
- Err(e) => Err(exceptions::Exception::py_err(format!("{:?}", e))),
- }
-}
-
-#[pymethods]
-impl I18nBackend {
- fn translate(&self, input: &PyBytes) -> String {
+ fn db_command(&mut self, py: Python, input: &PyBytes) -> PyResult {
let in_bytes = input.as_bytes();
- self.backend.translate(in_bytes)
+ let out_res = py.allow_threads(move || {
+ self.backend
+ .db_command(in_bytes)
+ .map_err(|e| DBError::py_err(e.localized_description(&self.backend.i18n())))
+ });
+ let out_string = out_res?;
+ let out_obj = PyBytes::new(py, out_string.as_bytes());
+ Ok(out_obj.into())
}
}
@@ -104,7 +93,6 @@ fn ankirspy(_py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::()?;
m.add_wrapped(wrap_pyfunction!(buildhash)).unwrap();
m.add_wrapped(wrap_pyfunction!(open_backend)).unwrap();
- m.add_wrapped(wrap_pyfunction!(open_i18n)).unwrap();
Ok(())
}