mirror of
https://github.com/ankitects/anki.git
synced 2025-11-10 14:47:12 -05:00
* Add crate csv
* Add start of csv importing on backend
* Add Menomosyne serializer
* Add csv and json importing on backend
* Add plaintext importing on frontend
* Add csv metadata extraction on backend
* Add csv importing with GUI
* Fix missing dfa file in build
Added compile_data_attr, then re-ran cargo/update.py.
* Don't use doubly buffered reader in csv
* Escape HTML entities if CSV is not HTML
Also use name 'is_html' consistently.
* Use decimal number as foreign ease (like '2.5')
* ForeignCard.ivl → ForeignCard.interval
* Only allow fixed set of CSV delimiters
* Map timestamp of ForeignCard to native due time
* Don't trim CSV records
* Document use of empty strings for defaults
* Avoid creating CardGenContexts for every note
This requires CardGenContext to be generic, so it works both with an
owned and borrowed notetype.
* Show all accepted file types in import file picker
* Add import_json_file()
* factor → ease_factor
* delimter_from_value → delimiter_from_value
* Map columns to fields, not the other way around
* Fallback to current config for csv metadata
* Add start of new import csv screen
* Temporary fix for compilation issue on Linux/Mac
* Disable jest bazel action for import-csv
Jest fails with an error code if no tests are available, but this would
not be noticable on Windows as Jest is not run there.
* Fix field mapping issue
* Revert "Temporary fix for compilation issue on Linux/Mac"
This reverts commit 21f8a26140.
* Add HtmlSwitch and move Switch to components
* Fix spacing and make selectors consistent
* Fix shortcut tooltip
* Place import button at the top with path
* Fix meta column indices
* Remove NotetypeForString
* Fix queue and type of foreign cards
* Support different dupe resolution strategies
* Allow dupe resolution selection when importing CSV
* Test import of unnormalized text
Close #1863.
* Fix logging of foreign notes
* Implement CSV exports
* Use db_scalar() in notes_table_len()
* Rework CSV metadata
- Notetypes and decks are either defined by a global id or by a column.
- If a notetype id is provided, its field map must also be specified.
- If a notetype column is provided, fields are now mapped by index
instead of name at import time. So the first non-meta column is used for
the first field of every note, regardless of notetype. This makes
importing easier and should improve compatiblity with files without a
notetype column.
- Ensure first field can be mapped to a column.
- Meta columns must be defined as `#[meta name]:[column index]` instead
of in the `#columns` tag.
- Column labels contain the raw names defined by the file and must be
prettified by the frontend.
* Adjust frontend to new backend column mapping
* Add force flags for is_html and delimiter
* Detect if CSV is HTML by field content
* Update dupe resolution labels
* Simplify selectors
* Fix coalescence of oneofs in TS
* Disable meta columns from selection
Plus a lot of refactoring.
* Make import button stick to the bottom
* Write delimiter and html flag into csv
* Refetch field map after notetype change
* Fix log labels for csv import
* Log notes whose deck/notetype was missing
* Fix hiding of empty log queues
* Implement adding tags to all notes of a csv
* Fix dupe resolution not being set in log
* Implement adding tags to updated notes of a csv
* Check first note field is not empty
* Temporary fix for build on Linux/Mac
* Fix inverted html check (dae)
* Remove unused ftl string
* Delimiter → Separator
* Remove commented-out line
* Don't accept .json files
* Tweak tag ftl strings
* Remove redundant blur call
* Strip sound and add spaces in csv export
* Export HTML by default
* Fix unset deck in Mnemosyne import
Also accept both numbers and strings for notetypes and decks in JSON.
* Make DupeResolution::Update the default
* Fix missing dot in extension
* Make column indices 1-based
* Remove StickContainer from TagEditor
Fixes line breaking, border and z index on ImportCsvPage.
* Assign different key combos to tag editors
* Log all updated duplicates
Add a log field for the true number of found notes.
* Show identical notes as skipped
* Split tag-editor into separate ts module (dae)
* Add progress for CSV export
* Add progress for text import
* Tidy-ups after tag-editor split (dae)
- import-csv no longer depends on editor
- remove some commented lines
130 lines
3.7 KiB
Rust
130 lines
3.7 KiB
Rust
// Copyright: Ankitects Pty Ltd and contributors
|
|
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
|
|
|
mod gather;
|
|
mod insert;
|
|
pub mod package;
|
|
pub mod text;
|
|
|
|
use std::marker::PhantomData;
|
|
|
|
pub use crate::backend_proto::import_response::{Log as NoteLog, Note as LogNote};
|
|
use crate::{
|
|
prelude::*,
|
|
text::{
|
|
newlines_to_spaces, strip_html_preserving_media_filenames, truncate_to_char_boundary,
|
|
CowMapping,
|
|
},
|
|
};
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
|
pub enum ImportProgress {
|
|
File,
|
|
Extracting,
|
|
Gathering,
|
|
Media(usize),
|
|
MediaCheck(usize),
|
|
Notes(usize),
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq)]
|
|
pub enum ExportProgress {
|
|
File,
|
|
Gathering,
|
|
Notes(usize),
|
|
Cards(usize),
|
|
Media(usize),
|
|
}
|
|
|
|
/// Wrapper around a progress function, usually passed by the [crate::backend::Backend],
|
|
/// to make repeated calls more ergonomic.
|
|
pub(crate) struct IncrementableProgress<P>(Box<dyn FnMut(P, bool) -> bool>);
|
|
|
|
impl<P> IncrementableProgress<P> {
|
|
/// `progress_fn: (progress, throttle) -> should_continue`
|
|
pub(crate) fn new(progress_fn: impl 'static + FnMut(P, bool) -> bool) -> Self {
|
|
Self(Box::new(progress_fn))
|
|
}
|
|
|
|
/// Returns an [Incrementor] with an `increment()` function for use in loops.
|
|
pub(crate) fn incrementor<'inc, 'progress: 'inc, 'map: 'inc>(
|
|
&'progress mut self,
|
|
mut count_map: impl 'map + FnMut(usize) -> P,
|
|
) -> Incrementor<'inc, impl FnMut(usize) -> Result<()> + 'inc> {
|
|
Incrementor::new(move |u| self.update(count_map(u), true))
|
|
}
|
|
|
|
/// Manually triggers an update.
|
|
/// Returns [AnkiError::Interrupted] if the operation should be cancelled.
|
|
pub(crate) fn call(&mut self, progress: P) -> Result<()> {
|
|
self.update(progress, false)
|
|
}
|
|
|
|
fn update(&mut self, progress: P, throttle: bool) -> Result<()> {
|
|
if (self.0)(progress, throttle) {
|
|
Ok(())
|
|
} else {
|
|
Err(AnkiError::Interrupted)
|
|
}
|
|
}
|
|
|
|
/// Stopgap for returning a progress fn compliant with the media code.
|
|
pub(crate) fn media_db_fn(
|
|
&mut self,
|
|
count_map: impl 'static + Fn(usize) -> P,
|
|
) -> Result<impl FnMut(usize) -> bool + '_> {
|
|
Ok(move |count| (self.0)(count_map(count), true))
|
|
}
|
|
}
|
|
|
|
pub(crate) struct Incrementor<'f, F: 'f + FnMut(usize) -> Result<()>> {
|
|
update_fn: F,
|
|
count: usize,
|
|
update_interval: usize,
|
|
_phantom: PhantomData<&'f ()>,
|
|
}
|
|
|
|
impl<'f, F: 'f + FnMut(usize) -> Result<()>> Incrementor<'f, F> {
|
|
fn new(update_fn: F) -> Self {
|
|
Self {
|
|
update_fn,
|
|
count: 0,
|
|
update_interval: 17,
|
|
_phantom: PhantomData,
|
|
}
|
|
}
|
|
|
|
/// Increments the progress counter, periodically triggering an update.
|
|
/// Returns [AnkiError::Interrupted] if the operation should be cancelled.
|
|
pub(crate) fn increment(&mut self) -> Result<()> {
|
|
self.count += 1;
|
|
if self.count % self.update_interval != 0 {
|
|
return Ok(());
|
|
}
|
|
(self.update_fn)(self.count)
|
|
}
|
|
|
|
pub(crate) fn count(&self) -> usize {
|
|
self.count
|
|
}
|
|
}
|
|
|
|
impl Note {
|
|
pub(crate) fn into_log_note(self) -> LogNote {
|
|
LogNote {
|
|
id: Some(self.id.into()),
|
|
fields: self
|
|
.into_fields()
|
|
.into_iter()
|
|
.map(|field| {
|
|
let mut reduced = strip_html_preserving_media_filenames(&field)
|
|
.map_cow(newlines_to_spaces)
|
|
.get_owned()
|
|
.unwrap_or(field);
|
|
truncate_to_char_boundary(&mut reduced, 80);
|
|
reduced
|
|
})
|
|
.collect(),
|
|
}
|
|
}
|
|
}
|