mirror of
https://github.com/ankitects/anki.git
synced 2025-09-18 22:12:21 -04:00
cf45cbf429
This PR replaces the existing Python-driven sync server with a new one in Rust. The new server supports both collection and media syncing, and is compatible with both the new protocol mentioned below, and older clients. A setting has been added to the preferences screen to point Anki to a local server, and a similar setting is likely to come to AnkiMobile soon. Documentation is available here: <https://docs.ankiweb.net/sync-server.html> In addition to the new server and refactoring, this PR also makes changes to the sync protocol. The existing sync protocol places payloads and metadata inside a multipart POST body, which causes a few headaches: - Legacy clients build the request in a non-deterministic order, meaning the entire request needs to be scanned to extract the metadata. - Reqwest's multipart API directly writes the multipart body, without exposing the resulting stream to us, making it harder to track the progress of the transfer. We've been relying on a patched version of reqwest for timeouts, which is a pain to keep up to date. To address these issues, the metadata is now sent in a HTTP header, with the data payload sent directly in the body. Instead of the slower gzip, we now use zstd. The old timeout handling code has been replaced with a new implementation that wraps the request and response body streams to track progress, allowing us to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout. The main other change to the protocol is that one-way syncs no longer need to downgrade the collection to schema 11 prior to sending.
96 lines
3.5 KiB
Rust
96 lines
3.5 KiB
Rust
// Copyright: Ankitects Pty Ltd and contributors
|
|
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
|
|
|
|
use std::{fs, io::ErrorKind, path::Path};
|
|
|
|
use snafu::ResultExt;
|
|
use tracing::info;
|
|
|
|
use crate::{
|
|
error,
|
|
error::{FileIoError, FileIoSnafu, FileOp},
|
|
io::write_file,
|
|
sync::{
|
|
error::{HttpResult, OrHttpErr},
|
|
http_server::media_manager::ServerMediaManager,
|
|
media::{
|
|
database::server::entry::upload::UploadedChangeResult, upload::MediaUploadResponse,
|
|
zip::unzip_and_validate_files,
|
|
},
|
|
},
|
|
};
|
|
|
|
impl ServerMediaManager {
|
|
pub fn process_uploaded_changes(
|
|
&mut self,
|
|
zip_data: Vec<u8>,
|
|
) -> HttpResult<MediaUploadResponse> {
|
|
let extracted = unzip_and_validate_files(&zip_data).or_bad_request("unzip files")?;
|
|
let folder = &self.media_folder;
|
|
let mut processed = 0;
|
|
let new_usn = self
|
|
.db
|
|
.with_transaction(|db, meta| {
|
|
for change in extracted {
|
|
match db.register_uploaded_change(meta, change)? {
|
|
UploadedChangeResult::FileAlreadyDeleted { filename } => {
|
|
info!(filename, "already deleted");
|
|
}
|
|
UploadedChangeResult::FileIdentical { filename, sha1 } => {
|
|
info!(filename, sha1 = hex::encode(sha1), "already have");
|
|
}
|
|
UploadedChangeResult::Added {
|
|
filename,
|
|
data,
|
|
sha1,
|
|
} => {
|
|
info!(filename, sha1 = hex::encode(sha1), "added");
|
|
add_or_replace_file(&folder.join(filename), data)?;
|
|
}
|
|
UploadedChangeResult::Replaced {
|
|
filename,
|
|
data,
|
|
old_sha1,
|
|
new_sha1,
|
|
} => {
|
|
info!(
|
|
filename,
|
|
old_sha1 = hex::encode(old_sha1),
|
|
new_sha1 = hex::encode(new_sha1),
|
|
"replaced"
|
|
);
|
|
add_or_replace_file(&folder.join(filename), data)?;
|
|
}
|
|
UploadedChangeResult::Removed { filename, sha1 } => {
|
|
info!(filename, sha1 = hex::encode(sha1), "removed");
|
|
remove_file(&folder.join(filename))?;
|
|
}
|
|
}
|
|
processed += 1;
|
|
}
|
|
Ok(())
|
|
})
|
|
.or_internal_err("handle uploaded change")?;
|
|
Ok(MediaUploadResponse {
|
|
processed,
|
|
current_usn: new_usn,
|
|
})
|
|
}
|
|
}
|
|
|
|
fn add_or_replace_file(path: &Path, data: Vec<u8>) -> error::Result<(), FileIoError> {
|
|
write_file(path, data).map_err(Into::into)
|
|
}
|
|
|
|
fn remove_file(path: &Path) -> error::Result<(), FileIoError> {
|
|
if let Err(err) = fs::remove_file(path) {
|
|
// if transaction was previously aborted, the file may have already been deleted
|
|
if err.kind() != ErrorKind::NotFound {
|
|
return Err(err).context(FileIoSnafu {
|
|
path,
|
|
op: FileOp::Remove,
|
|
});
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|