diff --git a/Cargo.lock b/Cargo.lock index 91efedab2..e275039a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,7 +56,6 @@ dependencies = [ "ammonia", "anki_i18n", "askama", - "async-compression", "async-trait", "blake3", "bytes", @@ -208,19 +207,6 @@ dependencies = [ "toml", ] -[[package]] -name = "async-compression" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5443ccbb270374a2b1055fc72da40e1f237809cd6bb0e97e66d264cd138473a6" -dependencies = [ - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", -] - [[package]] name = "async-trait" version = "0.1.50" diff --git a/cargo/crates.bzl b/cargo/crates.bzl index 30e18c474..3d8e16295 100644 --- a/cargo/crates.bzl +++ b/cargo/crates.bzl @@ -151,16 +151,6 @@ def raze_fetch_remote_crates(): build_file = Label("//cargo/remote:BUILD.askama_shared-0.11.1.bazel"), ) - maybe( - http_archive, - name = "raze__async_compression__0_3_8", - url = "https://crates.io/api/v1/crates/async-compression/0.3.8/download", - type = "tar.gz", - sha256 = "5443ccbb270374a2b1055fc72da40e1f237809cd6bb0e97e66d264cd138473a6", - strip_prefix = "async-compression-0.3.8", - build_file = Label("//cargo/remote:BUILD.async-compression-0.3.8.bazel"), - ) - maybe( http_archive, name = "raze__async_trait__0_1_50", diff --git a/cargo/licenses.json b/cargo/licenses.json index 902d520e9..0415fc2c3 100644 --- a/cargo/licenses.json +++ b/cargo/licenses.json @@ -152,15 +152,6 @@ "license_file": null, "description": "Shared code for Askama" }, - { - "name": "async-compression", - "version": "0.3.8", - "authors": "Wim Looman |Allen Bui ", - "repository": "https://github.com/Nemo157/async-compression", - "license": "Apache-2.0 OR MIT", - "license_file": null, - "description": "Adaptors between compression crates and Rust's modern asynchronous IO types." - }, { "name": "async-trait", "version": "0.1.50", diff --git a/cargo/remote/BUILD.async-compression-0.3.8.bazel b/cargo/remote/BUILD.async-compression-0.3.8.bazel deleted file mode 100644 index 3adfce2b4..000000000 --- a/cargo/remote/BUILD.async-compression-0.3.8.bazel +++ /dev/null @@ -1,84 +0,0 @@ -""" -@generated -cargo-raze crate build file. - -DO NOT EDIT! Replaced on runs of cargo-raze -""" - -# buildifier: disable=load -load("@bazel_skylib//lib:selects.bzl", "selects") - -# buildifier: disable=load -load( - "@rules_rust//rust:rust.bzl", - "rust_binary", - "rust_library", - "rust_test", -) - -package(default_visibility = [ - # Public for visibility by "@raze__crate__version//" targets. - # - # Prefer access through "//cargo", which limits external - # visibility to explicit Cargo.toml dependencies. - "//visibility:public", -]) - -licenses([ - "notice", # MIT from expression "MIT OR Apache-2.0" -]) - -# Generated Targets - -# Unsupported target "zlib_tokio_02_write" with type "example" omitted - -# Unsupported target "zstd_gzip" with type "example" omitted - -rust_library( - name = "async_compression", - srcs = glob(["**/*.rs"]), - crate_features = [ - "default", - "flate2", - "gzip", - "tokio", - ], - crate_root = "src/lib.rs", - crate_type = "lib", - data = [], - edition = "2018", - rustc_flags = [ - "--cap-lints=allow", - ], - tags = [ - "cargo-raze", - "manual", - ], - version = "0.3.8", - # buildifier: leave-alone - deps = [ - "@raze__flate2__1_0_20//:flate2", - "@raze__futures_core__0_3_15//:futures_core", - "@raze__memchr__2_4_0//:memchr", - "@raze__pin_project_lite__0_2_6//:pin_project_lite", - "@raze__tokio__1_7_1//:tokio", - ], -) - -# Unsupported target "brotli" with type "test" omitted - -# Unsupported target "bzip2" with type "test" omitted - -# Unsupported target "deflate" with type "test" omitted - -# Unsupported target "gzip" with type "test" omitted - -# Unsupported target "lzma" with type "test" omitted - -# Unsupported target "proptest" with type "test" omitted - -# Unsupported target "xz" with type "test" omitted - -# Unsupported target "zlib" with type "test" omitted - -# Unsupported target "zstd" with type "test" omitted diff --git a/ftl/core/sync.ftl b/ftl/core/sync.ftl index 0b2f99d4b..6874a09b1 100644 --- a/ftl/core/sync.ftl +++ b/ftl/core/sync.ftl @@ -41,6 +41,10 @@ sync-account-required = A free account is required to keep your collection synchronized. Please sign up for an account, then enter your details below. sync-sanity-check-failed = Please use the Check Database function, then sync again. If problems persist, please force a full sync in the preferences screen. sync-clock-off = Unable to sync - your clock is not set to the correct time. +sync-upload-too-large = + Your collection file is too large to send to AnkiWeb. You can reduce its + size by removing any unwanted decks (optionally exporting them first), and + then using Check Database to shrink the file size down. ({ $details }) ## Buttons diff --git a/rslib/BUILD.bazel b/rslib/BUILD.bazel index 03b6496fd..988f9f836 100644 --- a/rslib/BUILD.bazel +++ b/rslib/BUILD.bazel @@ -75,7 +75,6 @@ rust_library( ":build_script", "//rslib/cargo:ammonia", "//rslib/cargo:askama", - "//rslib/cargo:async_compression", "//rslib/cargo:blake3", "//rslib/cargo:bytes", "//rslib/cargo:chrono", diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index ab6b4ef7b..11ac0f24c 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -37,7 +37,6 @@ slog-term = "=2.6.0" anki_i18n = { path="i18n" } askama = "0.10.5" -async-compression = { version = "0.3.8", features = ["tokio", "gzip"] } blake3 = "0.3.8" bytes = "1.0.1" chrono = "0.4.19" diff --git a/rslib/cargo/BUILD.bazel b/rslib/cargo/BUILD.bazel index 38ec40354..5d3e72813 100644 --- a/rslib/cargo/BUILD.bazel +++ b/rslib/cargo/BUILD.bazel @@ -30,15 +30,6 @@ alias( ], ) -alias( - name = "async_compression", - actual = "@raze__async_compression__0_3_8//:async_compression", - tags = [ - "cargo-raze", - "manual", - ], -) - alias( name = "async_trait", actual = "@raze__async_trait__0_1_50//:async_trait", diff --git a/rslib/src/error/network.rs b/rslib/src/error/network.rs index e1bed1e8b..3a2177811 100644 --- a/rslib/src/error/network.rs +++ b/rslib/src/error/network.rs @@ -38,6 +38,7 @@ pub enum SyncErrorKind { ResyncRequired, DatabaseCheckRequired, SyncNotStarted, + UploadTooLarge, } impl AnkiError { @@ -152,6 +153,7 @@ impl SyncError { SyncErrorKind::ClockIncorrect => tr.sync_clock_off(), SyncErrorKind::DatabaseCheckRequired => tr.sync_sanity_check_failed(), SyncErrorKind::SyncNotStarted => "sync not started".into(), + SyncErrorKind::UploadTooLarge => tr.sync_upload_too_large(&self.info), } .into() } diff --git a/rslib/src/sync/http_client.rs b/rslib/src/sync/http_client.rs index 7d340f39f..d835bdda6 100644 --- a/rslib/src/sync/http_client.rs +++ b/rslib/src/sync/http_client.rs @@ -1,15 +1,23 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use std::{io::prelude::*, mem::MaybeUninit, path::Path, time::Duration}; +use std::{ + env, + io::{prelude::*, Cursor}, + mem::MaybeUninit, + path::Path, + time::Duration, +}; use async_trait::async_trait; use bytes::Bytes; use flate2::{write::GzEncoder, Compression}; use futures::{Stream, StreamExt}; +use lazy_static::lazy_static; use reqwest::{multipart, Body, Client, Response}; use serde::de::DeserializeOwned; use tempfile::NamedTempFile; +use tokio_util::io::ReaderStream; use super::{ http::{ @@ -22,7 +30,16 @@ use super::{ }; use crate::{error::SyncErrorKind, notes::guid, prelude::*, version::sync_client_version}; -// fixme: 100mb limit +lazy_static! { + // These limits are enforced server-side, but are made adjustable for users + // who are using a custom sync server. + static ref MAXIMUM_UPLOAD_MEGS_UNCOMPRESSED: usize = env::var("MAX_UPLOAD_MEGS_UNCOMP") + .map(|v| v.parse().expect("invalid upload limit")) + .unwrap_or(250); + static ref MAXIMUM_UPLOAD_MEGS_COMPRESSED: usize = env::var("MAX_UPLOAD_MEGS_COMP") + .map(|v| v.parse().expect("invalid upload limit")) + .unwrap_or(100); +} pub type FullSyncProgressFn = Box; @@ -119,23 +136,23 @@ impl SyncServer for HttpSyncClient { async fn full_upload(mut self: Box, col_path: &Path, _can_consume: bool) -> Result<()> { let file = tokio::fs::File::open(col_path).await?; let total_bytes = file.metadata().await?.len() as usize; + check_upload_limit(total_bytes, *MAXIMUM_UPLOAD_MEGS_UNCOMPRESSED)?; + let compressed_data: Vec = gzipped_data_from_tokio_file(file).await?; + let compressed_size = compressed_data.len(); + check_upload_limit(compressed_size, *MAXIMUM_UPLOAD_MEGS_COMPRESSED)?; let progress_fn = self .full_sync_progress_fn .take() .expect("progress func was not set"); - let wrap1 = ProgressWrapper { - reader: file, + let with_progress = ProgressWrapper { + reader: Cursor::new(compressed_data), progress_fn, progress: FullSyncProgress { transferred_bytes: 0, - total_bytes, + total_bytes: compressed_size, }, }; - let wrap2 = - tokio_util::io::ReaderStream::new(async_compression::tokio::bufread::GzipEncoder::new( - tokio_util::io::StreamReader::new(wrap1), - )); - let body = Body::wrap_stream(wrap2); + let body = Body::wrap_stream(with_progress); self.upload_inner(body).await?; Ok(()) @@ -174,6 +191,28 @@ impl SyncServer for HttpSyncClient { } } +async fn gzipped_data_from_tokio_file(file: tokio::fs::File) -> Result> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + let mut stream = ReaderStream::new(file); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + encoder.write_all(&chunk)?; + } + encoder.finish().map_err(Into::into) +} + +fn check_upload_limit(size: usize, limit_mb: usize) -> Result<()> { + let size_mb = size / 1024 / 1024; + if size_mb >= limit_mb { + Err(AnkiError::sync_error( + format!("{}MB > {}MB", size_mb, limit_mb), + SyncErrorKind::UploadTooLarge, + )) + } else { + Ok(()) + } +} + impl HttpSyncClient { pub fn new(hkey: Option, host_number: u32) -> HttpSyncClient { let timeouts = Timeouts::new(); diff --git a/rslib/src/sync/mod.rs b/rslib/src/sync/mod.rs index 83d62374d..c802f2bc2 100644 --- a/rslib/src/sync/mod.rs +++ b/rslib/src/sync/mod.rs @@ -645,7 +645,6 @@ impl Collection { let mut server = HttpSyncClient::new(Some(auth.hkey), auth.host_number); server.set_full_sync_progress_fn(Some(progress_fn)); self.full_upload_inner(Box::new(server)).await - // remote.upload(&col_path, progress_fn).await?; } pub(crate) async fn full_upload_inner(mut self, server: Box) -> Result<()> {