From cf45cbf429f68f089598c892b0027278f9cd230e Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Wed, 18 Jan 2023 12:43:46 +1000 Subject: [PATCH] Rework syncing code, and replace local sync server (#2329) This PR replaces the existing Python-driven sync server with a new one in Rust. The new server supports both collection and media syncing, and is compatible with both the new protocol mentioned below, and older clients. A setting has been added to the preferences screen to point Anki to a local server, and a similar setting is likely to come to AnkiMobile soon. Documentation is available here: In addition to the new server and refactoring, this PR also makes changes to the sync protocol. The existing sync protocol places payloads and metadata inside a multipart POST body, which causes a few headaches: - Legacy clients build the request in a non-deterministic order, meaning the entire request needs to be scanned to extract the metadata. - Reqwest's multipart API directly writes the multipart body, without exposing the resulting stream to us, making it harder to track the progress of the transfer. We've been relying on a patched version of reqwest for timeouts, which is a pain to keep up to date. To address these issues, the metadata is now sent in a HTTP header, with the data payload sent directly in the body. Instead of the slower gzip, we now use zstd. The old timeout handling code has been replaced with a new implementation that wraps the request and response body streams to track progress, allowing us to drop the git dependencies for reqwest, hyper-timeout and tokio-io-timeout. The main other change to the protocol is that one-way syncs no longer need to downgrade the collection to schema 11 prior to sending. --- .config/hakari.toml | 5 - .gitignore | 2 +- .rustfmt.toml | 2 + Cargo.lock | 617 +++++-- Cargo.toml | 4 - build/archives/Cargo.toml | 2 +- cargo/licenses.json | 492 ++++- docs/syncserver.md | 103 -- ftl/core/preferences.ftl | 4 +- proto/anki/sync.proto | 27 +- pylib/anki/_backend.py | 7 +- pylib/anki/_rsbridge.pyi | 1 + pylib/anki/collection.py | 9 +- pylib/anki/syncserver.py | 24 + pylib/anki/syncserver/__init__.py | 195 -- pylib/anki/syncserver/__main__.py | 6 - pylib/rsbridge/lib.rs | 8 + python/write_wheel.py | 6 +- qt/aqt/__init__.py | 15 +- qt/aqt/forms/preferences.ui | 19 + qt/aqt/mediasync.py | 13 +- qt/aqt/preferences.py | 2 + qt/aqt/profiles.py | 30 +- qt/aqt/sync.py | 32 +- rslib/Cargo.toml | 12 +- rslib/i18n_helpers/src/serialize.rs | 4 +- rslib/linkchecker/Cargo.toml | 2 +- rslib/src/backend/ankidroid/mod.rs | 7 +- rslib/src/backend/collection.rs | 5 +- rslib/src/backend/decks.rs | 2 +- rslib/src/backend/import_export.rs | 6 +- rslib/src/backend/media.rs | 22 +- rslib/src/backend/mod.rs | 4 +- rslib/src/backend/notes.rs | 6 +- rslib/src/backend/progress.rs | 9 +- rslib/src/backend/sync/mod.rs | 127 +- rslib/src/backend/sync/server.rs | 211 --- rslib/src/collection/backup.rs | 5 +- rslib/src/collection/mod.rs | 22 +- rslib/src/config/mod.rs | 7 +- rslib/src/decks/schema11.rs | 5 +- rslib/src/error/file_io.rs | 4 +- rslib/src/error/mod.rs | 2 +- rslib/src/error/network.rs | 40 +- .../package/apkg/import/media.rs | 9 +- .../import_export/package/apkg/import/mod.rs | 2 +- .../import_export/package/colpkg/export.rs | 5 +- .../import_export/package/colpkg/import.rs | 10 +- .../src/import_export/package/colpkg/tests.rs | 9 +- rslib/src/import_export/package/media.rs | 5 +- rslib/src/import_export/package/meta.rs | 7 +- rslib/src/io.rs | 9 + rslib/src/lib.rs | 2 +- rslib/src/log.rs | 24 +- rslib/src/media/check.rs | 30 +- rslib/src/media/files.rs | 67 +- rslib/src/media/mod.rs | 138 +- rslib/src/media/sync.rs | 839 --------- rslib/src/notes/mod.rs | 17 +- rslib/src/scheduler/filtered/custom_study.rs | 6 +- rslib/src/storage/card/mod.rs | 2 +- rslib/src/storage/graves/mod.rs | 2 +- rslib/src/storage/mod.rs | 3 +- rslib/src/storage/sqlite.rs | 8 + rslib/src/storage/sync.rs | 29 +- rslib/src/storage/sync_check.rs | 2 +- rslib/src/sync/collection/changes.rs | 327 ++++ rslib/src/sync/collection/chunks.rs | 431 +++++ rslib/src/sync/collection/download.rs | 64 + rslib/src/sync/collection/finish.rs | 43 + rslib/src/sync/collection/graves.rs | 71 + rslib/src/sync/collection/meta.rs | 170 ++ rslib/src/sync/collection/mod.rs | 17 + rslib/src/sync/collection/normal.rs | 182 ++ rslib/src/sync/collection/progress.rs | 39 + rslib/src/sync/collection/protocol.rs | 110 ++ rslib/src/sync/collection/sanity.rs | 129 ++ rslib/src/sync/collection/start.rs | 186 ++ rslib/src/sync/collection/status.rs | 58 + rslib/src/sync/collection/tests.rs | 752 ++++++++ rslib/src/sync/collection/upload.rs | 136 ++ rslib/src/sync/error.rs | 150 ++ rslib/src/sync/http.rs | 124 -- rslib/src/sync/http_client.rs | 499 ----- rslib/src/sync/http_client/full_sync.rs | 82 + rslib/src/sync/http_client/io_monitor.rs | 292 +++ rslib/src/sync/http_client/mod.rs | 124 ++ rslib/src/sync/http_client/protocol.rs | 139 ++ rslib/src/sync/http_server/handlers.rs | 244 +++ rslib/src/sync/http_server/logging.rs | 33 + .../http_server/media_manager/download.rs | 50 + .../src/sync/http_server/media_manager/mod.rs | 58 + .../sync/http_server/media_manager/upload.rs | 96 + rslib/src/sync/http_server/mod.rs | 176 ++ rslib/src/sync/http_server/routes.rs | 108 ++ rslib/src/sync/http_server/user.rs | 95 + rslib/src/sync/login.rs | 58 + rslib/src/sync/media/begin.rs | 34 + rslib/src/sync/media/changes.rs | 135 ++ .../media/database/client}/changetracker.rs | 56 +- .../media/database/client/mod.rs} | 289 +-- .../media/database/client}/schema.sql | 0 rslib/src/sync/media/database/mod.rs | 5 + .../media/database/server/entry/changes.rs | 28 + .../media/database/server/entry/changes.sql | 11 + .../media/database/server/entry/download.rs | 59 + .../media/database/server/entry/get_entry.sql | 7 + .../sync/media/database/server/entry/mod.rs | 119 ++ .../media/database/server/entry/set_entry.sql | 3 + .../media/database/server/entry/upload.rs | 102 ++ .../media/database/server/meta/get_meta.sql | 4 + .../sync/media/database/server/meta/mod.rs | 113 ++ .../media/database/server/meta/set_meta.sql | 4 + rslib/src/sync/media/database/server/mod.rs | 40 + .../sync/media/database/server/schema_v3.sql | 14 + .../sync/media/database/server/schema_v4.sql | 46 + rslib/src/sync/media/download.rs | 50 + rslib/src/sync/media/mod.rs | 32 + rslib/src/sync/media/progress.rs | 11 + rslib/src/sync/media/protocol.rs | 103 ++ rslib/src/sync/media/sanity.rs | 17 + rslib/src/sync/media/syncer.rs | 291 +++ rslib/src/sync/media/tests.rs | 292 +++ rslib/src/sync/media/upload.rs | 111 ++ rslib/src/sync/media/zip.rs | 156 ++ rslib/src/sync/mod.rs | 1617 +---------------- rslib/src/sync/request/header_and_stream.rs | 155 ++ rslib/src/sync/request/mod.rs | 187 ++ rslib/src/sync/request/multipart.rs | 98 + rslib/src/sync/response.rs | 90 + rslib/src/sync/server.rs | 233 --- rslib/src/sync/version.rs | 80 + rslib/src/tests.rs | 3 +- rslib/src/version.rs | 12 + tools/workspace-hack/Cargo.toml | 28 +- 135 files changed, 8490 insertions(+), 4540 deletions(-) delete mode 100644 docs/syncserver.md create mode 100644 pylib/anki/syncserver.py delete mode 100644 pylib/anki/syncserver/__init__.py delete mode 100644 pylib/anki/syncserver/__main__.py delete mode 100644 rslib/src/backend/sync/server.rs delete mode 100644 rslib/src/media/sync.rs create mode 100644 rslib/src/sync/collection/changes.rs create mode 100644 rslib/src/sync/collection/chunks.rs create mode 100644 rslib/src/sync/collection/download.rs create mode 100644 rslib/src/sync/collection/finish.rs create mode 100644 rslib/src/sync/collection/graves.rs create mode 100644 rslib/src/sync/collection/meta.rs create mode 100644 rslib/src/sync/collection/mod.rs create mode 100644 rslib/src/sync/collection/normal.rs create mode 100644 rslib/src/sync/collection/progress.rs create mode 100644 rslib/src/sync/collection/protocol.rs create mode 100644 rslib/src/sync/collection/sanity.rs create mode 100644 rslib/src/sync/collection/start.rs create mode 100644 rslib/src/sync/collection/status.rs create mode 100644 rslib/src/sync/collection/tests.rs create mode 100644 rslib/src/sync/collection/upload.rs create mode 100644 rslib/src/sync/error.rs delete mode 100644 rslib/src/sync/http.rs delete mode 100644 rslib/src/sync/http_client.rs create mode 100644 rslib/src/sync/http_client/full_sync.rs create mode 100644 rslib/src/sync/http_client/io_monitor.rs create mode 100644 rslib/src/sync/http_client/mod.rs create mode 100644 rslib/src/sync/http_client/protocol.rs create mode 100644 rslib/src/sync/http_server/handlers.rs create mode 100644 rslib/src/sync/http_server/logging.rs create mode 100644 rslib/src/sync/http_server/media_manager/download.rs create mode 100644 rslib/src/sync/http_server/media_manager/mod.rs create mode 100644 rslib/src/sync/http_server/media_manager/upload.rs create mode 100644 rslib/src/sync/http_server/mod.rs create mode 100644 rslib/src/sync/http_server/routes.rs create mode 100644 rslib/src/sync/http_server/user.rs create mode 100644 rslib/src/sync/login.rs create mode 100644 rslib/src/sync/media/begin.rs create mode 100644 rslib/src/sync/media/changes.rs rename rslib/src/{media => sync/media/database/client}/changetracker.rs (88%) rename rslib/src/{media/database.rs => sync/media/database/client/mod.rs} (53%) rename rslib/src/{media => sync/media/database/client}/schema.sql (100%) create mode 100644 rslib/src/sync/media/database/mod.rs create mode 100644 rslib/src/sync/media/database/server/entry/changes.rs create mode 100644 rslib/src/sync/media/database/server/entry/changes.sql create mode 100644 rslib/src/sync/media/database/server/entry/download.rs create mode 100644 rslib/src/sync/media/database/server/entry/get_entry.sql create mode 100644 rslib/src/sync/media/database/server/entry/mod.rs create mode 100644 rslib/src/sync/media/database/server/entry/set_entry.sql create mode 100644 rslib/src/sync/media/database/server/entry/upload.rs create mode 100644 rslib/src/sync/media/database/server/meta/get_meta.sql create mode 100644 rslib/src/sync/media/database/server/meta/mod.rs create mode 100644 rslib/src/sync/media/database/server/meta/set_meta.sql create mode 100644 rslib/src/sync/media/database/server/mod.rs create mode 100644 rslib/src/sync/media/database/server/schema_v3.sql create mode 100644 rslib/src/sync/media/database/server/schema_v4.sql create mode 100644 rslib/src/sync/media/download.rs create mode 100644 rslib/src/sync/media/mod.rs create mode 100644 rslib/src/sync/media/progress.rs create mode 100644 rslib/src/sync/media/protocol.rs create mode 100644 rslib/src/sync/media/sanity.rs create mode 100644 rslib/src/sync/media/syncer.rs create mode 100644 rslib/src/sync/media/tests.rs create mode 100644 rslib/src/sync/media/upload.rs create mode 100644 rslib/src/sync/media/zip.rs create mode 100644 rslib/src/sync/request/header_and_stream.rs create mode 100644 rslib/src/sync/request/mod.rs create mode 100644 rslib/src/sync/request/multipart.rs create mode 100644 rslib/src/sync/response.rs delete mode 100644 rslib/src/sync/server.rs create mode 100644 rslib/src/sync/version.rs diff --git a/.config/hakari.toml b/.config/hakari.toml index 6ff0af81a..1858c8428 100644 --- a/.config/hakari.toml +++ b/.config/hakari.toml @@ -1,8 +1,3 @@ hakari-package = "workspace-hack" dep-format-version = "2" resolver = "2" - -[traversal-excludes] -third-party = [ - { name = "reqwest", git = "https://github.com/ankitects/reqwest.git", rev = "7591444614de02b658ddab125efba7b2bb4e2335" }, -] diff --git a/.gitignore b/.gitignore index 0e9577c18..1d70eaee9 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ target /user.bazelrc .dmypy.json /.idea/ -/.vscode/ +/.vscode /.bazel /windows.bazelrc /out diff --git a/.rustfmt.toml b/.rustfmt.toml index a19194a0b..e4645a37f 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -2,5 +2,7 @@ # useful for manual invocation with 'cargo +nightly fmt' imports_granularity = "Crate" group_imports = "StdExternalCrate" + +# wrap_comments = true # imports_granularity = "Item" # imports_layout = "Vertical" diff --git a/Cargo.lock b/Cargo.lock index 1537749eb..9569a1452 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,7 +84,11 @@ dependencies = [ "ammonia", "anki_i18n", "ascii_percent_encoding", + "async-compression", + "async-stream", "async-trait", + "axum", + "axum-client-ip", "blake3", "bytes", "chrono", @@ -101,6 +105,7 @@ dependencies = [ "futures", "hex", "htmlescape", + "hyper", "id_tree", "itertools", "lazy_static", @@ -130,6 +135,7 @@ dependencies = [ "tempfile", "tokio", "tokio-util", + "tower-http", "tracing", "tracing-appender", "tracing-subscriber", @@ -138,6 +144,7 @@ dependencies = [ "unicode-normalization", "utime", "which", + "wiremock", "workspace-hack", "zip", "zstd 0.12.1+zstd.1.5.2", @@ -223,10 +230,66 @@ name = "ascii_percent_encoding" version = "0.0.0" [[package]] -name = "async-trait" -version = "0.1.59" +name = "assert-json-diff" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "async-channel" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-compression" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942c7cd7ae39e91bde4820d74132e9862e62c2f386c3aa90ccf55949f5bad63a" +dependencies = [ + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "zstd 0.11.2+zstd.1.5.2", + "zstd-safe 5.0.2+zstd.1.5.2", +] + +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "async-trait" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d1d8ab452a3936018a687b20e6f7cf5363d713b732b8884001317b0e48aa3" dependencies = [ "proc-macro2", "quote", @@ -250,6 +313,81 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "axum" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08b108ad2665fa3f6e6a517c3d80ec3e77d224c47d605167aefaa5d7ef97fa48" +dependencies = [ + "async-trait", + "axum-core", + "axum-macros", + "bitflags", + "bytes", + "futures-util", + "headers", + "http", + "http-body", + "hyper", + "itoa 1.0.5", + "matchit", + "memchr", + "mime", + "multer", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-client-ip" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfb5a3ddd6367075d50629546fb46710584016ae7704cd03b6d41cb5be82e5a" +dependencies = [ + "axum", + "forwarded-header-value", +] + +[[package]] +name = "axum-core" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b8558f5a0581152dc94dcd289132a1d377494bdeafcd41869b3258e3e2ad92" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4df0fc33ada14a338b799002f7e8657711422b25d4e16afb032708d6b185621" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "backtrace" version = "0.3.66" @@ -271,6 +409,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" + [[package]] name = "base64ct" version = "1.5.3" @@ -551,6 +695,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "concurrent-queue" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd7bef69dc86e3c610e4e7aed41035e2a7ed12e72dd7530f61327a6579a4390b" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "configure" version = "0.0.0" @@ -743,7 +896,7 @@ source = "git+https://github.com/ankitects/rust-csv.git?rev=1c9d3aab6f79a7d815c6 dependencies = [ "bstr", "csv-core", - "itoa 1.0.4", + "itoa 1.0.5", "ryu", "serde", ] @@ -800,6 +953,25 @@ dependencies = [ "syn", ] +[[package]] +name = "deadpool" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e" +dependencies = [ + "async-trait", + "deadpool-runtime", + "num_cpus", + "retain_mut", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaa37046cc0f6c3cc6090fbdbf73ef0b8ef4cfcc37f6befc0020f63e8cf121e1" + [[package]] name = "derive_more" version = "0.99.17" @@ -938,6 +1110,12 @@ dependencies = [ "libc", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "fallible-iterator" version = "0.2.0" @@ -1072,6 +1250,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "forwarded-header-value" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" +dependencies = [ + "nonempty", + "thiserror", +] + [[package]] name = "fs2" version = "0.4.3" @@ -1149,6 +1337,21 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +[[package]] +name = "futures-lite" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.25" @@ -1172,6 +1375,12 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + [[package]] name = "futures-util" version = "0.3.25" @@ -1308,6 +1517,31 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "headers" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" +dependencies = [ + "base64 0.13.1", + "bitflags", + "bytes", + "headers-core", + "http", + "httpdate", + "mime", + "sha1", +] + +[[package]] +name = "headers-core" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" +dependencies = [ + "http", +] + [[package]] name = "heck" version = "0.4.0" @@ -1389,7 +1623,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", - "itoa 1.0.4", + "itoa 1.0.5", ] [[package]] @@ -1403,6 +1637,33 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfe8eed0a9285ef776bb792479ea3834e8b94e13d615c2f66d03dd50a435a29" + +[[package]] +name = "http-types" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" +dependencies = [ + "anyhow", + "async-channel", + "base64 0.13.1", + "futures-lite", + "http", + "infer", + "pin-project-lite", + "rand 0.7.3", + "serde", + "serde_json", + "serde_qs", + "serde_urlencoded", + "url", +] + [[package]] name = "httparse" version = "1.8.0" @@ -1436,7 +1697,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.4", + "itoa 1.0.5", "pin-project-lite", "socket2", "tokio", @@ -1447,28 +1708,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.22.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ - "futures-util", + "http", "hyper", - "log", "rustls", "tokio", "tokio-rustls", - "webpki", -] - -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "git+https://github.com/ankitects/hyper-timeout.git?rev=0cb6f7d14c62819e37cd221736f8b0555e823712#0cb6f7d14c62819e37cd221736f8b0555e823712" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", ] [[package]] @@ -1543,6 +1791,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adab1eaa3408fb7f0c777a73e7465fd5656136fc93b670eb6df3c88c2c1344e3" +[[package]] +name = "infer" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" + [[package]] name = "inflections" version = "1.1.1" @@ -1632,9 +1886,9 @@ checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" [[package]] name = "itoa" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" [[package]] name = "jobserver" @@ -1684,9 +1938,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.138" +version = "0.2.139" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8" +checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libsqlite3-sys" @@ -1884,6 +2138,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" +[[package]] +name = "matchit" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b87248edafb776e59e6ee64a79086f65890d3510f2c656c000bf2a7e8a0aea40" + [[package]] name = "memchr" version = "2.5.0" @@ -1960,6 +2220,24 @@ dependencies = [ "windows-sys 0.42.0", ] +[[package]] +name = "multer" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed4198ce7a4cbd2a57af78d28c6fbb57d81ac5f1d6ad79ac6c5587419cbdf22" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "log", + "memchr", + "mime", + "spin 0.9.4", + "version_check", +] + [[package]] name = "multimap" version = "0.8.3" @@ -2021,6 +2299,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonempty" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2038,7 +2322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" dependencies = [ "arrayvec", - "itoa 1.0.4", + "itoa 1.0.5", ] [[package]] @@ -2062,11 +2346,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi 0.1.19", + "hermit-abi 0.2.6", "libc", ] @@ -2091,15 +2375,6 @@ dependencies = [ "syn", ] -[[package]] -name = "num_threads" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" -dependencies = [ - "libc", -] - [[package]] name = "object" version = "0.29.0" @@ -2111,9 +2386,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "oorandom" @@ -2211,6 +2486,12 @@ dependencies = [ "yasna", ] +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.12.1" @@ -2263,7 +2544,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03c64931a1a212348ec4f3b4362585eca7159d0d09cbdf4a7f74f02173596fd4" dependencies = [ - "base64", + "base64 0.13.1", ] [[package]] @@ -2460,7 +2741,7 @@ version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd39bc6cdc9355ad1dc5eeedefee696bb35c34caf21768741e81826c0bbd7225" dependencies = [ - "base64", + "base64 0.13.1", "indexmap", "line-wrap", "serde", @@ -2561,9 +2842,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.47" +version = "1.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +checksum = "57a8eca9f9c4ffde41714334dee777596264c7825420f521abc92b5b5deb63a5" dependencies = [ "unicode-ident", ] @@ -2709,9 +2990,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.21" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" dependencies = [ "proc-macro2", ] @@ -2887,31 +3168,33 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.3" -source = "git+https://github.com/ankitects/reqwest.git?rev=7591444614de02b658ddab125efba7b2bb4e2335#7591444614de02b658ddab125efba7b2bb4e2335" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68cc60575865c7831548863cc02356512e3f1dc2f3f82cb837d7fc4cc8f3c97c" dependencies = [ - "base64", + "base64 0.13.1", "bytes", "encoding_rs", "futures-core", "futures-util", + "h2", "http", "http-body", "hyper", "hyper-rustls", - "hyper-timeout", "hyper-tls", "ipnet", "js-sys", - "lazy_static", "log", "mime", "mime_guess", "native-tls", + "once_cell", "percent-encoding", "pin-project-lite", "rustls", "rustls-native-certs", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", @@ -2919,14 +3202,22 @@ dependencies = [ "tokio-native-tls", "tokio-rustls", "tokio-socks", + "tokio-util", + "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", "webpki-roots", - "winreg 0.7.0", + "winreg 0.10.1", ] +[[package]] +name = "retain_mut" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" + [[package]] name = "ring" version = "0.16.20" @@ -2936,7 +3227,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -3013,11 +3304,10 @@ dependencies = [ [[package]] name = "rustls" -version = "0.19.1" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" +checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" dependencies = [ - "base64", "log", "ring", "sct", @@ -3026,27 +3316,36 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.5.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" +checksum = "0167bac7a9f490495f3c33013e7722b53cb087ecbe082fb0c6387c96f634ea50" dependencies = [ "openssl-probe", - "rustls", + "rustls-pemfile", "schannel", "security-framework", ] [[package]] -name = "rustversion" -version = "1.0.9" +name = "rustls-pemfile" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" +checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +dependencies = [ + "base64 0.21.0", +] + +[[package]] +name = "rustversion" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" [[package]] name = "ryu" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" [[package]] name = "safemem" @@ -3087,9 +3386,9 @@ checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" [[package]] name = "sct" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" dependencies = [ "ring", "untrusted", @@ -3152,9 +3451,9 @@ checksum = "e25dfac463d778e353db5be2449d1cce89bd6fd23c9f1ea21310ce6e5a1b29c4" [[package]] name = "serde" -version = "1.0.149" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" +checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" dependencies = [ "serde_derive", ] @@ -3172,9 +3471,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.149" +version = "1.0.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" +checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" dependencies = [ "proc-macro2", "quote", @@ -3183,15 +3482,35 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.89" +version = "1.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "020ff22c755c2ed3f8cf162dbb41a7268d934702f3ed3631656ea597e08fc3db" +checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" dependencies = [ - "itoa 1.0.4", + "itoa 1.0.5", "ryu", "serde", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b04f22b563c91331a10074bda3dd5492e3cc39d56bd557e91c0af42b6c7341" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_qs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" +dependencies = [ + "percent-encoding", + "serde", + "thiserror", +] + [[package]] name = "serde_repr" version = "0.1.9" @@ -3231,7 +3550,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.4", + "itoa 1.0.5", "ryu", "serde", ] @@ -3368,6 +3687,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6002a767bff9e83f8eeecf883ecb8011875a21ae8da43bffb817a57e78cc09" + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -3436,15 +3761,21 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.105" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" +checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20518fe4a4c9acf048008599e464deb21beeae3d3578418951a189c235a7a9a8" + [[package]] name = "tar" version = "0.4.38" @@ -3510,18 +3841,18 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" [[package]] name = "thiserror" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" dependencies = [ "proc-macro2", "quote", @@ -3543,9 +3874,7 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" dependencies = [ - "itoa 1.0.4", - "libc", - "num_threads", + "itoa 1.0.5", "serde", "time-core", "time-macros", @@ -3602,9 +3931,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.23.1" +version = "1.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38a54aca0c15d014013256222ba0ebed095673f89345dd79119d912eb561b7a8" +checksum = "1d9f76183f91ecfb55e1d7d5602bd1d979e38a3a522fe900241cf195624d67ae" dependencies = [ "autocfg", "bytes", @@ -3620,15 +3949,6 @@ dependencies = [ "windows-sys 0.42.0", ] -[[package]] -name = "tokio-io-timeout" -version = "1.1.1" -source = "git+https://github.com/ankitects/tokio-io-timeout.git?rev=1ee0892217e9a76bba4bb369ec5fab8854935a3c#1ee0892217e9a76bba4bb369ec5fab8854935a3c" -dependencies = [ - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-macros" version = "1.8.2" @@ -3652,9 +3972,9 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.22.0" +version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" +checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ "rustls", "tokio", @@ -3696,6 +4016,48 @@ dependencies = [ "serde", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -3709,6 +4071,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -3950,9 +4313,9 @@ checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" [[package]] name = "unicode-ident" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" [[package]] name = "unicode-normalization" @@ -4033,6 +4396,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "walkdir" version = "2.3.2" @@ -4073,8 +4442,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" dependencies = [ "cfg-if", - "serde", - "serde_json", "wasm-bindgen-macro", ] @@ -4146,9 +4513,9 @@ dependencies = [ [[package]] name = "webpki" -version = "0.21.4" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" dependencies = [ "ring", "untrusted", @@ -4156,9 +4523,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.21.1" +version = "0.22.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" dependencies = [ "webpki", ] @@ -4317,13 +4684,35 @@ dependencies = [ [[package]] name = "winreg" -version = "0.7.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] +[[package]] +name = "wiremock" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12316b50eb725e22b2f6b9c4cbede5b7b89984274d113a7440c86e5c3fc6f99b" +dependencies = [ + "assert-json-diff", + "async-trait", + "base64 0.13.1", + "deadpool", + "futures", + "futures-timer", + "http-types", + "hyper", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "workspace-hack" version = "0.1.0" @@ -4334,29 +4723,43 @@ dependencies = [ "crossbeam-utils", "either", "flate2", + "futures-channel", + "futures-io", + "futures-task", + "futures-util", "getrandom 0.2.8", "hashbrown", "hmac", + "hyper", "indexmap", "log", "num-traits", "phf_shared 0.11.1", "prost", + "rand 0.7.3", "rand 0.8.5", "rand_core 0.6.4", "regex", + "regex-automata", "regex-syntax", + "reqwest", "scopeguard", "serde", + "serde_json", "sha2", "snafu", "syn", "time", "time-macros", "tokio", + "tokio-util", + "tracing", + "tracing-core", "url", "zip", + "zstd 0.11.2+zstd.1.5.2", "zstd 0.12.1+zstd.1.5.2", + "zstd-safe 5.0.2+zstd.1.5.2", "zstd-safe 6.0.2+zstd.1.5.2", "zstd-sys", ] diff --git a/Cargo.toml b/Cargo.toml index e39117f5f..4e5fef7a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,10 +24,6 @@ members = [ exclude = ["qt/bundle"] resolver = "2" -[patch.crates-io] -# If updating rev, hakari.toml needs updating too. -reqwest = { git = "https://github.com/ankitects/reqwest.git", rev = "7591444614de02b658ddab125efba7b2bb4e2335" } - # Apply mild optimizations to our dependencies in dev mode, which among other things # improves sha2 performance by about 21x. Opt 1 chosen due to # https://doc.rust-lang.org/cargo/reference/profiles.html#overrides-and-generics. This diff --git a/build/archives/Cargo.toml b/build/archives/Cargo.toml index edb4af65a..20f3069cc 100644 --- a/build/archives/Cargo.toml +++ b/build/archives/Cargo.toml @@ -24,5 +24,5 @@ rustls = ["reqwest/rustls-tls", "reqwest/rustls-tls-native-roots"] native-tls = ["reqwest/native-tls"] [dependencies.reqwest] -version = "=0.11.3" +version = "0.11.13" default-features = false diff --git a/cargo/licenses.json b/cargo/licenses.json index 72c6de27f..d05f5c6da 100644 --- a/cargo/licenses.json +++ b/cargo/licenses.json @@ -116,9 +116,54 @@ "license_file": null, "description": "Like percent_encoding, but does not encode non-ASCII characters." }, + { + "name": "assert-json-diff", + "version": "2.0.2", + "authors": "David Pedersen ", + "repository": "https://github.com/davidpdrsn/assert-json-diff.git", + "license": "MIT", + "license_file": null, + "description": "Easily compare two JSON values and get great output" + }, + { + "name": "async-channel", + "version": "1.8.0", + "authors": "Stjepan Glavina ", + "repository": "https://github.com/smol-rs/async-channel", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Async multi-producer multi-consumer channel" + }, + { + "name": "async-compression", + "version": "0.3.15", + "authors": "Wim Looman |Allen Bui ", + "repository": "https://github.com/Nemo157/async-compression", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Adaptors between compression crates and Rust's modern asynchronous IO types." + }, + { + "name": "async-stream", + "version": "0.3.3", + "authors": "Carl Lerche ", + "repository": "https://github.com/tokio-rs/async-stream", + "license": "MIT", + "license_file": null, + "description": "Asynchronous streams using async & await notation" + }, + { + "name": "async-stream-impl", + "version": "0.3.3", + "authors": "Carl Lerche ", + "repository": "https://github.com/tokio-rs/async-stream", + "license": "MIT", + "license_file": null, + "description": "proc macros for async-stream crate" + }, { "name": "async-trait", - "version": "0.1.59", + "version": "0.1.60", "authors": "David Tolnay ", "repository": "https://github.com/dtolnay/async-trait", "license": "Apache-2.0 OR MIT", @@ -134,6 +179,42 @@ "license_file": null, "description": "Automatic cfg for Rust compiler features" }, + { + "name": "axum", + "version": "0.6.1", + "authors": null, + "repository": "https://github.com/tokio-rs/axum", + "license": "MIT", + "license_file": null, + "description": "Web framework that focuses on ergonomics and modularity" + }, + { + "name": "axum-client-ip", + "version": "0.3.1", + "authors": null, + "repository": "https://github.com/imbolc/axum-client-ip", + "license": "MIT", + "license_file": null, + "description": "A client IP address extractor for Axum" + }, + { + "name": "axum-core", + "version": "0.3.0", + "authors": null, + "repository": "https://github.com/tokio-rs/axum", + "license": "MIT", + "license_file": null, + "description": "Core types and traits for axum" + }, + { + "name": "axum-macros", + "version": "0.3.0", + "authors": null, + "repository": "https://github.com/tokio-rs/axum", + "license": "MIT", + "license_file": null, + "description": "Macros for axum" + }, { "name": "backtrace", "version": "0.3.66", @@ -152,6 +233,15 @@ "license_file": null, "description": "encodes and decodes base64 as bytes or utf8" }, + { + "name": "base64", + "version": "0.21.0", + "authors": "Alice Maz |Marshall Pierce ", + "repository": "https://github.com/marshallpierce/rust-base64", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "encodes and decodes base64 as bytes or utf8" + }, { "name": "base64ct", "version": "1.5.3", @@ -296,6 +386,15 @@ "license_file": null, "description": "Beautiful diagnostic reporting for text-based programming languages" }, + { + "name": "concurrent-queue", + "version": "2.0.0", + "authors": "Stjepan Glavina |Taiki Endo |John Nunley ", + "repository": "https://github.com/smol-rs/concurrent-queue", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Concurrent multi-producer multi-consumer queue" + }, { "name": "constant_time_eq", "version": "0.1.5", @@ -440,6 +539,24 @@ "license_file": null, "description": "Implementation detail of the `cxx` crate." }, + { + "name": "deadpool", + "version": "0.9.5", + "authors": "Michael P. Jung ", + "repository": "https://github.com/bikeshedder/deadpool", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Dead simple async pool" + }, + { + "name": "deadpool-runtime", + "version": "0.1.2", + "authors": "Michael P. Jung ", + "repository": "https://github.com/bikeshedder/deadpool", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Dead simple async pool utitities for sync managers" + }, { "name": "digest", "version": "0.10.6", @@ -521,6 +638,15 @@ "license_file": null, "description": "Exposes errno functionality to stable Rust on DragonFlyBSD" }, + { + "name": "event-listener", + "version": "2.5.3", + "authors": "Stjepan Glavina ", + "repository": "https://github.com/smol-rs/event-listener", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Notify async tasks or threads" + }, { "name": "fallible-iterator", "version": "0.2.0", @@ -638,6 +764,15 @@ "license_file": null, "description": "Parser and serializer for the application/x-www-form-urlencoded syntax, as used by HTML forms." }, + { + "name": "forwarded-header-value", + "version": "0.1.1", + "authors": "James Brown ", + "repository": "https://github.com/EasyPost/rust-forwarded-header-value", + "license": "ISC", + "license_file": null, + "description": "Parser for values from the Forwarded header (RFC 7239)" + }, { "name": "futf", "version": "0.1.5", @@ -692,6 +827,15 @@ "license_file": null, "description": "The `AsyncRead`, `AsyncWrite`, `AsyncSeek`, and `AsyncBufRead` traits for the futures-rs library." }, + { + "name": "futures-lite", + "version": "1.12.0", + "authors": "Stjepan Glavina |Contributors to futures-rs", + "repository": "https://github.com/smol-rs/futures-lite", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Futures, streams, and async I/O combinators" + }, { "name": "futures-macro", "version": "0.3.25", @@ -719,6 +863,15 @@ "license_file": null, "description": "Tools for working with tasks." }, + { + "name": "futures-timer", + "version": "3.0.2", + "authors": "Alex Crichton ", + "repository": "https://github.com/async-rs/futures-timer", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Timeouts for futures." + }, { "name": "futures-util", "version": "0.3.25", @@ -746,6 +899,15 @@ "license_file": null, "description": "getopts-like option parsing." }, + { + "name": "getrandom", + "version": "0.1.16", + "authors": "The Rand Project Developers", + "repository": "https://github.com/rust-random/getrandom", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "A small cross-platform library for retrieving random data from system source" + }, { "name": "getrandom", "version": "0.2.8", @@ -791,6 +953,24 @@ "license_file": null, "description": "HashMap-like containers that hold their key-value pairs in a user controllable order" }, + { + "name": "headers", + "version": "0.3.8", + "authors": "Sean McArthur ", + "repository": "https://github.com/hyperium/headers", + "license": "MIT", + "license_file": null, + "description": "typed HTTP headers" + }, + { + "name": "headers-core", + "version": "0.2.0", + "authors": "Sean McArthur ", + "repository": "https://github.com/hyperium/headers", + "license": "MIT", + "license_file": null, + "description": "typed HTTP headers core trait" + }, { "name": "heck", "version": "0.4.0", @@ -800,15 +980,6 @@ "license_file": null, "description": "heck is a case conversion library." }, - { - "name": "hermit-abi", - "version": "0.1.19", - "authors": "Stefan Lankes", - "repository": "https://github.com/hermitcore/libhermit-rs", - "license": "Apache-2.0 OR MIT", - "license_file": null, - "description": "hermit-abi is small interface to call functions from the unikernel RustyHermit. It is used to build the target `x86_64-unknown-hermit`." - }, { "name": "hermit-abi", "version": "0.2.6", @@ -872,6 +1043,24 @@ "license_file": null, "description": "Trait representing an asynchronous, streaming, HTTP request or response body." }, + { + "name": "http-range-header", + "version": "0.3.0", + "authors": null, + "repository": "https://github.com/MarcusGrass/parse-range-headers", + "license": "MIT", + "license_file": null, + "description": "No-dep range header parser" + }, + { + "name": "http-types", + "version": "2.12.0", + "authors": "Yoshua Wuyts ", + "repository": "https://github.com/http-rs/http-types", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Common types for HTTP operations." + }, { "name": "httparse", "version": "1.8.0", @@ -910,22 +1099,13 @@ }, { "name": "hyper-rustls", - "version": "0.22.1", - "authors": "Joseph Birr-Pixton ", + "version": "0.23.2", + "authors": null, "repository": "https://github.com/ctz/hyper-rustls", "license": "Apache-2.0 OR ISC OR MIT", "license_file": null, "description": "Rustls+hyper integration for pure rust HTTPS" }, - { - "name": "hyper-timeout", - "version": "0.4.1", - "authors": "Herman J. Radtke III ", - "repository": "https://github.com/hjr3/hyper-timeout", - "license": "Apache-2.0 OR MIT", - "license_file": null, - "description": "A connect, read and write timeout aware connector to be used with hyper Client." - }, { "name": "hyper-tls", "version": "0.5.0", @@ -980,6 +1160,15 @@ "license_file": null, "description": "A hash table with consistent order and fast iteration." }, + { + "name": "infer", + "version": "0.2.3", + "authors": "Bojan ", + "repository": "https://github.com/bojand/infer", + "license": "MIT", + "license_file": null, + "description": "Small crate to infer file types based on its magic number signature" + }, { "name": "inflections", "version": "1.1.1", @@ -1054,7 +1243,7 @@ }, { "name": "itoa", - "version": "1.0.4", + "version": "1.0.5", "authors": "David Tolnay ", "repository": "https://github.com/dtolnay/itoa", "license": "Apache-2.0 OR MIT", @@ -1090,7 +1279,7 @@ }, { "name": "libc", - "version": "0.2.138", + "version": "0.2.139", "authors": "The Rust Project Developers", "repository": "https://github.com/rust-lang/libc", "license": "Apache-2.0 OR MIT", @@ -1187,6 +1376,15 @@ "license_file": null, "description": "A macro to evaluate, as a boolean, whether an expression matches a pattern." }, + { + "name": "matchit", + "version": "0.7.0", + "authors": "Ibraheem Ahmed ", + "repository": "https://github.com/ibraheemdev/matchit", + "license": "MIT", + "license_file": null, + "description": "A blazing fast URL router." + }, { "name": "memchr", "version": "2.5.0", @@ -1250,6 +1448,15 @@ "license_file": null, "description": "Lightweight non-blocking IO" }, + { + "name": "multer", + "version": "2.0.4", + "authors": "Rousan Ali ", + "repository": "https://github.com/rousan/multer-rs", + "license": "MIT", + "license_file": null, + "description": "An async parser for `multipart/form-data` content-type in Rust." + }, { "name": "multimap", "version": "0.8.3", @@ -1286,6 +1493,15 @@ "license_file": null, "description": "A byte-oriented, zero-copy, parser combinators library" }, + { + "name": "nonempty", + "version": "0.7.0", + "authors": "Alexis Sellier ", + "repository": "https://github.com/cloudhead/nonempty", + "license": "MIT", + "license_file": null, + "description": "Correct by construction non-empty vector" + }, { "name": "nu-ansi-term", "version": "0.46.0", @@ -1324,7 +1540,7 @@ }, { "name": "num_cpus", - "version": "1.14.0", + "version": "1.15.0", "authors": "Sean McArthur ", "repository": "https://github.com/seanmonstar/num_cpus", "license": "Apache-2.0 OR MIT", @@ -1349,15 +1565,6 @@ "license_file": null, "description": "Internal implementation details for ::num_enum (Procedural macros to make inter-operation between primitives and enums easier)" }, - { - "name": "num_threads", - "version": "0.1.6", - "authors": "Jacob Pratt ", - "repository": "https://github.com/jhpratt/num_threads", - "license": "Apache-2.0 OR MIT", - "license_file": null, - "description": "A minimal library that determines the number of running threads for the current process." - }, { "name": "object", "version": "0.29.0", @@ -1369,7 +1576,7 @@ }, { "name": "once_cell", - "version": "1.16.0", + "version": "1.17.0", "authors": "Aleksey Kladov ", "repository": "https://github.com/matklad/once_cell", "license": "Apache-2.0 OR MIT", @@ -1430,6 +1637,15 @@ "license_file": null, "description": "Provides a macro to simplify operator overloading." }, + { + "name": "parking", + "version": "2.0.0", + "authors": "Stjepan Glavina |The Rust Project Developers", + "repository": "https://github.com/stjepang/parking", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Thread parking and unparking" + }, { "name": "parking_lot", "version": "0.12.1", @@ -1648,7 +1864,7 @@ }, { "name": "proc-macro2", - "version": "1.0.47", + "version": "1.0.49", "authors": "David Tolnay |Alex Crichton ", "repository": "https://github.com/dtolnay/proc-macro2", "license": "Apache-2.0 OR MIT", @@ -1702,13 +1918,22 @@ }, { "name": "quote", - "version": "1.0.21", + "version": "1.0.23", "authors": "David Tolnay ", "repository": "https://github.com/dtolnay/quote", "license": "Apache-2.0 OR MIT", "license_file": null, "description": "Quasi-quoting macro quote!(...)" }, + { + "name": "rand", + "version": "0.7.3", + "authors": "The Rand Project Developers|The Rust Project Developers", + "repository": "https://github.com/rust-random/rand", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Random number generators and other randomness functionality." + }, { "name": "rand", "version": "0.8.5", @@ -1718,6 +1943,15 @@ "license_file": null, "description": "Random number generators and other randomness functionality." }, + { + "name": "rand_chacha", + "version": "0.2.2", + "authors": "The Rand Project Developers|The Rust Project Developers|The CryptoCorrosion Contributors", + "repository": "https://github.com/rust-random/rand", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "ChaCha random number generator" + }, { "name": "rand_chacha", "version": "0.3.1", @@ -1727,6 +1961,15 @@ "license_file": null, "description": "ChaCha random number generator" }, + { + "name": "rand_core", + "version": "0.5.1", + "authors": "The Rand Project Developers|The Rust Project Developers", + "repository": "https://github.com/rust-random/rand", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Core random number generator traits and tools for implementation." + }, { "name": "rand_core", "version": "0.6.4", @@ -1736,6 +1979,24 @@ "license_file": null, "description": "Core random number generator traits and tools for implementation." }, + { + "name": "rand_hc", + "version": "0.2.0", + "authors": "The Rand Project Developers", + "repository": "https://github.com/rust-random/rand", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "HC128 random number generator" + }, + { + "name": "rand_pcg", + "version": "0.2.1", + "authors": "The Rand Project Developers", + "repository": "https://github.com/rust-random/rand", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Selected PCG random number generators" + }, { "name": "redox_syscall", "version": "0.2.16", @@ -1783,13 +2044,22 @@ }, { "name": "reqwest", - "version": "0.11.3", + "version": "0.11.13", "authors": "Sean McArthur ", "repository": "https://github.com/seanmonstar/reqwest", "license": "Apache-2.0 OR MIT", "license_file": null, "description": "higher level HTTP client library" }, + { + "name": "retain_mut", + "version": "0.1.9", + "authors": "Xidorn Quan ", + "repository": "https://github.com/upsuper/retain_mut", + "license": "MIT", + "license_file": null, + "description": "Provide retain_mut method that has the same functionality as retain but gives mutable borrow to the predicate." + }, { "name": "ring", "version": "0.16.20", @@ -1837,25 +2107,34 @@ }, { "name": "rustls", - "version": "0.19.1", - "authors": "Joseph Birr-Pixton ", - "repository": "https://github.com/ctz/rustls", + "version": "0.20.8", + "authors": null, + "repository": "https://github.com/rustls/rustls", "license": "Apache-2.0 OR ISC OR MIT", "license_file": null, "description": "Rustls is a modern TLS library written in Rust." }, { "name": "rustls-native-certs", - "version": "0.5.0", + "version": "0.6.2", "authors": "Joseph Birr-Pixton ", "repository": "https://github.com/ctz/rustls-native-certs", "license": "Apache-2.0 OR ISC OR MIT", "license_file": null, "description": "rustls-native-certs allows rustls to use the platform native certificate store" }, + { + "name": "rustls-pemfile", + "version": "1.0.2", + "authors": null, + "repository": "https://github.com/rustls/pemfile", + "license": "Apache-2.0 OR ISC OR MIT", + "license_file": null, + "description": "Basic .pem file parser for keys and certificates" + }, { "name": "rustversion", - "version": "1.0.9", + "version": "1.0.11", "authors": "David Tolnay ", "repository": "https://github.com/dtolnay/rustversion", "license": "Apache-2.0 OR MIT", @@ -1864,7 +2143,7 @@ }, { "name": "ryu", - "version": "1.0.11", + "version": "1.0.12", "authors": "David Tolnay ", "repository": "https://github.com/dtolnay/ryu", "license": "Apache-2.0 OR BSL-1.0", @@ -1900,7 +2179,7 @@ }, { "name": "sct", - "version": "0.6.1", + "version": "0.7.0", "authors": "Joseph Birr-Pixton ", "repository": "https://github.com/ctz/sct.rs", "license": "Apache-2.0 OR ISC OR MIT", @@ -1936,7 +2215,7 @@ }, { "name": "serde", - "version": "1.0.149", + "version": "1.0.152", "authors": "Erick Tryzelaar |David Tolnay ", "repository": "https://github.com/serde-rs/serde", "license": "Apache-2.0 OR MIT", @@ -1954,7 +2233,7 @@ }, { "name": "serde_derive", - "version": "1.0.149", + "version": "1.0.152", "authors": "Erick Tryzelaar |David Tolnay ", "repository": "https://github.com/serde-rs/serde", "license": "Apache-2.0 OR MIT", @@ -1963,13 +2242,31 @@ }, { "name": "serde_json", - "version": "1.0.89", + "version": "1.0.91", "authors": "Erick Tryzelaar |David Tolnay ", "repository": "https://github.com/serde-rs/json", "license": "Apache-2.0 OR MIT", "license_file": null, "description": "A JSON serialization file format" }, + { + "name": "serde_path_to_error", + "version": "0.1.9", + "authors": "David Tolnay ", + "repository": "https://github.com/dtolnay/path-to-error", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Path to the element that failed to deserialize" + }, + { + "name": "serde_qs", + "version": "0.8.5", + "authors": "Sam Scott ", + "repository": "https://github.com/samscott89/serde_qs", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Querystrings for Serde" + }, { "name": "serde_repr", "version": "0.1.9", @@ -2114,6 +2411,15 @@ "license_file": null, "description": "Synchronization primitives based on spinning. They may contain data, are usable without `std`, and static initializers are available." }, + { + "name": "spin", + "version": "0.9.4", + "authors": "Mathijs van de Nes |John Ericson |Joshua Barretto ", + "repository": "https://github.com/mvdnes/spin-rs.git", + "license": "MIT", + "license_file": null, + "description": "Spin-based synchronization primitives" + }, { "name": "string_cache", "version": "0.8.4", @@ -2161,13 +2467,22 @@ }, { "name": "syn", - "version": "1.0.105", + "version": "1.0.107", "authors": "David Tolnay ", "repository": "https://github.com/dtolnay/syn", "license": "Apache-2.0 OR MIT", "license_file": null, "description": "Parser for Rust source code" }, + { + "name": "sync_wrapper", + "version": "0.1.1", + "authors": "Actyx AG ", + "repository": "https://github.com/Actyx/sync_wrapper", + "license": "Apache-2.0", + "license_file": null, + "description": "A tool for enlisting the compiler’s help in proving the absence of concurrency" + }, { "name": "tempfile", "version": "3.3.0", @@ -2197,7 +2512,7 @@ }, { "name": "thiserror", - "version": "1.0.37", + "version": "1.0.38", "authors": "David Tolnay ", "repository": "https://github.com/dtolnay/thiserror", "license": "Apache-2.0 OR MIT", @@ -2206,7 +2521,7 @@ }, { "name": "thiserror-impl", - "version": "1.0.37", + "version": "1.0.38", "authors": "David Tolnay ", "repository": "https://github.com/dtolnay/thiserror", "license": "Apache-2.0 OR MIT", @@ -2278,22 +2593,13 @@ }, { "name": "tokio", - "version": "1.23.0", + "version": "1.24.1", "authors": "Tokio Contributors ", "repository": "https://github.com/tokio-rs/tokio", "license": "MIT", "license_file": null, "description": "An event-driven, non-blocking I/O platform for writing asynchronous I/O backed applications." }, - { - "name": "tokio-io-timeout", - "version": "1.1.1", - "authors": "Steven Fackler ", - "repository": "https://github.com/sfackler/tokio-io-timeout", - "license": "Apache-2.0 OR MIT", - "license_file": null, - "description": "Tokio wrappers which apply timeouts to IO operations" - }, { "name": "tokio-macros", "version": "1.8.2", @@ -2314,7 +2620,7 @@ }, { "name": "tokio-rustls", - "version": "0.22.0", + "version": "0.23.4", "authors": "quininer kel ", "repository": "https://github.com/tokio-rs/tls", "license": "Apache-2.0 OR MIT", @@ -2348,6 +2654,33 @@ "license_file": null, "description": "A native Rust encoder and decoder of TOML-formatted files and streams. Provides implementations of the standard Serialize/Deserialize traits for TOML data to facilitate deserializing and serializing Rust structures." }, + { + "name": "tower", + "version": "0.4.13", + "authors": "Tower Maintainers ", + "repository": "https://github.com/tower-rs/tower", + "license": "MIT", + "license_file": null, + "description": "Tower is a library of modular and reusable components for building robust clients and servers." + }, + { + "name": "tower-http", + "version": "0.3.5", + "authors": "Tower Maintainers ", + "repository": "https://github.com/tower-rs/tower-http", + "license": "MIT", + "license_file": null, + "description": "Tower middleware and utilities for HTTP clients and servers" + }, + { + "name": "tower-layer", + "version": "0.3.2", + "authors": "Tower Maintainers ", + "repository": "https://github.com/tower-rs/tower", + "license": "MIT", + "license_file": null, + "description": "Decorates a `Service` to allow easy composition between `Service`s." + }, { "name": "tower-service", "version": "0.3.2", @@ -2539,7 +2872,7 @@ }, { "name": "unicode-ident", - "version": "1.0.5", + "version": "1.0.6", "authors": "David Tolnay ", "repository": "https://github.com/dtolnay/unicode-ident", "license": "(MIT OR Apache-2.0) AND Unicode-DFS-2016", @@ -2636,6 +2969,15 @@ "license_file": null, "description": "Tiny crate to check the version of the installed/running rustc." }, + { + "name": "waker-fn", + "version": "1.1.0", + "authors": "Stjepan Glavina ", + "repository": "https://github.com/stjepang/waker-fn", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "Convert closures into wakers" + }, { "name": "want", "version": "0.3.0", @@ -2645,6 +2987,15 @@ "license_file": null, "description": "Detect when another Future wants a result." }, + { + "name": "wasi", + "version": "0.9.0+wasi-snapshot-preview1", + "authors": "The Cranelift Project Developers", + "repository": "https://github.com/bytecodealliance/wasi", + "license": "Apache-2.0 OR Apache-2.0 WITH LLVM-exception OR MIT", + "license_file": null, + "description": "Experimental WASI API bindings for Rust" + }, { "name": "wasi", "version": "0.11.0+wasi-snapshot-preview1", @@ -2719,7 +3070,7 @@ }, { "name": "webpki", - "version": "0.21.4", + "version": "0.22.0", "authors": "Brian Smith ", "repository": "https://github.com/briansmith/webpki", "license": null, @@ -2728,9 +3079,9 @@ }, { "name": "webpki-roots", - "version": "0.21.1", + "version": "0.22.6", "authors": "Joseph Birr-Pixton ", - "repository": "https://github.com/ctz/webpki-roots", + "repository": "https://github.com/rustls/webpki-roots", "license": "MPL-2.0", "license_file": null, "description": "Mozilla's CA root certificates for use with webpki" @@ -2908,13 +3259,22 @@ }, { "name": "winreg", - "version": "0.7.0", + "version": "0.10.1", "authors": "Igor Shaula ", "repository": "https://github.com/gentoo90/winreg-rs", "license": "MIT", "license_file": null, "description": "Rust bindings to MS Windows Registry API" }, + { + "name": "wiremock", + "version": "0.5.17", + "authors": "Luca Palmieri ", + "repository": "https://github.com/LukeMathWalker/wiremock-rs", + "license": "Apache-2.0 OR MIT", + "license_file": null, + "description": "HTTP mocking to test Rust applications." + }, { "name": "workspace-hack", "version": "0.1.0", diff --git a/docs/syncserver.md b/docs/syncserver.md deleted file mode 100644 index 1644c9dcd..000000000 --- a/docs/syncserver.md +++ /dev/null @@ -1,103 +0,0 @@ -# Local sync server - -A local sync server is bundled with Anki. If you cannot or do not wish to -use AnkiWeb, you can run the server on a machine on your local network. - -Things to be aware of: - -- Media syncing is not currently supported. You will either need to disable - syncing of sounds and images in the preferences screen, sync your media via - AnkiWeb, or use some other solution. -- AnkiMobile does not yet provide an option for using a local sync server, - so for now this will only be usable with the computer version of Anki, and - AnkiDroid. -- This code is partly new, and while it has had some testing, it's possible - something has been missed. Please make backups, and report any bugs you run - into. -- The server runs over an unencrypted HTTP connection and does not require - authentication, so it is only suitable for use on a private network. -- This is an advanced feature, targeted at users who are comfortable with - networking and the command line. If you use this, the expectation is you - can resolve any setup/network/firewall issues you run into yourself, and - use of this is entirely at your own risk. - -## From source - -If you run Anki from git, you can run a sync server with: - -``` -./tools/runopt --syncserver -``` - -## From a packaged build - -From 2.1.39beta1+, the sync server is included in the packaged binaries. - -On Windows in a cmd.exe session: - -``` -"\program files\anki\anki-console.exe" --syncserver -``` - -Or MacOS, in Terminal.app: - -``` -/Applications/Anki.app/Contents/MacOS/AnkiMac --syncserver -``` - -Or Linux: - -``` -anki --syncserver -``` - -## Without Qt dependencies - -You can run the server without installing the GUI portion of Anki. Once Anki -2.1.39 is released, the following will work: - -``` -pip install anki[syncserver] -python -m anki.syncserver -``` - -## Server setup - -The server needs to store a copy of your collection in a folder. -By default it is ~/.syncserver; you can change this by defining -a `FOLDER` environmental variable. This should not be the same location -as your normal Anki data folder. - -You can also define `HOST` and `PORT`. - -## Client setup - -When the server starts, it will print the address it is listening on. -You need to set an environmental variable before starting your Anki -clients to tell them where to connect to. Eg: - -``` -set SYNC_ENDPOINT="http://10.0.0.5:8080/sync/" -anki -``` - -Currently any username and password will be accepted. If you wish to -keep using AnkiWeb for media, sync once with AnkiWeb first, then switch -to your local endpoint - collection syncs will be local, and media syncs -will continue to go to AnkiWeb. - -## Contributing - -Authentication shouldn't be too hard to add - login() and request() in -http_client.rs can be used as a reference. A PR that accepts a password in an -env var, and generates a stable hkey based on it would be welcome. - -Once that is done, basic multi-profile support could be implemented by moving -the col object into an array or dict, and fetching the relevant collection based -on the user's authentication. - -Because this server is bundled with Anki, simplicity is a design goal - it is -targeted at individual/family use, only makes use of Python libraries the GUI is -already using, and does not require a configuration file. PRs that deviate from -this are less likely to be merged, so please consider reaching out first if you -are thinking of starting work on a larger change. diff --git a/ftl/core/preferences.ftl b/ftl/core/preferences.ftl index 367ad74ba..a47b5f86e 100644 --- a/ftl/core/preferences.ftl +++ b/ftl/core/preferences.ftl @@ -8,7 +8,7 @@ preferences-interface-language = Interface language: preferences-interrupt-current-audio-when-answering = Interrupt current audio when answering preferences-learn-ahead-limit = Learn ahead limit preferences-mins = mins -preferences-network = Network +preferences-network = Syncing preferences-next-day-starts-at = Next day starts at preferences-note-media-is-not-backed-up = Note: Media is not backed up. Please create a periodic backup of your Anki folder to be safe. preferences-on-next-sync-force-changes-in = On next sync, force changes in one direction @@ -49,3 +49,5 @@ preferences-minutes-between-backups = Minutes between automatic backups: preferences-reduce-motion = Reduce motion preferences-reduce-motion-tooltip = Disable various animations and transitions of the user interface preferences-collapse-toolbar = Hide top bar during review +preferences-custom-sync-url = Self-hosted sync server +preferences-custom-sync-url-disclaimer = For advanced users - please see the manual diff --git a/proto/anki/sync.proto b/proto/anki/sync.proto index 4658ef480..c4d1bffe9 100644 --- a/proto/anki/sync.proto +++ b/proto/anki/sync.proto @@ -19,18 +19,17 @@ service SyncService { rpc FullUpload(SyncAuth) returns (generic.Empty); rpc FullDownload(SyncAuth) returns (generic.Empty); rpc AbortSync(generic.Empty) returns (generic.Empty); - - rpc SyncServerMethod(SyncServerMethodRequest) returns (generic.Json); } message SyncAuth { string hkey = 1; - uint32 host_number = 2; + optional string endpoint = 2; } message SyncLoginRequest { string username = 1; string password = 2; + optional string endpoint = 3; } message SyncStatusResponse { @@ -40,6 +39,7 @@ message SyncStatusResponse { FULL_SYNC = 2; } Required required = 1; + optional string new_endpoint = 4; } message SyncCollectionResponse { @@ -56,24 +56,5 @@ message SyncCollectionResponse { uint32 host_number = 1; string server_message = 2; ChangesRequired required = 3; -} - -message SyncServerMethodRequest { - enum Method { - HOST_KEY = 0; - META = 1; - START = 2; - APPLY_GRAVES = 3; - APPLY_CHANGES = 4; - CHUNK = 5; - APPLY_CHUNK = 6; - SANITY_CHECK = 7; - FINISH = 8; - ABORT = 9; - // caller must reopen after these two are called - FULL_UPLOAD = 10; - FULL_DOWNLOAD = 11; - } - Method method = 1; - bytes data = 2; + optional string new_endpoint = 4; } diff --git a/pylib/anki/_backend.py b/pylib/anki/_backend.py index fbe8b5139..e62dd5e36 100644 --- a/pylib/anki/_backend.py +++ b/pylib/anki/_backend.py @@ -11,7 +11,6 @@ from weakref import ref from markdown import markdown import anki.buildinfo -import anki.lang from anki import _rsbridge, backend_pb2, i18n_pb2 from anki._backend_generated import RustBackendGenerated from anki._fluent import GeneratedTranslations @@ -72,6 +71,8 @@ class RustBackend(RustBackendGenerated): server: bool = False, ) -> None: # pick up global defaults if not provided + import anki.lang + if langs is None: langs = [anki.lang.current_lang] @@ -81,6 +82,10 @@ class RustBackend(RustBackendGenerated): ) self._backend = _rsbridge.open_backend(init_msg.SerializeToString()) + @staticmethod + def syncserver() -> None: + _rsbridge.syncserver() + def db_query( self, sql: str, args: Sequence[ValueForDB], first_row_only: bool ) -> list[DBRow]: diff --git a/pylib/anki/_rsbridge.pyi b/pylib/anki/_rsbridge.pyi index 3b112f3a9..164fe58b9 100644 --- a/pylib/anki/_rsbridge.pyi +++ b/pylib/anki/_rsbridge.pyi @@ -1,6 +1,7 @@ def buildhash() -> str: ... def open_backend(data: bytes) -> Backend: ... def initialize_logging(log_file: str | None) -> Backend: ... +def syncserver() -> None: ... class Backend: @classmethod diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index c9bc72363..7f9239d78 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -16,6 +16,7 @@ from anki import ( stats_pb2, ) from anki._legacy import DeprecatedNamesMixin, deprecated +from anki.sync_pb2 import SyncLoginRequest # protobuf we publicly export - listed first to avoid circular imports HelpPage = links_pb2.HelpPageLinkRequest.HelpPage @@ -1189,8 +1190,12 @@ class Collection(DeprecatedNamesMixin): def full_download(self, auth: SyncAuth) -> None: self._backend.full_download(auth) - def sync_login(self, username: str, password: str) -> SyncAuth: - return self._backend.sync_login(username=username, password=password) + def sync_login( + self, username: str, password: str, endpoint: str | None + ) -> SyncAuth: + return self._backend.sync_login( + SyncLoginRequest(username=username, password=password, endpoint=endpoint) + ) def sync_collection(self, auth: SyncAuth) -> SyncOutput: return self._backend.sync_collection(auth) diff --git a/pylib/anki/syncserver.py b/pylib/anki/syncserver.py new file mode 100644 index 000000000..72edb90bd --- /dev/null +++ b/pylib/anki/syncserver.py @@ -0,0 +1,24 @@ +# Copyright: Ankitects Pty Ltd and contributors +# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + + +def run_sync_server() -> None: + import sys + from os import environ as env + from os.path import expanduser + + from anki._backend import RustBackend + + env["SYNC_BASE"] = env.get("SYNC_BASE", expanduser("~/.syncserver")) + env["RUST_LOG"] = env.get("RUST_LOG", "anki=info") + + try: + RustBackend.syncserver() + except Exception as exc: + print("Sync server failed:", exc) + sys.exit(1) + sys.exit(0) + + +if __name__ == "__main__": + run_sync_server() diff --git a/pylib/anki/syncserver/__init__.py b/pylib/anki/syncserver/__init__.py deleted file mode 100644 index 72bfd500f..000000000 --- a/pylib/anki/syncserver/__init__.py +++ /dev/null @@ -1,195 +0,0 @@ -# Copyright: Ankitects Pty Ltd and contributors -# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -# -# Please see /docs/syncserver.md -# - -from __future__ import annotations - -import gzip -import os -import socket -import sys -import time -from http import HTTPStatus -from io import BytesIO -from tempfile import NamedTemporaryFile -from typing import Iterable, Optional - -try: - import flask - from waitress.server import create_server -except ImportError as error: - print(error, "- to use the server, 'pip install anki[syncserver]'") - sys.exit(1) - - -from flask import Response - -from anki.collection import Collection -from anki.sync_pb2 import SyncServerMethodRequest - -Method = SyncServerMethodRequest.Method # pylint: disable=no-member - -app = flask.Flask(__name__, root_path="/fake") -col: Collection -trace = os.getenv("TRACE") - - -def get_request_data() -> bytes: - buf = BytesIO() - flask.request.files["data"].save(buf) - buf.seek(0) - zip = gzip.GzipFile(mode="rb", fileobj=buf) - return zip.read() - - -def get_request_data_into_file() -> bytes: - "Returns the utf8 path to the resulting file." - # this could be optimized to stream the data into a file - # in the future - data = get_request_data() - tempobj = NamedTemporaryFile(dir=folder(), delete=False) - tempobj.write(data) - tempobj.close() - return tempobj.name.encode("utf8") - - -def handle_sync_request(method_str: str) -> Response: - method = get_method(method_str) - if method is None: - raise Exception(f"unknown method: {method_str}") - - if method == Method.FULL_UPLOAD: - data = get_request_data_into_file() - else: - data = get_request_data() - if trace: - print("-->", data) - - full = method in (Method.FULL_UPLOAD, Method.FULL_DOWNLOAD) - if full: - col.close_for_full_sync() - try: - outdata = col._backend.sync_server_method(method=method, data=data) - except Exception as error: - if method == Method.META: - # if parallel syncing requests come in, block them - print("exception in meta", error) - return flask.make_response("Conflict", 409) - else: - raise - finally: - if full: - after_full_sync() - - resp = None - if method == Method.FULL_UPLOAD: - # upload call expects a raw string literal returned - outdata = b"OK" - elif method == Method.FULL_DOWNLOAD: - path = outdata.decode("utf8") - - def stream_reply() -> Iterable[bytes]: - with open(path, "rb") as file: - while chunk := file.read(16 * 1024): - yield chunk - os.unlink(path) - - resp = Response(stream_reply()) - else: - if trace: - print("<--", outdata) - - if not resp: - resp = flask.make_response(outdata) - resp.headers["Content-Type"] = "application/binary" - return resp - - -def after_full_sync() -> None: - # the server methods do not reopen the collection after a full sync, - # so we need to - col.reopen(after_full_sync=False) - col.db.rollback() - - -def get_method( - method_str: str, -) -> SyncServerMethodRequest.Method.V | None: # pylint: disable=no-member - if method_str == "hostKey": - return Method.HOST_KEY - elif method_str == "meta": - return Method.META - elif method_str == "start": - return Method.START - elif method_str == "applyGraves": - return Method.APPLY_GRAVES - elif method_str == "applyChanges": - return Method.APPLY_CHANGES - elif method_str == "chunk": - return Method.CHUNK - elif method_str == "applyChunk": - return Method.APPLY_CHUNK - elif method_str == "sanityCheck2": - return Method.SANITY_CHECK - elif method_str == "finish": - return Method.FINISH - elif method_str == "abort": - return Method.ABORT - elif method_str == "upload": - return Method.FULL_UPLOAD - elif method_str == "download": - return Method.FULL_DOWNLOAD - else: - return None - - -@app.route("/", methods=["POST"]) -def handle_request(pathin: str) -> Response: - path = pathin - print(int(time.time()), flask.request.remote_addr, path) - - if path.startswith("sync/"): - return handle_sync_request(path.split("/", maxsplit=1)[1]) - else: - return flask.make_response("not found", HTTPStatus.NOT_FOUND) - - -def folder() -> str: - folder = os.getenv("FOLDER", os.path.expanduser("~/.syncserver")) - if not os.path.exists(folder): - print("creating", folder) - os.mkdir(folder) - return folder - - -def col_path() -> str: - return os.path.join(folder(), "collection.server.anki2") - - -def serve() -> None: - global col # pylint: disable=invalid-name - - col = Collection(col_path(), server=True) - # don't hold an outer transaction open - col.db.rollback() - host = os.getenv("HOST", "0.0.0.0") - port = int(os.getenv("PORT", "8080")) - - server = create_server( - app, - host=host, - port=port, - clear_untrusted_proxy_headers=True, - ) - - effective_port = server.effective_port # type: ignore - print(f"Sync server listening on http://{host}:{effective_port}/sync/") - if host == "0.0.0.0": - ip = socket.gethostbyname(socket.gethostname()) - print(f"Replace 0.0.0.0 with your machine's IP address (perhaps {ip})") - print( - "For more info, see https://github.com/ankitects/anki/blob/master/docs/syncserver.md" - ) - server.run() diff --git a/pylib/anki/syncserver/__main__.py b/pylib/anki/syncserver/__main__.py deleted file mode 100644 index 01e371ae6..000000000 --- a/pylib/anki/syncserver/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright: Ankitects Pty Ltd and contributors -# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html - -from anki.syncserver import serve - -serve() diff --git a/pylib/rsbridge/lib.rs b/pylib/rsbridge/lib.rs index 825e773df..1cfab5791 100644 --- a/pylib/rsbridge/lib.rs +++ b/pylib/rsbridge/lib.rs @@ -4,6 +4,7 @@ use anki::{ backend::{init_backend, Backend as RustBackend}, log::set_global_logger, + sync::http_server::SimpleServer, }; use pyo3::{ create_exception, exceptions::PyException, prelude::*, types::PyBytes, wrap_pyfunction, @@ -26,6 +27,12 @@ fn initialize_logging(path: Option<&str>) -> PyResult<()> { set_global_logger(path).map_err(|e| PyException::new_err(e.to_string())) } +#[pyfunction] +fn syncserver() -> PyResult<()> { + set_global_logger(None).unwrap(); + SimpleServer::run().map_err(|e| PyException::new_err(format!("{e:?}"))) +} + #[pyfunction] fn open_backend(init_msg: &PyBytes) -> PyResult { match init_backend(init_msg.as_bytes()) { @@ -76,6 +83,7 @@ fn _rsbridge(_py: Python, m: &PyModule) -> PyResult<()> { m.add_wrapped(wrap_pyfunction!(buildhash)).unwrap(); m.add_wrapped(wrap_pyfunction!(open_backend)).unwrap(); m.add_wrapped(wrap_pyfunction!(initialize_logging)).unwrap(); + m.add_wrapped(wrap_pyfunction!(syncserver)).unwrap(); Ok(()) } diff --git a/python/write_wheel.py b/python/write_wheel.py index 897f781a4..0a9382830 100644 --- a/python/write_wheel.py +++ b/python/write_wheel.py @@ -6,6 +6,7 @@ from __future__ import annotations import sys from email.message import EmailMessage from pathlib import Path +from typing import Sequence from zipfile import ZIP_DEFLATED, ZipInfo from wheel.wheelfile import WheelFile @@ -158,11 +159,10 @@ else: contents: dict[str, str] = {} merge_sources(contents, src_root, exclude) merge_sources(contents, generated_root, exclude) +all_requires: Sequence[str | ExtraRequires] if name == "anki": - all_requires = extract_requirements(Path("python/requirements.anki.in")) + [ - ExtraRequires("syncserver", ["flask", "waitress"]), - ] + all_requires = extract_requirements(Path("python/requirements.anki.in")) entrypoints = None top_level = None else: diff --git a/qt/aqt/__init__.py b/qt/aqt/__init__.py index c5f54ae2b..3ebacdbe0 100644 --- a/qt/aqt/__init__.py +++ b/qt/aqt/__init__.py @@ -16,18 +16,17 @@ except UnicodeEncodeError as exc: print("Please Google 'how to change locale on [your Linux distro]'") sys.exit(1) +# if sync server enabled, bypass the rest of the startup +if "--syncserver" in sys.argv: + from anki.syncserver import run_sync_server + + # does not return + run_sync_server() + from .package import packaged_build_setup packaged_build_setup() -# syncserver needs to be run before Qt loaded -if "--syncserver" in sys.argv: - from anki.syncserver import serve - - serve() - sys.exit(0) - - import argparse import builtins import cProfile diff --git a/qt/aqt/forms/preferences.ui b/qt/aqt/forms/preferences.ui index c550fd45e..8c7200706 100644 --- a/qt/aqt/forms/preferences.ui +++ b/qt/aqt/forms/preferences.ui @@ -492,6 +492,24 @@ + + + + + + preferences_custom_sync_url_disclaimer + + + + + + + preferences_custom_sync_url + + + + + @@ -722,6 +740,7 @@ fullSync syncDeauth media_log + custom_sync_url minutes_between_backups daily_backups weekly_backups diff --git a/qt/aqt/mediasync.py b/qt/aqt/mediasync.py index 04fcf1a7b..d0796aae7 100644 --- a/qt/aqt/mediasync.py +++ b/qt/aqt/mediasync.py @@ -12,12 +12,12 @@ import aqt import aqt.forms import aqt.main from anki.collection import Progress -from anki.errors import Interrupted, NetworkError +from anki.errors import Interrupted from anki.types import assert_exhaustive from anki.utils import int_time from aqt import gui_hooks from aqt.qt import QDialog, QDialogButtonBox, QPushButton, QTextCursor, QTimer, qconnect -from aqt.utils import disable_help_button, showWarning, tr +from aqt.utils import disable_help_button, tr LogEntry = Union[Progress.MediaSync, str] @@ -92,14 +92,13 @@ class MediaSyncer: if isinstance(exc, Interrupted): self._log_and_notify(tr.sync_media_aborted()) return - elif isinstance(exc, NetworkError): - # avoid popups for network errors + else: + # Avoid popups for errors; they can cause a deadlock if + # a modal window happens to be active, or a duplicate auth + # failed message if the password is changed. self._log_and_notify(str(exc)) return - self._log_and_notify(tr.sync_media_failed()) - showWarning(str(exc)) - def entries(self) -> list[LogEntryWithTime]: return self._log diff --git a/qt/aqt/preferences.py b/qt/aqt/preferences.py index 0b9048ff6..51926f3a8 100644 --- a/qt/aqt/preferences.py +++ b/qt/aqt/preferences.py @@ -174,6 +174,7 @@ class Preferences(QDialog): self.form.syncUser.setText(self.prof.get("syncUser", "")) qconnect(self.form.syncDeauth.clicked, self.sync_logout) self.form.syncDeauth.setText(tr.sync_log_out_button()) + self.form.custom_sync_url.setText(self.mw.pm.custom_sync_url()) def on_media_log(self) -> None: self.mw.media_syncer.show_sync_log() @@ -201,6 +202,7 @@ class Preferences(QDialog): ) if self.form.fullSync.isChecked(): self.mw.col.mod_schema(check=False) + self.mw.pm.set_custom_sync_url(self.form.custom_sync_url.text()) # Global preferences ###################################################################### diff --git a/qt/aqt/profiles.py b/qt/aqt/profiles.py index b7277057a..a534f1bf0 100644 --- a/qt/aqt/profiles.py +++ b/qt/aqt/profiles.py @@ -606,15 +606,33 @@ create table if not exists profiles return self.profile["autoSync"] def sync_auth(self) -> SyncAuth | None: - hkey = self.profile.get("syncKey") - if not hkey: + if not (hkey := self.profile.get("syncKey")): return None - return SyncAuth(hkey=hkey, host_number=self.profile.get("hostNum", 0)) + return SyncAuth(hkey=hkey, endpoint=self.sync_endpoint()) def clear_sync_auth(self) -> None: - self.profile["syncKey"] = None - self.profile["syncUser"] = None - self.profile["hostNum"] = 0 + self.set_sync_key(None) + self.set_sync_username(None) + self.set_host_number(None) + self.set_current_sync_url(None) + + def sync_endpoint(self) -> str | None: + return self._current_sync_url() or self.custom_sync_url() or None + + def _current_sync_url(self) -> str | None: + """The last endpoint the server redirected us to.""" + return self.profile.get("currentSyncUrl") + + def set_current_sync_url(self, url: str | None) -> None: + self.profile["currentSyncUrl"] = url + + def custom_sync_url(self) -> str | None: + """A custom server provided by the user.""" + return self.profile.get("customSyncUrl") + + def set_custom_sync_url(self, url: str | None) -> None: + self.set_current_sync_url(None) + self.profile["customSyncUrl"] = url def auto_sync_media_minutes(self) -> int: return self.profile.get("autoSyncMediaMinutes", 15) diff --git a/qt/aqt/sync.py b/qt/aqt/sync.py index 94f95b117..34f7a2dc1 100644 --- a/qt/aqt/sync.py +++ b/qt/aqt/sync.py @@ -12,6 +12,7 @@ import aqt.main from anki.errors import Interrupted, SyncError, SyncErrorKind from anki.lang import without_unicode_isolation from anki.sync import SyncOutput, SyncStatus +from anki.sync_pb2 import SyncAuth from anki.utils import plat_desc from aqt import gui_hooks from aqt.qt import ( @@ -43,13 +44,15 @@ def get_sync_status( callback(SyncStatus(required=SyncStatus.NO_CHANGES)) # pylint:disable=no-member return - def on_future_done(fut: Future) -> None: + def on_future_done(fut: Future[SyncStatus]) -> None: try: out = fut.result() except Exception as e: # swallow errors print("sync status check failed:", str(e)) return + if out.new_endpoint: + mw.pm.set_current_sync_url(out.new_endpoint) callback(out) mw.taskman.run_in_background(lambda: mw.col.sync_status(auth), on_future_done) @@ -93,18 +96,20 @@ def sync_collection(mw: aqt.main.AnkiQt, on_done: Callable[[], None]) -> None: qconnect(timer.timeout, on_timer) timer.start(150) - def on_future_done(fut: Future) -> None: + def on_future_done(fut: Future[SyncOutput]) -> None: mw.col.db.begin() # scheduler version may have changed mw.col._load_scheduler() timer.stop() try: - out: SyncOutput = fut.result() + out = fut.result() except Exception as err: handle_sync_error(mw, err) return on_done() mw.pm.set_host_number(out.host_number) + if out.new_endpoint: + mw.pm.set_current_sync_url(out.new_endpoint) if out.server_message: showText(out.server_message) if out.required == out.NO_CHANGES: @@ -161,7 +166,7 @@ def confirm_full_download(mw: aqt.main.AnkiQt, on_done: Callable[[], None]) -> N mw.closeAllWindows(lambda: full_download(mw, on_done)) -def on_full_sync_timer(mw: aqt.main.AnkiQt) -> None: +def on_full_sync_timer(mw: aqt.main.AnkiQt, label: str) -> None: progress = mw.col.latest_progress() if not progress.HasField("full_sync"): return @@ -169,8 +174,6 @@ def on_full_sync_timer(mw: aqt.main.AnkiQt) -> None: if sync_progress.transferred == sync_progress.total: label = tr.sync_checking() - else: - label = None mw.progress.update( value=sync_progress.transferred, max=sync_progress.total, @@ -183,8 +186,10 @@ def on_full_sync_timer(mw: aqt.main.AnkiQt) -> None: def full_download(mw: aqt.main.AnkiQt, on_done: Callable[[], None]) -> None: + label = tr.sync_downloading_from_ankiweb() + def on_timer() -> None: - on_full_sync_timer(mw) + on_full_sync_timer(mw, label) timer = QTimer(mw) qconnect(timer.timeout, on_timer) @@ -212,7 +217,6 @@ def full_download(mw: aqt.main.AnkiQt, on_done: Callable[[], None]) -> None: mw.taskman.with_progress( download, on_future_done, - label=tr.sync_downloading_from_ankiweb(), ) @@ -220,8 +224,10 @@ def full_upload(mw: aqt.main.AnkiQt, on_done: Callable[[], None]) -> None: gui_hooks.collection_will_temporarily_close(mw.col) mw.col.close_for_full_sync() + label = tr.sync_uploading_to_ankiweb() + def on_timer() -> None: - on_full_sync_timer(mw) + on_full_sync_timer(mw, label) timer = QTimer(mw) qconnect(timer.timeout, on_timer) @@ -242,7 +248,6 @@ def full_upload(mw: aqt.main.AnkiQt, on_done: Callable[[], None]) -> None: mw.taskman.with_progress( lambda: mw.col.full_upload(mw.pm.sync_auth()), on_future_done, - label=tr.sync_uploading_to_ankiweb(), ) @@ -259,7 +264,7 @@ def sync_login( if username and password: break - def on_future_done(fut: Future) -> None: + def on_future_done(fut: Future[SyncAuth]) -> None: try: auth = fut.result() except SyncError as e: @@ -273,14 +278,15 @@ def sync_login( handle_sync_error(mw, err) return - mw.pm.set_host_number(auth.host_number) mw.pm.set_sync_key(auth.hkey) mw.pm.set_sync_username(username) on_success() mw.taskman.with_progress( - lambda: mw.col.sync_login(username=username, password=password), + lambda: mw.col.sync_login( + username=username, password=password, endpoint=mw.pm.sync_endpoint() + ), on_future_done, ) diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index 0666248d7..48d878ea4 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -31,11 +31,12 @@ prost-build = "0.11.4" which = "4.3.0" [dev-dependencies] +async-stream = "0.3.3" env_logger = "0.10.0" -tokio = { version = "1.23", features = ["macros"] } +wiremock = "0.5.17" [dependencies.reqwest] -version = "=0.11.3" +version = "0.11.13" default-features = false features = ["json", "socks", "stream", "multipart"] @@ -51,7 +52,10 @@ unicase = "=2.6.0" criterion = { version = "0.4.0", optional = true } ammonia = "3.3.0" +async-compression = { version = "0.3.15", features = ["zstd", "tokio"] } async-trait = "0.1.59" +axum = { version = "0.6.1", features = ["multipart", "macros", "headers"] } +axum-client-ip = "0.3.1" blake3 = "1.3.3" bytes = "1.3.0" chrono = { version = "0.4.19", default-features = false, features = ["std", "clock"] } @@ -65,6 +69,7 @@ fnv = "1.0.7" futures = "0.3.25" hex = "0.4.3" htmlescape = "0.3.1" +hyper = "0.14.23" id_tree = "1.8.0" itertools = "0.10.5" lazy_static = "1.4.0" @@ -90,8 +95,9 @@ sha1 = "0.10.5" snafu = { version = "0.7.3", features = ["backtraces"] } strum = { version = "0.24.1", features = ["derive"] } tempfile = "3.3.0" -tokio = { version = "1.23", features = ["fs", "rt-multi-thread"] } +tokio = { version = "1.23", features = ["fs", "rt-multi-thread", "macros"] } tokio-util = { version = "0.7.4", features = ["io"] } +tower-http = { version = "0.3.5", features = ["trace"] } tracing = { version = "0.1.37", features = ["max_level_trace", "release_max_level_debug"] } tracing-appender = "0.2.2" tracing-subscriber = { version = "0.3.16", features = ["fmt", "env-filter"] } diff --git a/rslib/i18n_helpers/src/serialize.rs b/rslib/i18n_helpers/src/serialize.rs index a0e5d8021..7cd9d8806 100644 --- a/rslib/i18n_helpers/src/serialize.rs +++ b/rslib/i18n_helpers/src/serialize.rs @@ -3,7 +3,9 @@ // copied from https://github.com/projectfluent/fluent-rs/pull/241 -use std::fmt::{self, Error, Write}; +use std::fmt::{ + Error, Write, {self}, +}; use fluent_syntax::{ast::*, parser::Slice}; diff --git a/rslib/linkchecker/Cargo.toml b/rslib/linkchecker/Cargo.toml index 20bd5ea52..7c265f97a 100644 --- a/rslib/linkchecker/Cargo.toml +++ b/rslib/linkchecker/Cargo.toml @@ -28,5 +28,5 @@ rustls = ["reqwest/rustls-tls", "reqwest/rustls-tls-native-roots"] native-tls = ["reqwest/native-tls"] [dependencies.reqwest] -version = "=0.11.3" +version = "0.11.13" default-features = false diff --git a/rslib/src/backend/ankidroid/mod.rs b/rslib/src/backend/ankidroid/mod.rs index da190a564..26881a059 100644 --- a/rslib/src/backend/ankidroid/mod.rs +++ b/rslib/src/backend/ankidroid/mod.rs @@ -12,13 +12,14 @@ use super::{ pub(super) use crate::pb::ankidroid::ankidroid_service::Service as AnkidroidService; use crate::{ backend::ankidroid::db::{execute_for_row_count, insert_for_id}, + pb, pb::{ - self as pb, ankidroid::{DbResponse, GetActiveSequenceNumbersResponse, GetNextResultPageRequest}, - generic::{self, Empty, Int32, Json}, + generic, + generic::{Empty, Int32, Json}, }, prelude::*, - scheduler::timing::{self, fixed_offset_from_minutes}, + scheduler::{timing, timing::fixed_offset_from_minutes}, }; impl AnkidroidService for Backend { diff --git a/rslib/src/backend/collection.rs b/rslib/src/backend/collection.rs index 4b6cbddd4..8ba1fc4a7 100644 --- a/rslib/src/backend/collection.rs +++ b/rslib/src/backend/collection.rs @@ -8,10 +8,7 @@ use tracing::error; use super::{progress::Progress, Backend}; pub(super) use crate::pb::collection::collection_service::Service as CollectionService; use crate::{ - backend::progress::progress_to_proto, - collection::CollectionBuilder, - pb::{self as pb}, - prelude::*, + backend::progress::progress_to_proto, collection::CollectionBuilder, pb, prelude::*, storage::SchemaVersion, }; diff --git a/rslib/src/backend/decks.rs b/rslib/src/backend/decks.rs index 162d9631c..72605935d 100644 --- a/rslib/src/backend/decks.rs +++ b/rslib/src/backend/decks.rs @@ -7,7 +7,7 @@ use super::Backend; pub(super) use crate::pb::decks::decks_service::Service as DecksService; use crate::{ decks::{DeckSchema11, FilteredSearchOrder}, - pb::{self as pb}, + pb, prelude::*, scheduler::filtered::FilteredDeckForUpdate, }; diff --git a/rslib/src/backend/import_export.rs b/rslib/src/backend/import_export.rs index 86cecc05d..4a857a87e 100644 --- a/rslib/src/backend/import_export.rs +++ b/rslib/src/backend/import_export.rs @@ -7,10 +7,8 @@ use super::{progress::Progress, Backend}; pub(super) use crate::pb::import_export::importexport_service::Service as ImportExportService; use crate::{ import_export::{package::import_colpkg, ExportProgress, ImportProgress, NoteLog}, - pb::{ - import_export::{export_limit, ExportLimit}, - {self as pb}, - }, + pb, + pb::import_export::{export_limit, ExportLimit}, prelude::*, search::SearchNode, }; diff --git a/rslib/src/backend/media.rs b/rslib/src/backend/media.rs index e241e293c..d30cc6a69 100644 --- a/rslib/src/backend/media.rs +++ b/rslib/src/backend/media.rs @@ -3,11 +3,7 @@ use super::{progress::Progress, Backend}; pub(super) use crate::pb::media::media_service::Service as MediaService; -use crate::{ - media::{check::MediaChecker, MediaManager}, - pb, - prelude::*, -}; +use crate::{media::check::MediaChecker, pb, prelude::*}; impl MediaService for Backend { // media @@ -18,7 +14,7 @@ impl MediaService for Backend { let progress_fn = move |progress| handler.update(Progress::MediaCheck(progress as u32), true); self.with_col(|col| { - let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; + let mgr = col.media()?; col.transact_no_undo(|ctx| { let mut checker = MediaChecker::new(ctx, &mgr, progress_fn); let mut output = checker.check()?; @@ -41,19 +37,17 @@ impl MediaService for Backend { input: pb::media::TrashMediaFilesRequest, ) -> Result { self.with_col(|col| { - let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; - let mut ctx = mgr.dbctx(); - mgr.remove_files(&mut ctx, &input.fnames) + let mgr = col.media()?; + mgr.remove_files(&input.fnames) }) .map(Into::into) } fn add_media_file(&self, input: pb::media::AddMediaFileRequest) -> Result { self.with_col(|col| { - let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; - let mut ctx = mgr.dbctx(); + let mgr = col.media()?; Ok(mgr - .add_file(&mut ctx, &input.desired_name, &input.data)? + .add_file(&input.desired_name, &input.data)? .to_string() .into()) }) @@ -65,7 +59,7 @@ impl MediaService for Backend { move |progress| handler.update(Progress::MediaCheck(progress as u32), true); self.with_col(|col| { - let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; + let mgr = col.media()?; let mut checker = MediaChecker::new(col, &mgr, progress_fn); checker.empty_trash() }) @@ -77,7 +71,7 @@ impl MediaService for Backend { let progress_fn = move |progress| handler.update(Progress::MediaCheck(progress as u32), true); self.with_col(|col| { - let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; + let mgr = col.media()?; let mut checker = MediaChecker::new(col, &mgr, progress_fn); checker.restore_trash() }) diff --git a/rslib/src/backend/mod.rs b/rslib/src/backend/mod.rs index 1f681f925..4bba04f36 100644 --- a/rslib/src/backend/mod.rs +++ b/rslib/src/backend/mod.rs @@ -38,9 +38,7 @@ use std::{ use once_cell::sync::OnceCell; use progress::AbortHandleSlot; use prost::Message; -use tokio::runtime::{ - Runtime, {self}, -}; +use tokio::{runtime, runtime::Runtime}; use self::{ ankidroid::AnkidroidService, diff --git a/rslib/src/backend/notes.rs b/rslib/src/backend/notes.rs index 62dfa8935..695d16621 100644 --- a/rslib/src/backend/notes.rs +++ b/rslib/src/backend/notes.rs @@ -5,11 +5,7 @@ use std::collections::HashSet; use super::Backend; pub(super) use crate::pb::notes::notes_service::Service as NotesService; -use crate::{ - cloze::add_cloze_numbers_in_string, - pb::{self as pb}, - prelude::*, -}; +use crate::{cloze::add_cloze_numbers_in_string, pb, prelude::*}; impl NotesService for Backend { fn new_note(&self, input: pb::notetypes::NotetypeId) -> Result { diff --git a/rslib/src/backend/progress.rs b/rslib/src/backend/progress.rs index c69ca4ac9..0d10e9cc5 100644 --- a/rslib/src/backend/progress.rs +++ b/rslib/src/backend/progress.rs @@ -10,9 +10,14 @@ use crate::{ dbcheck::DatabaseCheckProgress, i18n::I18n, import_export::{ExportProgress, ImportProgress}, - media::sync::MediaSyncProgress, pb, - sync::{FullSyncProgress, NormalSyncProgress, SyncStage}, + sync::{ + collection::{ + normal::NormalSyncProgress, + progress::{FullSyncProgress, SyncStage}, + }, + media::progress::MediaSyncProgress, + }, }; pub(super) struct ThrottlingProgressHandler { diff --git a/rslib/src/backend/sync/mod.rs b/rslib/src/backend/sync/mod.rs index a367dfa04..dc629c55a 100644 --- a/rslib/src/backend/sync/mod.rs +++ b/rslib/src/backend/sync/mod.rs @@ -1,22 +1,27 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -mod server; - use std::sync::Arc; use futures::future::{AbortHandle, AbortRegistration, Abortable}; +use pb::sync::sync_status_response::Required; +use reqwest::Url; use tracing::warn; use super::{progress::AbortHandleSlot, Backend}; pub(super) use crate::pb::sync::sync_service::Service as SyncService; use crate::{ - media::MediaManager, pb, + pb::sync::SyncStatusResponse, prelude::*, sync::{ - get_remote_sync_meta, http::SyncRequest, sync_abort, sync_login, FullSyncProgress, - LocalServer, NormalSyncProgress, SyncActionRequired, SyncAuth, SyncMeta, SyncOutput, + collection::{ + normal::{ClientSyncState, NormalSyncProgress, SyncActionRequired, SyncOutput}, + progress::{sync_abort, FullSyncProgress}, + status::online_sync_status_check, + }, + http_client::HttpSyncClient, + login::{sync_login, SyncAuth}, }, }; @@ -24,17 +29,16 @@ use crate::{ pub(super) struct SyncState { remote_sync_status: RemoteSyncStatus, media_sync_abort: Option, - http_sync_server: Option, } #[derive(Default, Debug)] pub(super) struct RemoteSyncStatus { pub last_check: TimestampSecs, - pub last_response: pb::sync::sync_status_response::Required, + pub last_response: Required, } impl RemoteSyncStatus { - pub(super) fn update(&mut self, required: pb::sync::sync_status_response::Required) { + pub(super) fn update(&mut self, required: Required) { self.last_check = TimestampSecs::now(); self.last_response = required } @@ -45,6 +49,7 @@ impl From for pb::sync::SyncCollectionResponse { pb::sync::SyncCollectionResponse { host_number: o.host_number, server_message: o.server_message, + new_endpoint: o.new_endpoint, required: match o.required { SyncActionRequired::NoChanges => { pb::sync::sync_collection_response::ChangesRequired::NoChanges as i32 @@ -69,12 +74,20 @@ impl From for pb::sync::SyncCollectionResponse { } } -impl From for SyncAuth { - fn from(a: pb::sync::SyncAuth) -> Self { - SyncAuth { - hkey: a.hkey, - host_number: a.host_number, - } +impl TryFrom for SyncAuth { + type Error = AnkiError; + + fn try_from(value: pb::sync::SyncAuth) -> std::result::Result { + Ok(SyncAuth { + hkey: value.hkey, + endpoint: value + .endpoint + .map(|v| { + Url::try_from(v.as_str()) + .or_invalid("Invalid sync server specified. Please check the preferences.") + }) + .transpose()?, + }) } } @@ -123,14 +136,6 @@ impl SyncService for Backend { self.full_sync_inner(input, false)?; Ok(().into()) } - - fn sync_server_method( - &self, - input: pb::sync::SyncServerMethodRequest, - ) -> Result { - let req = SyncRequest::from_method_and_data(input.method(), input.data)?; - self.sync_server_method_inner(req).map(Into::into) - } } impl Backend { @@ -160,7 +165,8 @@ impl Backend { Ok((guard, abort_reg)) } - pub(super) fn sync_media_inner(&self, input: pb::sync::SyncAuth) -> Result<()> { + pub(super) fn sync_media_inner(&self, auth: pb::sync::SyncAuth) -> Result<()> { + let auth = auth.try_into()?; // mark media sync as active let (abort_handle, abort_reg) = AbortHandle::new_pair(); { @@ -173,20 +179,13 @@ impl Backend { } } - // get required info from collection - let mut guard = self.col.lock().unwrap(); - let col = guard.as_mut().unwrap(); - let folder = col.media_folder.clone(); - let db = col.media_db.clone(); - drop(guard); - // start the sync + let mgr = self.col.lock().unwrap().as_mut().unwrap().media()?; let mut handler = self.new_progress_handler(); let progress_fn = move |progress| handler.update(progress, true); - let mgr = MediaManager::new(&folder, &db)?; let rt = self.runtime_handle(); - let sync_fut = mgr.sync_media(progress_fn, input.host_number, &input.hkey); + let sync_fut = mgr.sync_media(progress_fn, auth); let abortable_sync = Abortable::new(sync_fut, abort_reg); let result = rt.block_on(abortable_sync); @@ -226,7 +225,7 @@ impl Backend { let (_guard, abort_reg) = self.sync_abort_handle()?; let rt = self.runtime_handle(); - let sync_fut = sync_login(&input.username, &input.password); + let sync_fut = sync_login(input.username, input.password, input.endpoint); let abortable_sync = Abortable::new(sync_fut, abort_reg); let ret = match rt.block_on(abortable_sync) { Ok(sync_result) => sync_result, @@ -234,7 +233,7 @@ impl Backend { }; ret.map(|a| pb::sync::SyncAuth { hkey: a.hkey, - host_number: a.host_number, + endpoint: None, }) } @@ -243,8 +242,8 @@ impl Backend { input: pb::sync::SyncAuth, ) -> Result { // any local changes mean we can skip the network round-trip - let req = self.with_col(|col| col.get_local_sync_status())?; - if req != pb::sync::sync_status_response::Required::NoChanges { + let req = self.with_col(|col| col.sync_status_offline())?; + if req != Required::NoChanges { return Ok(req.into()); } @@ -257,11 +256,12 @@ impl Backend { } // fetch and cache result + let auth = input.try_into()?; let rt = self.runtime_handle(); let time_at_check_begin = TimestampSecs::now(); - let remote: SyncMeta = rt.block_on(get_remote_sync_meta(input.into()))?; - let response = self.with_col(|col| col.get_sync_status(remote).map(Into::into))?; - + let local = self.with_col(|col| col.sync_meta())?; + let mut client = HttpSyncClient::new(auth); + let state = rt.block_on(online_sync_status_check(local, &mut client))?; { let mut guard = self.state.lock().unwrap(); // On startup, the sync status check will block on network access, and then automatic syncing begins, @@ -269,21 +269,21 @@ impl Backend { // so we discard it if stale. if guard.sync.remote_sync_status.last_check < time_at_check_begin { guard.sync.remote_sync_status.last_check = time_at_check_begin; - guard.sync.remote_sync_status.last_response = response; + guard.sync.remote_sync_status.last_response = state.required.into(); } } - Ok(response.into()) + Ok(state.into()) } pub(super) fn sync_collection_inner( &self, input: pb::sync::SyncAuth, ) -> Result { + let auth: SyncAuth = input.try_into()?; let (_guard, abort_reg) = self.sync_abort_handle()?; let rt = self.runtime_handle(); - let input_copy = input.clone(); let ret = self.with_col(|col| { let mut handler = self.new_progress_handler(); @@ -291,7 +291,7 @@ impl Backend { handler.update(progress, throttle); }; - let sync_fut = col.normal_sync(input.into(), progress_fn); + let sync_fut = col.normal_sync(auth.clone(), progress_fn); let abortable_sync = Abortable::new(sync_fut, abort_reg); match rt.block_on(abortable_sync) { @@ -301,7 +301,7 @@ impl Backend { col.storage.rollback_trx()?; // and tell AnkiWeb to clean up let _handle = std::thread::spawn(move || { - let _ = rt.block_on(sync_abort(input_copy.hkey, input_copy.host_number)); + let _ = rt.block_on(sync_abort(auth)); }); Err(AnkiError::Interrupted) @@ -320,6 +320,7 @@ impl Backend { } pub(super) fn full_sync_inner(&self, input: pb::sync::SyncAuth, upload: bool) -> Result<()> { + let auth = input.try_into()?; self.abort_media_sync_and_wait(); let rt = self.runtime_handle(); @@ -336,16 +337,16 @@ impl Backend { let builder = col_inner.as_builder(); let mut handler = self.new_progress_handler(); - let progress_fn = move |progress: FullSyncProgress, throttle: bool| { + let progress_fn = Box::new(move |progress: FullSyncProgress, throttle: bool| { handler.update(progress, throttle); - }; + }); let result = if upload { - let sync_fut = col_inner.full_upload(input.into(), Box::new(progress_fn)); + let sync_fut = col_inner.full_upload(auth, progress_fn); let abortable_sync = Abortable::new(sync_fut, abort_reg); rt.block_on(abortable_sync) } else { - let sync_fut = col_inner.full_download(input.into(), Box::new(progress_fn)); + let sync_fut = col_inner.full_download(auth, progress_fn); let abortable_sync = Abortable::new(sync_fut, abort_reg); rt.block_on(abortable_sync) }; @@ -361,7 +362,7 @@ impl Backend { .unwrap() .sync .remote_sync_status - .update(pb::sync::sync_status_response::Required::NoChanges); + .update(Required::NoChanges); } sync_result } @@ -369,3 +370,31 @@ impl Backend { } } } + +impl From for SyncStatusResponse { + fn from(r: Required) -> Self { + SyncStatusResponse { + required: r.into(), + new_endpoint: None, + } + } +} + +impl From for SyncStatusResponse { + fn from(r: ClientSyncState) -> Self { + SyncStatusResponse { + required: Required::from(r.required).into(), + new_endpoint: r.new_endpoint, + } + } +} + +impl From for Required { + fn from(r: SyncActionRequired) -> Self { + match r { + SyncActionRequired::NoChanges => Required::NoChanges, + SyncActionRequired::FullSyncRequired { .. } => Required::FullSync, + SyncActionRequired::NormalSyncRequired => Required::NormalSync, + } + } +} diff --git a/rslib/src/backend/sync/server.rs b/rslib/src/backend/sync/server.rs deleted file mode 100644 index 7687e747e..000000000 --- a/rslib/src/backend/sync/server.rs +++ /dev/null @@ -1,211 +0,0 @@ -// Copyright: Ankitects Pty Ltd and contributors -// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html - -use std::{path::PathBuf, sync::MutexGuard}; - -use tokio::runtime::Runtime; - -use crate::{ - backend::{Backend, BackendState}, - error::SyncErrorKind, - prelude::*, - sync::{ - http::{ - ApplyChangesRequest, ApplyChunkRequest, ApplyGravesRequest, HostKeyRequest, - HostKeyResponse, MetaRequest, SanityCheckRequest, StartRequest, SyncRequest, - }, - Chunk, Graves, LocalServer, SanityCheckResponse, SanityCheckStatus, SyncMeta, SyncServer, - UnchunkedChanges, SYNC_VERSION_MAX, SYNC_VERSION_MIN, - }, -}; - -impl Backend { - fn with_sync_server(&self, func: F) -> Result - where - F: FnOnce(&mut LocalServer) -> Result, - { - let mut state_guard = self.state.lock().unwrap(); - let out = func( - state_guard - .sync - .http_sync_server - .as_mut() - .ok_or_else(|| AnkiError::sync_error("", SyncErrorKind::SyncNotStarted))?, - ); - if out.is_err() { - self.abort_and_restore_collection(Some(state_guard)) - } - out - } - - /// Gives out a dummy hkey - auth should be implemented at a higher layer. - fn host_key(&self, _input: HostKeyRequest) -> Result { - Ok(HostKeyResponse { - key: "unimplemented".into(), - }) - } - - fn meta(&self, input: MetaRequest) -> Result { - if input.sync_version < SYNC_VERSION_MIN || input.sync_version > SYNC_VERSION_MAX { - return Ok(SyncMeta { - server_message: "Your Anki version is either too old, or too new.".into(), - should_continue: false, - ..Default::default() - }); - } - let server = self.col_into_server()?; - let rt = Runtime::new().unwrap(); - let meta = rt.block_on(server.meta())?; - self.server_into_col(server); - - Ok(meta) - } - - /// Takes the collection from the backend, places it into a server, and returns it. - fn col_into_server(&self) -> Result { - self.col - .lock() - .unwrap() - .take() - .map(LocalServer::new) - .ok_or(AnkiError::CollectionNotOpen) - } - - fn server_into_col(&self, server: LocalServer) { - let col = server.into_col(); - let mut col_guard = self.col.lock().unwrap(); - assert!(col_guard.replace(col).is_none()); - } - - fn take_server(&self, state_guard: Option>) -> Result { - let mut state_guard = state_guard.unwrap_or_else(|| self.state.lock().unwrap()); - state_guard - .sync - .http_sync_server - .take() - .ok_or_else(|| AnkiError::sync_error("", SyncErrorKind::SyncNotStarted)) - } - - fn start(&self, input: StartRequest) -> Result { - // place col into new server - let server = self.col_into_server()?; - let mut state_guard = self.state.lock().unwrap(); - assert!(state_guard.sync.http_sync_server.replace(server).is_none()); - drop(state_guard); - - self.with_sync_server(|server| { - let rt = Runtime::new().unwrap(); - rt.block_on(server.start( - input.client_usn, - input.local_is_newer, - input.deprecated_client_graves, - )) - }) - } - - fn apply_graves(&self, input: ApplyGravesRequest) -> Result<()> { - self.with_sync_server(|server| { - let rt = Runtime::new().unwrap(); - rt.block_on(server.apply_graves(input.chunk)) - }) - } - - fn apply_changes(&self, input: ApplyChangesRequest) -> Result { - self.with_sync_server(|server| { - let rt = Runtime::new().unwrap(); - rt.block_on(server.apply_changes(input.changes)) - }) - } - - fn chunk(&self) -> Result { - self.with_sync_server(|server| { - let rt = Runtime::new().unwrap(); - rt.block_on(server.chunk()) - }) - } - - fn apply_chunk(&self, input: ApplyChunkRequest) -> Result<()> { - self.with_sync_server(|server| { - let rt = Runtime::new().unwrap(); - rt.block_on(server.apply_chunk(input.chunk)) - }) - } - - fn sanity_check(&self, input: SanityCheckRequest) -> Result { - self.with_sync_server(|server| { - let rt = Runtime::new().unwrap(); - rt.block_on(server.sanity_check(input.client)) - }) - .map(|out| { - if out.status != SanityCheckStatus::Ok { - // sanity check failures are an implicit abort - self.abort_and_restore_collection(None); - } - out - }) - } - - fn finish(&self) -> Result { - let out = self.with_sync_server(|server| { - let rt = Runtime::new().unwrap(); - rt.block_on(server.finish()) - }); - self.server_into_col(self.take_server(None)?); - out - } - - fn abort(&self) -> Result<()> { - self.abort_and_restore_collection(None); - Ok(()) - } - - fn abort_and_restore_collection(&self, state_guard: Option>) { - if let Ok(mut server) = self.take_server(state_guard) { - let rt = Runtime::new().unwrap(); - // attempt to roll back - if let Err(abort_err) = rt.block_on(server.abort()) { - println!("abort failed: {:?}", abort_err); - } - self.server_into_col(server); - } - } - - /// Caller must re-open collection after this request. Provided file will be - /// consumed. - fn upload(&self, input: PathBuf) -> Result<()> { - // spool input into a file - let server = Box::new(self.col_into_server()?); - // then process upload - let rt = Runtime::new().unwrap(); - rt.block_on(server.full_upload(&input, true)) - } - - /// Caller must re-open collection after this request, and is responsible - /// for cleaning up the returned file. - fn download(&self) -> Result> { - let server = Box::new(self.col_into_server()?); - let rt = Runtime::new().unwrap(); - let file = rt.block_on(server.full_download(None))?; - let path = file.into_temp_path().keep()?; - Ok(path.to_str().expect("path was not in utf8").into()) - } - - pub(crate) fn sync_server_method_inner(&self, req: SyncRequest) -> Result> { - use serde_json::to_vec; - match req { - SyncRequest::HostKey(v) => to_vec(&self.host_key(v)?), - SyncRequest::Meta(v) => to_vec(&self.meta(v)?), - SyncRequest::Start(v) => to_vec(&self.start(v)?), - SyncRequest::ApplyGraves(v) => to_vec(&self.apply_graves(v)?), - SyncRequest::ApplyChanges(v) => to_vec(&self.apply_changes(v)?), - SyncRequest::Chunk => to_vec(&self.chunk()?), - SyncRequest::ApplyChunk(v) => to_vec(&self.apply_chunk(v)?), - SyncRequest::SanityCheck(v) => to_vec(&self.sanity_check(v)?), - SyncRequest::Finish => to_vec(&self.finish()?), - SyncRequest::Abort => to_vec(&self.abort()?), - SyncRequest::FullUpload(v) => to_vec(&self.upload(v)?), - SyncRequest::FullDownload => return self.download(), - } - .map_err(Into::into) - } -} diff --git a/rslib/src/collection/backup.rs b/rslib/src/collection/backup.rs index a50c4131e..f52414c1c 100644 --- a/rslib/src/collection/backup.rs +++ b/rslib/src/collection/backup.rs @@ -5,9 +5,8 @@ use std::{ ffi::OsStr, fs::{read_dir, remove_file, DirEntry}, path::{Path, PathBuf}, - thread::{ - JoinHandle, {self}, - }, + thread, + thread::JoinHandle, time::SystemTime, }; diff --git a/rslib/src/collection/mod.rs b/rslib/src/collection/mod.rs index 6cf8f429b..b682c219b 100644 --- a/rslib/src/collection/mod.rs +++ b/rslib/src/collection/mod.rs @@ -33,6 +33,7 @@ pub struct CollectionBuilder { media_db: Option, server: Option, tr: Option, + check_integrity: bool, // temporary option for AnkiDroid force_schema11: Option, } @@ -56,7 +57,13 @@ impl CollectionBuilder { let media_folder = self.media_folder.clone().unwrap_or_default(); let media_db = self.media_db.clone().unwrap_or_default(); let force_schema11 = self.force_schema11.unwrap_or_default(); - let storage = SqliteStorage::open_or_create(&col_path, &tr, server, force_schema11)?; + let storage = SqliteStorage::open_or_create( + &col_path, + &tr, + server, + self.check_integrity, + force_schema11, + )?; let col = Collection { storage, col_path, @@ -95,6 +102,11 @@ impl CollectionBuilder { self.force_schema11 = Some(force); self } + + pub fn set_check_integrity(&mut self, check_integrity: bool) -> &mut Self { + self.check_integrity = check_integrity; + self + } } #[cfg(test)] @@ -147,7 +159,13 @@ impl Collection { builder } - pub(crate) fn close(self, desired_version: Option) -> Result<()> { + // A count of all changed rows since the collection was opened, which can be used to detect + // if the collection was modified or not. + pub fn changes_since_open(&self) -> u64 { + self.storage.db.changes() + } + + pub fn close(self, desired_version: Option) -> Result<()> { self.storage.close(desired_version) } diff --git a/rslib/src/config/mod.rs b/rslib/src/config/mod.rs index 09ce82544..050cb40e0 100644 --- a/rslib/src/config/mod.rs +++ b/rslib/src/config/mod.rs @@ -12,7 +12,6 @@ pub(crate) mod undo; use serde::{de::DeserializeOwned, Serialize}; use serde_repr::{Deserialize_repr, Serialize_repr}; use strum::IntoStaticStr; -use tracing::warn; pub use self::{ bool::BoolKey, deck::DeckConfigKey, notetype::get_aux_notetype_config_key, @@ -108,10 +107,8 @@ impl Collection { match self.storage.get_config_value(key) { Ok(Some(val)) => Some(val), Ok(None) => None, - Err(e) => { - warn!(key, ?e, "error accessing config key"); - None - } + // If the key is missing or invalid, we use the default value. + Err(_) => None, } } diff --git a/rslib/src/decks/schema11.rs b/rslib/src/decks/schema11.rs index 5a7206913..deb9604a1 100644 --- a/rslib/src/decks/schema11.rs +++ b/rslib/src/decks/schema11.rs @@ -23,8 +23,9 @@ pub enum DeckSchema11 { // serde doesn't support integer/bool enum tags, so we manually pick the correct variant mod dynfix { - use serde::de::{ - Deserialize, Deserializer, {self}, + use serde::{ + de, + de::{Deserialize, Deserializer}, }; use serde_json::{Map, Value}; diff --git a/rslib/src/error/file_io.rs b/rslib/src/error/file_io.rs index c80dbe037..2200a179a 100644 --- a/rslib/src/error/file_io.rs +++ b/rslib/src/error/file_io.rs @@ -8,7 +8,7 @@ use snafu::Snafu; /// Wrapper for [std::io::Error] with additional information on the attempted /// operation. #[derive(Debug, Snafu)] -#[snafu(visibility(pub))] +#[snafu(visibility(pub), display("{op:?} {path:?}"))] pub struct FileIoError { pub path: PathBuf, pub op: FileOp, @@ -29,6 +29,7 @@ pub enum FileOp { Open, Create, Write, + Remove, CopyFrom(PathBuf), Persist, Sync, @@ -52,6 +53,7 @@ impl FileIoError { FileOp::Read => "read".into(), FileOp::Create => "create file in".into(), FileOp::Write => "write".into(), + FileOp::Remove => "remove".into(), FileOp::CopyFrom(p) => format!("copy from '{}' to", p.to_string_lossy()), FileOp::Persist => "persist".into(), FileOp::Sync => "sync".into(), diff --git a/rslib/src/error/mod.rs b/rslib/src/error/mod.rs index a155ef83b..89cfeddff 100644 --- a/rslib/src/error/mod.rs +++ b/rslib/src/error/mod.rs @@ -5,7 +5,7 @@ mod db; mod file_io; mod filtered; mod invalid_input; -mod network; +pub(crate) mod network; mod not_found; mod search; diff --git a/rslib/src/error/network.rs b/rslib/src/error/network.rs index 0b418e1bc..715e2144d 100644 --- a/rslib/src/error/network.rs +++ b/rslib/src/error/network.rs @@ -1,13 +1,17 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +use std::error::Error; + use anki_i18n::I18n; use reqwest::StatusCode; use snafu::Snafu; use super::AnkiError; +use crate::sync::{collection::sanity::SanityCheckCounts, error::HttpError}; #[derive(Debug, PartialEq, Eq, Snafu)] +#[snafu(visibility(pub(crate)))] pub struct NetworkError { pub info: String, pub kind: NetworkErrorKind, @@ -40,6 +44,10 @@ pub enum SyncErrorKind { DatabaseCheckRequired, SyncNotStarted, UploadTooLarge, + SanityCheckFailed { + client: Option, + server: Option, + }, } impl AnkiError { @@ -172,7 +180,9 @@ impl SyncError { SyncErrorKind::AuthFailed => tr.sync_wrong_pass(), SyncErrorKind::ResyncRequired => tr.sync_resync_required(), SyncErrorKind::ClockIncorrect => tr.sync_clock_off(), - SyncErrorKind::DatabaseCheckRequired => tr.sync_sanity_check_failed(), + SyncErrorKind::DatabaseCheckRequired | SyncErrorKind::SanityCheckFailed { .. } => { + tr.sync_sanity_check_failed() + } SyncErrorKind::SyncNotStarted => "sync not started".into(), SyncErrorKind::UploadTooLarge => tr.sync_upload_too_large(&self.info), } @@ -192,3 +202,31 @@ impl NetworkError { format!("{}\n\n{}", summary, details) } } + +// This needs rethinking; we should be attaching error context as errors are encountered +// instead of trying to determine the problem later. +impl From for AnkiError { + fn from(err: HttpError) -> Self { + if let Some(source) = &err.source { + if let Some(err) = source.downcast_ref::() { + if let Some(status) = err.status() { + let kind = match status { + StatusCode::CONFLICT => SyncErrorKind::Conflict, + StatusCode::NOT_IMPLEMENTED => SyncErrorKind::ClientTooOld, + StatusCode::FORBIDDEN => SyncErrorKind::AuthFailed, + StatusCode::INTERNAL_SERVER_ERROR => SyncErrorKind::ServerError, + StatusCode::BAD_REQUEST => SyncErrorKind::DatabaseCheckRequired, + _ => SyncErrorKind::Other, + }; + let info = format!("{:?}", err); + // in the future we should chain the error instead of discarding it + return AnkiError::sync_error(info, kind); + } else if let Some(source) = err.source() { + let info = format!("{:?}", source); + return AnkiError::sync_error(info, SyncErrorKind::Other); + } + } + } + AnkiError::sync_error(format!("{:?}", err), SyncErrorKind::Other) + } +} diff --git a/rslib/src/import_export/package/apkg/import/media.rs b/rslib/src/import_export/package/apkg/import/media.rs index 653aa73dc..0ba168d2f 100644 --- a/rslib/src/import_export/package/apkg/import/media.rs +++ b/rslib/src/import_export/package/apkg/import/media.rs @@ -38,7 +38,9 @@ impl Context<'_> { } let db_progress_fn = self.progress.media_db_fn(ImportProgress::MediaCheck)?; - let existing_sha1s = self.media_manager.all_checksums(db_progress_fn)?; + let existing_sha1s = self + .media_manager + .all_checksums_after_checking(db_progress_fn)?; prepare_media( media_entries, @@ -50,9 +52,8 @@ impl Context<'_> { pub(super) fn copy_media(&mut self, media_map: &mut MediaUseMap) -> Result<()> { let mut incrementor = self.progress.incrementor(ImportProgress::Media); - let mut dbctx = self.media_manager.dbctx(); let mut copier = MediaCopier::new(false); - self.media_manager.transact(&mut dbctx, |dbctx| { + self.media_manager.transact(|_db| { for entry in media_map.used_entries() { incrementor.increment()?; entry.copy_and_ensure_sha1_set( @@ -61,7 +62,7 @@ impl Context<'_> { &mut copier, )?; self.media_manager - .add_entry(dbctx, &entry.name, entry.sha1.unwrap())?; + .add_entry(&entry.name, entry.sha1.unwrap())?; } Ok(()) }) diff --git a/rslib/src/import_export/package/apkg/import/mod.rs b/rslib/src/import_export/package/apkg/import/mod.rs index 192bb5f1d..0ede571c6 100644 --- a/rslib/src/import_export/package/apkg/import/mod.rs +++ b/rslib/src/import_export/package/apkg/import/mod.rs @@ -59,7 +59,7 @@ impl<'a> Context<'a> { ) -> Result { let mut progress = IncrementableProgress::new(progress_fn); progress.call(ImportProgress::Extracting)?; - let media_manager = MediaManager::new(&target_col.media_folder, &target_col.media_db)?; + let media_manager = target_col.media()?; let meta = Meta::from_archive(&mut archive)?; let data = ExchangeData::gather_from_archive( &mut archive, diff --git a/rslib/src/import_export/package/colpkg/export.rs b/rslib/src/import_export/package/colpkg/export.rs index 9f7ee8516..eb5ce637b 100644 --- a/rslib/src/import_export/package/colpkg/export.rs +++ b/rslib/src/import_export/package/colpkg/export.rs @@ -6,9 +6,8 @@ use std::{ collections::HashMap, ffi::OsStr, fs::File, - io::{ - Read, Write, {self}, - }, + io, + io::{Read, Write}, path::{Path, PathBuf}, }; diff --git a/rslib/src/import_export/package/colpkg/import.rs b/rslib/src/import_export/package/colpkg/import.rs index 5af758323..a8c1a6fbd 100644 --- a/rslib/src/import_export/package/colpkg/import.rs +++ b/rslib/src/import_export/package/colpkg/import.rs @@ -3,17 +3,13 @@ use std::{ fs::File, - io::{ - Write, {self}, - }, + io, + io::Write, path::{Path, PathBuf}, }; use zip::{read::ZipFile, ZipArchive}; -use zstd::{ - stream::copy_decode, - {self}, -}; +use zstd::{self, stream::copy_decode}; use crate::{ collection::CollectionBuilder, diff --git a/rslib/src/import_export/package/colpkg/tests.rs b/rslib/src/import_export/package/colpkg/tests.rs index 9ac76f2da..e4e3f1ecc 100644 --- a/rslib/src/import_export/package/colpkg/tests.rs +++ b/rslib/src/import_export/package/colpkg/tests.rs @@ -27,11 +27,10 @@ fn collection_with_media(dir: &Path, name: &str) -> Result { let mut note = nt.new_note(); col.add_note(&mut note, DeckId(1))?; // add sample media - let mgr = MediaManager::new(&col.media_folder, &col.media_db)?; - let mut ctx = mgr.dbctx(); - mgr.add_file(&mut ctx, "1", b"1")?; - mgr.add_file(&mut ctx, "2", b"2")?; - mgr.add_file(&mut ctx, "3", b"3")?; + let mgr = col.media()?; + mgr.add_file("1", b"1")?; + mgr.add_file("2", b"2")?; + mgr.add_file("3", b"3")?; Ok(col) } diff --git a/rslib/src/import_export/package/media.rs b/rslib/src/import_export/package/media.rs index e41a78731..2faa083d0 100644 --- a/rslib/src/import_export/package/media.rs +++ b/rslib/src/import_export/package/media.rs @@ -4,9 +4,8 @@ use std::{ borrow::Cow, collections::HashMap, - fs::{ - File, {self}, - }, + fs, + fs::File, io, path::{Path, PathBuf}, }; diff --git a/rslib/src/import_export/package/meta.rs b/rslib/src/import_export/package/meta.rs index a001d3d1c..c5beea630 100644 --- a/rslib/src/import_export/package/meta.rs +++ b/rslib/src/import_export/package/meta.rs @@ -1,12 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use std::{ - fs::File, - io::{ - Read, {self}, - }, -}; +use std::{fs::File, io, io::Read}; use prost::Message; use zip::ZipArchive; diff --git a/rslib/src/io.rs b/rslib/src/io.rs index 2aee5b3b9..0f4ce278e 100644 --- a/rslib/src/io.rs +++ b/rslib/src/io.rs @@ -31,6 +31,15 @@ pub(crate) fn write_file(path: impl AsRef, contents: impl AsRef<[u8]>) -> }) } +/// See [std::fs::remove_file]. +#[allow(dead_code)] +pub(crate) fn remove_file(path: impl AsRef) -> Result<()> { + std::fs::remove_file(&path).context(FileIoSnafu { + path: path.as_ref(), + op: FileOp::Remove, + }) +} + /// See [std::fs::create_dir]. pub(crate) fn create_dir(path: impl AsRef) -> Result<()> { std::fs::create_dir(&path).context(FileIoSnafu { diff --git a/rslib/src/lib.rs b/rslib/src/lib.rs index 9ef072d26..bf7582263 100644 --- a/rslib/src/lib.rs +++ b/rslib/src/lib.rs @@ -36,7 +36,7 @@ pub mod search; pub mod serde; mod stats; pub mod storage; -mod sync; +pub mod sync; pub mod tags; pub mod template; pub mod template_filters; diff --git a/rslib/src/log.rs b/rslib/src/log.rs index b2b6fae54..ecaa858ad 100644 --- a/rslib/src/log.rs +++ b/rslib/src/log.rs @@ -14,16 +14,20 @@ const LOG_ROTATE_BYTES: u64 = 50 * 1024 * 1024; /// Enable logging to the console, and optionally also to a file. pub fn set_global_logger(path: Option<&str>) -> Result<()> { - let file_writer = if let Some(path) = path { - Some(Layer::new().with_writer(get_appender(path)?)) - } else { - None - }; - let subscriber = tracing_subscriber::registry() - .with(fmt::layer()) - .with(file_writer) - .with(EnvFilter::from_default_env()); - set_global_default(subscriber).or_invalid("global subscriber already set")?; + static ONCE: OnceCell<()> = OnceCell::new(); + ONCE.get_or_try_init(|| -> Result<()> { + let file_writer = if let Some(path) = path { + Some(Layer::new().with_writer(get_appender(path)?)) + } else { + None + }; + let subscriber = tracing_subscriber::registry() + .with(fmt::layer().with_target(false)) + .with(file_writer) + .with(EnvFilter::from_default_env()); + set_global_default(subscriber).or_invalid("global subscriber already set")?; + Ok(()) + })?; Ok(()) } diff --git a/rslib/src/media/check.rs b/rslib/src/media/check.rs index 68e4d29b7..08c769fdd 100644 --- a/rslib/src/media/check.rs +++ b/rslib/src/media/check.rs @@ -16,14 +16,11 @@ use crate::{ error::{AnkiError, DbErrorKind, Result}, latex::extract_latex_expanding_clozes, media::{ - database::MediaDatabaseContext, - files::{ - data_for_file, filename_if_normalized, normalize_nfc_filename, trash_folder, - MEDIA_SYNC_FILESIZE_LIMIT, - }, + files::{data_for_file, filename_if_normalized, normalize_nfc_filename, trash_folder}, MediaManager, }, notes::Note, + sync::media::MAX_INDIVIDUAL_MEDIA_FILE_SIZE, text::{extract_media_refs, normalize_to_nfc, MediaRef, REMOTE_FILENAME}, }; @@ -74,9 +71,7 @@ where } pub fn check(&mut self) -> Result { - let mut ctx = self.mgr.dbctx(); - - let folder_check = self.check_media_folder(&mut ctx)?; + let folder_check = self.check_media_folder()?; let referenced_files = self.check_media_references(&folder_check.renamed)?; let (unused, missing) = find_unused_and_missing(folder_check.files, referenced_files); let (trash_count, trash_bytes) = self.files_in_trash()?; @@ -186,7 +181,7 @@ where /// - Renames files with invalid names /// - Notes folders/oversized files /// - Gathers a list of all files - fn check_media_folder(&mut self, ctx: &mut MediaDatabaseContext) -> Result { + fn check_media_folder(&mut self) -> Result { let mut out = MediaFolderCheck::default(); for dentry in self.mgr.media_folder.read_dir()? { let dentry = dentry?; @@ -211,7 +206,7 @@ where // ignore large files and zero byte files let metadata = dentry.metadata()?; - if metadata.len() > MEDIA_SYNC_FILESIZE_LIMIT as u64 { + if metadata.len() > MAX_INDIVIDUAL_MEDIA_FILE_SIZE as u64 { out.oversize.push(disk_fname.to_string()); continue; } @@ -224,7 +219,7 @@ where } else { match data_for_file(&self.mgr.media_folder, disk_fname)? { Some(data) => { - let norm_name = self.normalize_file(ctx, disk_fname, data)?; + let norm_name = self.normalize_file(disk_fname, data)?; out.renamed .insert(disk_fname.to_string(), norm_name.to_string()); out.files.push(norm_name.into_owned()); @@ -242,14 +237,9 @@ where } /// Write file data to normalized location, moving old file to trash. - fn normalize_file<'a>( - &mut self, - ctx: &mut MediaDatabaseContext, - disk_fname: &'a str, - data: Vec, - ) -> Result> { + fn normalize_file<'a>(&mut self, disk_fname: &'a str, data: Vec) -> Result> { // add a copy of the file using the correct name - let fname = self.mgr.add_file(ctx, disk_fname, &data)?; + let fname = self.mgr.add_file(disk_fname, &data)?; debug!(from = disk_fname, to = &fname.as_ref(), "renamed"); assert_ne!(fname.as_ref(), disk_fname); @@ -336,9 +326,7 @@ where let fname_os = dentry.file_name(); let fname = fname_os.to_string_lossy(); if let Some(data) = data_for_file(&trash, fname.as_ref())? { - let _new_fname = - self.mgr - .add_file(&mut self.mgr.dbctx(), fname.as_ref(), &data)?; + let _new_fname = self.mgr.add_file(fname.as_ref(), &data)?; } else { debug!(?fname, "file disappeared while restoring trash"); } diff --git a/rslib/src/media/files.rs b/rslib/src/media/files.rs index fc4bcae38..99a52b8f2 100644 --- a/rslib/src/media/files.rs +++ b/rslib/src/media/files.rs @@ -20,17 +20,9 @@ use crate::{ error::{FileIoError, FileIoSnafu, FileOp}, io::{create_dir, open_file, write_file}, prelude::*, + sync::media::MAX_MEDIA_FILENAME_LENGTH, }; -/// The maximum length we allow a filename to be. When combined -/// with the rest of the path, the full path needs to be under ~240 chars -/// on some platforms, and some filesystems like eCryptFS will increase -/// the length of the filename. -pub(super) static MAX_FILENAME_LENGTH: usize = 120; - -/// Media syncing does not support files over 100MiB. -pub(super) static MEDIA_SYNC_FILESIZE_LIMIT: usize = 100 * 1024 * 1024; - lazy_static! { static ref WINDOWS_DEVICE_NAME: Regex = Regex::new( r#"(?xi) @@ -56,7 +48,7 @@ lazy_static! { "# ) .unwrap(); - pub(super) static ref NONSYNCABLE_FILENAME: Regex = Regex::new( + pub(crate) static ref NONSYNCABLE_FILENAME: Regex = Regex::new( r#"(?xi) ^ (:? @@ -119,7 +111,7 @@ pub(crate) fn normalize_nfc_filename(mut fname: Cow) -> Cow { fname = format!("{}_", fname.as_ref()).into(); } - if let Cow::Owned(o) = truncate_filename(fname.as_ref(), MAX_FILENAME_LENGTH) { + if let Cow::Owned(o) = truncate_filename(fname.as_ref(), MAX_MEDIA_FILENAME_LENGTH) { fname = o.into(); } @@ -198,7 +190,7 @@ where /// Convert foo.jpg into foo-abcde12345679.jpg pub(crate) fn add_hash_suffix_to_file_stem(fname: &str, hash: &Sha1Hash) -> String { // when appending a hash to make unique, it will be 40 bytes plus the hyphen. - let max_len = MAX_FILENAME_LENGTH - 40 - 1; + let max_len = MAX_MEDIA_FILENAME_LENGTH - 40 - 1; let (stem, ext) = split_and_truncate_filename(fname, max_len); @@ -308,14 +300,14 @@ pub(crate) fn sha1_of_data(data: &[u8]) -> Sha1Hash { hasher.finalize().into() } -pub(super) fn mtime_as_i64>(path: P) -> io::Result { +pub(crate) fn mtime_as_i64>(path: P) -> io::Result { Ok(path .as_ref() .metadata()? .modified()? .duration_since(time::UNIX_EPOCH) .unwrap() - .as_secs() as i64) + .as_millis() as i64) } pub fn remove_files(media_folder: &Path, files: &[S]) -> Result<()> @@ -375,7 +367,7 @@ pub(super) fn trash_folder(media_folder: &Path) -> Result { } } -pub(super) struct AddedFile { +pub struct AddedFile { pub fname: String, pub sha1: Sha1Hash, pub mtime: i64, @@ -386,7 +378,7 @@ pub(super) struct AddedFile { /// /// Because AnkiWeb did not previously enforce file name limits and invalid /// characters, we'll need to rename the file if it is not valid. -pub(super) fn add_file_from_ankiweb( +pub(crate) fn add_file_from_ankiweb( media_folder: &Path, fname: &str, data: &[u8], @@ -424,7 +416,7 @@ pub(super) fn add_file_from_ankiweb( }) } -pub(super) fn data_for_file(media_folder: &Path, fname: &str) -> Result>> { +pub(crate) fn data_for_file(media_folder: &Path, fname: &str) -> Result>> { let mut file = match open_file(&media_folder.join(fname)) { Err(e) if e.is_not_found() => return Ok(None), res => res?, @@ -440,9 +432,12 @@ mod test { use tempfile::tempdir; - use crate::media::files::{ - add_data_to_folder_uniquely, add_hash_suffix_to_file_stem, normalize_filename, - remove_files, sha1_of_data, truncate_filename, MAX_FILENAME_LENGTH, + use crate::{ + media::files::{ + add_data_to_folder_uniquely, add_hash_suffix_to_file_stem, normalize_filename, + remove_files, sha1_of_data, truncate_filename, + }, + sync::media::MAX_MEDIA_FILENAME_LENGTH, }; #[test] @@ -456,9 +451,12 @@ mod test { assert_eq!(normalize_filename("test.").as_ref(), "test._"); assert_eq!(normalize_filename("test ").as_ref(), "test _"); - let expected_stem_len = MAX_FILENAME_LENGTH - ".jpg".len() - 1; + let expected_stem_len = MAX_MEDIA_FILENAME_LENGTH - ".jpg".len() - 1; assert_eq!( - normalize_filename(&format!("{}.jpg", "x".repeat(MAX_FILENAME_LENGTH * 2))), + normalize_filename(&format!( + "{}.jpg", + "x".repeat(MAX_MEDIA_FILENAME_LENGTH * 2) + )), "x".repeat(expected_stem_len) + ".jpg" ); } @@ -516,29 +514,32 @@ mod test { #[test] fn truncation() { - let one_less = "x".repeat(MAX_FILENAME_LENGTH - 1); + let one_less = "x".repeat(MAX_MEDIA_FILENAME_LENGTH - 1); assert_eq!( - truncate_filename(&one_less, MAX_FILENAME_LENGTH), + truncate_filename(&one_less, MAX_MEDIA_FILENAME_LENGTH), Cow::Borrowed(&one_less) ); - let equal = "x".repeat(MAX_FILENAME_LENGTH); + let equal = "x".repeat(MAX_MEDIA_FILENAME_LENGTH); assert_eq!( - truncate_filename(&equal, MAX_FILENAME_LENGTH), + truncate_filename(&equal, MAX_MEDIA_FILENAME_LENGTH), Cow::Borrowed(&equal) ); - let equal = format!("{}.jpg", "x".repeat(MAX_FILENAME_LENGTH - 4)); + let equal = format!("{}.jpg", "x".repeat(MAX_MEDIA_FILENAME_LENGTH - 4)); assert_eq!( - truncate_filename(&equal, MAX_FILENAME_LENGTH), + truncate_filename(&equal, MAX_MEDIA_FILENAME_LENGTH), Cow::Borrowed(&equal) ); - let one_more = "x".repeat(MAX_FILENAME_LENGTH + 1); + let one_more = "x".repeat(MAX_MEDIA_FILENAME_LENGTH + 1); assert_eq!( - truncate_filename(&one_more, MAX_FILENAME_LENGTH), - Cow::::Owned("x".repeat(MAX_FILENAME_LENGTH - 2)) + truncate_filename(&one_more, MAX_MEDIA_FILENAME_LENGTH), + Cow::::Owned("x".repeat(MAX_MEDIA_FILENAME_LENGTH - 2)) ); assert_eq!( - truncate_filename(&" ".repeat(MAX_FILENAME_LENGTH + 1), MAX_FILENAME_LENGTH), - Cow::::Owned(format!("{}_", " ".repeat(MAX_FILENAME_LENGTH - 2))) + truncate_filename( + &" ".repeat(MAX_MEDIA_FILENAME_LENGTH + 1), + MAX_MEDIA_FILENAME_LENGTH + ), + Cow::::Owned(format!("{}_", " ".repeat(MAX_MEDIA_FILENAME_LENGTH - 2))) ); } } diff --git a/rslib/src/media/mod.rs b/rslib/src/media/mod.rs index 58a4afadb..c24f509ab 100644 --- a/rslib/src/media/mod.rs +++ b/rslib/src/media/mod.rs @@ -1,35 +1,41 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html +pub mod check; +pub mod files; + use std::{ borrow::Cow, collections::HashMap, path::{Path, PathBuf}, }; -use rusqlite::Connection; - -use self::changetracker::ChangeTracker; use crate::{ - media::{ - database::{open_or_create, MediaDatabaseContext, MediaEntry}, - files::{add_data_to_folder_uniquely, mtime_as_i64, remove_files, sha1_of_data}, - sync::{MediaSyncProgress, MediaSyncer}, - }, + io::create_dir_all, + media::files::{add_data_to_folder_uniquely, mtime_as_i64, remove_files, sha1_of_data}, prelude::*, + sync::{ + http_client::HttpSyncClient, + login::SyncAuth, + media::{ + database::client::{changetracker::ChangeTracker, MediaDatabase, MediaEntry}, + progress::MediaSyncProgress, + syncer::MediaSyncer, + }, + }, }; -pub mod changetracker; -pub mod check; -pub mod database; -pub mod files; -pub mod sync; - pub type Sha1Hash = [u8; 20]; +impl Collection { + pub fn media(&self) -> Result { + MediaManager::new(&self.media_folder, &self.media_db) + } +} + pub struct MediaManager { - db: Connection, - media_folder: PathBuf, + pub(crate) db: MediaDatabase, + pub(crate) media_folder: PathBuf, } impl MediaManager { @@ -38,10 +44,11 @@ impl MediaManager { P: Into, P2: AsRef, { - let db = open_or_create(media_db.as_ref())?; + let media_folder = media_folder.into(); + create_dir_all(&media_folder)?; Ok(MediaManager { - db, - media_folder: media_folder.into(), + db: MediaDatabase::new(media_db.as_ref())?, + media_folder, }) } @@ -51,26 +58,21 @@ impl MediaManager { /// appended to the name. /// /// Also notes the file in the media database. - pub fn add_file<'a>( - &self, - ctx: &mut MediaDatabaseContext, - desired_name: &'a str, - data: &[u8], - ) -> Result> { + pub fn add_file<'a>(&self, desired_name: &'a str, data: &[u8]) -> Result> { let data_hash = sha1_of_data(data); - self.transact(ctx, |ctx| { + self.transact(|db| { let chosen_fname = add_data_to_folder_uniquely(&self.media_folder, desired_name, data, data_hash)?; let file_mtime = mtime_as_i64(self.media_folder.join(chosen_fname.as_ref()))?; - let existing_entry = ctx.get_entry(&chosen_fname)?; + let existing_entry = db.get_entry(&chosen_fname)?; let new_sha1 = Some(data_hash); let entry_update_required = existing_entry.map(|e| e.sha1 != new_sha1).unwrap_or(true); if entry_update_required { - ctx.set_entry(&MediaEntry { + db.set_entry(&MediaEntry { fname: chosen_fname.to_string(), sha1: new_sha1, mtime: file_mtime, @@ -82,18 +84,18 @@ impl MediaManager { }) } - pub fn remove_files(&self, ctx: &mut MediaDatabaseContext, filenames: &[S]) -> Result<()> + pub fn remove_files(&self, filenames: &[S]) -> Result<()> where S: AsRef + std::fmt::Debug, { - self.transact(ctx, |ctx| { + self.transact(|db| { remove_files(&self.media_folder, filenames)?; for fname in filenames { - if let Some(mut entry) = ctx.get_entry(fname.as_ref())? { + if let Some(mut entry) = db.get_entry(fname.as_ref())? { entry.sha1 = None; entry.mtime = 0; entry.sync_required = true; - ctx.set_entry(&entry)?; + db.set_entry(&entry)?; } } Ok(()) @@ -102,21 +104,17 @@ impl MediaManager { /// Opens a transaction and manages folder mtime, so user should perform not /// only db ops, but also all file ops inside the closure. - pub(crate) fn transact( - &self, - ctx: &mut MediaDatabaseContext, - func: impl FnOnce(&mut MediaDatabaseContext) -> Result, - ) -> Result { + pub(crate) fn transact(&self, func: impl FnOnce(&MediaDatabase) -> Result) -> Result { let start_folder_mtime = mtime_as_i64(&self.media_folder)?; - ctx.transact(|ctx| { - let out = func(ctx)?; + self.db.transact(|db| { + let out = func(db)?; - let mut meta = ctx.get_meta()?; + let mut meta = db.get_meta()?; if meta.folder_mtime == start_folder_mtime { // if media db was in sync with folder prior to this add, // we can keep it in sync meta.folder_mtime = mtime_as_i64(&self.media_folder)?; - ctx.set_meta(&meta)?; + db.set_meta(&meta)?; } else { // otherwise, leave it alone so that other pending changes // get picked up later @@ -127,15 +125,10 @@ impl MediaManager { } /// Set entry for a newly added file. Caller must ensure transaction. - pub(crate) fn add_entry( - &self, - ctx: &mut MediaDatabaseContext, - fname: impl Into, - sha1: [u8; 20], - ) -> Result<()> { + pub(crate) fn add_entry(&self, fname: impl Into, sha1: [u8; 20]) -> Result<()> { let fname = fname.into(); let mtime = mtime_as_i64(self.media_folder.join(&fname))?; - ctx.set_entry(&MediaEntry { + self.db.set_entry(&MediaEntry { fname, mtime, sha1: Some(sha1), @@ -144,55 +137,38 @@ impl MediaManager { } /// Sync media. - pub async fn sync_media<'a, F>( - &'a self, - progress: F, - host_number: u32, - hkey: &'a str, - ) -> Result<()> + pub async fn sync_media(self, progress: F, auth: SyncAuth) -> Result<()> where F: FnMut(MediaSyncProgress) -> bool, { - let mut syncer = MediaSyncer::new(self, progress, host_number); - syncer.sync(hkey).await + let client = HttpSyncClient::new(auth); + let mut syncer = MediaSyncer::new(self, progress, client)?; + syncer.sync().await } - pub fn dbctx(&self) -> MediaDatabaseContext { - MediaDatabaseContext::new(&self.db) - } - - pub fn all_checksums( + pub fn all_checksums_after_checking( &self, progress: impl FnMut(usize) -> bool, ) -> Result> { - let mut dbctx = self.dbctx(); - ChangeTracker::new(&self.media_folder, progress).register_changes(&mut dbctx)?; - dbctx.all_checksums() + ChangeTracker::new(&self.media_folder, progress).register_changes(&self.db)?; + self.db.all_registered_checksums() } pub fn checksum_getter(&self) -> impl FnMut(&str) -> Result> + '_ { - let mut dbctx = self.dbctx(); - move |fname: &str| { - dbctx + |fname: &str| { + self.db .get_entry(fname) .map(|opt| opt.and_then(|entry| entry.sha1)) } } pub fn register_changes(&self, progress: &mut impl FnMut(usize) -> bool) -> Result<()> { - ChangeTracker::new(&self.media_folder, progress).register_changes(&mut self.dbctx()) - } -} - -#[cfg(test)] -mod test { - use super::*; - - impl MediaManager { - /// All checksums without registering changes first. - pub(crate) fn all_checksums_as_is(&self) -> HashMap { - let mut dbctx = self.dbctx(); - dbctx.all_checksums().unwrap() - } + ChangeTracker::new(&self.media_folder, progress).register_changes(&self.db) + } + + /// All checksums without registering changes first. + #[cfg(test)] + pub(crate) fn all_checksums_as_is(&self) -> HashMap { + self.db.all_registered_checksums().unwrap() } } diff --git a/rslib/src/media/sync.rs b/rslib/src/media/sync.rs deleted file mode 100644 index 228ce96d7..000000000 --- a/rslib/src/media/sync.rs +++ /dev/null @@ -1,839 +0,0 @@ -// Copyright: Ankitects Pty Ltd and contributors -// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html - -use std::{ - borrow::Cow, - collections::HashMap, - io, - io::{Read, Write}, - path::Path, - time, -}; - -use bytes::Bytes; -use reqwest::{multipart, Client, Response}; -use serde_derive::{Deserialize, Serialize}; -use serde_tuple::Serialize_tuple; -use time::Duration; -use tracing::debug; -use version::sync_client_version; - -use crate::{ - error::{AnkiError, Result, SyncErrorKind}, - media::{ - changetracker::ChangeTracker, - database::{MediaDatabaseContext, MediaDatabaseMetadata, MediaEntry}, - files::{ - add_file_from_ankiweb, data_for_file, mtime_as_i64, normalize_filename, AddedFile, - }, - MediaManager, - }, - sync::Timeouts, - version, -}; - -static SYNC_MAX_FILES: usize = 25; -static SYNC_MAX_BYTES: usize = (2.5 * 1024.0 * 1024.0) as usize; -static SYNC_SINGLE_FILE_MAX_BYTES: usize = 100 * 1024 * 1024; - -#[derive(Debug, Default, Clone, Copy)] -pub struct MediaSyncProgress { - pub checked: usize, - pub downloaded_files: usize, - pub downloaded_deletions: usize, - pub uploaded_files: usize, - pub uploaded_deletions: usize, -} - -pub struct MediaSyncer<'a, P> -where - P: FnMut(MediaSyncProgress) -> bool, -{ - mgr: &'a MediaManager, - ctx: MediaDatabaseContext<'a>, - skey: Option, - client: Client, - progress_cb: P, - progress: MediaSyncProgress, - endpoint: String, -} - -#[derive(Debug, Deserialize)] -struct SyncBeginResult { - data: Option, - err: String, -} - -#[derive(Debug, Deserialize)] -struct SyncBeginResponse { - #[serde(rename = "sk")] - sync_key: String, - usn: i32, -} - -#[derive(Debug, Clone, Copy)] -enum LocalState { - NotInDb, - InDbNotPending, - InDbAndPending, -} - -#[derive(PartialEq, Eq, Debug)] -enum RequiredChange { - // none also covers the case where we'll later upload - None, - Download, - Delete, - RemovePending, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct RecordBatchRequest { - last_usn: i32, -} - -#[derive(Debug, Deserialize)] -struct RecordBatchResult { - data: Option>, - err: String, -} - -#[derive(Debug, Deserialize)] -struct ServerMediaRecord { - fname: String, - usn: i32, - sha1: String, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -struct ZipRequest<'a> { - files: &'a [&'a String], -} - -#[derive(Serialize_tuple)] -struct UploadEntry<'a> { - fname: &'a str, - in_zip_name: Option, -} - -#[derive(Deserialize, Debug)] -struct UploadResult { - data: Option, - err: String, -} - -#[derive(Deserialize, Debug)] -struct UploadReply { - processed: usize, - current_usn: i32, -} - -#[derive(Serialize)] -struct FinalizeRequest { - local: u32, -} - -#[derive(Debug, Deserialize)] -struct FinalizeResponse { - data: Option, - err: String, -} - -fn media_sync_endpoint(host_number: u32) -> String { - if let Ok(endpoint) = std::env::var("SYNC_ENDPOINT_MEDIA") { - endpoint - } else { - let suffix = if host_number > 0 { - format!("{}", host_number) - } else { - "".to_string() - }; - format!("https://sync{}.ankiweb.net/msync/", suffix) - } -} - -impl

MediaSyncer<'_, P> -where - P: FnMut(MediaSyncProgress) -> bool, -{ - pub fn new(mgr: &MediaManager, progress_cb: P, host_number: u32) -> MediaSyncer<'_, P> { - let timeouts = Timeouts::new(); - let client = Client::builder() - .connect_timeout(Duration::from_secs(timeouts.connect_secs)) - .timeout(Duration::from_secs(timeouts.request_secs)) - .io_timeout(Duration::from_secs(timeouts.io_secs)) - .build() - .unwrap(); - let endpoint = media_sync_endpoint(host_number); - let ctx = mgr.dbctx(); - MediaSyncer { - mgr, - ctx, - skey: None, - client, - progress_cb, - progress: Default::default(), - endpoint, - } - } - - fn skey(&self) -> &str { - self.skey.as_ref().unwrap() - } - - pub async fn sync(&mut self, hkey: &str) -> Result<()> { - self.sync_inner(hkey).await.map_err(|e| { - debug!("sync error: {:?}", e); - e - }) - } - - #[allow(clippy::useless_let_if_seq)] - async fn sync_inner(&mut self, hkey: &str) -> Result<()> { - self.register_changes()?; - - let meta = self.ctx.get_meta()?; - let client_usn = meta.last_sync_usn; - - debug!("begin media sync"); - let (sync_key, server_usn) = self.sync_begin(hkey).await?; - self.skey = Some(sync_key); - debug!("server usn was {}", server_usn); - - let mut actions_performed = false; - - // need to fetch changes from server? - if client_usn != server_usn { - debug!("differs from local usn {}, fetching changes", client_usn); - self.fetch_changes(meta).await?; - actions_performed = true; - } - - // need to send changes to server? - let changes_pending = !self.ctx.get_pending_uploads(1)?.is_empty(); - if changes_pending { - self.send_changes().await?; - actions_performed = true; - } - - if actions_performed { - self.finalize_sync().await?; - } - - self.fire_progress_cb()?; - - debug!("media sync complete"); - - Ok(()) - } - - /// Make sure media DB is up to date. - fn register_changes(&mut self) -> Result<()> { - // make borrow checker happy - let progress = &mut self.progress; - let progress_cb = &mut self.progress_cb; - - let progress = |checked| { - progress.checked = checked; - (progress_cb)(*progress) - }; - - ChangeTracker::new(self.mgr.media_folder.as_path(), progress) - .register_changes(&mut self.ctx) - } - - async fn sync_begin(&self, hkey: &str) -> Result<(String, i32)> { - let url = format!("{}begin", self.endpoint); - - let resp = self - .client - .get(&url) - .query(&[("k", hkey), ("v", sync_client_version())]) - .send() - .await? - .error_for_status()?; - - let reply: SyncBeginResult = resp.json().await?; - - if let Some(data) = reply.data { - Ok((data.sync_key, data.usn)) - } else { - Err(AnkiError::server_message(reply.err)) - } - } - - async fn fetch_changes(&mut self, mut meta: MediaDatabaseMetadata) -> Result<()> { - let mut last_usn = meta.last_sync_usn; - loop { - debug!(start_usn = last_usn, "fetching record batch"); - - let batch = self.fetch_record_batch(last_usn).await?; - if batch.is_empty() { - debug!("empty batch, done"); - break; - } - last_usn = batch.last().unwrap().usn; - - self.progress.checked += batch.len(); - self.fire_progress_cb()?; - - let (to_download, to_delete, to_remove_pending) = - determine_required_changes(&mut self.ctx, &batch)?; - - // file removal - self.mgr.remove_files(&mut self.ctx, to_delete.as_slice())?; - self.progress.downloaded_deletions += to_delete.len(); - self.fire_progress_cb()?; - - // file download - let mut downloaded = vec![]; - let mut dl_fnames = to_download.as_slice(); - while !dl_fnames.is_empty() { - let batch: Vec<_> = dl_fnames - .iter() - .take(SYNC_MAX_FILES) - .map(ToOwned::to_owned) - .collect(); - let zip_data = self.fetch_zip(batch.as_slice()).await?; - let download_batch = - extract_into_media_folder(self.mgr.media_folder.as_path(), zip_data)? - .into_iter(); - let len = download_batch.len(); - dl_fnames = &dl_fnames[len..]; - downloaded.extend(download_batch); - - self.progress.downloaded_files += len; - self.fire_progress_cb()?; - } - - // then update the DB - let dirmod = mtime_as_i64(&self.mgr.media_folder)?; - self.ctx.transact(|ctx| { - record_clean(ctx, &to_remove_pending)?; - record_removals(ctx, &to_delete)?; - record_additions(ctx, downloaded)?; - - // update usn - meta.last_sync_usn = last_usn; - meta.folder_mtime = dirmod; - ctx.set_meta(&meta)?; - - Ok(()) - })?; - } - Ok(()) - } - - async fn send_changes(&mut self) -> Result<()> { - loop { - let pending: Vec = self.ctx.get_pending_uploads(SYNC_MAX_FILES as u32)?; - if pending.is_empty() { - break; - } - - let zip_data = zip_files(&mut self.ctx, &self.mgr.media_folder, &pending)?; - if zip_data.is_none() { - self.progress.checked += pending.len(); - self.fire_progress_cb()?; - // discard zip info and retry batch - not particularly efficient, - // but this is a corner case - continue; - } - - let reply = self.send_zip_data(zip_data.unwrap()).await?; - - let (processed_files, processed_deletions): (Vec<_>, Vec<_>) = pending - .iter() - .take(reply.processed) - .partition(|e| e.sha1.is_some()); - - self.progress.uploaded_files += processed_files.len(); - self.progress.uploaded_deletions += processed_deletions.len(); - self.fire_progress_cb()?; - - let fnames: Vec<_> = processed_files - .iter() - .chain(processed_deletions.iter()) - .map(|e| &e.fname) - .collect(); - let fname_cnt = fnames.len() as i32; - self.ctx.transact(|ctx| { - record_clean(ctx, fnames.as_slice())?; - let mut meta = ctx.get_meta()?; - if meta.last_sync_usn + fname_cnt == reply.current_usn { - meta.last_sync_usn = reply.current_usn; - ctx.set_meta(&meta)?; - } else { - debug!( - "server usn {} is not {}, skipping usn update", - reply.current_usn, - meta.last_sync_usn + fname_cnt - ); - } - Ok(()) - })?; - } - - Ok(()) - } - - async fn finalize_sync(&mut self) -> Result<()> { - let url = format!("{}mediaSanity", self.endpoint); - let local = self.ctx.count()?; - - let obj = FinalizeRequest { local }; - let resp = ankiweb_json_request(&self.client, &url, &obj, self.skey(), false).await?; - let resp: FinalizeResponse = resp.json().await?; - - if let Some(data) = resp.data { - if data == "OK" { - Ok(()) - } else { - self.ctx.transact(|ctx| ctx.force_resync())?; - Err(AnkiError::sync_error("", SyncErrorKind::ResyncRequired)) - } - } else { - Err(AnkiError::server_message(resp.err)) - } - } - - fn fire_progress_cb(&mut self) -> Result<()> { - if (self.progress_cb)(self.progress) { - Ok(()) - } else { - Err(AnkiError::Interrupted) - } - } - - async fn fetch_record_batch(&self, last_usn: i32) -> Result> { - let url = format!("{}mediaChanges", self.endpoint); - - let req = RecordBatchRequest { last_usn }; - let resp = ankiweb_json_request(&self.client, &url, &req, self.skey(), false).await?; - let res: RecordBatchResult = resp.json().await?; - - if let Some(batch) = res.data { - Ok(batch) - } else { - Err(AnkiError::server_message(res.err)) - } - } - - async fn fetch_zip(&self, files: &[&String]) -> Result { - let url = format!("{}downloadFiles", self.endpoint); - - debug!("requesting files: {:?}", files); - - let req = ZipRequest { files }; - let resp = ankiweb_json_request(&self.client, &url, &req, self.skey(), true).await?; - resp.bytes().await.map_err(Into::into) - } - - async fn send_zip_data(&self, data: Vec) -> Result { - let url = format!("{}uploadChanges", self.endpoint); - - let resp = ankiweb_bytes_request(&self.client, &url, data, self.skey(), true).await?; - let res: UploadResult = resp.json().await?; - - if let Some(reply) = res.data { - Ok(reply) - } else { - Err(AnkiError::server_message(res.err)) - } - } -} - -fn determine_required_change( - local_sha1: &str, - remote_sha1: &str, - local_state: LocalState, -) -> RequiredChange { - use LocalState as L; - use RequiredChange as R; - - match (local_sha1, remote_sha1, local_state) { - // both deleted, not in local DB - ("", "", L::NotInDb) => R::None, - // both deleted, in local DB - ("", "", _) => R::Delete, - // added on server, add even if local deletion pending - ("", _, _) => R::Download, - // deleted on server but added locally; upload later - (_, "", L::InDbAndPending) => R::None, - // deleted on server and not pending sync - (_, "", _) => R::Delete, - // if pending but the same as server, don't need to upload - (lsum, rsum, L::InDbAndPending) if lsum == rsum => R::RemovePending, - (lsum, rsum, _) => { - if lsum == rsum { - // not pending and same as server, nothing to do - R::None - } else { - // differs from server, favour server - R::Download - } - } - } -} - -/// Get a list of server filenames and the actions required on them. -/// Returns filenames in (to_download, to_delete). -fn determine_required_changes<'a>( - ctx: &mut MediaDatabaseContext, - records: &'a [ServerMediaRecord], -) -> Result<(Vec<&'a String>, Vec<&'a String>, Vec<&'a String>)> { - let mut to_download = vec![]; - let mut to_delete = vec![]; - let mut to_remove_pending = vec![]; - - for remote in records { - let (local_sha1, local_state) = match ctx.get_entry(&remote.fname)? { - Some(entry) => ( - match entry.sha1 { - Some(arr) => hex::encode(arr), - None => "".to_string(), - }, - if entry.sync_required { - LocalState::InDbAndPending - } else { - LocalState::InDbNotPending - }, - ), - None => ("".to_string(), LocalState::NotInDb), - }; - - let req_change = determine_required_change(&local_sha1, &remote.sha1, local_state); - debug!( - fname = &remote.fname, - lsha = local_sha1.chars().take(8).collect::(), - rsha = remote.sha1.chars().take(8).collect::(), - state = ?local_state, - action = ?req_change, - "determine action" - ); - match req_change { - RequiredChange::Download => to_download.push(&remote.fname), - RequiredChange::Delete => to_delete.push(&remote.fname), - RequiredChange::RemovePending => to_remove_pending.push(&remote.fname), - RequiredChange::None => (), - }; - } - - Ok((to_download, to_delete, to_remove_pending)) -} - -async fn ankiweb_json_request( - client: &Client, - url: &str, - json: &T, - skey: &str, - timeout_long: bool, -) -> Result -where - T: serde::Serialize, -{ - let req_json = serde_json::to_string(json)?; - let part = multipart::Part::text(req_json); - ankiweb_request(client, url, part, skey, timeout_long).await -} - -async fn ankiweb_bytes_request( - client: &Client, - url: &str, - bytes: Vec, - skey: &str, - timeout_long: bool, -) -> Result { - let part = multipart::Part::bytes(bytes); - ankiweb_request(client, url, part, skey, timeout_long).await -} - -async fn ankiweb_request( - client: &Client, - url: &str, - data_part: multipart::Part, - skey: &str, - timeout_long: bool, -) -> Result { - let data_part = data_part.file_name("data"); - - let form = multipart::Form::new() - .part("data", data_part) - .text("sk", skey.to_string()); - - let mut req = client.post(url).multipart(form); - - if timeout_long { - req = req.timeout(Duration::from_secs(60 * 60)); - } - - req.send().await?.error_for_status().map_err(Into::into) -} - -fn extract_into_media_folder(media_folder: &Path, zip: Bytes) -> Result> { - let reader = io::Cursor::new(zip); - let mut zip = zip::ZipArchive::new(reader)?; - - let meta_file = zip.by_name("_meta")?; - let fmap: HashMap = serde_json::from_reader(meta_file)?; - let mut output = Vec::with_capacity(fmap.len()); - - for i in 0..zip.len() { - let mut file = zip.by_index(i)?; - let name = file.name(); - if name == "_meta" { - continue; - } - - let real_name = fmap - .get(name) - .ok_or_else(|| AnkiError::sync_error("malformed zip", SyncErrorKind::Other))?; - - let mut data = Vec::with_capacity(file.size() as usize); - file.read_to_end(&mut data)?; - - let added = add_file_from_ankiweb(media_folder, real_name, &data)?; - - output.push(added); - } - - Ok(output) -} - -fn record_removals(ctx: &mut MediaDatabaseContext, removals: &[&String]) -> Result<()> { - for &fname in removals { - debug!(fname, "mark removed"); - ctx.remove_entry(fname)?; - } - - Ok(()) -} - -fn record_additions(ctx: &mut MediaDatabaseContext, additions: Vec) -> Result<()> { - for file in additions { - if let Some(renamed) = file.renamed_from { - // the file AnkiWeb sent us wasn't normalized, so we need to record - // the old file name as a deletion - debug!("marking non-normalized file as deleted: {}", renamed); - let mut entry = MediaEntry { - fname: renamed, - sha1: None, - mtime: 0, - sync_required: true, - }; - ctx.set_entry(&entry)?; - // and upload the new filename to ankiweb - debug!("marking renamed file as needing upload: {}", file.fname); - entry = MediaEntry { - fname: file.fname.to_string(), - sha1: Some(file.sha1), - mtime: file.mtime, - sync_required: true, - }; - ctx.set_entry(&entry)?; - } else { - // a normal addition - let entry = MediaEntry { - fname: file.fname.to_string(), - sha1: Some(file.sha1), - mtime: file.mtime, - sync_required: false, - }; - debug!( - fname = &entry.fname, - sha1 = hex::encode(&entry.sha1.as_ref().unwrap()[0..4]), - "mark added" - ); - ctx.set_entry(&entry)?; - } - } - - Ok(()) -} - -fn record_clean(ctx: &mut MediaDatabaseContext, clean: &[&String]) -> Result<()> { - for &fname in clean { - if let Some(mut entry) = ctx.get_entry(fname)? { - if entry.sync_required { - entry.sync_required = false; - debug!(fname = &entry.fname, "mark clean"); - ctx.set_entry(&entry)?; - } - } - } - - Ok(()) -} - -fn zip_files<'a>( - ctx: &mut MediaDatabaseContext, - media_folder: &Path, - files: &'a [MediaEntry], -) -> Result>> { - let buf = vec![]; - let mut invalid_entries = vec![]; - - let w = io::Cursor::new(buf); - let mut zip = zip::ZipWriter::new(w); - - let options = - zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); - - let mut accumulated_size = 0; - let mut entries = vec![]; - - for (idx, file) in files.iter().enumerate() { - if accumulated_size > SYNC_MAX_BYTES { - break; - } - - #[cfg(target_vendor = "apple")] - { - use unicode_normalization::is_nfc; - if !is_nfc(&file.fname) { - // older Anki versions stored non-normalized filenames in the DB; clean them up - debug!(fname = file.fname, "clean up non-nfc entry"); - invalid_entries.push(&file.fname); - continue; - } - } - - let file_data = if file.sha1.is_some() { - match data_for_file(media_folder, &file.fname) { - Ok(data) => data, - Err(e) => { - debug!("error accessing {}: {}", &file.fname, e); - invalid_entries.push(&file.fname); - continue; - } - } - } else { - // uploading deletion - None - }; - - if let Some(data) = &file_data { - let normalized = normalize_filename(&file.fname); - if let Cow::Owned(o) = normalized { - debug!("media check required: {} should be {}", &file.fname, o); - invalid_entries.push(&file.fname); - continue; - } - - if data.is_empty() { - invalid_entries.push(&file.fname); - continue; - } - if data.len() > SYNC_SINGLE_FILE_MAX_BYTES { - invalid_entries.push(&file.fname); - continue; - } - accumulated_size += data.len(); - zip.start_file(format!("{}", idx), options)?; - zip.write_all(data)?; - } - - debug!( - fname = &file.fname, - kind = if file_data.is_some() { - "addition " - } else { - "removal" - }, - "will upload" - ); - - entries.push(UploadEntry { - fname: &file.fname, - in_zip_name: if file_data.is_some() { - Some(format!("{}", idx)) - } else { - None - }, - }); - } - - if !invalid_entries.is_empty() { - // clean up invalid entries; we'll build a new zip - ctx.transact(|ctx| { - for fname in invalid_entries { - ctx.remove_entry(fname)?; - } - Ok(()) - })?; - return Ok(None); - } - - let meta = serde_json::to_string(&entries)?; - zip.start_file("_meta", options)?; - zip.write_all(meta.as_bytes())?; - - let w = zip.finish()?; - - Ok(Some(w.into_inner())) -} - -#[cfg(test)] -mod test { - use tempfile::tempdir; - use tokio::runtime::Runtime; - - use crate::{ - error::Result, - io::{create_dir, write_file}, - media::{ - sync::{determine_required_change, LocalState, MediaSyncProgress, RequiredChange}, - MediaManager, - }, - }; - - async fn test_sync(hkey: &str) -> Result<()> { - let dir = tempdir()?; - let media_dir = dir.path().join("media"); - create_dir(&media_dir)?; - let media_db = dir.path().join("media.db"); - - write_file(media_dir.join("test.file").as_path(), "hello")?; - - let progress = |progress: MediaSyncProgress| { - println!("got progress: {:?}", progress); - true - }; - - let mgr = MediaManager::new(&media_dir, &media_db)?; - mgr.sync_media(progress, 0, hkey).await?; - - Ok(()) - } - - #[test] - fn sync() { - let hkey = match std::env::var("TEST_HKEY") { - Ok(s) => s, - Err(_) => { - return; - } - }; - - let rt = Runtime::new().unwrap(); - rt.block_on(test_sync(&hkey)).unwrap() - } - - #[test] - fn required_change() { - use determine_required_change as d; - use LocalState as L; - use RequiredChange as R; - assert_eq!(d("", "", L::NotInDb), R::None); - assert_eq!(d("", "", L::InDbNotPending), R::Delete); - assert_eq!(d("", "1", L::InDbAndPending), R::Download); - assert_eq!(d("1", "", L::InDbAndPending), R::None); - assert_eq!(d("1", "", L::InDbNotPending), R::Delete); - assert_eq!(d("1", "1", L::InDbNotPending), R::None); - assert_eq!(d("1", "1", L::InDbAndPending), R::RemovePending); - assert_eq!(d("a", "b", L::InDbAndPending), R::Download); - assert_eq!(d("a", "b", L::InDbNotPending), R::Download); - } -} diff --git a/rslib/src/notes/mod.rs b/rslib/src/notes/mod.rs index 3e26e8283..dfb512499 100644 --- a/rslib/src/notes/mod.rs +++ b/rslib/src/notes/mod.rs @@ -120,7 +120,7 @@ impl Note { pub(crate) fn new(notetype: &Notetype) -> Self { Note { id: NoteId(0), - guid: guid(), + guid: base91_u64(), notetype_id: notetype.id, mtime: TimestampSecs(0), usn: Usn(0), @@ -297,20 +297,25 @@ pub(crate) fn field_checksum(text: &str) -> u32 { u32::from_be_bytes(digest[..4].try_into().unwrap()) } -pub(crate) fn guid() -> String { +pub(crate) fn base91_u64() -> String { anki_base91(rand::random()) } -fn anki_base91(mut n: u64) -> String { - let table = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\ -0123456789!#$%&()*+,-./:;<=>?@[]^_`{|}~"; +fn anki_base91(n: u64) -> String { + to_base_n( + n, + b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\ +0123456789!#$%&()*+,-./:;<=>?@[]^_`{|}~", + ) +} + +pub fn to_base_n(mut n: u64, table: &[u8]) -> String { let mut buf = String::new(); while n > 0 { let (q, r) = n.div_rem(&(table.len() as u64)); buf.push(table[r as usize] as char); n = q; } - buf.chars().rev().collect() } diff --git a/rslib/src/scheduler/filtered/custom_study.rs b/rslib/src/scheduler/filtered/custom_study.rs index ec3738503..e737ac80c 100644 --- a/rslib/src/scheduler/filtered/custom_study.rs +++ b/rslib/src/scheduler/filtered/custom_study.rs @@ -8,10 +8,8 @@ use crate::{ config::DeckConfigKey, decks::{FilteredDeck, FilteredSearchOrder, FilteredSearchTerm}, error::{CustomStudyError, FilteredDeckError}, - pb::{ - scheduler::custom_study_request::{cram::CramKind, Cram, Value as CustomStudyValue}, - {self as pb}, - }, + pb, + pb::scheduler::custom_study_request::{cram::CramKind, Cram, Value as CustomStudyValue}, prelude::*, search::{JoinSearches, Negated, PropertyKind, RatingKind, SearchNode, StateKind}, }; diff --git a/rslib/src/storage/card/mod.rs b/rslib/src/storage/card/mod.rs index e06b5d373..163c51d5d 100644 --- a/rslib/src/storage/card/mod.rs +++ b/rslib/src/storage/card/mod.rs @@ -763,7 +763,7 @@ mod test { fn add_card() { let tr = I18n::template_only(); let storage = - SqliteStorage::open_or_create(Path::new(":memory:"), &tr, false, false).unwrap(); + SqliteStorage::open_or_create(Path::new(":memory:"), &tr, false, false, false).unwrap(); let mut card = Card::default(); storage.add_card(&mut card).unwrap(); let id1 = card.id; diff --git a/rslib/src/storage/graves/mod.rs b/rslib/src/storage/graves/mod.rs index 5d4db21c0..8fdc6db0b 100644 --- a/rslib/src/storage/graves/mod.rs +++ b/rslib/src/storage/graves/mod.rs @@ -7,7 +7,7 @@ use num_enum::TryFromPrimitive; use rusqlite::params; use super::SqliteStorage; -use crate::{prelude::*, sync::Graves}; +use crate::{prelude::*, sync::collection::graves::Graves}; #[derive(TryFromPrimitive)] #[repr(u8)] diff --git a/rslib/src/storage/mod.rs b/rslib/src/storage/mod.rs index 22b960677..15d553aa8 100644 --- a/rslib/src/storage/mod.rs +++ b/rslib/src/storage/mod.rs @@ -19,10 +19,9 @@ mod upgrades; use std::fmt::Write; pub(crate) use sqlite::SqliteStorage; -pub(crate) use sync::open_and_check_sqlite_file; #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum SchemaVersion { +pub enum SchemaVersion { V11, V18, } diff --git a/rslib/src/storage/sqlite.rs b/rslib/src/storage/sqlite.rs index 7a829548d..17f0b1f0c 100644 --- a/rslib/src/storage/sqlite.rs +++ b/rslib/src/storage/sqlite.rs @@ -208,6 +208,7 @@ impl SqliteStorage { path: &Path, tr: &I18n, server: bool, + check_integrity: bool, force_schema11: bool, ) -> Result { let db = open_or_create_collection_db(path)?; @@ -227,6 +228,13 @@ impl SqliteStorage { return Err(AnkiError::db_error("", kind)); } + if check_integrity { + match db.pragma_query_value(None, "integrity_check", |row| row.get::<_, String>(0)) { + Ok(s) => require!(s == "ok", "corrupt: {s}"), + Err(e) => return Err(e.into()), + }; + } + let upgrade = ver != SCHEMA_MAX_VERSION; if create || upgrade { db.execute("begin exclusive", [])?; diff --git a/rslib/src/storage/sync.rs b/rslib/src/storage/sync.rs index 5d95cf45f..ec0ff45d1 100644 --- a/rslib/src/storage/sync.rs +++ b/rslib/src/storage/sync.rs @@ -1,9 +1,7 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -use std::path::Path; - -use rusqlite::{params, types::FromSql, Connection, ToSql}; +use rusqlite::{params, types::FromSql, ToSql}; use super::*; use crate::prelude::*; @@ -49,9 +47,9 @@ impl SqliteStorage { &self, table: &str, ids: &[I], - new_usn: Option, + server_usn_if_client: Option, ) -> Result<()> { - if let Some(new_usn) = new_usn { + if let Some(new_usn) = server_usn_if_client { let mut stmt = self .db .prepare_cached(&format!("update {} set usn=? where id=?", table))?; @@ -63,18 +61,13 @@ impl SqliteStorage { } } -/// Return error if file is unreadable, fails the sqlite -/// integrity check, or is not in the 'delete' journal mode. -/// On success, returns the opened DB. -pub(crate) fn open_and_check_sqlite_file(path: &Path) -> Result { - let db = Connection::open(path)?; - match db.pragma_query_value(None, "integrity_check", |row| row.get::<_, String>(0)) { - Ok(s) => require!(s == "ok", "corrupt: {s}"), - Err(e) => return Err(e.into()), - }; - match db.pragma_query_value(None, "journal_mode", |row| row.get::<_, String>(0)) { - Ok(s) if s == "delete" => Ok(db), - Ok(s) => invalid_input!("corrupt: {s}"), - Err(e) => Err(e.into()), +impl Usn { + /// Used when gathering pending objects during sync. + pub(crate) fn pending_object_clause(self) -> &'static str { + if self.0 == -1 { + "usn = ?" + } else { + "usn >= ?" + } } } diff --git a/rslib/src/storage/sync_check.rs b/rslib/src/storage/sync_check.rs index d52d679e6..7c8066c8d 100644 --- a/rslib/src/storage/sync_check.rs +++ b/rslib/src/storage/sync_check.rs @@ -5,7 +5,7 @@ use super::*; use crate::{ error::SyncErrorKind, prelude::*, - sync::{SanityCheckCounts, SanityCheckDueCounts}, + sync::collection::sanity::{SanityCheckCounts, SanityCheckDueCounts}, }; impl SqliteStorage { diff --git a/rslib/src/sync/collection/changes.rs b/rslib/src/sync/collection/changes.rs new file mode 100644 index 000000000..17812723c --- /dev/null +++ b/rslib/src/sync/collection/changes.rs @@ -0,0 +1,327 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +//! The current sync protocol sends changed notetypes, decks, tags and config +//! all in a single request. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use serde_tuple::Serialize_tuple; +use tracing::{debug, trace}; + +use crate::{ + deckconfig::DeckConfSchema11, + decks::DeckSchema11, + error::SyncErrorKind, + notetype::NotetypeSchema11, + prelude::*, + sync::{ + collection::{ + normal::{ClientSyncState, NormalSyncProgress, NormalSyncer}, + protocol::SyncProtocol, + start::ServerSyncState, + }, + request::IntoSyncRequest, + }, + tags::Tag, +}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct ApplyChangesRequest { + pub changes: UnchunkedChanges, +} + +#[derive(Serialize, Deserialize, Debug, Default)] +pub struct UnchunkedChanges { + #[serde(rename = "models")] + notetypes: Vec, + #[serde(rename = "decks")] + decks_and_config: DecksAndConfig, + tags: Vec, + + // the following are only sent if local is newer + #[serde(skip_serializing_if = "Option::is_none", rename = "conf")] + config: Option>, + #[serde(skip_serializing_if = "Option::is_none", rename = "crt")] + creation_stamp: Option, +} + +#[derive(Serialize_tuple, Deserialize, Debug, Default)] +pub struct DecksAndConfig { + decks: Vec, + config: Vec, +} + +impl NormalSyncer<'_, F> +where + F: FnMut(NormalSyncProgress, bool), +{ + // This was assumed to a cheap operation when originally written - it didn't anticipate + // the large deck trees and note types some users would create. They should be chunked + // in the future, like other objects. Syncing tags explicitly is also probably of limited + // usefulness. + pub(in crate::sync) async fn process_unchunked_changes( + &mut self, + state: &ClientSyncState, + ) -> Result<()> { + debug!("gathering local changes"); + let local = self.col.local_unchunked_changes( + state.pending_usn, + Some(state.server_usn), + state.local_is_newer, + )?; + + debug!( + notetypes = local.notetypes.len(), + decks = local.decks_and_config.decks.len(), + deck_config = local.decks_and_config.config.len(), + tags = local.tags.len(), + "sending" + ); + + self.progress.local_update += local.notetypes.len() + + local.decks_and_config.decks.len() + + local.decks_and_config.config.len() + + local.tags.len(); + let remote = self + .server + .apply_changes(ApplyChangesRequest { changes: local }.try_into_sync_request()?) + .await? + .json()?; + self.fire_progress_cb(true); + + debug!( + notetypes = remote.notetypes.len(), + decks = remote.decks_and_config.decks.len(), + deck_config = remote.decks_and_config.config.len(), + tags = remote.tags.len(), + "received" + ); + + self.progress.remote_update += remote.notetypes.len() + + remote.decks_and_config.decks.len() + + remote.decks_and_config.config.len() + + remote.tags.len(); + + self.col.apply_changes(remote, state.server_usn)?; + self.fire_progress_cb(true); + Ok(()) + } +} + +impl Collection { + // Local->remote unchunked changes + //---------------------------------------------------------------- + + pub(in crate::sync) fn local_unchunked_changes( + &mut self, + pending_usn: Usn, + server_usn_if_client: Option, + local_is_newer: bool, + ) -> Result { + let mut changes = UnchunkedChanges { + notetypes: self.changed_notetypes(pending_usn, server_usn_if_client)?, + decks_and_config: DecksAndConfig { + decks: self.changed_decks(pending_usn, server_usn_if_client)?, + config: self.changed_deck_config(pending_usn, server_usn_if_client)?, + }, + tags: self.changed_tags(pending_usn, server_usn_if_client)?, + ..Default::default() + }; + if local_is_newer { + changes.config = Some(self.changed_config()?); + changes.creation_stamp = Some(self.storage.creation_stamp()?); + } + + Ok(changes) + } + + fn changed_notetypes( + &mut self, + pending_usn: Usn, + server_usn_if_client: Option, + ) -> Result> { + let ids = self + .storage + .objects_pending_sync("notetypes", pending_usn)?; + self.storage + .maybe_update_object_usns("notetypes", &ids, server_usn_if_client)?; + self.state.notetype_cache.clear(); + ids.into_iter() + .map(|id| { + self.storage.get_notetype(id).map(|opt| { + let mut nt: NotetypeSchema11 = opt.unwrap().into(); + nt.usn = server_usn_if_client.unwrap_or(nt.usn); + nt + }) + }) + .collect() + } + + fn changed_decks( + &mut self, + pending_usn: Usn, + server_usn_if_client: Option, + ) -> Result> { + let ids = self.storage.objects_pending_sync("decks", pending_usn)?; + self.storage + .maybe_update_object_usns("decks", &ids, server_usn_if_client)?; + self.state.deck_cache.clear(); + ids.into_iter() + .map(|id| { + self.storage.get_deck(id).map(|opt| { + let mut deck = opt.unwrap(); + deck.usn = server_usn_if_client.unwrap_or(deck.usn); + deck.into() + }) + }) + .collect() + } + + fn changed_deck_config( + &self, + pending_usn: Usn, + server_usn_if_client: Option, + ) -> Result> { + let ids = self + .storage + .objects_pending_sync("deck_config", pending_usn)?; + self.storage + .maybe_update_object_usns("deck_config", &ids, server_usn_if_client)?; + ids.into_iter() + .map(|id| { + self.storage.get_deck_config(id).map(|opt| { + let mut conf: DeckConfSchema11 = opt.unwrap().into(); + conf.usn = server_usn_if_client.unwrap_or(conf.usn); + conf + }) + }) + .collect() + } + + fn changed_tags( + &self, + pending_usn: Usn, + server_usn_if_client: Option, + ) -> Result> { + let changed = self.storage.tags_pending_sync(pending_usn)?; + if let Some(usn) = server_usn_if_client { + self.storage.update_tag_usns(&changed, usn)?; + } + Ok(changed) + } + + /// Currently this is all config, as legacy clients overwrite the local items + /// with the provided value. + fn changed_config(&self) -> Result> { + let conf = self.storage.get_all_config()?; + self.storage.clear_config_usns()?; + Ok(conf) + } + + // Remote->local unchunked changes + //---------------------------------------------------------------- + + pub(in crate::sync) fn apply_changes( + &mut self, + remote: UnchunkedChanges, + latest_usn: Usn, + ) -> Result<()> { + self.merge_notetypes(remote.notetypes, latest_usn)?; + self.merge_decks(remote.decks_and_config.decks, latest_usn)?; + self.merge_deck_config(remote.decks_and_config.config)?; + self.merge_tags(remote.tags, latest_usn)?; + if let Some(crt) = remote.creation_stamp { + self.set_creation_stamp(crt)?; + } + if let Some(config) = remote.config { + self.storage + .set_all_config(config, latest_usn, TimestampSecs::now())?; + } + + Ok(()) + } + + fn merge_notetypes(&mut self, notetypes: Vec, latest_usn: Usn) -> Result<()> { + for nt in notetypes { + let mut nt: Notetype = nt.into(); + let proceed = if let Some(existing_nt) = self.storage.get_notetype(nt.id)? { + if existing_nt.mtime_secs <= nt.mtime_secs { + if (existing_nt.fields.len() != nt.fields.len()) + || (existing_nt.templates.len() != nt.templates.len()) + { + return Err(AnkiError::sync_error( + "notetype schema changed", + SyncErrorKind::ResyncRequired, + )); + } + true + } else { + false + } + } else { + true + }; + if proceed { + self.ensure_notetype_name_unique(&mut nt, latest_usn)?; + self.storage.add_or_update_notetype_with_existing_id(&nt)?; + self.state.notetype_cache.remove(&nt.id); + } + } + Ok(()) + } + + fn merge_decks(&mut self, decks: Vec, latest_usn: Usn) -> Result<()> { + for deck in decks { + let proceed = if let Some(existing_deck) = self.storage.get_deck(deck.id())? { + existing_deck.mtime_secs <= deck.common().mtime + } else { + true + }; + if proceed { + let mut deck = deck.into(); + self.ensure_deck_name_unique(&mut deck, latest_usn)?; + self.storage.add_or_update_deck_with_existing_id(&deck)?; + self.state.deck_cache.remove(&deck.id); + } + } + Ok(()) + } + + fn merge_deck_config(&self, dconf: Vec) -> Result<()> { + for conf in dconf { + let proceed = if let Some(existing_conf) = self.storage.get_deck_config(conf.id)? { + existing_conf.mtime_secs <= conf.mtime + } else { + true + }; + if proceed { + let conf = conf.into(); + self.storage + .add_or_update_deck_config_with_existing_id(&conf)?; + } + } + Ok(()) + } + + fn merge_tags(&mut self, tags: Vec, latest_usn: Usn) -> Result<()> { + for tag in tags { + self.register_tag(&mut Tag::new(tag, latest_usn))?; + } + Ok(()) + } +} + +pub fn server_apply_changes( + req: ApplyChangesRequest, + col: &mut Collection, + state: &mut ServerSyncState, +) -> Result { + let server_changes = + col.local_unchunked_changes(state.client_usn, None, !state.client_is_newer)?; + trace!(?req.changes, ?server_changes); + col.apply_changes(req.changes, state.server_usn)?; + Ok(server_changes) +} diff --git a/rslib/src/sync/collection/chunks.rs b/rslib/src/sync/collection/chunks.rs new file mode 100644 index 000000000..a3d9e2004 --- /dev/null +++ b/rslib/src/sync/collection/chunks.rs @@ -0,0 +1,431 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use serde_tuple::Serialize_tuple; +use tracing::debug; + +use crate::{ + card::{Card, CardQueue, CardType}, + notes::Note, + prelude::*, + revlog::RevlogEntry, + serde::deserialize_int_from_number, + storage::card::data::{card_data_string, CardData}, + sync::{ + collection::{ + normal::{ClientSyncState, NormalSyncProgress, NormalSyncer}, + protocol::{EmptyInput, SyncProtocol}, + start::ServerSyncState, + }, + request::IntoSyncRequest, + }, + tags::{join_tags, split_tags}, +}; + +pub(in crate::sync) struct ChunkableIds { + revlog: Vec, + cards: Vec, + notes: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Default)] +pub struct Chunk { + #[serde(default)] + pub done: bool, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub revlog: Vec, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub cards: Vec, + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub notes: Vec, +} + +#[derive(Serialize_tuple, Deserialize, Debug)] +pub struct NoteEntry { + pub id: NoteId, + pub guid: String, + #[serde(rename = "mid")] + pub ntid: NotetypeId, + #[serde(rename = "mod")] + pub mtime: TimestampSecs, + pub usn: Usn, + pub tags: String, + pub fields: String, + pub sfld: String, // always empty + pub csum: String, // always empty + pub flags: u32, + pub data: String, +} + +#[derive(Serialize_tuple, Deserialize, Debug)] +pub struct CardEntry { + pub id: CardId, + pub nid: NoteId, + pub did: DeckId, + pub ord: u16, + #[serde(deserialize_with = "deserialize_int_from_number")] + pub mtime: TimestampSecs, + pub usn: Usn, + pub ctype: CardType, + pub queue: CardQueue, + #[serde(deserialize_with = "deserialize_int_from_number")] + pub due: i32, + #[serde(deserialize_with = "deserialize_int_from_number")] + pub ivl: u32, + pub factor: u16, + pub reps: u32, + pub lapses: u32, + pub left: u32, + #[serde(deserialize_with = "deserialize_int_from_number")] + pub odue: i32, + pub odid: DeckId, + pub flags: u8, + pub data: String, +} + +impl NormalSyncer<'_, F> +where + F: FnMut(NormalSyncProgress, bool), +{ + pub(in crate::sync) async fn process_chunks_from_server( + &mut self, + state: &ClientSyncState, + ) -> Result<()> { + loop { + let chunk = self.server.chunk(EmptyInput::request()).await?.json()?; + + debug!( + done = chunk.done, + cards = chunk.cards.len(), + notes = chunk.notes.len(), + revlog = chunk.revlog.len(), + "received" + ); + + self.progress.remote_update += + chunk.cards.len() + chunk.notes.len() + chunk.revlog.len(); + + let done = chunk.done; + self.col.apply_chunk(chunk, state.pending_usn)?; + + self.fire_progress_cb(true); + + if done { + return Ok(()); + } + } + } + + pub(in crate::sync) async fn send_chunks_to_server( + &mut self, + state: &ClientSyncState, + ) -> Result<()> { + let mut ids = self.col.get_chunkable_ids(state.pending_usn)?; + + loop { + let chunk: Chunk = self.col.get_chunk(&mut ids, Some(state.server_usn))?; + let done = chunk.done; + + debug!( + done = chunk.done, + cards = chunk.cards.len(), + notes = chunk.notes.len(), + revlog = chunk.revlog.len(), + "sending" + ); + + self.progress.local_update += + chunk.cards.len() + chunk.notes.len() + chunk.revlog.len(); + + self.server + .apply_chunk(ApplyChunkRequest { chunk }.try_into_sync_request()?) + .await?; + + self.fire_progress_cb(true); + + if done { + return Ok(()); + } + } + } +} + +impl Collection { + // Remote->local chunks + //---------------------------------------------------------------- + + /// pending_usn is used to decide whether the local objects are newer. + /// If the provided objects are not modified locally, the USN inside + /// the individual objects is used. + pub(in crate::sync) fn apply_chunk(&mut self, chunk: Chunk, pending_usn: Usn) -> Result<()> { + self.merge_revlog(chunk.revlog)?; + self.merge_cards(chunk.cards, pending_usn)?; + self.merge_notes(chunk.notes, pending_usn) + } + + fn merge_revlog(&self, entries: Vec) -> Result<()> { + for entry in entries { + self.storage.add_revlog_entry(&entry, false)?; + } + Ok(()) + } + + fn merge_cards(&self, entries: Vec, pending_usn: Usn) -> Result<()> { + for entry in entries { + self.add_or_update_card_if_newer(entry, pending_usn)?; + } + Ok(()) + } + + fn add_or_update_card_if_newer(&self, entry: CardEntry, pending_usn: Usn) -> Result<()> { + let proceed = if let Some(existing_card) = self.storage.get_card(entry.id)? { + !existing_card.usn.is_pending_sync(pending_usn) || existing_card.mtime < entry.mtime + } else { + true + }; + if proceed { + let card = entry.into(); + self.storage.add_or_update_card(&card)?; + } + Ok(()) + } + + fn merge_notes(&mut self, entries: Vec, pending_usn: Usn) -> Result<()> { + for entry in entries { + self.add_or_update_note_if_newer(entry, pending_usn)?; + } + Ok(()) + } + + fn add_or_update_note_if_newer(&mut self, entry: NoteEntry, pending_usn: Usn) -> Result<()> { + let proceed = if let Some(existing_note) = self.storage.get_note(entry.id)? { + !existing_note.usn.is_pending_sync(pending_usn) || existing_note.mtime < entry.mtime + } else { + true + }; + if proceed { + let mut note: Note = entry.into(); + let nt = self + .get_notetype(note.notetype_id)? + .or_invalid("note missing notetype")?; + note.prepare_for_update(&nt, false)?; + self.storage.add_or_update_note(¬e)?; + } + Ok(()) + } + + // Local->remote chunks + //---------------------------------------------------------------- + + pub(in crate::sync) fn get_chunkable_ids(&self, pending_usn: Usn) -> Result { + Ok(ChunkableIds { + revlog: self.storage.objects_pending_sync("revlog", pending_usn)?, + cards: self.storage.objects_pending_sync("cards", pending_usn)?, + notes: self.storage.objects_pending_sync("notes", pending_usn)?, + }) + } + + /// Fetch a chunk of ids from `ids`, returning the referenced objects. + pub(in crate::sync) fn get_chunk( + &self, + ids: &mut ChunkableIds, + server_usn_if_client: Option, + ) -> Result { + // get a bunch of IDs + let mut limit = CHUNK_SIZE as i32; + let mut revlog_ids = vec![]; + let mut card_ids = vec![]; + let mut note_ids = vec![]; + let mut chunk = Chunk::default(); + while limit > 0 { + let last_limit = limit; + if let Some(id) = ids.revlog.pop() { + revlog_ids.push(id); + limit -= 1; + } + if let Some(id) = ids.notes.pop() { + note_ids.push(id); + limit -= 1; + } + if let Some(id) = ids.cards.pop() { + card_ids.push(id); + limit -= 1; + } + if limit == last_limit { + // all empty + break; + } + } + if limit > 0 { + chunk.done = true; + } + + // remove pending status + if !self.server { + self.storage + .maybe_update_object_usns("revlog", &revlog_ids, server_usn_if_client)?; + self.storage + .maybe_update_object_usns("cards", &card_ids, server_usn_if_client)?; + self.storage + .maybe_update_object_usns("notes", ¬e_ids, server_usn_if_client)?; + } + + // the fetch associated objects, and return + chunk.revlog = revlog_ids + .into_iter() + .map(|id| { + self.storage.get_revlog_entry(id).map(|e| { + let mut e = e.unwrap(); + e.usn = server_usn_if_client.unwrap_or(e.usn); + e + }) + }) + .collect::>()?; + chunk.cards = card_ids + .into_iter() + .map(|id| { + self.storage.get_card(id).map(|e| { + let mut e: CardEntry = e.unwrap().into(); + e.usn = server_usn_if_client.unwrap_or(e.usn); + e + }) + }) + .collect::>()?; + chunk.notes = note_ids + .into_iter() + .map(|id| { + self.storage.get_note(id).map(|e| { + let mut e: NoteEntry = e.unwrap().into(); + e.usn = server_usn_if_client.unwrap_or(e.usn); + e + }) + }) + .collect::>()?; + + Ok(chunk) + } +} + +impl From for Card { + fn from(e: CardEntry) -> Self { + let CardData { + original_position, + custom_data, + } = CardData::from_str(&e.data); + Card { + id: e.id, + note_id: e.nid, + deck_id: e.did, + template_idx: e.ord, + mtime: e.mtime, + usn: e.usn, + ctype: e.ctype, + queue: e.queue, + due: e.due, + interval: e.ivl, + ease_factor: e.factor, + reps: e.reps, + lapses: e.lapses, + remaining_steps: e.left, + original_due: e.odue, + original_deck_id: e.odid, + flags: e.flags, + original_position, + custom_data, + } + } +} + +impl From for CardEntry { + fn from(e: Card) -> Self { + CardEntry { + id: e.id, + nid: e.note_id, + did: e.deck_id, + ord: e.template_idx, + mtime: e.mtime, + usn: e.usn, + ctype: e.ctype, + queue: e.queue, + due: e.due, + ivl: e.interval, + factor: e.ease_factor, + reps: e.reps, + lapses: e.lapses, + left: e.remaining_steps, + odue: e.original_due, + odid: e.original_deck_id, + flags: e.flags, + data: card_data_string(&e), + } + } +} + +impl From for Note { + fn from(e: NoteEntry) -> Self { + let fields = e.fields.split('\x1f').map(ToString::to_string).collect(); + Note::new_from_storage( + e.id, + e.guid, + e.ntid, + e.mtime, + e.usn, + split_tags(&e.tags).map(ToString::to_string).collect(), + fields, + None, + None, + ) + } +} + +impl From for NoteEntry { + fn from(e: Note) -> Self { + NoteEntry { + id: e.id, + fields: e.fields().iter().join("\x1f"), + guid: e.guid, + ntid: e.notetype_id, + mtime: e.mtime, + usn: e.usn, + tags: join_tags(&e.tags), + sfld: String::new(), + csum: String::new(), + flags: 0, + data: String::new(), + } + } +} + +pub fn server_chunk(col: &mut Collection, state: &mut ServerSyncState) -> Result { + if state.server_chunk_ids.is_none() { + state.server_chunk_ids = Some(col.get_chunkable_ids(state.client_usn)?); + } + col.get_chunk(state.server_chunk_ids.as_mut().unwrap(), None) +} + +pub fn server_apply_chunk( + req: ApplyChunkRequest, + col: &mut Collection, + state: &mut ServerSyncState, +) -> Result<()> { + col.apply_chunk(req.chunk, state.client_usn) +} + +impl Usn { + pub(crate) fn is_pending_sync(self, pending_usn: Usn) -> bool { + if pending_usn.0 == -1 { + self.0 == -1 + } else { + self.0 >= pending_usn.0 + } + } +} + +pub const CHUNK_SIZE: usize = 250; + +#[derive(Serialize, Deserialize, Debug)] +pub struct ApplyChunkRequest { + pub chunk: Chunk, +} diff --git a/rslib/src/sync/collection/download.rs b/rslib/src/sync/collection/download.rs new file mode 100644 index 000000000..bdbb49557 --- /dev/null +++ b/rslib/src/sync/collection/download.rs @@ -0,0 +1,64 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::{ + collection::CollectionBuilder, + io::{atomic_rename, new_tempfile_in_parent_of, read_file, write_file}, + prelude::*, + storage::SchemaVersion, + sync::{ + collection::{ + progress::FullSyncProgressFn, + protocol::{EmptyInput, SyncProtocol}, + }, + error::{HttpResult, OrHttpErr}, + http_client::HttpSyncClient, + login::SyncAuth, + }, +}; + +impl Collection { + /// Download collection from AnkiWeb. Caller must re-open afterwards. + pub async fn full_download( + self, + auth: SyncAuth, + progress_fn: FullSyncProgressFn, + ) -> Result<()> { + let mut server = HttpSyncClient::new(auth); + server.set_full_sync_progress_fn(Some(progress_fn)); + self.full_download_with_server(server).await + } + + pub(crate) async fn full_download_with_server(self, server: HttpSyncClient) -> Result<()> { + let col_path = self.col_path.clone(); + let _col_folder = col_path.parent().or_invalid("couldn't get col_folder")?; + self.close(None)?; + let out_data = server.download(EmptyInput::request()).await?.data; + // check file ok + let temp_file = new_tempfile_in_parent_of(&col_path)?; + write_file(temp_file.path(), out_data)?; + let col = CollectionBuilder::new(temp_file.path()) + .set_check_integrity(true) + .build()?; + col.storage.db.execute_batch("update col set ls=mod")?; + col.close(None)?; + atomic_rename(temp_file, &col_path, true)?; + Ok(()) + } +} + +pub fn server_download( + col: &mut Option, + schema_version: SchemaVersion, +) -> HttpResult> { + let col_path = { + let mut col = col.take().or_internal_err("take col")?; + let path = col.col_path.clone(); + col.transact_no_undo(|col| col.storage.increment_usn()) + .or_internal_err("incr usn")?; + col.close(Some(schema_version)).or_internal_err("close")?; + path + }; + let data = read_file(col_path).or_internal_err("read col")?; + Ok(data) +} diff --git a/rslib/src/sync/collection/finish.rs b/rslib/src/sync/collection/finish.rs new file mode 100644 index 000000000..25d73cd01 --- /dev/null +++ b/rslib/src/sync/collection/finish.rs @@ -0,0 +1,43 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::{ + prelude::*, + sync::collection::{ + normal::{ClientSyncState, NormalSyncProgress, NormalSyncer}, + protocol::{EmptyInput, SyncProtocol}, + }, +}; + +impl NormalSyncer<'_, F> +where + F: FnMut(NormalSyncProgress, bool), +{ + pub(in crate::sync) async fn finalize(&mut self, state: &ClientSyncState) -> Result<()> { + let new_server_mtime = self.server.finish(EmptyInput::request()).await?.json()?; + self.col.finalize_sync(state, new_server_mtime) + } +} + +impl Collection { + fn finalize_sync( + &self, + state: &ClientSyncState, + new_server_mtime: TimestampMillis, + ) -> Result<()> { + self.storage.set_last_sync(new_server_mtime)?; + let mut usn = state.server_usn; + usn.0 += 1; + self.storage.set_usn(usn)?; + self.storage.set_modified_time(new_server_mtime) + } +} + +pub fn server_finish(col: &mut Collection) -> Result { + let now = TimestampMillis::now(); + col.storage.set_last_sync(now)?; + col.storage.increment_usn()?; + col.storage.commit_rust_trx()?; + col.storage.set_modified_time(now)?; + Ok(now) +} diff --git a/rslib/src/sync/collection/graves.rs b/rslib/src/sync/collection/graves.rs new file mode 100644 index 000000000..8213b3e13 --- /dev/null +++ b/rslib/src/sync/collection/graves.rs @@ -0,0 +1,71 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use serde::{Deserialize, Serialize}; + +use crate::{ + prelude::*, + sync::collection::{chunks::CHUNK_SIZE, start::ServerSyncState}, +}; + +#[derive(Serialize, Deserialize, Debug, Default, Clone)] +pub struct ApplyGravesRequest { + pub chunk: Graves, +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone)] +pub struct Graves { + pub(crate) cards: Vec, + pub(crate) decks: Vec, + pub(crate) notes: Vec, +} + +impl Graves { + pub(in crate::sync) fn take_chunk(&mut self) -> Option { + let mut limit = CHUNK_SIZE; + let mut out = Graves::default(); + while limit > 0 && !self.cards.is_empty() { + out.cards.push(self.cards.pop().unwrap()); + limit -= 1; + } + while limit > 0 && !self.notes.is_empty() { + out.notes.push(self.notes.pop().unwrap()); + limit -= 1; + } + while limit > 0 && !self.decks.is_empty() { + out.decks.push(self.decks.pop().unwrap()); + limit -= 1; + } + if limit == CHUNK_SIZE { + None + } else { + Some(out) + } + } +} + +impl Collection { + pub fn apply_graves(&self, graves: Graves, latest_usn: Usn) -> Result<()> { + for nid in graves.notes { + self.storage.remove_note(nid)?; + self.storage.add_note_grave(nid, latest_usn)?; + } + for cid in graves.cards { + self.storage.remove_card(cid)?; + self.storage.add_card_grave(cid, latest_usn)?; + } + for did in graves.decks { + self.storage.remove_deck(did)?; + self.storage.add_deck_grave(did, latest_usn)?; + } + Ok(()) + } +} + +pub fn server_apply_graves( + req: ApplyGravesRequest, + col: &mut Collection, + state: &mut ServerSyncState, +) -> Result<()> { + col.apply_graves(req.chunk, state.server_usn) +} diff --git a/rslib/src/sync/collection/meta.rs b/rslib/src/sync/collection/meta.rs new file mode 100644 index 000000000..9b62dc261 --- /dev/null +++ b/rslib/src/sync/collection/meta.rs @@ -0,0 +1,170 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use ammonia::Url; +use axum::http::StatusCode; +use serde::{Deserialize, Serialize}; +use tracing::debug; + +use crate::{ + config::SchedulerVersion, + prelude::*, + sync::{ + collection::{ + normal::{ClientSyncState, SyncActionRequired}, + protocol::SyncProtocol, + }, + error::{HttpError, HttpResult, HttpSnafu, OrHttpErr}, + http_client::HttpSyncClient, + request::{IntoSyncRequest, SyncRequest}, + version::{ + SYNC_VERSION_09_V2_SCHEDULER, SYNC_VERSION_10_V2_TIMEZONE, SYNC_VERSION_MAX, + SYNC_VERSION_MIN, + }, + }, + version::sync_client_version, +}; + +#[derive(Serialize, Deserialize, Debug, Default)] +pub struct SyncMeta { + #[serde(rename = "mod")] + pub modified: TimestampMillis, + #[serde(rename = "scm")] + pub schema: TimestampMillis, + pub usn: Usn, + #[serde(rename = "ts")] + pub current_time: TimestampSecs, + #[serde(rename = "msg")] + pub server_message: String, + #[serde(rename = "cont")] + pub should_continue: bool, + /// Used by clients prior to sync version 11 + #[serde(rename = "hostNum")] + pub host_number: u32, + #[serde(default)] + pub empty: bool, + #[serde(skip)] + pub v2_scheduler_or_later: bool, + #[serde(skip)] + pub v2_timezone: bool, +} + +impl SyncMeta { + pub(in crate::sync) fn compared_to_remote( + &self, + remote: SyncMeta, + new_endpoint: Option, + ) -> ClientSyncState { + let local = self; + let required = if remote.modified == local.modified { + SyncActionRequired::NoChanges + } else if remote.schema != local.schema { + let upload_ok = !local.empty || remote.empty; + let download_ok = !remote.empty || local.empty; + SyncActionRequired::FullSyncRequired { + upload_ok, + download_ok, + } + } else { + SyncActionRequired::NormalSyncRequired + }; + + ClientSyncState { + required, + local_is_newer: local.modified > remote.modified, + usn_at_last_sync: local.usn, + server_usn: remote.usn, + pending_usn: Usn(-1), + server_message: remote.server_message, + host_number: remote.host_number, + new_endpoint, + } + } +} + +impl HttpSyncClient { + /// Fetch server meta. Returns a new endpoint if one was provided. + pub(in crate::sync) async fn meta_with_redirect( + &mut self, + ) -> Result<(SyncMeta, Option)> { + let mut new_endpoint = None; + let response = match self.meta(MetaRequest::request()).await { + Ok(remote) => remote, + Err(HttpError { + code: StatusCode::PERMANENT_REDIRECT, + context, + .. + }) => { + debug!(endpoint = context, "redirect to new location"); + let url = Url::try_from(context.as_str()) + .or_bad_request("couldn't parse new location")?; + new_endpoint = Some(context); + self.endpoint = url; + self.meta(MetaRequest::request()).await? + } + err => err?, + }; + let remote = response.json()?; + Ok((remote, new_endpoint)) + } +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct MetaRequest { + #[serde(rename = "v")] + pub sync_version: u8, + #[serde(rename = "cv")] + pub client_version: String, +} + +impl Collection { + pub fn sync_meta(&self) -> Result { + let stamps = self.storage.get_collection_timestamps()?; + Ok(SyncMeta { + modified: stamps.collection_change, + schema: stamps.schema_change, + // server=true is used for the client case as well, as we + // want the actual usn and not -1 + usn: self.storage.usn(true)?, + current_time: TimestampSecs::now(), + server_message: "".into(), + should_continue: true, + host_number: 0, + empty: !self.storage.have_at_least_one_card()?, + v2_scheduler_or_later: self.scheduler_version() == SchedulerVersion::V2, + v2_timezone: self.get_creation_utc_offset().is_some(), + }) + } +} + +pub fn server_meta(req: MetaRequest, col: &mut Collection) -> HttpResult { + if !matches!(req.sync_version, SYNC_VERSION_MIN..=SYNC_VERSION_MAX) { + return HttpSnafu { + // old clients expected this code + code: StatusCode::NOT_IMPLEMENTED, + context: "unsupported version", + source: None, + } + .fail(); + } + let mut meta = col.sync_meta().or_internal_err("sync meta")?; + if meta.v2_scheduler_or_later && req.sync_version < SYNC_VERSION_09_V2_SCHEDULER { + meta.server_message = "Your client does not support the v2 scheduler".into(); + meta.should_continue = false; + } else if meta.v2_timezone && req.sync_version < SYNC_VERSION_10_V2_TIMEZONE { + meta.server_message = "Your client does not support the new timezone handling.".into(); + meta.should_continue = false; + } + Ok(meta) +} + +impl MetaRequest { + pub fn request() -> SyncRequest { + MetaRequest { + sync_version: SYNC_VERSION_MAX, + client_version: sync_client_version().into(), + } + .try_into_sync_request() + .expect("infallible meta request") + } +} diff --git a/rslib/src/sync/collection/mod.rs b/rslib/src/sync/collection/mod.rs new file mode 100644 index 000000000..618e2f787 --- /dev/null +++ b/rslib/src/sync/collection/mod.rs @@ -0,0 +1,17 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +pub mod changes; +pub mod chunks; +pub mod download; +pub mod finish; +pub mod graves; +pub mod meta; +pub mod normal; +pub mod progress; +pub mod protocol; +pub mod sanity; +pub mod start; +pub mod status; +pub mod tests; +pub mod upload; diff --git a/rslib/src/sync/collection/normal.rs b/rslib/src/sync/collection/normal.rs new file mode 100644 index 000000000..d476f0ee8 --- /dev/null +++ b/rslib/src/sync/collection/normal.rs @@ -0,0 +1,182 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use tracing::debug; + +use crate::{ + collection::Collection, + error, + error::{AnkiError, SyncError, SyncErrorKind}, + prelude::Usn, + sync::{ + collection::{ + progress::SyncStage, + protocol::{EmptyInput, SyncProtocol}, + status::online_sync_status_check, + }, + http_client::HttpSyncClient, + login::SyncAuth, + }, +}; + +pub struct NormalSyncer<'a, F> { + pub(in crate::sync) col: &'a mut Collection, + pub(in crate::sync) server: HttpSyncClient, + pub(in crate::sync) progress: NormalSyncProgress, + pub(in crate::sync) progress_fn: F, +} + +#[derive(Default, Debug, Clone, Copy)] +pub struct NormalSyncProgress { + pub stage: SyncStage, + pub local_update: usize, + pub local_remove: usize, + pub remote_update: usize, + pub remote_remove: usize, +} + +#[derive(PartialEq, Eq, Debug, Clone, Copy)] +pub enum SyncActionRequired { + NoChanges, + FullSyncRequired { upload_ok: bool, download_ok: bool }, + NormalSyncRequired, +} + +#[derive(Debug)] +pub struct ClientSyncState { + pub required: SyncActionRequired, + pub server_message: String, + pub host_number: u32, + pub new_endpoint: Option, + + pub(in crate::sync) local_is_newer: bool, + pub(in crate::sync) usn_at_last_sync: Usn, + // latest server usn; local -1 entries will be rewritten to this + pub(in crate::sync) server_usn: Usn, + // -1 in client case; used to locate pending entries + pub(in crate::sync) pending_usn: Usn, +} + +impl NormalSyncer<'_, F> +where + F: FnMut(NormalSyncProgress, bool), +{ + pub fn new(col: &mut Collection, server: HttpSyncClient, progress_fn: F) -> NormalSyncer<'_, F> + where + F: FnMut(NormalSyncProgress, bool), + { + NormalSyncer { + col, + server, + progress: NormalSyncProgress::default(), + progress_fn, + } + } + + pub(in crate::sync) fn fire_progress_cb(&mut self, throttle: bool) { + (self.progress_fn)(self.progress, throttle) + } + + pub async fn sync(&mut self) -> error::Result { + debug!("fetching meta..."); + self.fire_progress_cb(false); + let local = self.col.sync_meta()?; + let state = online_sync_status_check(local, &mut self.server).await?; + debug!(?state, "fetched"); + match state.required { + SyncActionRequired::NoChanges => Ok(state.into()), + SyncActionRequired::FullSyncRequired { .. } => Ok(state.into()), + SyncActionRequired::NormalSyncRequired => { + self.col.discard_undo_and_study_queues(); + let timing = self.col.timing_today()?; + self.col.unbury_if_day_rolled_over(timing)?; + self.col.storage.begin_trx()?; + match self.normal_sync_inner(state).await { + Ok(success) => { + self.col.storage.commit_trx()?; + Ok(success) + } + Err(e) => { + self.col.storage.rollback_trx()?; + + let _ = self.server.abort(EmptyInput::request()).await; + + if let AnkiError::SyncError { + source: + SyncError { + kind: SyncErrorKind::SanityCheckFailed { client, server }, + .. + }, + } = &e + { + debug!(?client, ?server, "sanity check failed"); + self.col.set_schema_modified()?; + } + + Err(e) + } + } + } + } + } + + /// Sync. Caller must have created a transaction, and should call + /// abort on failure. + async fn normal_sync_inner(&mut self, mut state: ClientSyncState) -> error::Result { + self.progress.stage = SyncStage::Syncing; + self.fire_progress_cb(false); + + debug!("start"); + self.start_and_process_deletions(&state).await?; + debug!("unchunked changes"); + self.process_unchunked_changes(&state).await?; + debug!("begin stream from server"); + self.process_chunks_from_server(&state).await?; + debug!("begin stream to server"); + self.send_chunks_to_server(&state).await?; + + self.progress.stage = SyncStage::Finalizing; + self.fire_progress_cb(false); + + debug!("sanity check"); + self.sanity_check().await?; + debug!("finalize"); + self.finalize(&state).await?; + state.required = SyncActionRequired::NoChanges; + Ok(state.into()) + } +} + +#[derive(Debug)] +pub struct SyncOutput { + pub required: SyncActionRequired, + pub server_message: String, + pub host_number: u32, + pub new_endpoint: Option, +} + +impl From for SyncOutput { + fn from(s: ClientSyncState) -> Self { + SyncOutput { + required: s.required, + server_message: s.server_message, + host_number: s.host_number, + new_endpoint: s.new_endpoint, + } + } +} + +impl Collection { + pub async fn normal_sync( + &mut self, + auth: SyncAuth, + progress_fn: F, + ) -> error::Result + where + F: FnMut(NormalSyncProgress, bool), + { + NormalSyncer::new(self, HttpSyncClient::new(auth), progress_fn) + .sync() + .await + } +} diff --git a/rslib/src/sync/collection/progress.rs b/rslib/src/sync/collection/progress.rs new file mode 100644 index 000000000..0e2ea52ba --- /dev/null +++ b/rslib/src/sync/collection/progress.rs @@ -0,0 +1,39 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::{ + error, + sync::{ + collection::protocol::{EmptyInput, SyncProtocol}, + http_client::HttpSyncClient, + login::SyncAuth, + }, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SyncStage { + Connecting, + Syncing, + Finalizing, +} + +impl Default for SyncStage { + fn default() -> Self { + SyncStage::Connecting + } +} + +#[derive(Debug, Default, Clone, Copy)] +pub struct FullSyncProgress { + pub transferred_bytes: usize, + pub total_bytes: usize, +} + +pub async fn sync_abort(auth: SyncAuth) -> error::Result<()> { + HttpSyncClient::new(auth) + .abort(EmptyInput::request()) + .await? + .json() +} + +pub type FullSyncProgressFn = Box; diff --git a/rslib/src/sync/collection/protocol.rs b/rslib/src/sync/collection/protocol.rs new file mode 100644 index 000000000..e9a005990 --- /dev/null +++ b/rslib/src/sync/collection/protocol.rs @@ -0,0 +1,110 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::marker::PhantomData; + +use ammonia::Url; +use async_trait::async_trait; +use serde_derive::{Deserialize, Serialize}; +use strum::IntoStaticStr; + +use crate::{ + prelude::TimestampMillis, + sync::{ + collection::{ + changes::{ApplyChangesRequest, UnchunkedChanges}, + chunks::{ApplyChunkRequest, Chunk}, + graves::{ApplyGravesRequest, Graves}, + meta::{MetaRequest, SyncMeta}, + sanity::{SanityCheckRequest, SanityCheckResponse}, + start::StartRequest, + upload::UploadResponse, + }, + error::HttpResult, + login::{HostKeyRequest, HostKeyResponse}, + request::{IntoSyncRequest, SyncRequest}, + response::SyncResponse, + }, +}; + +#[derive(IntoStaticStr, Deserialize, PartialEq, Eq, Debug)] +#[serde(rename_all = "camelCase")] +#[strum(serialize_all = "camelCase")] +pub enum SyncMethod { + HostKey, + Meta, + Start, + ApplyGraves, + ApplyChanges, + Chunk, + ApplyChunk, + SanityCheck2, + Finish, + Abort, + Upload, + Download, +} + +pub trait AsSyncEndpoint: Into<&'static str> { + fn as_sync_endpoint(&self, base: &Url) -> Url; +} + +impl AsSyncEndpoint for SyncMethod { + fn as_sync_endpoint(&self, base: &Url) -> Url { + base.join("sync/").unwrap().join(self.into()).unwrap() + } +} + +#[async_trait] +pub trait SyncProtocol: Send + Sync + 'static { + async fn host_key( + &self, + req: SyncRequest, + ) -> HttpResult>; + async fn meta(&self, req: SyncRequest) -> HttpResult>; + async fn start(&self, req: SyncRequest) -> HttpResult>; + async fn apply_graves( + &self, + req: SyncRequest, + ) -> HttpResult>; + async fn apply_changes( + &self, + req: SyncRequest, + ) -> HttpResult>; + async fn chunk(&self, req: SyncRequest) -> HttpResult>; + async fn apply_chunk( + &self, + req: SyncRequest, + ) -> HttpResult>; + async fn sanity_check( + &self, + req: SyncRequest, + ) -> HttpResult>; + async fn finish( + &self, + req: SyncRequest, + ) -> HttpResult>; + async fn abort(&self, req: SyncRequest) -> HttpResult>; + async fn upload(&self, req: SyncRequest>) -> HttpResult>; + async fn download(&self, req: SyncRequest) -> HttpResult>>; +} + +/// The sync protocol expects '{}' to be sent in requests without args. +/// Serde serializes/deserializes empty structs as 'null', so we add an empty value +/// to cause it to produce a map instead. This only applies to inputs; empty outputs +/// are returned as ()/null. +#[derive(Serialize, Deserialize, Default)] +#[serde(deny_unknown_fields)] +pub struct EmptyInput { + #[serde(default)] + _pad: PhantomData<()>, +} + +impl EmptyInput { + pub(crate) fn request() -> SyncRequest { + Self::default() + .try_into_sync_request() + // should be infallible + .expect("empty input into request") + } +} diff --git a/rslib/src/sync/collection/sanity.rs b/rslib/src/sync/collection/sanity.rs new file mode 100644 index 000000000..181219bbf --- /dev/null +++ b/rslib/src/sync/collection/sanity.rs @@ -0,0 +1,129 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use serde::{Deserialize, Serialize}; +use serde_tuple::Serialize_tuple; +use tracing::{debug, info}; + +use crate::{ + error::SyncErrorKind, + prelude::*, + serde::default_on_invalid, + sync::{ + collection::{ + normal::{NormalSyncProgress, NormalSyncer}, + protocol::SyncProtocol, + }, + request::IntoSyncRequest, + }, +}; + +#[derive(Serialize, Deserialize, Debug)] +pub struct SanityCheckResponse { + pub status: SanityCheckStatus, + #[serde(rename = "c", default, deserialize_with = "default_on_invalid")] + pub client: Option, + #[serde(rename = "s", default, deserialize_with = "default_on_invalid")] + pub server: Option, +} + +#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum SanityCheckStatus { + Ok, + Bad, +} + +#[derive(Serialize_tuple, Deserialize, Debug, PartialEq, Eq)] +pub struct SanityCheckCounts { + pub counts: SanityCheckDueCounts, + pub cards: u32, + pub notes: u32, + pub revlog: u32, + pub graves: u32, + #[serde(rename = "models")] + pub notetypes: u32, + pub decks: u32, + pub deck_config: u32, +} + +#[derive(Serialize_tuple, Deserialize, Debug, Default, PartialEq, Eq)] +pub struct SanityCheckDueCounts { + pub new: u32, + pub learn: u32, + pub review: u32, +} + +impl NormalSyncer<'_, F> +where + F: FnMut(NormalSyncProgress, bool), +{ + /// Caller should force full sync after rolling back. + pub(in crate::sync) async fn sanity_check(&mut self) -> Result<()> { + let local_counts = self.col.storage.sanity_check_info()?; + + debug!("gathered local counts; waiting for server reply"); + let SanityCheckResponse { + status, + client, + server, + } = self + .server + .sanity_check( + SanityCheckRequest { + client: local_counts, + } + .try_into_sync_request()?, + ) + .await? + .json()?; + debug!("got server reply"); + if status != SanityCheckStatus::Ok { + Err(AnkiError::sync_error( + "", + SyncErrorKind::SanityCheckFailed { client, server }, + )) + } else { + Ok(()) + } + } +} + +pub fn server_sanity_check( + SanityCheckRequest { mut client }: SanityCheckRequest, + col: &mut Collection, +) -> Result { + let mut server = match col.storage.sanity_check_info() { + Ok(info) => info, + Err(err) => { + info!(?client, ?err, "sanity check failed"); + return Ok(SanityCheckResponse { + status: SanityCheckStatus::Bad, + client: Some(client), + server: None, + }); + } + }; + + client.counts = Default::default(); + // clients on schema 17 and below may send duplicate + // deletion markers, so we can't compare graves until + // the minimum syncing version is schema 18. + client.graves = 0; + server.graves = 0; + Ok(SanityCheckResponse { + status: if client == server { + SanityCheckStatus::Ok + } else { + info!(?client, ?server, "sanity check failed"); + SanityCheckStatus::Bad + }, + client: Some(client), + server: Some(server), + }) +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct SanityCheckRequest { + pub client: SanityCheckCounts, +} diff --git a/rslib/src/sync/collection/start.rs b/rslib/src/sync/collection/start.rs new file mode 100644 index 000000000..682e34191 --- /dev/null +++ b/rslib/src/sync/collection/start.rs @@ -0,0 +1,186 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use serde::{Deserialize, Deserializer, Serialize}; +use tracing::debug; + +use crate::{ + prelude::*, + sync::{ + collection::{ + chunks::ChunkableIds, + graves::{ApplyGravesRequest, Graves}, + normal::{ClientSyncState, NormalSyncProgress, NormalSyncer}, + protocol::SyncProtocol, + }, + request::IntoSyncRequest, + }, +}; + +impl NormalSyncer<'_, F> +where + F: FnMut(NormalSyncProgress, bool), +{ + pub(in crate::sync) async fn start_and_process_deletions( + &mut self, + state: &ClientSyncState, + ) -> Result<()> { + let remote: Graves = self + .server + .start( + StartRequest { + client_usn: state.usn_at_last_sync, + local_is_newer: state.local_is_newer, + deprecated_client_graves: None, + } + .try_into_sync_request()?, + ) + .await? + .json()?; + + debug!( + cards = remote.cards.len(), + notes = remote.notes.len(), + decks = remote.decks.len(), + "removed on remote" + ); + + let mut local = self.col.storage.pending_graves(state.pending_usn)?; + self.col + .storage + .update_pending_grave_usns(state.server_usn)?; + + debug!( + cards = local.cards.len(), + notes = local.notes.len(), + decks = local.decks.len(), + "locally removed " + ); + + while let Some(chunk) = local.take_chunk() { + debug!("sending graves chunk"); + self.progress.local_remove += chunk.cards.len() + chunk.notes.len() + chunk.decks.len(); + self.server + .apply_graves(ApplyGravesRequest { chunk }.try_into_sync_request()?) + .await?; + self.fire_progress_cb(true); + } + + self.progress.remote_remove = remote.cards.len() + remote.notes.len() + remote.decks.len(); + self.col.apply_graves(remote, state.server_usn)?; + self.fire_progress_cb(true); + debug!("applied server graves"); + + Ok(()) + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct StartRequest { + #[serde(rename = "minUsn")] + pub client_usn: Usn, + #[serde(rename = "lnewer")] + pub local_is_newer: bool, + /// Used by old clients, and still used by AnkiDroid. + #[serde(rename = "graves", default, deserialize_with = "legacy_graves")] + pub deprecated_client_graves: Option, +} + +pub fn server_start( + req: StartRequest, + col: &mut Collection, + state: &mut ServerSyncState, +) -> Result { + state.server_usn = col.usn()?; + state.client_usn = req.client_usn; + state.client_is_newer = req.local_is_newer; + + col.discard_undo_and_study_queues(); + col.storage.begin_rust_trx()?; + + // make sure any pending cards have been unburied first if necessary + let timing = col.timing_today()?; + col.unbury_if_day_rolled_over(timing)?; + + // fetch local graves + let server_graves = col.storage.pending_graves(state.client_usn)?; + // handle AnkiDroid using old protocol + if let Some(graves) = req.deprecated_client_graves { + col.apply_graves(graves, state.server_usn)?; + } + + Ok(server_graves) +} + +/// The current sync protocol is stateful, so unfortunately we need to +/// retain a bunch of information across requests. These are set either +/// on start, or on subsequent methods. +pub struct ServerSyncState { + /// The session key. This is sent on every http request, but is ignored for methods + /// where there is not active sync state. + pub skey: String, + + pub(in crate::sync) server_usn: Usn, + pub(in crate::sync) client_usn: Usn, + /// Only used to determine whether we should send our + /// config to client. + pub(in crate::sync) client_is_newer: bool, + /// Set on the first call to chunk() + pub(in crate::sync) server_chunk_ids: Option, +} + +impl ServerSyncState { + pub fn new(skey: impl Into) -> Self { + Self { + skey: skey.into(), + server_usn: Default::default(), + client_usn: Default::default(), + client_is_newer: false, + server_chunk_ids: None, + } + } +} + +pub(crate) fn legacy_graves<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum GraveType { + Normal(Graves), + Legacy(StringGraves), + Null, + } + match GraveType::deserialize(deserializer)? { + GraveType::Normal(normal) => Ok(Some(normal)), + GraveType::Legacy(stringly) => Ok(Some(Graves { + cards: string_list_to_ids(stringly.cards)?, + decks: string_list_to_ids(stringly.decks)?, + notes: string_list_to_ids(stringly.notes)?, + })), + GraveType::Null => Ok(None), + } +} + +// old AnkiMobile versions +#[derive(Deserialize)] +struct StringGraves { + cards: Vec, + decks: Vec, + notes: Vec, +} + +fn string_list_to_ids(list: Vec) -> Result, E> +where + T: From, + E: serde::de::Error, +{ + list.into_iter() + .map(|s| { + s.parse::() + .map_err(serde::de::Error::custom) + .map(Into::into) + }) + .collect::, E>>() +} diff --git a/rslib/src/sync/collection/status.rs b/rslib/src/sync/collection/status.rs new file mode 100644 index 000000000..829dd9413 --- /dev/null +++ b/rslib/src/sync/collection/status.rs @@ -0,0 +1,58 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use tracing::debug; + +use crate::{ + error::SyncErrorKind, + pb::sync::sync_status_response, + prelude::*, + sync::{ + collection::{meta::SyncMeta, normal::ClientSyncState}, + http_client::HttpSyncClient, + }, +}; + +impl Collection { + /// Checks local collection only. If local collection is clean but changes are pending + /// on AnkiWeb, NoChanges will be returned. + pub fn sync_status_offline(&mut self) -> Result { + let stamps = self.storage.get_collection_timestamps()?; + let required = if stamps.schema_changed_since_sync() { + sync_status_response::Required::FullSync + } else if stamps.collection_changed_since_sync() { + sync_status_response::Required::NormalSync + } else { + sync_status_response::Required::NoChanges + }; + + Ok(required) + } +} + +/// Should be called if a call to sync_status_offline() returns NoChanges, to check +/// if AnkiWeb has pending changes. Caller should persist new endpoint if returned. +/// +/// This routine is outside of the collection, as we don't want to block collection access +/// for a potentially slow network request that happens in the background. +pub async fn online_sync_status_check( + local: SyncMeta, + server: &mut HttpSyncClient, +) -> Result { + let (remote, new_endpoint) = server.meta_with_redirect().await?; + debug!(?remote, "meta"); + debug!(?local, "meta"); + if !remote.should_continue { + debug!(remote.server_message, "server says abort"); + return Err(AnkiError::sync_error( + remote.server_message, + SyncErrorKind::ServerMessage, + )); + } + let delta = remote.current_time.0 - local.current_time.0; + if delta.abs() > 300 { + debug!(delta, "clock off"); + return Err(AnkiError::sync_error("", SyncErrorKind::ClockIncorrect)); + } + Ok(local.compared_to_remote(remote, new_endpoint)) +} diff --git a/rslib/src/sync/collection/tests.rs b/rslib/src/sync/collection/tests.rs new file mode 100644 index 000000000..0b2f19eb7 --- /dev/null +++ b/rslib/src/sync/collection/tests.rs @@ -0,0 +1,752 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +#![cfg(test)] + +use std::future::Future; + +use axum::http::StatusCode; +use once_cell::sync::Lazy; +use reqwest::Url; +use serde_json::json; +use tempfile::{tempdir, TempDir}; +use tokio::sync::{Mutex, MutexGuard}; +use tracing::{Instrument, Span}; +use wiremock::{ + matchers::{method, path}, + Mock, MockServer, ResponseTemplate, +}; + +use crate::{ + card::CardQueue, + collection::CollectionBuilder, + deckconfig::DeckConfig, + decks::DeckKind, + error::{SyncError, SyncErrorKind}, + log::set_global_logger, + notetype::all_stock_notetypes, + prelude::*, + revlog::RevlogEntry, + search::SortMode, + sync::{ + collection::{ + graves::ApplyGravesRequest, + meta::MetaRequest, + normal::{NormalSyncProgress, NormalSyncer, SyncActionRequired, SyncOutput}, + progress::FullSyncProgress, + protocol::{EmptyInput, SyncProtocol}, + start::StartRequest, + upload::{UploadResponse, CORRUPT_MESSAGE}, + }, + http_client::HttpSyncClient, + http_server::SimpleServer, + login::{HostKeyRequest, SyncAuth}, + request::IntoSyncRequest, + }, +}; + +struct TestAuth { + username: String, + password: String, + host_key: String, +} + +static AUTH: Lazy = Lazy::new(|| { + if let Ok(auth) = std::env::var("TEST_AUTH") { + let mut auth = auth.split(':'); + TestAuth { + username: auth.next().unwrap().into(), + password: auth.next().unwrap().into(), + host_key: auth.next().unwrap().into(), + } + } else { + TestAuth { + username: "user".to_string(), + password: "pass".to_string(), + host_key: "b2619aa1529dfdc4248e6edbf3c1b2a2b014cf6d".to_string(), + } + } +}); + +pub(in crate::sync) async fn with_active_server(op: F) -> Result<()> +where + F: FnOnce(HttpSyncClient) -> O, + O: Future>, +{ + let _ = set_global_logger(None); + // start server + let base_folder = tempdir()?; + std::env::set_var("SYNC_USER1", "user:pass"); + let (addr, server_fut) = SimpleServer::make_server(None, base_folder.path()).unwrap(); + tokio::spawn(server_fut.instrument(Span::current())); + // when not using ephemeral servers, tests need to be serialized + static LOCK: Lazy> = Lazy::new(|| Mutex::new(())); + let _lock: MutexGuard<()>; + // setup client to connect to it + let endpoint = if let Ok(endpoint) = std::env::var("TEST_ENDPOINT") { + _lock = LOCK.lock().await; + endpoint + } else { + format!("http://{}/", addr) + }; + let endpoint = Url::try_from(endpoint.as_str()).unwrap(); + let auth = SyncAuth { + hkey: AUTH.host_key.clone(), + endpoint: Some(endpoint), + }; + let client = HttpSyncClient::new(auth); + op(client).await +} + +fn unwrap_sync_err_kind(err: AnkiError) -> SyncErrorKind { + let AnkiError::SyncError { source: SyncError { kind, .. } } = err else { + panic!("not sync err: {err:?}"); + }; + kind +} + +fn norm_progress(_: NormalSyncProgress, _: bool) {} + +fn full_progress(_: FullSyncProgress, _: bool) {} + +#[tokio::test] +async fn host_key() -> Result<()> { + with_active_server(|mut client| async move { + let err = client + .host_key( + HostKeyRequest { + username: "bad".to_string(), + password: "bad".to_string(), + } + .try_into_sync_request()?, + ) + .await + .unwrap_err(); + assert_eq!(err.code, StatusCode::FORBIDDEN); + assert_eq!( + unwrap_sync_err_kind(AnkiError::from(err)), + SyncErrorKind::AuthFailed + ); + // hkey should be automatically set after successful login + client.sync_key = String::new(); + let resp = client + .host_key( + HostKeyRequest { + username: AUTH.username.clone(), + password: AUTH.password.clone(), + } + .try_into_sync_request()?, + ) + .await? + .json()?; + assert_eq!(resp.key, *AUTH.host_key); + Ok(()) + }) + .await +} + +#[tokio::test] +async fn meta() -> Result<()> { + with_active_server(|client| async move { + // unsupported sync version + assert_eq!( + SyncProtocol::meta( + &client, + MetaRequest { + sync_version: 0, + client_version: "".to_string(), + } + .try_into_sync_request()?, + ) + .await + .unwrap_err() + .code, + StatusCode::NOT_IMPLEMENTED + ); + + Ok(()) + }) + .await +} + +#[tokio::test] +async fn aborting_is_idempotent() -> Result<()> { + with_active_server(|mut client| async move { + // abort is a no-op if no sync in progress + client.abort(EmptyInput::request()).await?; + + // start a sync + let _graves = client + .start( + StartRequest { + client_usn: Default::default(), + local_is_newer: false, + deprecated_client_graves: None, + } + .try_into_sync_request()?, + ) + .await?; + + // an abort request with the wrong key is ignored + let orig_key = client.skey().to_string(); + client.set_skey("aabbccdd".into()); + client.abort(EmptyInput::request()).await?; + + // it should succeed with the correct key + client.set_skey(orig_key); + client.abort(EmptyInput::request()).await?; + Ok(()) + }) + .await +} + +#[tokio::test] +async fn new_syncs_cancel_old_ones() -> Result<()> { + with_active_server(|mut client| async move { + let ctx = SyncTestContext::new(client.partial_clone()); + + // start a sync + let req = StartRequest { + client_usn: Default::default(), + local_is_newer: false, + deprecated_client_graves: None, + } + .try_into_sync_request()?; + let _ = client.start(req.clone()).await?; + + // a new sync aborts the previous one + let orig_key = client.skey().to_string(); + client.set_skey("1".into()); + let _ = client.start(req.clone()).await?; + + // old sync can no longer proceed + client.set_skey(orig_key); + let graves_req = ApplyGravesRequest::default().try_into_sync_request()?; + assert_eq!( + client + .apply_graves(graves_req.clone()) + .await + .unwrap_err() + .code, + StatusCode::CONFLICT + ); + + // with the correct key, it can continue + client.set_skey("1".into()); + client.apply_graves(graves_req.clone()).await?; + // but a full upload will break the lock + ctx.full_upload(ctx.col1()).await; + assert_eq!( + client + .apply_graves(graves_req.clone()) + .await + .unwrap_err() + .code, + StatusCode::CONFLICT + ); + + // likewise with download + let _ = client.start(req.clone()).await?; + ctx.full_download(ctx.col1()).await; + assert_eq!( + client + .apply_graves(graves_req.clone()) + .await + .unwrap_err() + .code, + StatusCode::CONFLICT + ); + + Ok(()) + }) + .await +} + +#[tokio::test] +async fn sync_roundtrip() -> Result<()> { + with_active_server(|client| async move { + let ctx = SyncTestContext::new(client); + upload_download(&ctx).await?; + regular_sync(&ctx).await?; + Ok(()) + }) + .await +} + +#[tokio::test] +async fn sanity_check_should_roll_back_and_force_full_sync() -> Result<()> { + with_active_server(|client| async move { + let ctx = SyncTestContext::new(client); + upload_download(&ctx).await?; + + let mut col1 = ctx.col1(); + + // add a deck but don't mark it as requiring a sync, which will trigger the sanity + // check to fail + let mut deck = col1.get_or_create_normal_deck("unsynced deck")?; + col1.add_or_update_deck(&mut deck)?; + col1.storage + .db + .execute("update decks set usn=0 where id=?", [deck.id])?; + + // the sync should fail + let err = NormalSyncer::new(&mut col1, ctx.cloned_client(), norm_progress) + .sync() + .await + .unwrap_err(); + assert!(matches!( + err, + AnkiError::SyncError { + source: SyncError { + kind: SyncErrorKind::SanityCheckFailed { .. }, + .. + } + } + )); + + // the server should have rolled back + let mut col2 = ctx.col2(); + let out = ctx.normal_sync(&mut col2).await; + assert_eq!(out.required, SyncActionRequired::NoChanges); + + // and the client should have forced a one-way sync + let out = ctx.normal_sync(&mut col1).await; + assert_eq!( + out.required, + SyncActionRequired::FullSyncRequired { + upload_ok: true, + download_ok: true, + } + ); + + Ok(()) + }) + .await +} + +#[tokio::test] +async fn sync_errors_should_prompt_db_check() -> Result<()> { + with_active_server(|client| async move { + let ctx = SyncTestContext::new(client); + upload_download(&ctx).await?; + + let mut col1 = ctx.col1(); + + // Add a a new notetype, and a note that uses it, but don't mark the notetype as + // requiring a sync, which will cause the sync to fail as the note is added. + let mut nt = all_stock_notetypes(&col1.tr).remove(0); + nt.name = "new".into(); + col1.add_notetype(&mut nt, false)?; + let mut note = nt.new_note(); + note.set_field(0, "test")?; + col1.add_note(&mut note, DeckId(1))?; + col1.storage.db.execute("update notetypes set usn=0", [])?; + + // the sync should fail + let err = NormalSyncer::new(&mut col1, ctx.cloned_client(), norm_progress) + .sync() + .await + .unwrap_err(); + let AnkiError::SyncError { source: SyncError { info: _, kind } } = err else { panic!() }; + assert_eq!(kind, SyncErrorKind::DatabaseCheckRequired); + + // the server should have rolled back + let mut col2 = ctx.col2(); + let out = ctx.normal_sync(&mut col2).await; + assert_eq!(out.required, SyncActionRequired::NoChanges); + + // and the client should be able to sync again without a forced one-way sync + let err = NormalSyncer::new(&mut col1, ctx.cloned_client(), norm_progress) + .sync() + .await + .unwrap_err(); + let AnkiError::SyncError { source: SyncError { info: _, kind } } = err else { panic!() }; + assert_eq!(kind, SyncErrorKind::DatabaseCheckRequired); + + Ok(()) + }) + .await +} + +/// Old AnkiMobile versions sent grave ids as strings +#[tokio::test] +async fn string_grave_ids_are_handled() -> Result<()> { + with_active_server(|client| async move { + let req = json!({ + "minUsn": 0, + "lnewer": false, + "graves": { + "cards": vec!["1"], + "decks": vec!["2", "3"], + "notes": vec!["4"], + } + }); + let req = serde_json::to_vec(&req) + .unwrap() + .try_into_sync_request() + .unwrap(); + // should not return err 400 + client.start(req.into_output_type()).await.unwrap(); + client.abort(EmptyInput::request()).await?; + Ok(()) + }) + .await?; + // a missing value should be handled + with_active_server(|client| async move { + let req = json!({ + "minUsn": 0, + "lnewer": false, + }); + let req = serde_json::to_vec(&req) + .unwrap() + .try_into_sync_request() + .unwrap(); + client.start(req.into_output_type()).await.unwrap(); + client.abort(EmptyInput::request()).await?; + Ok(()) + }) + .await +} + +#[tokio::test] +async fn invalid_uploads_should_be_handled() -> Result<()> { + with_active_server(|client| async move { + let mut ctx = SyncTestContext::new(client); + ctx.client + .set_full_sync_progress_fn(Some(Box::new(full_progress))); + let res = ctx + .client + .upload(b"fake data".to_vec().try_into_sync_request()?) + .await?; + assert_eq!( + res.upload_response(), + UploadResponse::Err(CORRUPT_MESSAGE.into()) + ); + Ok(()) + }) + .await +} + +#[tokio::test] +async fn meta_redirect_is_handled() -> Result<()> { + with_active_server(|client| async move { + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/sync/meta")) + .respond_with( + ResponseTemplate::new(308).insert_header("location", client.endpoint.as_str()), + ) + .mount(&mock_server) + .await; + // starting from in-sync state + let mut ctx = SyncTestContext::new(client); + upload_download(&ctx).await?; + // add another note to trigger a normal sync + let mut col1 = ctx.col1(); + col1_setup(&mut col1); + // switch to bad endpoint + let orig_url = ctx.client.endpoint.to_string(); + ctx.client.endpoint = Url::try_from(mock_server.uri().as_str()).unwrap(); + // sync should succeed + let out = ctx.normal_sync(&mut col1).await; + // client should have received new endpoint + assert_eq!(out.new_endpoint, Some(orig_url)); + // client should not have tried the old endpoint more than once + assert_eq!(mock_server.received_requests().await.unwrap().len(), 1); + Ok(()) + }) + .await +} + +pub(in crate::sync) struct SyncTestContext { + pub folder: TempDir, + pub client: HttpSyncClient, +} + +impl SyncTestContext { + pub fn new(client: HttpSyncClient) -> Self { + Self { + folder: tempdir().expect("create temp dir"), + client, + } + } + + pub fn col1(&self) -> Collection { + let base = self.folder.path(); + CollectionBuilder::new(base.join("col1.anki2")) + .set_media_paths(base.join("col1.media"), base.join("col1.media.db")) + .build() + .unwrap() + } + + pub fn col2(&self) -> Collection { + let base = self.folder.path(); + CollectionBuilder::new(base.join("col2.anki2")) + .set_media_paths(base.join("col2.media"), base.join("col2.media.db")) + .build() + .unwrap() + } + + async fn normal_sync(&self, col: &mut Collection) -> SyncOutput { + NormalSyncer::new(col, self.cloned_client(), norm_progress) + .sync() + .await + .unwrap() + } + + async fn full_upload(&self, col: Collection) { + col.full_upload_with_server(self.cloned_client()) + .await + .unwrap() + } + + async fn full_download(&self, col: Collection) { + col.full_download_with_server(self.cloned_client()) + .await + .unwrap() + } + + fn cloned_client(&self) -> HttpSyncClient { + let mut client = self.client.partial_clone(); + client.set_full_sync_progress_fn(Some(Box::new(full_progress))); + client + } +} + +// Setup + full syncs +///////////////////// + +fn col1_setup(col: &mut Collection) { + let nt = col.get_notetype_by_name("Basic").unwrap().unwrap(); + let mut note = nt.new_note(); + note.set_field(0, "1").unwrap(); + col.add_note(&mut note, DeckId(1)).unwrap(); +} + +async fn upload_download(ctx: &SyncTestContext) -> Result<()> { + let mut col1 = ctx.col1(); + col1_setup(&mut col1); + + let out = ctx.normal_sync(&mut col1).await; + assert!(matches!( + out.required, + SyncActionRequired::FullSyncRequired { .. } + )); + + ctx.full_upload(col1).await; + + // another collection + let mut col2 = ctx.col2(); + + // won't allow ankiweb clobber + let out = ctx.normal_sync(&mut col2).await; + assert_eq!( + out.required, + SyncActionRequired::FullSyncRequired { + upload_ok: false, + download_ok: true, + } + ); + + // fetch so we're in sync + ctx.full_download(col2).await; + + Ok(()) +} + +// Regular syncs +///////////////////// + +async fn regular_sync(ctx: &SyncTestContext) -> Result<()> { + // add a deck + let mut col1 = ctx.col1(); + let mut col2 = ctx.col2(); + + let mut deck = col1.get_or_create_normal_deck("new deck")?; + + // give it a new option group + let mut dconf = DeckConfig { + name: "new dconf".into(), + ..Default::default() + }; + col1.add_or_update_deck_config(&mut dconf)?; + if let DeckKind::Normal(deck) = &mut deck.kind { + deck.config_id = dconf.id.0; + } + col1.add_or_update_deck(&mut deck)?; + + // and a new notetype + let mut nt = all_stock_notetypes(&col1.tr).remove(0); + nt.name = "new".into(); + col1.add_notetype(&mut nt, false)?; + + // add another note+card+tag + let mut note = nt.new_note(); + note.set_field(0, "2")?; + note.tags.push("tag".into()); + col1.add_note(&mut note, deck.id)?; + + // mock revlog entry + col1.storage.add_revlog_entry( + &RevlogEntry { + id: RevlogId(123), + cid: CardId(456), + usn: Usn(-1), + interval: 10, + ..Default::default() + }, + true, + )?; + + // config + creation + col1.set_config("test", &"test1")?; + // bumping this will affect 'last studied at' on decks at the moment + // col1.storage.set_creation_stamp(TimestampSecs(12345))?; + + // and sync our changes + let remote_meta = ctx + .client + .meta(MetaRequest::request()) + .await + .unwrap() + .json() + .unwrap(); + let out = col1.sync_meta()?.compared_to_remote(remote_meta, None); + assert_eq!(out.required, SyncActionRequired::NormalSyncRequired); + + let out = ctx.normal_sync(&mut col1).await; + assert_eq!(out.required, SyncActionRequired::NoChanges); + + // sync the other collection + let out = ctx.normal_sync(&mut col2).await; + assert_eq!(out.required, SyncActionRequired::NoChanges); + + let ntid = nt.id; + let deckid = deck.id; + let dconfid = dconf.id; + let noteid = note.id; + let cardid = col1.search_cards(note.id, SortMode::NoOrder)?[0]; + let revlogid = RevlogId(123); + + let compare_sides = |col1: &mut Collection, col2: &mut Collection| -> Result<()> { + assert_eq!( + col1.get_notetype(ntid)?.unwrap(), + col2.get_notetype(ntid)?.unwrap() + ); + assert_eq!( + col1.get_deck(deckid)?.unwrap(), + col2.get_deck(deckid)?.unwrap() + ); + assert_eq!( + col1.get_deck_config(dconfid, false)?.unwrap(), + col2.get_deck_config(dconfid, false)?.unwrap() + ); + assert_eq!( + col1.storage.get_note(noteid)?.unwrap(), + col2.storage.get_note(noteid)?.unwrap() + ); + assert_eq!( + col1.storage.get_card(cardid)?.unwrap(), + col2.storage.get_card(cardid)?.unwrap() + ); + assert_eq!( + col1.storage.get_revlog_entry(revlogid)?, + col2.storage.get_revlog_entry(revlogid)?, + ); + assert_eq!( + col1.storage.get_all_config()?, + col2.storage.get_all_config()? + ); + assert_eq!( + col1.storage.creation_stamp()?, + col2.storage.creation_stamp()? + ); + + // server doesn't send tag usns, so we can only compare tags, not usns, + // as the usns may not match + assert_eq!( + col1.storage + .all_tags()? + .into_iter() + .map(|t| t.name) + .collect::>(), + col2.storage + .all_tags()? + .into_iter() + .map(|t| t.name) + .collect::>() + ); + std::thread::sleep(std::time::Duration::from_millis(1)); + Ok(()) + }; + + // make sure everything has been transferred across + compare_sides(&mut col1, &mut col2)?; + + // make some modifications + let mut note = col2.storage.get_note(note.id)?.unwrap(); + note.set_field(1, "new")?; + note.tags.push("tag2".into()); + col2.update_note(&mut note)?; + + col2.get_and_update_card(cardid, |card| { + card.queue = CardQueue::Review; + Ok(()) + })?; + + let mut deck = col2.storage.get_deck(deck.id)?.unwrap(); + deck.name = NativeDeckName::from_native_str("newer"); + col2.add_or_update_deck(&mut deck)?; + + let mut nt = col2.storage.get_notetype(nt.id)?.unwrap(); + nt.name = "newer".into(); + col2.update_notetype(&mut nt, false)?; + + // sync the changes back + let out = ctx.normal_sync(&mut col2).await; + assert_eq!(out.required, SyncActionRequired::NoChanges); + let out = ctx.normal_sync(&mut col1).await; + assert_eq!(out.required, SyncActionRequired::NoChanges); + + // should still match + compare_sides(&mut col1, &mut col2)?; + + // deletions should sync too + for table in &["cards", "notes", "decks"] { + assert_eq!( + col1.storage + .db_scalar::(&format!("select count() from {}", table))?, + 2 + ); + } + + // fixme: inconsistent usn arg + std::thread::sleep(std::time::Duration::from_millis(1)); + col1.remove_cards_and_orphaned_notes(&[cardid])?; + let usn = col1.usn()?; + col1.remove_note_only_undoable(noteid, usn)?; + col1.remove_decks_and_child_decks(&[deckid])?; + + let out = ctx.normal_sync(&mut col1).await; + assert_eq!(out.required, SyncActionRequired::NoChanges); + let out = ctx.normal_sync(&mut col2).await; + assert_eq!(out.required, SyncActionRequired::NoChanges); + + for table in &["cards", "notes", "decks"] { + assert_eq!( + col2.storage + .db_scalar::(&format!("select count() from {}", table))?, + 1 + ); + } + + // removing things like a notetype forces a full sync + std::thread::sleep(std::time::Duration::from_millis(1)); + col2.remove_notetype(ntid)?; + let out = ctx.normal_sync(&mut col2).await; + assert!(matches!( + out.required, + SyncActionRequired::FullSyncRequired { .. } + )); + Ok(()) +} diff --git a/rslib/src/sync/collection/upload.rs b/rslib/src/sync/collection/upload.rs new file mode 100644 index 000000000..294e144bc --- /dev/null +++ b/rslib/src/sync/collection/upload.rs @@ -0,0 +1,136 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::{fs, io::Write}; + +use axum::response::{IntoResponse, Response}; +use flate2::{write::GzEncoder, Compression}; +use futures::StreamExt; +use tokio_util::io::ReaderStream; + +use crate::{ + collection::CollectionBuilder, + error::SyncErrorKind, + io::{atomic_rename, new_tempfile_in_parent_of, write_file}, + prelude::*, + storage::SchemaVersion, + sync::{ + collection::{progress::FullSyncProgressFn, protocol::SyncProtocol}, + error::{HttpResult, OrHttpErr}, + http_client::HttpSyncClient, + login::SyncAuth, + request::{IntoSyncRequest, MAXIMUM_SYNC_PAYLOAD_BYTES_UNCOMPRESSED}, + }, +}; + +/// Old clients didn't display a useful message on HTTP 400, and were expected to show the error message +/// returned by the server. +pub const CORRUPT_MESSAGE: &str = + "Your upload was corrupt. Please use Check Database, or restore from backup."; + +impl Collection { + /// Upload collection to AnkiWeb. Caller must re-open afterwards. + pub async fn full_upload(self, auth: SyncAuth, progress_fn: FullSyncProgressFn) -> Result<()> { + let mut server = HttpSyncClient::new(auth); + server.set_full_sync_progress_fn(Some(progress_fn)); + self.full_upload_with_server(server).await + } + + pub(crate) async fn full_upload_with_server(mut self, server: HttpSyncClient) -> Result<()> { + self.before_upload()?; + let col_path = self.col_path.clone(); + self.close(Some(SchemaVersion::V18))?; + let col_data = fs::read(&col_path)?; + + let total_bytes = col_data.len(); + if server.endpoint.as_str().contains("ankiweb") { + check_upload_limit( + total_bytes, + *MAXIMUM_SYNC_PAYLOAD_BYTES_UNCOMPRESSED as usize, + )?; + } + + match server + .upload(col_data.try_into_sync_request()?) + .await? + .upload_response() + { + UploadResponse::Ok => Ok(()), + UploadResponse::Err(msg) => { + Err(AnkiError::sync_error(msg, SyncErrorKind::ServerMessage)) + } + } + } +} + +/// Collection must already be open, and will be replaced on success. +pub fn handle_received_upload( + col: &mut Option, + new_data: Vec, +) -> HttpResult { + let max_bytes = *MAXIMUM_SYNC_PAYLOAD_BYTES_UNCOMPRESSED as usize; + if new_data.len() >= max_bytes { + return Ok(UploadResponse::Err("collection exceeds size limit".into())); + } + let path = col + .as_ref() + .or_internal_err("col was closed")? + .col_path + .clone(); + // write to temp file + let temp_file = new_tempfile_in_parent_of(&path).or_internal_err("temp file")?; + write_file(temp_file.path(), &new_data).or_internal_err("temp file")?; + // check the collection is valid + if let Err(err) = CollectionBuilder::new(temp_file.path()) + .set_check_integrity(true) + .build() + { + tracing::info!(?err, "uploaded file was corrupt/failed to open"); + return Ok(UploadResponse::Err(CORRUPT_MESSAGE.into())); + } + // close collection and rename + if let Some(col) = col.take() { + col.close(None) + .or_internal_err("closing current collection")?; + } + atomic_rename(temp_file, &path, true).or_internal_err("rename upload")?; + Ok(UploadResponse::Ok) +} + +impl IntoResponse for UploadResponse { + fn into_response(self) -> Response { + match self { + // the legacy protocol expects this exact string + UploadResponse::Ok => "OK".to_string(), + UploadResponse::Err(e) => e, + } + .into_response() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum UploadResponse { + Ok, + Err(String), +} + +pub fn check_upload_limit(size: usize, limit: usize) -> Result<()> { + if size >= limit { + Err(AnkiError::sync_error( + format!("{size} > {limit}"), + SyncErrorKind::UploadTooLarge, + )) + } else { + Ok(()) + } +} + +pub async fn gzipped_data_from_vec(vec: Vec) -> Result> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + let mut stream = ReaderStream::new(&vec[..]); + while let Some(chunk) = stream.next().await { + let chunk = chunk?; + encoder.write_all(&chunk)?; + } + encoder.finish().map_err(Into::into) +} diff --git a/rslib/src/sync/error.rs b/rslib/src/sync/error.rs new file mode 100644 index 000000000..a06182368 --- /dev/null +++ b/rslib/src/sync/error.rs @@ -0,0 +1,150 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use axum::{ + http::StatusCode, + response::{IntoResponse, Redirect, Response}, +}; +use snafu::{OptionExt, Snafu}; + +pub type HttpResult = std::result::Result; + +#[derive(Debug, Snafu)] +#[snafu(visibility(pub))] +pub struct HttpError { + pub code: StatusCode, + pub context: String, + // snafu's automatic error conversion only supports Option if + // the whatever trait is derived, and deriving whatever means we + // can't have extra fields like `code`. Even without Option, the + // error conversion requires us to manually box the error, so we end + // up having to disable the default behaviour and add the error to the + // snafu ourselves + #[snafu(source(false))] + pub source: Option>, +} + +impl HttpError { + pub fn new_without_source(code: StatusCode, context: impl Into) -> Self { + Self { + code, + context: context.into(), + source: None, + } + } + + /// Compatibility with ensure!() macro + pub fn fail(self) -> Result { + Err(self) + } +} + +impl IntoResponse for HttpError { + fn into_response(self) -> Response { + let HttpError { + code, + context, + source, + } = self; + if code.is_server_error() && code != StatusCode::NOT_IMPLEMENTED { + tracing::error!(context, ?source, httpstatus = code.as_u16(),); + } else { + tracing::info!(context, ?source, httpstatus = code.as_u16(),); + } + if code == StatusCode::PERMANENT_REDIRECT { + Redirect::permanent(&context).into_response() + } else { + (code, code.as_str().to_string()).into_response() + } + } +} + +pub trait OrHttpErr { + type Value; + + fn or_http_err( + self, + code: StatusCode, + context: impl Into, + ) -> Result; + + fn or_bad_request(self, context: impl Into) -> Result + where + Self: Sized, + { + self.or_http_err(StatusCode::BAD_REQUEST, context) + } + + fn or_internal_err(self, context: impl Into) -> Result + where + Self: Sized, + { + self.or_http_err(StatusCode::INTERNAL_SERVER_ERROR, context) + } + + fn or_forbidden(self, context: impl Into) -> Result + where + Self: Sized, + { + self.or_http_err(StatusCode::FORBIDDEN, context) + } + + fn or_conflict(self, context: impl Into) -> Result + where + Self: Sized, + { + self.or_http_err(StatusCode::CONFLICT, context) + } + + fn or_not_found(self, context: impl Into) -> Result + where + Self: Sized, + { + self.or_http_err(StatusCode::NOT_FOUND, context) + } + + fn or_permanent_redirect(self, context: impl Into) -> Result + where + Self: Sized, + { + self.or_http_err(StatusCode::PERMANENT_REDIRECT, context) + } +} + +impl OrHttpErr for Result +where + E: std::error::Error + Send + Sync + 'static, +{ + type Value = T; + + fn or_http_err( + self, + code: StatusCode, + context: impl Into, + ) -> Result { + self.map_err(|err| { + HttpSnafu { + code, + context: context.into(), + source: Some(Box::new(err) as _), + } + .build() + }) + } +} + +impl OrHttpErr for Option { + type Value = T; + + fn or_http_err( + self, + code: StatusCode, + context: impl Into, + ) -> Result { + self.context(HttpSnafu { + code, + context, + source: None, + }) + } +} diff --git a/rslib/src/sync/http.rs b/rslib/src/sync/http.rs deleted file mode 100644 index 88e3a9469..000000000 --- a/rslib/src/sync/http.rs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright: Ankitects Pty Ltd and contributors -// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html - -use std::path::PathBuf; - -use serde::{Deserialize, Serialize}; - -use super::{Chunk, Graves, SanityCheckCounts, UnchunkedChanges}; -use crate::{io::read_file, pb::sync::sync_server_method_request::Method, prelude::*}; -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -pub enum SyncRequest { - HostKey(HostKeyRequest), - Meta(MetaRequest), - Start(StartRequest), - ApplyGraves(ApplyGravesRequest), - ApplyChanges(ApplyChangesRequest), - Chunk, - ApplyChunk(ApplyChunkRequest), - #[serde(rename = "sanityCheck2")] - SanityCheck(SanityCheckRequest), - Finish, - Abort, - #[serde(rename = "upload")] - FullUpload(PathBuf), - #[serde(rename = "download")] - FullDownload, -} - -impl SyncRequest { - /// Return method name and payload bytes. - pub(crate) fn into_method_and_data(self) -> Result<(&'static str, Vec)> { - use serde_json::to_vec; - Ok(match self { - SyncRequest::HostKey(v) => ("hostKey", to_vec(&v)?), - SyncRequest::Meta(v) => ("meta", to_vec(&v)?), - SyncRequest::Start(v) => ("start", to_vec(&v)?), - SyncRequest::ApplyGraves(v) => ("applyGraves", to_vec(&v)?), - SyncRequest::ApplyChanges(v) => ("applyChanges", to_vec(&v)?), - SyncRequest::Chunk => ("chunk", b"{}".to_vec()), - SyncRequest::ApplyChunk(v) => ("applyChunk", to_vec(&v)?), - SyncRequest::SanityCheck(v) => ("sanityCheck2", to_vec(&v)?), - SyncRequest::Finish => ("finish", b"{}".to_vec()), - SyncRequest::Abort => ("abort", b"{}".to_vec()), - SyncRequest::FullUpload(v) => { - // fixme: stream in the data instead, in a different call - ("upload", read_file(&v)?) - } - SyncRequest::FullDownload => ("download", b"{}".to_vec()), - }) - } - - pub(crate) fn from_method_and_data(method: Method, data: Vec) -> Result { - use serde_json::from_slice; - Ok(match method { - Method::HostKey => SyncRequest::HostKey(from_slice(&data)?), - Method::Meta => SyncRequest::Meta(from_slice(&data)?), - Method::Start => SyncRequest::Start(from_slice(&data)?), - Method::ApplyGraves => SyncRequest::ApplyGraves(from_slice(&data)?), - Method::ApplyChanges => SyncRequest::ApplyChanges(from_slice(&data)?), - Method::Chunk => SyncRequest::Chunk, - Method::ApplyChunk => SyncRequest::ApplyChunk(from_slice(&data)?), - Method::SanityCheck => SyncRequest::SanityCheck(from_slice(&data)?), - Method::Finish => SyncRequest::Finish, - Method::Abort => SyncRequest::Abort, - Method::FullUpload => { - let path = PathBuf::from(String::from_utf8(data).expect("path was not in utf8")); - SyncRequest::FullUpload(path) - } - Method::FullDownload => SyncRequest::FullDownload, - }) - } -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct HostKeyRequest { - #[serde(rename = "u")] - pub username: String, - #[serde(rename = "p")] - pub password: String, -} -#[derive(Serialize, Deserialize, Debug)] -pub struct HostKeyResponse { - pub key: String, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct MetaRequest { - #[serde(rename = "v")] - pub sync_version: u8, - #[serde(rename = "cv")] - pub client_version: String, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct StartRequest { - #[serde(rename = "minUsn")] - pub client_usn: Usn, - #[serde(rename = "lnewer")] - pub local_is_newer: bool, - /// Unfortunately AnkiDroid is still using this - #[serde(rename = "graves", default)] - pub deprecated_client_graves: Option, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct ApplyGravesRequest { - pub chunk: Graves, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct ApplyChangesRequest { - pub changes: UnchunkedChanges, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct ApplyChunkRequest { - pub chunk: Chunk, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct SanityCheckRequest { - pub client: SanityCheckCounts, -} diff --git a/rslib/src/sync/http_client.rs b/rslib/src/sync/http_client.rs deleted file mode 100644 index 6bc7e4cbe..000000000 --- a/rslib/src/sync/http_client.rs +++ /dev/null @@ -1,499 +0,0 @@ -// Copyright: Ankitects Pty Ltd and contributors -// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html - -use std::{ - env, - io::{prelude::*, Cursor}, - mem::MaybeUninit, - path::Path, - time::Duration, -}; - -use async_trait::async_trait; -use bytes::Bytes; -use flate2::{write::GzEncoder, Compression}; -use futures::{Stream, StreamExt}; -use lazy_static::lazy_static; -use reqwest::{multipart, Body, Client, Response}; -use serde::de::DeserializeOwned; -use tempfile::NamedTempFile; -use tokio_util::io::ReaderStream; - -use super::{ - http::{ - ApplyChangesRequest, ApplyChunkRequest, ApplyGravesRequest, HostKeyRequest, - HostKeyResponse, MetaRequest, SanityCheckRequest, StartRequest, SyncRequest, - }, - server::SyncServer, - Chunk, FullSyncProgress, Graves, SanityCheckCounts, SanityCheckResponse, SyncMeta, - UnchunkedChanges, SYNC_VERSION_MAX, -}; -use crate::{ - error::SyncErrorKind, - io::{new_tempfile, new_tempfile_in}, - notes::guid, - prelude::*, - version::sync_client_version, -}; - -lazy_static! { - // These limits are enforced server-side, but are made adjustable for users - // who are using a custom sync server. - static ref MAXIMUM_UPLOAD_MEGS_UNCOMPRESSED: usize = env::var("MAX_UPLOAD_MEGS_UNCOMP") - .map(|v| v.parse().expect("invalid upload limit")) - .unwrap_or(250); - static ref MAXIMUM_UPLOAD_MEGS_COMPRESSED: usize = env::var("MAX_UPLOAD_MEGS_COMP") - .map(|v| v.parse().expect("invalid upload limit")) - .unwrap_or(100); -} - -pub type FullSyncProgressFn = Box; - -pub struct HttpSyncClient { - hkey: Option, - skey: String, - client: Client, - endpoint: String, - full_sync_progress_fn: Option, -} - -pub struct Timeouts { - pub connect_secs: u64, - pub request_secs: u64, - pub io_secs: u64, -} - -impl Timeouts { - pub fn new() -> Self { - let io_secs = if env::var("LONG_IO_TIMEOUT").is_ok() { - 3600 - } else { - 300 - }; - Timeouts { - connect_secs: 30, - /// This is smaller than the I/O limit because it is just a - /// default - some longer-running requests override it. - request_secs: 60, - io_secs, - } - } -} - -#[async_trait(?Send)] -impl SyncServer for HttpSyncClient { - async fn meta(&self) -> Result { - let input = SyncRequest::Meta(MetaRequest { - sync_version: SYNC_VERSION_MAX, - client_version: sync_client_version().to_string(), - }); - self.json_request(input).await - } - - async fn start( - &mut self, - client_usn: Usn, - local_is_newer: bool, - deprecated_client_graves: Option, - ) -> Result { - let input = SyncRequest::Start(StartRequest { - client_usn, - local_is_newer, - deprecated_client_graves, - }); - self.json_request(input).await - } - - async fn apply_graves(&mut self, chunk: Graves) -> Result<()> { - let input = SyncRequest::ApplyGraves(ApplyGravesRequest { chunk }); - self.json_request(input).await - } - - async fn apply_changes(&mut self, changes: UnchunkedChanges) -> Result { - let input = SyncRequest::ApplyChanges(ApplyChangesRequest { changes }); - self.json_request(input).await - } - - async fn chunk(&mut self) -> Result { - let input = SyncRequest::Chunk; - self.json_request(input).await - } - - async fn apply_chunk(&mut self, chunk: Chunk) -> Result<()> { - let input = SyncRequest::ApplyChunk(ApplyChunkRequest { chunk }); - self.json_request(input).await - } - - async fn sanity_check(&mut self, client: SanityCheckCounts) -> Result { - let input = SyncRequest::SanityCheck(SanityCheckRequest { client }); - self.json_request(input).await - } - - async fn finish(&mut self) -> Result { - let input = SyncRequest::Finish; - self.json_request(input).await - } - - async fn abort(&mut self) -> Result<()> { - let input = SyncRequest::Abort; - self.json_request(input).await - } - - async fn full_upload(mut self: Box, col_path: &Path, _can_consume: bool) -> Result<()> { - let file = tokio::fs::File::open(col_path).await?; - let total_bytes = file.metadata().await?.len() as usize; - check_upload_limit(total_bytes, *MAXIMUM_UPLOAD_MEGS_UNCOMPRESSED)?; - let compressed_data: Vec = gzipped_data_from_tokio_file(file).await?; - let compressed_size = compressed_data.len(); - check_upload_limit(compressed_size, *MAXIMUM_UPLOAD_MEGS_COMPRESSED)?; - let progress_fn = self - .full_sync_progress_fn - .take() - .expect("progress func was not set"); - let with_progress = ProgressWrapper { - reader: Cursor::new(compressed_data), - progress_fn, - progress: FullSyncProgress { - transferred_bytes: 0, - total_bytes: compressed_size, - }, - }; - let body = Body::wrap_stream(with_progress); - self.upload_inner(body).await?; - - Ok(()) - } - - /// Download collection into a temporary file, returning it. Caller should - /// persist the file in the correct path after checking it. Progress func - /// must be set first. The caller should pass the collection's folder in as - /// the temp folder if it wishes to atomically .persist() it. - async fn full_download( - mut self: Box, - col_folder: Option<&Path>, - ) -> Result { - let mut temp_file = if let Some(folder) = col_folder { - new_tempfile_in(folder) - } else { - new_tempfile() - }?; - let (size, mut stream) = self.download_inner().await?; - let mut progress = FullSyncProgress { - transferred_bytes: 0, - total_bytes: size, - }; - let mut progress_fn = self - .full_sync_progress_fn - .take() - .expect("progress func was not set"); - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - temp_file.write_all(&chunk)?; - progress.transferred_bytes += chunk.len(); - progress_fn(progress, true); - } - progress_fn(progress, false); - Ok(temp_file) - } -} - -async fn gzipped_data_from_tokio_file(file: tokio::fs::File) -> Result> { - let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); - let mut stream = ReaderStream::new(file); - while let Some(chunk) = stream.next().await { - let chunk = chunk?; - encoder.write_all(&chunk)?; - } - encoder.finish().map_err(Into::into) -} - -fn check_upload_limit(size: usize, limit_mb: usize) -> Result<()> { - let size_mb = size / 1024 / 1024; - if size_mb >= limit_mb { - Err(AnkiError::sync_error( - format!("{}MB > {}MB", size_mb, limit_mb), - SyncErrorKind::UploadTooLarge, - )) - } else { - Ok(()) - } -} - -impl HttpSyncClient { - pub fn new(hkey: Option, host_number: u32) -> HttpSyncClient { - let timeouts = Timeouts::new(); - let client = Client::builder() - .connect_timeout(Duration::from_secs(timeouts.connect_secs)) - .timeout(Duration::from_secs(timeouts.request_secs)) - .io_timeout(Duration::from_secs(timeouts.io_secs)) - .build() - .unwrap(); - let skey = guid(); - let endpoint = sync_endpoint(host_number); - HttpSyncClient { - hkey, - skey, - client, - endpoint, - full_sync_progress_fn: None, - } - } - - pub fn set_full_sync_progress_fn(&mut self, func: Option) { - self.full_sync_progress_fn = func; - } - - async fn json_request(&self, req: SyncRequest) -> Result - where - T: DeserializeOwned, - { - let (method, req_json) = req.into_method_and_data()?; - self.request_bytes(method, &req_json, false) - .await? - .json() - .await - .map_err(Into::into) - } - - async fn request_bytes( - &self, - method: &str, - req: &[u8], - timeout_long: bool, - ) -> Result { - let mut gz = GzEncoder::new(Vec::new(), Compression::fast()); - gz.write_all(req)?; - let part = multipart::Part::bytes(gz.finish()?); - let resp = self.request(method, part, timeout_long).await?; - resp.error_for_status().map_err(Into::into) - } - - async fn request( - &self, - method: &str, - data_part: multipart::Part, - timeout_long: bool, - ) -> Result { - let data_part = data_part.file_name("data"); - - let mut form = multipart::Form::new() - .part("data", data_part) - .text("c", "1"); - if let Some(hkey) = &self.hkey { - form = form.text("k", hkey.clone()).text("s", self.skey.clone()); - } - - let url = format!("{}{}", self.endpoint, method); - let mut req = self.client.post(&url).multipart(form); - - if timeout_long { - req = req.timeout(Duration::from_secs(60 * 60)); - } - - req.send().await?.error_for_status().map_err(Into::into) - } - - pub(crate) async fn login>(&mut self, username: S, password: S) -> Result<()> { - let input = SyncRequest::HostKey(HostKeyRequest { - username: username.into(), - password: password.into(), - }); - let output: HostKeyResponse = self.json_request(input).await?; - self.hkey = Some(output.key); - - Ok(()) - } - - pub(crate) fn hkey(&self) -> &str { - self.hkey.as_ref().unwrap() - } - - async fn download_inner( - &self, - ) -> Result<( - usize, - impl Stream>, - )> { - let resp: Response = self.request_bytes("download", b"{}", true).await?; - let len = resp.content_length().unwrap_or_default(); - Ok((len as usize, resp.bytes_stream())) - } - - async fn upload_inner(&self, body: Body) -> Result<()> { - let data_part = multipart::Part::stream(body); - let resp = self.request("upload", data_part, true).await?; - resp.error_for_status_ref()?; - let text = resp.text().await?; - if text != "OK" { - Err(AnkiError::sync_error(text, SyncErrorKind::Other)) - } else { - Ok(()) - } - } -} - -use std::pin::Pin; - -use futures::{ - ready, - task::{Context, Poll}, -}; -use pin_project::pin_project; -use tokio::io::{AsyncRead, ReadBuf}; - -#[pin_project] -struct ProgressWrapper { - #[pin] - reader: S, - progress_fn: P, - progress: FullSyncProgress, -} - -impl Stream for ProgressWrapper -where - S: AsyncRead, - P: FnMut(FullSyncProgress, bool), -{ - type Item = std::result::Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let mut buf = [MaybeUninit::::uninit(); 8192]; - let mut buf = ReadBuf::uninit(&mut buf); - let this = self.project(); - let res = ready!(this.reader.poll_read(cx, &mut buf)); - match res { - Ok(()) => { - let filled = buf.filled().to_vec(); - Poll::Ready(if filled.is_empty() { - (this.progress_fn)(*this.progress, false); - None - } else { - this.progress.transferred_bytes += filled.len(); - (this.progress_fn)(*this.progress, true); - Some(Ok(Bytes::from(filled))) - }) - } - Err(e) => Poll::Ready(Some(Err(e))), - } - } -} - -fn sync_endpoint(host_number: u32) -> String { - if let Ok(endpoint) = env::var("SYNC_ENDPOINT") { - endpoint - } else { - let suffix = if host_number > 0 { - format!("{}", host_number) - } else { - "".to_string() - }; - format!("https://sync{}.ankiweb.net/sync/", suffix) - } -} - -#[cfg(test)] -mod test { - use tokio::runtime::Runtime; - - use super::*; - use crate::{ - error::{SyncError, SyncErrorKind}, - sync::SanityCheckDueCounts, - }; - - async fn http_client_inner(username: String, password: String) -> Result<()> { - let mut syncer = Box::new(HttpSyncClient::new(None, 0)); - - assert!(matches!( - syncer.login("nosuchuser", "nosuchpass").await, - Err(AnkiError::SyncError { - source: SyncError { - kind: SyncErrorKind::AuthFailed, - .. - } - }) - )); - - assert!(syncer.login(&username, &password).await.is_ok()); - - let _meta = syncer.meta().await?; - - // aborting before a start is a conflict - assert!(matches!( - syncer.abort().await, - Err(AnkiError::SyncError { - source: SyncError { - kind: SyncErrorKind::Conflict, - .. - } - }) - )); - - let _graves = syncer.start(Usn(1), true, None).await?; - - // aborting should now work - syncer.abort().await?; - - // start again, and continue - let _graves = syncer.start(Usn(1), true, None).await?; - - syncer.apply_graves(Graves::default()).await?; - - let _changes = syncer.apply_changes(UnchunkedChanges::default()).await?; - let _chunk = syncer.chunk().await?; - syncer - .apply_chunk(Chunk { - done: true, - ..Default::default() - }) - .await?; - - let _out = syncer - .sanity_check(SanityCheckCounts { - counts: SanityCheckDueCounts { - new: 0, - learn: 0, - review: 0, - }, - cards: 0, - notes: 0, - revlog: 0, - graves: 0, - notetypes: 0, - decks: 0, - deck_config: 0, - }) - .await?; - - // failed sanity check will have cleaned up; can't finish - // syncer.finish().await?; - - syncer.set_full_sync_progress_fn(Some(Box::new(|progress, _throttle| { - println!("progress: {:?}", progress); - }))); - let out_path = syncer.full_download(None).await?; - - let mut syncer = Box::new(HttpSyncClient::new(None, 0)); - syncer.set_full_sync_progress_fn(Some(Box::new(|progress, _throttle| { - println!("progress {:?}", progress); - }))); - syncer.full_upload(out_path.path(), false).await?; - - Ok(()) - } - - #[test] - fn http_client() -> Result<()> { - let user = match env::var("TEST_SYNC_USER") { - Ok(s) => s, - Err(_) => { - return Ok(()); - } - }; - let pass = env::var("TEST_SYNC_PASS").unwrap(); - env_logger::init(); - - let rt = Runtime::new().unwrap(); - rt.block_on(http_client_inner(user, pass)) - } -} diff --git a/rslib/src/sync/http_client/full_sync.rs b/rslib/src/sync/http_client/full_sync.rs new file mode 100644 index 000000000..157012bcc --- /dev/null +++ b/rslib/src/sync/http_client/full_sync.rs @@ -0,0 +1,82 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::{future::Future, time::Duration}; + +use tokio::{select, time::interval}; + +use crate::sync::{ + collection::{ + progress::{FullSyncProgress, FullSyncProgressFn}, + protocol::{EmptyInput, SyncMethod}, + upload::UploadResponse, + }, + error::HttpResult, + http_client::{io_monitor::IoMonitor, HttpSyncClient}, + request::SyncRequest, + response::SyncResponse, +}; + +impl HttpSyncClient { + pub fn set_full_sync_progress_fn(&mut self, func: Option) { + *self.full_sync_progress_fn.lock().unwrap() = func; + } + + fn full_sync_progress_monitor(&self, sending: bool) -> (IoMonitor, impl Future) { + let mut progress = FullSyncProgress { + transferred_bytes: 0, + total_bytes: 0, + }; + let mut progress_fn = self + .full_sync_progress_fn + .lock() + .unwrap() + .take() + .expect("progress func was not set"); + let io_monitor = IoMonitor::new(); + let io_monitor2 = io_monitor.clone(); + let update_progress = async move { + let mut interval = interval(Duration::from_millis(100)); + loop { + interval.tick().await; + let guard = io_monitor2.0.lock().unwrap(); + progress.total_bytes = if sending { + guard.total_bytes_to_send + } else { + guard.total_bytes_to_receive + } as usize; + progress.transferred_bytes = if sending { + guard.bytes_sent + } else { + guard.bytes_received + } as usize; + progress_fn(progress, true) + } + }; + (io_monitor, update_progress) + } + + pub(super) async fn download_inner( + &self, + req: SyncRequest, + ) -> HttpResult>> { + let (io_monitor, progress_fut) = self.full_sync_progress_monitor(false); + let output = self.request_ext(SyncMethod::Download, req, io_monitor); + select! { + _ = progress_fut => unreachable!(), + out = output => out + } + } + + pub(super) async fn upload_inner( + &self, + req: SyncRequest>, + ) -> HttpResult> { + let (io_monitor, progress_fut) = self.full_sync_progress_monitor(true); + let output = self.request_ext(SyncMethod::Upload, req, io_monitor); + select! { + _ = progress_fut => unreachable!(), + out = output => out + } + } +} diff --git a/rslib/src/sync/http_client/io_monitor.rs b/rslib/src/sync/http_client/io_monitor.rs new file mode 100644 index 000000000..6356f4918 --- /dev/null +++ b/rslib/src/sync/http_client/io_monitor.rs @@ -0,0 +1,292 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::{ + io::{Cursor, ErrorKind}, + sync::{Arc, Mutex}, + time::Duration, +}; + +use bytes::Bytes; +use futures::{Stream, StreamExt, TryStreamExt}; +use reqwest::{ + header::{CONTENT_TYPE, LOCATION}, + Body, RequestBuilder, Response, StatusCode, +}; +use tokio::{ + io::AsyncReadExt, + select, + time::{interval, Instant}, +}; +use tokio_util::io::{ReaderStream, StreamReader}; + +use crate::{ + error::Result, + sync::{ + error::{HttpError, HttpResult, HttpSnafu, OrHttpErr}, + request::header_and_stream::{decode_zstd_body_stream, encode_zstd_body_stream}, + response::ORIGINAL_SIZE, + }, +}; + +/// Serves two purposes: +/// - allows us to monitor data sending/receiving and abort if +/// the transfer stalls +/// - allows us to monitor amount of data moving, to provide progress +/// reporting +#[derive(Clone)] +pub struct IoMonitor(pub Arc>); + +impl IoMonitor { + pub fn new() -> Self { + Self(Arc::new(Mutex::new(IoMonitorInner { + last_activity: Instant::now(), + bytes_sent: 0, + total_bytes_to_send: 0, + bytes_received: 0, + total_bytes_to_receive: 0, + }))) + } + + pub fn wrap_stream( + &self, + sending: bool, + total_bytes: u32, + stream: S, + ) -> impl Stream> + Send + Sync + 'static + where + S: Stream> + Send + Sync + 'static, + E: std::error::Error + Send + Sync + 'static, + { + let inner = self.0.clone(); + { + let mut inner = inner.lock().unwrap(); + inner.last_activity = Instant::now(); + if sending { + inner.total_bytes_to_send += total_bytes + } else { + inner.total_bytes_to_receive += total_bytes + } + } + stream.map(move |res| match res { + Ok(bytes) => { + let mut inner = inner.lock().unwrap(); + inner.last_activity = Instant::now(); + if sending { + inner.bytes_sent += bytes.len() as u32; + } else { + inner.bytes_received += bytes.len() as u32; + } + Ok(bytes) + } + err => err.or_http_err(StatusCode::SEE_OTHER, "stream failure"), + }) + } + + /// Returns if no I/O activity observed for `stall_time`. + pub async fn timeout(&self, stall_time: Duration) { + let poll_interval = Duration::from_millis(if cfg!(test) { 10 } else { 1000 }); + let mut interval = interval(poll_interval); + loop { + let now = interval.tick().await; + let last_activity = self.0.lock().unwrap().last_activity; + if now.duration_since(last_activity) > stall_time { + return; + } + } + } + + /// Takes care of encoding provided request data and setting content type to binary, and returns + /// the decompressed response body. + pub async fn zstd_request_with_timeout( + &self, + request: RequestBuilder, + request_body: Vec, + stall_duration: Duration, + ) -> HttpResult> { + let request_total = request_body.len() as u32; + let request_body_stream = encode_zstd_body_stream(self.wrap_stream( + true, + request_total, + ReaderStream::new(Cursor::new(request_body)), + )); + let response_body_stream = async move { + let resp = request + .header(CONTENT_TYPE, "application/octet-stream") + .body(Body::wrap_stream(request_body_stream)) + .send() + .await? + .error_for_status()?; + map_redirect_to_error(&resp)?; + let response_total = resp + .headers() + .get(&ORIGINAL_SIZE) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + .or_bad_request("missing original size")?; + let response_stream = self.wrap_stream( + false, + response_total, + decode_zstd_body_stream(resp.bytes_stream()), + ); + let mut reader = + StreamReader::new(response_stream.map_err(|e| { + std::io::Error::new(ErrorKind::ConnectionAborted, format!("{e}")) + })); + let mut buf = Vec::with_capacity(response_total as usize); + reader + .read_to_end(&mut buf) + .await + .or_http_err(StatusCode::SEE_OTHER, "reading stream")?; + Ok::<_, HttpError>(buf) + }; + select! { + // happy path + data = response_body_stream => Ok(data?), + // timeout + _ = self.timeout(stall_duration) => { + HttpSnafu { + code: StatusCode::REQUEST_TIMEOUT, + context: "timeout monitor", + source: None, + }.fail() + } + } + } +} + +/// Reqwest can't retry a redirected request as the body has been consumed, so we need +/// to bubble it up to the sync driver to retry. +fn map_redirect_to_error(resp: &Response) -> HttpResult<()> { + if resp.status() == StatusCode::PERMANENT_REDIRECT { + let location = resp + .headers() + .get(LOCATION) + .or_bad_request("missing location header")?; + let location = String::from_utf8(location.as_bytes().to_vec()) + .or_bad_request("location was not in utf8")?; + None.or_permanent_redirect(location)?; + } + Ok(()) +} + +#[derive(Debug)] +pub struct IoMonitorInner { + last_activity: Instant, + pub bytes_sent: u32, + pub total_bytes_to_send: u32, + pub bytes_received: u32, + pub total_bytes_to_receive: u32, +} + +impl IoMonitor {} + +#[cfg(test)] +mod test { + use async_stream::stream; + use futures::{pin_mut, StreamExt}; + use tokio::{select, time::sleep}; + use wiremock::{ + matchers::{method, path}, + Mock, MockServer, ResponseTemplate, + }; + + use super::*; + use crate::sync::error::HttpError; + + /// Longer delays on Windows + fn millis(millis: u64) -> Duration { + Duration::from_millis(millis * if cfg!(windows) { 10 } else { 1 }) + } + + #[tokio::test] + async fn can_fail_before_any_bytes() { + let monitor = IoMonitor::new(); + let stream = monitor.wrap_stream( + true, + 0, + stream! { + sleep(millis(2000)).await; + yield Ok::<_, HttpError>(Bytes::from("1")) + }, + ); + pin_mut!(stream); + select! { + _ = stream.next() => panic!("expected failure"), + _ = monitor.timeout(millis(100)) => () + }; + } + + #[tokio::test] + async fn fails_when_data_stops_moving() { + let monitor = IoMonitor::new(); + let stream = monitor.wrap_stream( + true, + 0, + stream! { + for _ in 0..10 { + sleep(millis(10)).await; + yield Ok::<_, HttpError>(Bytes::from("1")) + } + sleep(millis(50)).await; + yield Ok::<_, HttpError>(Bytes::from("1")) + }, + ); + pin_mut!(stream); + for _ in 0..10 { + select! { + _ = stream.next() => (), + _ = monitor.timeout(millis(20)) => panic!("expected success") + }; + } + select! { + _ = stream.next() => panic!("expected timeout"), + _ = monitor.timeout(millis(20)) => () + }; + } + + #[tokio::test] + async fn connect_timeout_works() { + let monitor = IoMonitor::new(); + let req = monitor.zstd_request_with_timeout( + reqwest::Client::new().post("http://0.0.0.1"), + vec![], + millis(50), + ); + req.await.unwrap_err(); + } + + #[tokio::test] + async fn http_success() { + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/")) + .respond_with(ResponseTemplate::new(200).insert_header(ORIGINAL_SIZE.clone(), "0")) + .mount(&mock_server) + .await; + let monitor = IoMonitor::new(); + let req = monitor.zstd_request_with_timeout( + reqwest::Client::new().post(mock_server.uri()), + vec![], + millis(10), + ); + req.await.unwrap(); + } + + #[tokio::test] + async fn delay_before_reply_fails() { + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/")) + .respond_with(ResponseTemplate::new(200).set_delay(millis(50))) + .mount(&mock_server) + .await; + let monitor = IoMonitor::new(); + let req = monitor.zstd_request_with_timeout( + reqwest::Client::new().post(mock_server.uri()), + vec![], + millis(10), + ); + req.await.unwrap_err(); + } +} diff --git a/rslib/src/sync/http_client/mod.rs b/rslib/src/sync/http_client/mod.rs new file mode 100644 index 000000000..c5741a3e0 --- /dev/null +++ b/rslib/src/sync/http_client/mod.rs @@ -0,0 +1,124 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +pub(crate) mod full_sync; +pub(crate) mod io_monitor; +mod protocol; + +use std::{sync::Mutex, time::Duration}; + +use reqwest::{Client, Error, StatusCode, Url}; + +use crate::{ + notes, + sync::{ + collection::{progress::FullSyncProgressFn, protocol::AsSyncEndpoint}, + error::{HttpError, HttpResult, HttpSnafu}, + http_client::io_monitor::IoMonitor, + login::SyncAuth, + request::{ + header_and_stream::{SyncHeader, SYNC_HEADER_NAME}, + SyncRequest, + }, + response::SyncResponse, + }, +}; + +pub struct HttpSyncClient { + /// Set to the empty string for initial login + pub sync_key: String, + session_key: String, + client: Client, + pub endpoint: Url, + full_sync_progress_fn: Mutex>, +} + +impl HttpSyncClient { + pub fn new(auth: SyncAuth) -> HttpSyncClient { + HttpSyncClient { + sync_key: auth.hkey, + session_key: simple_session_id(), + client: Client::new(), + endpoint: auth + .endpoint + .unwrap_or_else(|| Url::try_from("https://sync.ankiweb.net/").unwrap()), + full_sync_progress_fn: Mutex::new(None), + } + } + + #[cfg(test)] + pub fn partial_clone(&self) -> Self { + Self { + sync_key: self.sync_key.clone(), + session_key: self.session_key.clone(), + client: self.client.clone(), + endpoint: self.endpoint.clone(), + full_sync_progress_fn: Mutex::new(None), + } + } + + async fn request( + &self, + method: impl AsSyncEndpoint, + request: SyncRequest, + ) -> HttpResult> { + self.request_ext(method, request, IoMonitor::new()).await + } + + async fn request_ext( + &self, + method: impl AsSyncEndpoint, + request: SyncRequest, + io_monitor: IoMonitor, + ) -> HttpResult> { + let header = SyncHeader { + sync_version: request.sync_version, + sync_key: self.sync_key.clone(), + client_ver: request.client_version, + session_key: self.session_key.clone(), + }; + let data = request.data; + let url = method.as_sync_endpoint(&self.endpoint); + let request = self + .client + .post(url) + .header(&SYNC_HEADER_NAME, serde_json::to_string(&header).unwrap()); + io_monitor + .zstd_request_with_timeout(request, data, Duration::from_secs(30)) + .await + .map(SyncResponse::from_vec) + } + + #[cfg(test)] + pub(crate) fn endpoint(&self) -> &Url { + &self.endpoint + } + + #[cfg(test)] + pub(crate) fn set_skey(&mut self, skey: String) { + self.session_key = skey; + } + + #[cfg(test)] + pub(crate) fn skey(&self) -> &str { + &self.session_key + } +} + +impl From for HttpError { + fn from(err: Error) -> Self { + HttpSnafu { + // we should perhaps make this Optional instead + code: err.status().unwrap_or(StatusCode::SEE_OTHER), + context: "from reqwest", + source: Some(Box::new(err) as _), + } + .build() + } +} + +fn simple_session_id() -> String { + let table = b"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ\ +0123456789"; + notes::to_base_n(rand::random::() as u64, table) +} diff --git a/rslib/src/sync/http_client/protocol.rs b/rslib/src/sync/http_client/protocol.rs new file mode 100644 index 000000000..cbb955437 --- /dev/null +++ b/rslib/src/sync/http_client/protocol.rs @@ -0,0 +1,139 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use async_trait::async_trait; + +use crate::{ + prelude::TimestampMillis, + sync::{ + collection::{ + changes::{ApplyChangesRequest, UnchunkedChanges}, + chunks::{ApplyChunkRequest, Chunk}, + graves::{ApplyGravesRequest, Graves}, + meta::{MetaRequest, SyncMeta}, + protocol::{EmptyInput, SyncMethod, SyncProtocol}, + sanity::{SanityCheckRequest, SanityCheckResponse}, + start::StartRequest, + upload::UploadResponse, + }, + error::HttpResult, + http_client::HttpSyncClient, + login::{HostKeyRequest, HostKeyResponse}, + media::{ + begin::{SyncBeginRequest, SyncBeginResponse}, + changes::{MediaChangesRequest, MediaChangesResponse}, + download::DownloadFilesRequest, + protocol::{JsonResult, MediaSyncMethod, MediaSyncProtocol}, + sanity, upload, + }, + request::SyncRequest, + response::SyncResponse, + }, +}; + +#[async_trait] +impl SyncProtocol for HttpSyncClient { + async fn host_key( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.request(SyncMethod::HostKey, req).await + } + + async fn meta(&self, req: SyncRequest) -> HttpResult> { + self.request(SyncMethod::Meta, req).await + } + + async fn start(&self, req: SyncRequest) -> HttpResult> { + self.request(SyncMethod::Start, req).await + } + + async fn apply_graves( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.request(SyncMethod::ApplyGraves, req).await + } + + async fn apply_changes( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.request(SyncMethod::ApplyChanges, req).await + } + + async fn chunk(&self, req: SyncRequest) -> HttpResult> { + self.request(SyncMethod::Chunk, req).await + } + + async fn apply_chunk( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.request(SyncMethod::ApplyChunk, req).await + } + + async fn sanity_check( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.request(SyncMethod::SanityCheck2, req).await + } + + async fn finish( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.request(SyncMethod::Finish, req).await + } + + async fn abort(&self, req: SyncRequest) -> HttpResult> { + self.request(SyncMethod::Abort, req).await + } + + async fn upload(&self, req: SyncRequest>) -> HttpResult> { + self.upload_inner(req).await + } + + async fn download(&self, req: SyncRequest) -> HttpResult>> { + self.download_inner(req).await + } +} + +#[async_trait] +impl MediaSyncProtocol for HttpSyncClient { + async fn begin( + &self, + req: SyncRequest, + ) -> HttpResult>> { + self.request(MediaSyncMethod::Begin, req).await + } + + async fn media_changes( + &self, + req: SyncRequest, + ) -> HttpResult>> { + self.request(MediaSyncMethod::MediaChanges, req).await + } + + async fn upload_changes( + &self, + req: SyncRequest>, + ) -> HttpResult>> { + self.request(MediaSyncMethod::UploadChanges, req).await + } + + async fn download_files( + &self, + req: SyncRequest, + ) -> HttpResult>> { + self.request(MediaSyncMethod::DownloadFiles, req).await + } + + async fn media_sanity_check( + &self, + req: SyncRequest, + ) -> HttpResult>> { + self.request(MediaSyncMethod::MediaSanity, req).await + } +} diff --git a/rslib/src/sync/http_server/handlers.rs b/rslib/src/sync/http_server/handlers.rs new file mode 100644 index 000000000..248fb78c6 --- /dev/null +++ b/rslib/src/sync/http_server/handlers.rs @@ -0,0 +1,244 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::sync::Arc; + +use async_trait::async_trait; +use media::{sanity::MediaSanityCheckResponse, upload::MediaUploadResponse}; + +use crate::{ + prelude::*, + sync::{ + collection::{ + changes::{server_apply_changes, ApplyChangesRequest, UnchunkedChanges}, + chunks::{server_apply_chunk, server_chunk, ApplyChunkRequest, Chunk}, + download::server_download, + finish::server_finish, + graves::{server_apply_graves, ApplyGravesRequest, Graves}, + meta::{server_meta, MetaRequest, SyncMeta}, + protocol::{EmptyInput, SyncProtocol}, + sanity::{ + server_sanity_check, SanityCheckRequest, SanityCheckResponse, SanityCheckStatus, + }, + start::{server_start, StartRequest}, + upload::{handle_received_upload, UploadResponse}, + }, + error::{HttpResult, OrHttpErr}, + http_server::SimpleServer, + login::{HostKeyRequest, HostKeyResponse}, + media, + media::{ + begin::{SyncBeginRequest, SyncBeginResponse}, + changes::{MediaChangesRequest, MediaChangesResponse}, + download::DownloadFilesRequest, + protocol::{JsonResult, MediaSyncProtocol}, + }, + request::SyncRequest, + response::SyncResponse, + }, +}; + +#[async_trait] +impl SyncProtocol for Arc { + async fn host_key( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.get_host_key(req.json()?) + } + + async fn meta(&self, req: SyncRequest) -> HttpResult> { + self.with_authenticated_user(req, |user, req| { + let req = req.json()?; + user.with_col(|col| server_meta(req, col)) + }) + .await + .and_then(SyncResponse::try_from_obj) + } + + async fn start(&self, req: SyncRequest) -> HttpResult> { + self.with_authenticated_user(req, |user, req| { + let skey = req.skey()?; + let req = req.json()?; + user.start_new_sync(skey)?; + user.with_sync_state(skey, |col, state| server_start(req, col, state)) + .and_then(SyncResponse::try_from_obj) + }) + .await + } + + async fn apply_graves( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.with_authenticated_user(req, |user, req| { + let skey = req.skey()?; + let req = req.json()?; + user.with_sync_state(skey, |col, state| server_apply_graves(req, col, state)) + .and_then(SyncResponse::try_from_obj) + }) + .await + } + + async fn apply_changes( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.with_authenticated_user(req, |user, req| { + let skey = req.skey()?; + let req = req.json()?; + user.with_sync_state(skey, |col, state| server_apply_changes(req, col, state)) + .and_then(SyncResponse::try_from_obj) + }) + .await + } + + async fn chunk(&self, req: SyncRequest) -> HttpResult> { + self.with_authenticated_user(req, |user, req| { + let skey = req.skey()?; + let _ = req.json()?; + user.with_sync_state(skey, server_chunk) + .and_then(SyncResponse::try_from_obj) + }) + .await + } + + async fn apply_chunk( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.with_authenticated_user(req, |user, req| { + let skey = req.skey()?; + let req = req.json()?; + user.with_sync_state(skey, |col, state| server_apply_chunk(req, col, state)) + .and_then(SyncResponse::try_from_obj) + }) + .await + } + + async fn sanity_check( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.with_authenticated_user(req, |user, req| { + let skey = req.skey()?; + let req = req.json()?; + let resp = user.with_sync_state(skey, |col, _state| server_sanity_check(req, col))?; + if resp.status == SanityCheckStatus::Bad { + // don't wait for an abort to roll back + let _ = user.col.take(); + } + SyncResponse::try_from_obj(resp) + }) + .await + } + + async fn finish( + &self, + req: SyncRequest, + ) -> HttpResult> { + self.with_authenticated_user(req, |user, req| { + let _ = req.json()?; + let now = user.with_sync_state(req.skey()?, |col, _state| server_finish(col))?; + user.sync_state = None; + SyncResponse::try_from_obj(now) + }) + .await + } + + async fn abort(&self, req: SyncRequest) -> HttpResult> { + self.with_authenticated_user(req, |user, req| { + let _ = req.json()?; + user.abort_stateful_sync_if_active(); + SyncResponse::try_from_obj(()) + }) + .await + } + + async fn upload(&self, req: SyncRequest>) -> HttpResult> { + self.with_authenticated_user(req, |user, req| { + user.abort_stateful_sync_if_active(); + user.ensure_col_open()?; + handle_received_upload(&mut user.col, req.data).map(SyncResponse::from_upload_response) + }) + .await + } + + async fn download(&self, req: SyncRequest) -> HttpResult>> { + self.with_authenticated_user(req, |user, req| { + let schema_version = req.sync_version.collection_schema(); + let _ = req.json()?; + user.abort_stateful_sync_if_active(); + user.ensure_col_open()?; + server_download(&mut user.col, schema_version).map(SyncResponse::from_vec) + }) + .await + } +} + +#[async_trait] +impl MediaSyncProtocol for Arc { + async fn begin( + &self, + req: SyncRequest, + ) -> HttpResult>> { + let hkey = req.sync_key.clone(); + self.with_authenticated_user(req, |user, req| { + let req = req.json()?; + if req.client_version.is_empty() { + None.or_bad_request("missing client version")?; + } + SyncResponse::try_from_obj(JsonResult::ok(SyncBeginResponse { + usn: user.media.last_usn()?, + host_key: hkey, + })) + }) + .await + } + + async fn media_changes( + &self, + req: SyncRequest, + ) -> HttpResult>> { + self.with_authenticated_user(req, |user, req| { + SyncResponse::try_from_obj(JsonResult::ok( + user.media.media_changes_chunk(req.json()?.last_usn)?, + )) + }) + .await + } + + async fn upload_changes( + &self, + req: SyncRequest>, + ) -> HttpResult>> { + self.with_authenticated_user(req, |user, req| { + SyncResponse::try_from_obj(JsonResult::ok( + user.media.process_uploaded_changes(req.data)?, + )) + }) + .await + } + + async fn download_files( + &self, + req: SyncRequest, + ) -> HttpResult>> { + self.with_authenticated_user(req, |user, req| { + Ok(SyncResponse::from_vec( + user.media.zip_files_for_download(req.json()?.files)?, + )) + }) + .await + } + + async fn media_sanity_check( + &self, + req: SyncRequest, + ) -> HttpResult>> { + self.with_authenticated_user(req, |user, req| { + SyncResponse::try_from_obj(JsonResult::ok(user.media.sanity_check(req.json()?.local)?)) + }) + .await + } +} diff --git a/rslib/src/sync/http_server/logging.rs b/rslib/src/sync/http_server/logging.rs new file mode 100644 index 000000000..9100e10ed --- /dev/null +++ b/rslib/src/sync/http_server/logging.rs @@ -0,0 +1,33 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::time::Duration; + +use axum::{body::Body, http::Request, response::Response, Router}; +use tower_http::trace::TraceLayer; +use tracing::{info_span, Span}; + +pub fn with_logging_layer(router: Router) -> Router { + router.layer( + TraceLayer::new_for_http() + .make_span_with(|request: &Request| { + info_span!( + "request", + uri = request.uri().path(), + ip = tracing::field::Empty, + uid = tracing::field::Empty, + client = tracing::field::Empty, + session = tracing::field::Empty, + ) + }) + .on_request(()) + .on_response(|response: &Response, latency: Duration, _span: &Span| { + tracing::info!( + elap_ms = latency.as_millis() as u32, + httpstatus = response.status().as_u16(), + "finished" + ); + }) + .on_failure(()), + ) +} diff --git a/rslib/src/sync/http_server/media_manager/download.rs b/rslib/src/sync/http_server/media_manager/download.rs new file mode 100644 index 000000000..7e4442bd8 --- /dev/null +++ b/rslib/src/sync/http_server/media_manager/download.rs @@ -0,0 +1,50 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::{fs, io::ErrorKind}; + +use snafu::ResultExt; + +use crate::{ + error::{FileIoSnafu, FileOp}, + sync::{ + error::{HttpResult, OrHttpErr}, + http_server::media_manager::ServerMediaManager, + media::{database::server::entry::MediaEntry, zip::zip_files_for_download}, + }, +}; + +impl ServerMediaManager { + pub fn zip_files_for_download(&mut self, files: Vec) -> HttpResult> { + let entries = self.db.get_entries_for_download(&files)?; + let filenames_with_data = self.gather_file_data(&entries)?; + zip_files_for_download(filenames_with_data).or_internal_err("zip files") + } + + /// Mutable for the missing file case. + fn gather_file_data(&mut self, entries: &[MediaEntry]) -> HttpResult)>> { + let mut out = vec![]; + for entry in entries { + let path = self.media_folder.join(&entry.nfc_filename); + match fs::read(&path) { + Ok(data) => out.push((entry.nfc_filename.clone(), data)), + Err(err) if err.kind() == ErrorKind::NotFound => { + self.db + .forget_missing_file(entry) + .or_internal_err("forget missing")?; + None.or_conflict(format!( + "requested a file that doesn't exist: {}", + entry.nfc_filename + ))?; + } + Err(err) => Err(err) + .context(FileIoSnafu { + path, + op: FileOp::Read, + }) + .or_internal_err("gather file data")?, + } + } + Ok(out) + } +} diff --git a/rslib/src/sync/http_server/media_manager/mod.rs b/rslib/src/sync/http_server/media_manager/mod.rs new file mode 100644 index 000000000..c95e980a3 --- /dev/null +++ b/rslib/src/sync/http_server/media_manager/mod.rs @@ -0,0 +1,58 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +pub mod download; +pub mod upload; + +use std::path::{Path, PathBuf}; + +use crate::{ + io::create_dir_all, + prelude::*, + sync::{ + error::{HttpResult, OrHttpErr}, + media::{ + changes::MediaChange, database::server::ServerMediaDatabase, + sanity::MediaSanityCheckResponse, + }, + }, +}; + +pub(crate) struct ServerMediaManager { + pub media_folder: PathBuf, + pub db: ServerMediaDatabase, +} + +impl ServerMediaManager { + pub(crate) fn new(user_folder: &Path) -> HttpResult { + let media_folder = user_folder.join("media"); + create_dir_all(&media_folder).or_internal_err("media folder create")?; + Ok(Self { + media_folder, + db: ServerMediaDatabase::new(&user_folder.join("media.db")) + .or_internal_err("open media db")?, + }) + } + + pub fn last_usn(&self) -> HttpResult { + self.db.last_usn().or_internal_err("get last usn") + } + + pub fn media_changes_chunk(&self, after_usn: Usn) -> HttpResult> { + self.db + .media_changes_chunk(after_usn) + .or_internal_err("changes chunk") + } + + pub fn sanity_check(&self, client_file_count: u32) -> HttpResult { + let server = self + .db + .nonempty_file_count() + .or_internal_err("get nonempty count")?; + Ok(if server == client_file_count { + MediaSanityCheckResponse::Ok + } else { + MediaSanityCheckResponse::SanityCheckFailed + }) + } +} diff --git a/rslib/src/sync/http_server/media_manager/upload.rs b/rslib/src/sync/http_server/media_manager/upload.rs new file mode 100644 index 000000000..c2e393368 --- /dev/null +++ b/rslib/src/sync/http_server/media_manager/upload.rs @@ -0,0 +1,96 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::{fs, io::ErrorKind, path::Path}; + +use snafu::ResultExt; +use tracing::info; + +use crate::{ + error, + error::{FileIoError, FileIoSnafu, FileOp}, + io::write_file, + sync::{ + error::{HttpResult, OrHttpErr}, + http_server::media_manager::ServerMediaManager, + media::{ + database::server::entry::upload::UploadedChangeResult, upload::MediaUploadResponse, + zip::unzip_and_validate_files, + }, + }, +}; + +impl ServerMediaManager { + pub fn process_uploaded_changes( + &mut self, + zip_data: Vec, + ) -> HttpResult { + let extracted = unzip_and_validate_files(&zip_data).or_bad_request("unzip files")?; + let folder = &self.media_folder; + let mut processed = 0; + let new_usn = self + .db + .with_transaction(|db, meta| { + for change in extracted { + match db.register_uploaded_change(meta, change)? { + UploadedChangeResult::FileAlreadyDeleted { filename } => { + info!(filename, "already deleted"); + } + UploadedChangeResult::FileIdentical { filename, sha1 } => { + info!(filename, sha1 = hex::encode(sha1), "already have"); + } + UploadedChangeResult::Added { + filename, + data, + sha1, + } => { + info!(filename, sha1 = hex::encode(sha1), "added"); + add_or_replace_file(&folder.join(filename), data)?; + } + UploadedChangeResult::Replaced { + filename, + data, + old_sha1, + new_sha1, + } => { + info!( + filename, + old_sha1 = hex::encode(old_sha1), + new_sha1 = hex::encode(new_sha1), + "replaced" + ); + add_or_replace_file(&folder.join(filename), data)?; + } + UploadedChangeResult::Removed { filename, sha1 } => { + info!(filename, sha1 = hex::encode(sha1), "removed"); + remove_file(&folder.join(filename))?; + } + } + processed += 1; + } + Ok(()) + }) + .or_internal_err("handle uploaded change")?; + Ok(MediaUploadResponse { + processed, + current_usn: new_usn, + }) + } +} + +fn add_or_replace_file(path: &Path, data: Vec) -> error::Result<(), FileIoError> { + write_file(path, data).map_err(Into::into) +} + +fn remove_file(path: &Path) -> error::Result<(), FileIoError> { + if let Err(err) = fs::remove_file(path) { + // if transaction was previously aborted, the file may have already been deleted + if err.kind() != ErrorKind::NotFound { + return Err(err).context(FileIoSnafu { + path, + op: FileOp::Remove, + }); + } + } + Ok(()) +} diff --git a/rslib/src/sync/http_server/mod.rs b/rslib/src/sync/http_server/mod.rs new file mode 100644 index 000000000..21f04a9ac --- /dev/null +++ b/rslib/src/sync/http_server/mod.rs @@ -0,0 +1,176 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +mod handlers; +mod logging; +mod media_manager; +mod routes; +mod user; + +use std::{ + collections::HashMap, + env, + future::Future, + net::{SocketAddr, TcpListener}, + path::{Path, PathBuf}, + pin::Pin, + sync::{Arc, Mutex}, +}; + +use axum::{extract::DefaultBodyLimit, Router}; +use snafu::{whatever, OptionExt, ResultExt, Whatever}; +use tracing::Span; + +use crate::{ + error, + io::create_dir_all, + media::files::sha1_of_data, + sync::{ + error::{HttpResult, OrHttpErr}, + http_server::{ + logging::with_logging_layer, + media_manager::ServerMediaManager, + routes::{collection_sync_router, media_sync_router}, + user::User, + }, + login::{HostKeyRequest, HostKeyResponse}, + request::{SyncRequest, MAXIMUM_SYNC_PAYLOAD_BYTES}, + response::SyncResponse, + }, +}; + +pub struct SimpleServer { + state: Mutex, +} + +pub struct SimpleServerInner { + /// hkey->user + users: HashMap, +} + +impl SimpleServerInner { + fn new_from_env(base_folder: &Path) -> error::Result { + let mut idx = 1; + let mut users: HashMap = Default::default(); + loop { + let envvar = format!("SYNC_USER{idx}"); + match std::env::var(&envvar) { + Ok(val) => { + let hkey = derive_hkey(&val); + let (name, _) = val.split_once(':').with_whatever_context(|| { + format!("{envvar} should be in 'username:password' format.") + })?; + let folder = base_folder.join(name); + create_dir_all(&folder).whatever_context("creating SYNC_BASE")?; + let media = + ServerMediaManager::new(&folder).whatever_context("opening media")?; + users.insert( + hkey, + User { + name: name.into(), + col: None, + sync_state: None, + media, + folder, + }, + ); + idx += 1; + } + Err(_) => break, + } + } + if users.is_empty() { + whatever!("No users defined; SYNC_USER1 env var should be set."); + } + Ok(Self { users }) + } +} + +// This is not what AnkiWeb does, but should suffice for this use case. +fn derive_hkey(user_and_pass: &str) -> String { + hex::encode(sha1_of_data(user_and_pass.as_bytes())) +} + +impl SimpleServer { + pub(in crate::sync) async fn with_authenticated_user( + &self, + req: SyncRequest, + op: F, + ) -> HttpResult + where + F: FnOnce(&mut User, SyncRequest) -> HttpResult, + { + let mut state = self.state.lock().unwrap(); + let user = state + .users + .get_mut(&req.sync_key) + .or_forbidden("invalid hkey")?; + Span::current().record("uid", &user.name); + Span::current().record("client", &req.client_version); + Span::current().record("session", &req.session_key); + op(user, req) + } + + pub(in crate::sync) fn get_host_key( + &self, + request: HostKeyRequest, + ) -> HttpResult> { + let state = self.state.lock().unwrap(); + let key = derive_hkey(&format!("{}:{}", request.username, request.password)); + if state.users.contains_key(&key) { + SyncResponse::try_from_obj(HostKeyResponse { key }) + } else { + None.or_forbidden("invalid user/pass in get_host_key") + } + } + + pub fn new(base_folder: &Path) -> error::Result { + let inner = SimpleServerInner::new_from_env(base_folder)?; + Ok(SimpleServer { + state: Mutex::new(inner), + }) + } + + pub fn make_server( + address: Option<&str>, + base_folder: &Path, + ) -> error::Result<(SocketAddr, ServerFuture), Whatever> { + let server = + Arc::new(SimpleServer::new(base_folder).whatever_context("unable to create server")?); + let address = address.unwrap_or("127.0.0.1:0"); + let listener = TcpListener::bind(address) + .with_whatever_context(|_| format!("couldn't bind to {address}"))?; + let addr = listener.local_addr().unwrap(); + let server = with_logging_layer( + Router::new() + .nest("/sync", collection_sync_router()) + .nest("/msync", media_sync_router()) + .with_state(server) + .layer(DefaultBodyLimit::max(*MAXIMUM_SYNC_PAYLOAD_BYTES)), + ); + let future = axum::Server::from_tcp(listener) + .whatever_context("listen failed")? + .serve(server.into_make_service_with_connect_info::()) + .with_graceful_shutdown(async { + let _ = tokio::signal::ctrl_c().await; + }); + tracing::info!(%addr, "listening"); + Ok((addr, Box::pin(future))) + } + + #[snafu::report] + #[tokio::main] + pub async fn run() -> error::Result<(), Whatever> { + let host = env::var("SYNC_HOST").unwrap_or_else(|_| "0.0.0.0".into()); + let port = env::var("SYNC_PORT").unwrap_or_else(|_| "8080".into()); + let base_folder = + PathBuf::from(env::var("SYNC_BASE").whatever_context("missing SYNC_BASE")?); + + let addr = format!("{host}:{port}"); + let (_addr, server_fut) = SimpleServer::make_server(Some(&addr), &base_folder)?; + server_fut.await.whatever_context("await server")?; + Ok(()) + } +} + +pub type ServerFuture = Pin> + Send>>; diff --git a/rslib/src/sync/http_server/routes.rs b/rslib/src/sync/http_server/routes.rs new file mode 100644 index 000000000..95cfca69c --- /dev/null +++ b/rslib/src/sync/http_server/routes.rs @@ -0,0 +1,108 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use axum::{ + extract::{Path, Query, State}, + response::Response, + routing::{get, post}, + Router, +}; + +use crate::sync::{ + collection::protocol::{SyncMethod, SyncProtocol}, + error::{HttpResult, OrHttpErr}, + media::{ + begin::{SyncBeginQuery, SyncBeginRequest}, + protocol::{MediaSyncMethod, MediaSyncProtocol}, + }, + request::{IntoSyncRequest, SyncRequest}, + version::SyncVersion, +}; + +macro_rules! sync_method { + ($server:ident, $req:ident, $method:ident) => {{ + let sync_version = $req.sync_version; + let obj = $server.$method($req.into_output_type()).await?; + obj.make_response(sync_version) + }}; +} + +async fn sync_handler( + Path(method): Path, + State(server): State

, + request: SyncRequest>, +) -> HttpResult { + Ok(match method { + SyncMethod::HostKey => sync_method!(server, request, host_key), + SyncMethod::Meta => sync_method!(server, request, meta), + SyncMethod::Start => sync_method!(server, request, start), + SyncMethod::ApplyGraves => sync_method!(server, request, apply_graves), + SyncMethod::ApplyChanges => sync_method!(server, request, apply_changes), + SyncMethod::Chunk => sync_method!(server, request, chunk), + SyncMethod::ApplyChunk => sync_method!(server, request, apply_chunk), + SyncMethod::SanityCheck2 => sync_method!(server, request, sanity_check), + SyncMethod::Finish => sync_method!(server, request, finish), + SyncMethod::Abort => sync_method!(server, request, abort), + SyncMethod::Upload => sync_method!(server, request, upload), + SyncMethod::Download => sync_method!(server, request, download), + }) +} + +pub fn collection_sync_router() -> Router

{ + Router::new().route("/:method", post(sync_handler::

)) +} + +/// The Rust code used to send a GET with query params, which was inconsistent with the +/// rest of our code - map the request into our standard structure. +async fn media_begin_get( + Query(req): Query, + server: State

, +) -> HttpResult { + let host_key = req.host_key; + let mut req = SyncBeginRequest { + client_version: req.client_version, + } + .try_into_sync_request() + .or_bad_request("convert begin")?; + req.sync_key = host_key; + req.sync_version = SyncVersion::multipart(); + media_begin_post(server, req).await +} + +/// Older clients would send client info in the multipart instead of the inner JSON; +/// Inject it into the json if provided. +async fn media_begin_post( + server: State

, + mut req: SyncRequest, +) -> HttpResult { + if let Some(ver) = &req.media_client_version { + req.data = serde_json::to_vec(&SyncBeginRequest { + client_version: ver.clone(), + }) + .or_internal_err("serialize begin request")?; + } + media_sync_handler(Path(MediaSyncMethod::Begin), server, req.into_output_type()).await +} + +async fn media_sync_handler( + Path(method): Path, + State(server): State

, + request: SyncRequest>, +) -> HttpResult { + Ok(match method { + MediaSyncMethod::Begin => sync_method!(server, request, begin), + MediaSyncMethod::MediaChanges => sync_method!(server, request, media_changes), + MediaSyncMethod::UploadChanges => sync_method!(server, request, upload_changes), + MediaSyncMethod::DownloadFiles => sync_method!(server, request, download_files), + MediaSyncMethod::MediaSanity => sync_method!(server, request, media_sanity_check), + }) +} + +pub fn media_sync_router() -> Router

{ + Router::new() + .route( + "/begin", + get(media_begin_get::

).post(media_begin_post::

), + ) + .route("/:method", post(media_sync_handler::

)) +} diff --git a/rslib/src/sync/http_server/user.rs b/rslib/src/sync/http_server/user.rs new file mode 100644 index 000000000..52e478903 --- /dev/null +++ b/rslib/src/sync/http_server/user.rs @@ -0,0 +1,95 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::path::PathBuf; + +use tracing::info; + +use crate::{ + collection::{Collection, CollectionBuilder}, + error, + sync::{ + collection::start::ServerSyncState, + error::{HttpResult, OrHttpErr}, + http_server::media_manager::ServerMediaManager, + }, +}; + +pub(in crate::sync) struct User { + pub name: String, + pub col: Option, + pub sync_state: Option, + pub media: ServerMediaManager, + pub folder: PathBuf, +} + +impl User { + /// Run op with access to the collection. If a sync is active, it's aborted. + pub(crate) fn with_col(&mut self, op: F) -> HttpResult + where + F: FnOnce(&mut Collection) -> HttpResult, + { + self.abort_stateful_sync_if_active(); + self.ensure_col_open()?; + op(self.col.as_mut().unwrap()) + } + + /// Run op with the existing sync state created by start_new_sync(). If there is no + /// existing state, or the current state's key does not match, abort the request with + /// a conflict. + pub(crate) fn with_sync_state(&mut self, skey: &str, op: F) -> HttpResult + where + F: FnOnce(&mut Collection, &mut ServerSyncState) -> error::Result, + { + match &self.sync_state { + None => None.or_conflict("no active sync")?, + Some(state) => { + if state.skey != skey { + None.or_conflict("active sync with different key")?; + } + } + }; + + self.ensure_col_open()?; + let state = self.sync_state.as_mut().unwrap(); + let col = self.col.as_mut().or_internal_err("open col")?; + // Failures in a sync op are usually caused by referential integrity issues (eg they've sent + // a note without sending its associated notetype). Returning HTTP 400 will inform the client that + // a DB check+full sync is required to fix the issue. + op(col, state) + .map_err(|e| { + self.col = None; + self.sync_state = None; + e + }) + .or_bad_request("op failed in sync_state") + } + + pub(crate) fn abort_stateful_sync_if_active(&mut self) { + if self.sync_state.is_some() { + info!("aborting active sync"); + self.sync_state = None; + self.col = None; + } + } + + pub(crate) fn start_new_sync(&mut self, skey: &str) -> HttpResult<()> { + self.abort_stateful_sync_if_active(); + self.sync_state = Some(ServerSyncState::new(skey)); + Ok(()) + } + + pub(crate) fn ensure_col_open(&mut self) -> HttpResult<()> { + if self.col.is_none() { + self.col = Some(self.open_collection()?); + } + Ok(()) + } + + fn open_collection(&mut self) -> HttpResult { + CollectionBuilder::new(self.folder.join("collection.anki2")) + .set_server(true) + .build() + .or_internal_err("open collection") + } +} diff --git a/rslib/src/sync/login.rs b/rslib/src/sync/login.rs new file mode 100644 index 000000000..7be27c3ba --- /dev/null +++ b/rslib/src/sync/login.rs @@ -0,0 +1,58 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use reqwest::Url; +use serde_derive::{Deserialize, Serialize}; + +use crate::{ + prelude::*, + sync::{ + collection::protocol::SyncProtocol, http_client::HttpSyncClient, request::IntoSyncRequest, + }, +}; + +#[derive(Clone, Default)] +pub struct SyncAuth { + pub hkey: String, + pub endpoint: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct HostKeyRequest { + #[serde(rename = "u")] + pub username: String, + #[serde(rename = "p")] + pub password: String, +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct HostKeyResponse { + pub key: String, +} + +pub async fn sync_login>( + username: S, + password: S, + endpoint: Option, +) -> Result { + let auth = crate::pb::sync::SyncAuth { + endpoint, + ..Default::default() + } + .try_into()?; + let client = HttpSyncClient::new(auth); + let resp = client + .host_key( + HostKeyRequest { + username: username.into(), + password: password.into(), + } + .try_into_sync_request()?, + ) + .await? + .json()?; + Ok(SyncAuth { + hkey: resp.key, + endpoint: None, + }) +} diff --git a/rslib/src/sync/media/begin.rs b/rslib/src/sync/media/begin.rs new file mode 100644 index 000000000..3cf7913f1 --- /dev/null +++ b/rslib/src/sync/media/begin.rs @@ -0,0 +1,34 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use serde_derive::{Deserialize, Serialize}; + +use crate::prelude::*; + +// The old Rust code sent the host key in a query string +#[derive(Debug, Serialize, Deserialize)] +pub struct SyncBeginQuery { + #[serde(rename = "k")] + pub host_key: String, + #[serde(rename = "v")] + pub client_version: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct SyncBeginRequest { + /// Older clients provide this in the multipart wrapper; our router will + /// inject the value in if necessary. The route handler should check that + /// a value has actually been provided. + #[serde(rename = "v", default)] + pub client_version: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct SyncBeginResponse { + pub usn: Usn, + /// The server used to send back a session key used for following requests, + /// but this is no longer required. To avoid breaking older clients, the host + /// key is returned in its place. + #[serde(rename = "sk")] + pub host_key: String, +} diff --git a/rslib/src/sync/media/changes.rs b/rslib/src/sync/media/changes.rs new file mode 100644 index 000000000..53a8336a6 --- /dev/null +++ b/rslib/src/sync/media/changes.rs @@ -0,0 +1,135 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use serde_derive::{Deserialize, Serialize}; +use serde_tuple::Serialize_tuple; +use tracing::debug; + +use crate::{error, prelude::Usn, sync::media::database::client::MediaDatabase}; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MediaChangesRequest { + pub last_usn: Usn, +} + +pub type MediaChangesResponse = Vec; + +#[derive(Debug, Serialize_tuple, Deserialize)] +pub struct MediaChange { + pub fname: String, + pub usn: Usn, + pub sha1: String, +} + +#[derive(Debug, Clone, Copy)] +pub enum LocalState { + NotInDb, + InDbNotPending, + InDbAndPending, +} + +#[derive(PartialEq, Eq, Debug)] +pub enum RequiredChange { + // none also covers the case where we'll later upload + None, + Download, + Delete, + RemovePending, +} + +pub fn determine_required_change( + local_sha1: &str, + remote_sha1: &str, + local_state: LocalState, +) -> RequiredChange { + match (local_sha1, remote_sha1, local_state) { + // both deleted, not in local DB + ("", "", LocalState::NotInDb) => RequiredChange::None, + // both deleted, in local DB + ("", "", _) => RequiredChange::Delete, + // added on server, add even if local deletion pending + ("", _, _) => RequiredChange::Download, + // deleted on server but added locally; upload later + (_, "", LocalState::InDbAndPending) => RequiredChange::None, + // deleted on server and not pending sync + (_, "", _) => RequiredChange::Delete, + // if pending but the same as server, don't need to upload + (lsum, rsum, LocalState::InDbAndPending) if lsum == rsum => RequiredChange::RemovePending, + (lsum, rsum, _) => { + if lsum == rsum { + // not pending and same as server, nothing to do + RequiredChange::None + } else { + // differs from server, favour server + RequiredChange::Download + } + } + } +} + +/// Get a list of server filenames and the actions required on them. +/// Returns filenames in (to_download, to_delete). +pub fn determine_required_changes( + ctx: &MediaDatabase, + records: Vec, +) -> error::Result<(Vec, Vec, Vec)> { + let mut to_download = vec![]; + let mut to_delete = vec![]; + let mut to_remove_pending = vec![]; + + for remote in records { + let (local_sha1, local_state) = match ctx.get_entry(&remote.fname)? { + Some(entry) => ( + match entry.sha1 { + Some(arr) => hex::encode(arr), + None => "".to_string(), + }, + if entry.sync_required { + LocalState::InDbAndPending + } else { + LocalState::InDbNotPending + }, + ), + None => ("".to_string(), LocalState::NotInDb), + }; + + let req_change = determine_required_change(&local_sha1, &remote.sha1, local_state); + debug!( + fname = &remote.fname, + lsha = local_sha1.chars().take(8).collect::(), + rsha = remote.sha1.chars().take(8).collect::(), + state = ?local_state, + action = ?req_change, + "determine action" + ); + match req_change { + RequiredChange::Download => to_download.push(remote.fname), + RequiredChange::Delete => to_delete.push(remote.fname), + RequiredChange::RemovePending => to_remove_pending.push(remote.fname), + RequiredChange::None => (), + }; + } + + Ok((to_download, to_delete, to_remove_pending)) +} + +#[cfg(test)] +mod test { + + #[test] + fn required_change() { + use crate::sync::media::changes::{ + determine_required_change as d, LocalState as L, RequiredChange as R, + }; + assert_eq!(d("", "", L::NotInDb), R::None); + assert_eq!(d("", "", L::InDbNotPending), R::Delete); + assert_eq!(d("", "1", L::InDbAndPending), R::Download); + assert_eq!(d("1", "", L::InDbAndPending), R::None); + assert_eq!(d("1", "", L::InDbNotPending), R::Delete); + assert_eq!(d("1", "1", L::InDbNotPending), R::None); + assert_eq!(d("1", "1", L::InDbAndPending), R::RemovePending); + assert_eq!(d("a", "b", L::InDbAndPending), R::Download); + assert_eq!(d("a", "b", L::InDbNotPending), R::Download); + } +} diff --git a/rslib/src/media/changetracker.rs b/rslib/src/sync/media/database/client/changetracker.rs similarity index 88% rename from rslib/src/media/changetracker.rs rename to rslib/src/sync/media/database/client/changetracker.rs index e020ad2f1..3c2601b12 100644 --- a/rslib/src/media/changetracker.rs +++ b/rslib/src/sync/media/database/client/changetracker.rs @@ -7,14 +7,12 @@ use tracing::debug; use crate::{ io::read_dir_files, - media::{ - database::{MediaDatabaseContext, MediaEntry}, - files::{ - filename_if_normalized, mtime_as_i64, sha1_of_file, MEDIA_SYNC_FILESIZE_LIMIT, - NONSYNCABLE_FILENAME, - }, - }, + media::files::{filename_if_normalized, mtime_as_i64, sha1_of_file, NONSYNCABLE_FILENAME}, prelude::*, + sync::media::{ + database::client::{MediaDatabase, MediaEntry}, + MAX_INDIVIDUAL_MEDIA_FILE_SIZE, + }, }; struct FilesystemEntry { @@ -24,7 +22,7 @@ struct FilesystemEntry { is_new: bool, } -pub(super) struct ChangeTracker<'a, F> +pub(crate) struct ChangeTracker<'a, F> where F: FnMut(usize) -> bool, { @@ -37,7 +35,7 @@ impl ChangeTracker<'_, F> where F: FnMut(usize) -> bool, { - pub(super) fn new(media_folder: &Path, progress: F) -> ChangeTracker<'_, F> { + pub(crate) fn new(media_folder: &Path, progress: F) -> ChangeTracker<'_, F> { ChangeTracker { media_folder, progress_cb: progress, @@ -53,7 +51,7 @@ where } } - pub(super) fn register_changes(&mut self, ctx: &mut MediaDatabaseContext) -> Result<()> { + pub(crate) fn register_changes(&mut self, ctx: &MediaDatabase) -> Result<()> { ctx.transact(|ctx| { // folder mtime unchanged? let dirmod = mtime_as_i64(self.media_folder)?; @@ -125,7 +123,7 @@ where // ignore large files and zero byte files let metadata = dentry.metadata()?; - if metadata.len() > MEDIA_SYNC_FILESIZE_LIMIT as u64 { + if metadata.len() > MAX_INDIVIDUAL_MEDIA_FILE_SIZE as u64 { continue; } if metadata.len() == 0 { @@ -184,7 +182,7 @@ where /// Skip files where the mod time differed, but checksums are the same. fn add_updated_entries( &mut self, - ctx: &mut MediaDatabaseContext, + ctx: &MediaDatabase, entries: Vec, ) -> Result<()> { for fentry in entries { @@ -217,11 +215,7 @@ where } /// Remove deleted files from the media DB. - fn remove_deleted_files( - &mut self, - ctx: &mut MediaDatabaseContext, - removed: Vec, - ) -> Result<()> { + fn remove_deleted_files(&mut self, ctx: &MediaDatabase, removed: Vec) -> Result<()> { for fname in removed { ctx.set_entry(&MediaEntry { fname, @@ -246,12 +240,12 @@ mod test { use tempfile::tempdir; + use super::*; use crate::{ error::Result, io::{create_dir, write_file}, - media::{ - changetracker::ChangeTracker, database::MediaEntry, files::sha1_of_data, MediaManager, - }, + media::{files::sha1_of_data, MediaManager}, + sync::media::database::client::MediaEntry, }; // helper @@ -273,9 +267,7 @@ mod test { let media_db = dir.path().join("media.db"); let mgr = MediaManager::new(&media_dir, media_db)?; - let mut ctx = mgr.dbctx(); - - assert_eq!(ctx.count()?, 0); + assert_eq!(mgr.db.count()?, 0); // add a file and check it's picked up let f1 = media_dir.join("file.jpg"); @@ -283,11 +275,11 @@ mod test { change_mtime(&media_dir); - let progress_cb = |_n| true; + let mut progress_cb = |_n| true; - ChangeTracker::new(&mgr.media_folder, progress_cb).register_changes(&mut ctx)?; + mgr.register_changes(&mut progress_cb)?; - let mut entry = ctx.transact(|ctx| { + let mut entry = mgr.db.transact(|ctx| { assert_eq!(ctx.count()?, 1); assert!(!ctx.get_pending_uploads(1)?.is_empty()); let mut entry = ctx.get_entry("file.jpg")?.unwrap(); @@ -320,9 +312,9 @@ mod test { Ok(entry) })?; - ChangeTracker::new(&mgr.media_folder, progress_cb).register_changes(&mut ctx)?; + ChangeTracker::new(&mgr.media_folder, progress_cb).register_changes(&mgr.db)?; - ctx.transact(|ctx| { + mgr.db.transact(|ctx| { assert_eq!(ctx.count()?, 1); assert!(!ctx.get_pending_uploads(1)?.is_empty()); assert_eq!( @@ -353,12 +345,12 @@ mod test { change_mtime(&media_dir); - ChangeTracker::new(&mgr.media_folder, progress_cb).register_changes(&mut ctx)?; + ChangeTracker::new(&mgr.media_folder, progress_cb).register_changes(&mgr.db)?; - assert_eq!(ctx.count()?, 0); - assert!(!ctx.get_pending_uploads(1)?.is_empty()); + assert_eq!(mgr.db.count()?, 0); + assert!(!mgr.db.get_pending_uploads(1)?.is_empty()); assert_eq!( - ctx.get_entry("file.jpg")?.unwrap(), + mgr.db.get_entry("file.jpg")?.unwrap(), MediaEntry { fname: "file.jpg".into(), sha1: None, diff --git a/rslib/src/media/database.rs b/rslib/src/sync/media/database/client/mod.rs similarity index 53% rename from rslib/src/media/database.rs rename to rslib/src/sync/media/database/client/mod.rs index a323a86c5..2bdcc85f6 100644 --- a/rslib/src/media/database.rs +++ b/rslib/src/sync/media/database/client/mod.rs @@ -3,45 +3,16 @@ use std::{collections::HashMap, path::Path}; -use rusqlite::{params, Connection, OptionalExtension, Row, Statement}; +use rusqlite::{params, Connection, OptionalExtension, Row}; +use tracing::debug; -use crate::prelude::*; +use crate::{ + error, + media::{files::AddedFile, Sha1Hash}, + prelude::{Usn, *}, +}; -fn trace(s: &str) { - println!("sql: {}", s) -} - -pub(super) fn open_or_create>(path: P) -> Result { - let mut db = Connection::open(path)?; - - if std::env::var("TRACESQL").is_ok() { - db.trace(Some(trace)); - } - - db.pragma_update(None, "page_size", 4096)?; - db.pragma_update(None, "legacy_file_format", false)?; - db.pragma_update_and_check(None, "journal_mode", "wal", |_| Ok(()))?; - - initial_db_setup(&mut db)?; - - Ok(db) -} - -fn initial_db_setup(db: &mut Connection) -> Result<()> { - // tables already exist? - if db - .prepare("select null from sqlite_master where type = 'table' and name = 'media'")? - .exists([])? - { - return Ok(()); - } - - db.execute("begin", [])?; - db.execute_batch(include_str!("schema.sql"))?; - db.execute_batch("commit; vacuum; analyze;")?; - - Ok(()) -} +pub mod changetracker; #[derive(Debug, PartialEq, Eq)] pub struct MediaEntry { @@ -57,42 +28,25 @@ pub struct MediaEntry { #[derive(Debug, PartialEq, Eq)] pub struct MediaDatabaseMetadata { pub folder_mtime: i64, - pub last_sync_usn: i32, + pub last_sync_usn: Usn, } -/// Helper to prepare a statement, or return a previously prepared one. -macro_rules! cached_sql { - ( $label:expr, $db:expr, $sql:expr ) => {{ - if $label.is_none() { - $label = Some($db.prepare($sql)?); - } - $label.as_mut().unwrap() - }}; +pub struct MediaDatabase { + db: Connection, } -pub struct MediaDatabaseContext<'a> { - db: &'a Connection, - - get_entry_stmt: Option>, - update_entry_stmt: Option>, - remove_entry_stmt: Option>, -} - -impl MediaDatabaseContext<'_> { - pub(super) fn new(db: &Connection) -> MediaDatabaseContext { - MediaDatabaseContext { - db, - get_entry_stmt: None, - update_entry_stmt: None, - remove_entry_stmt: None, - } +impl MediaDatabase { + pub(crate) fn new(db_path: &Path) -> error::Result { + Ok(MediaDatabase { + db: open_or_create(db_path)?, + }) } /// Execute the provided closure in a transaction, rolling back if /// an error is returned. - pub(super) fn transact(&mut self, func: F) -> Result + pub(crate) fn transact(&self, func: F) -> error::Result where - F: FnOnce(&mut MediaDatabaseContext) -> Result, + F: FnOnce(&MediaDatabase) -> error::Result, { self.begin()?; @@ -111,65 +65,59 @@ impl MediaDatabaseContext<'_> { res } - fn begin(&mut self) -> Result<()> { + fn begin(&self) -> error::Result<()> { self.db.execute_batch("begin immediate").map_err(Into::into) } - fn commit(&mut self) -> Result<()> { + fn commit(&self) -> error::Result<()> { self.db.execute_batch("commit").map_err(Into::into) } - fn rollback(&mut self) -> Result<()> { + fn rollback(&self) -> error::Result<()> { self.db.execute_batch("rollback").map_err(Into::into) } - pub(super) fn get_entry(&mut self, fname: &str) -> Result> { - let stmt = cached_sql!( - self.get_entry_stmt, - self.db, - " -select fname, csum, mtime, dirty from media where fname=?" - ); - - stmt.query_row(params![fname], row_to_entry) + pub(crate) fn get_entry(&self, fname: &str) -> error::Result> { + self.db + .prepare_cached( + " +select fname, csum, mtime, dirty from media where fname=?", + )? + .query_row(params![fname], row_to_entry) .optional() .map_err(Into::into) } - pub(super) fn set_entry(&mut self, entry: &MediaEntry) -> Result<()> { - let stmt = cached_sql!( - self.update_entry_stmt, - self.db, - " -insert or replace into media (fname, csum, mtime, dirty) -values (?, ?, ?, ?)" - ); - + pub(crate) fn set_entry(&self, entry: &MediaEntry) -> error::Result<()> { let sha1_str = entry.sha1.map(hex::encode); - stmt.execute(params![ - entry.fname, - sha1_str, - entry.mtime, - entry.sync_required - ])?; + self.db + .prepare_cached( + " +insert or replace into media (fname, csum, mtime, dirty) +values (?, ?, ?, ?)", + )? + .execute(params![ + entry.fname, + sha1_str, + entry.mtime, + entry.sync_required + ])?; Ok(()) } - pub(super) fn remove_entry(&mut self, fname: &str) -> Result<()> { - let stmt = cached_sql!( - self.remove_entry_stmt, - self.db, - " -delete from media where fname=?" - ); - - stmt.execute(params![fname])?; + pub(crate) fn remove_entry(&self, fname: &str) -> error::Result<()> { + self.db + .prepare_cached( + " +delete from media where fname=?", + )? + .execute(params![fname])?; Ok(()) } - pub(super) fn get_meta(&mut self) -> Result { + pub(crate) fn get_meta(&self) -> error::Result { let mut stmt = self.db.prepare("select dirMod, lastUsn from meta")?; stmt.query_row([], |row| { @@ -181,14 +129,14 @@ delete from media where fname=?" .map_err(Into::into) } - pub(super) fn set_meta(&mut self, meta: &MediaDatabaseMetadata) -> Result<()> { + pub(crate) fn set_meta(&self, meta: &MediaDatabaseMetadata) -> error::Result<()> { let mut stmt = self.db.prepare("update meta set dirMod = ?, lastUsn = ?")?; stmt.execute(params![meta.folder_mtime, meta.last_sync_usn])?; Ok(()) } - pub(super) fn count(&mut self) -> Result { + pub(crate) fn count(&self) -> error::Result { self.db .query_row( "select count(*) from media where csum is not null", @@ -198,11 +146,11 @@ delete from media where fname=?" .map_err(Into::into) } - pub(super) fn get_pending_uploads(&mut self, max_entries: u32) -> Result> { + pub(crate) fn get_pending_uploads(&self, max_entries: u32) -> error::Result> { let mut stmt = self .db .prepare("select fname from media where dirty=1 limit ?")?; - let results: Result> = stmt + let results: error::Result> = stmt .query_and_then(params![max_entries], |row| { let fname = row.get_ref_unwrap(0).as_str()?; Ok(self.get_entry(fname)?.unwrap()) @@ -212,7 +160,7 @@ delete from media where fname=?" results } - pub(super) fn all_mtimes(&mut self) -> Result> { + pub(crate) fn all_mtimes(&self) -> error::Result> { let mut stmt = self .db .prepare("select fname, mtime from media where csum is not null")?; @@ -223,18 +171,81 @@ delete from media where fname=?" } /// Returns all filenames and checksums, where the checksum is not null. - pub(super) fn all_checksums(&mut self) -> Result> { + pub(crate) fn all_registered_checksums(&self) -> error::Result> { self.db .prepare("SELECT fname, csum FROM media WHERE csum IS NOT NULL")? .query_and_then([], row_to_name_and_checksum)? .collect() } - pub(super) fn force_resync(&mut self) -> Result<()> { + pub(crate) fn force_resync(&self) -> error::Result<()> { self.db .execute_batch("delete from media; update meta set lastUsn = 0, dirMod = 0") .map_err(Into::into) } + + pub(crate) fn record_clean(&self, clean: &[String]) -> error::Result<()> { + for fname in clean { + if let Some(mut entry) = self.get_entry(fname)? { + if entry.sync_required { + entry.sync_required = false; + debug!(fname = &entry.fname, "mark clean"); + self.set_entry(&entry)?; + } + } + } + Ok(()) + } + + pub fn record_additions(&self, additions: Vec) -> error::Result<()> { + for file in additions { + if let Some(renamed) = file.renamed_from { + // the file AnkiWeb sent us wasn't normalized, so we need to record + // the old file name as a deletion + debug!("marking non-normalized file as deleted: {}", renamed); + let mut entry = MediaEntry { + fname: renamed, + sha1: None, + mtime: 0, + sync_required: true, + }; + self.set_entry(&entry)?; + // and upload the new filename to ankiweb + debug!("marking renamed file as needing upload: {}", file.fname); + entry = MediaEntry { + fname: file.fname.to_string(), + sha1: Some(file.sha1), + mtime: file.mtime, + sync_required: true, + }; + self.set_entry(&entry)?; + } else { + // a normal addition + let entry = MediaEntry { + fname: file.fname.to_string(), + sha1: Some(file.sha1), + mtime: file.mtime, + sync_required: false, + }; + debug!( + fname = &entry.fname, + sha1 = hex::encode(&entry.sha1.as_ref().unwrap()[0..4]), + "mark added" + ); + self.set_entry(&entry)?; + } + } + + Ok(()) + } + + pub fn record_removals(&self, removals: &[String]) -> error::Result<()> { + for fname in removals { + debug!(fname, "mark removed"); + self.remove_entry(fname)?; + } + Ok(()) + } } fn row_to_entry(row: &Row) -> rusqlite::Result { @@ -258,7 +269,7 @@ fn row_to_entry(row: &Row) -> rusqlite::Result { }) } -fn row_to_name_and_checksum(row: &Row) -> Result<(String, Sha1Hash)> { +fn row_to_name_and_checksum(row: &Row) -> error::Result<(String, Sha1Hash)> { let file_name = row.get(0)?; let sha1_str: String = row.get(1)?; let mut sha1 = [0; 20]; @@ -268,22 +279,60 @@ fn row_to_name_and_checksum(row: &Row) -> Result<(String, Sha1Hash)> { Ok((file_name, sha1)) } +fn trace(s: &str) { + println!("sql: {}", s) +} + +pub(crate) fn open_or_create>(path: P) -> error::Result { + let mut db = Connection::open(path)?; + + if std::env::var("TRACESQL").is_ok() { + db.trace(Some(trace)); + } + + db.pragma_update(None, "page_size", 4096)?; + db.pragma_update(None, "legacy_file_format", false)?; + db.pragma_update_and_check(None, "journal_mode", "wal", |_| Ok(()))?; + + initial_db_setup(&mut db)?; + + Ok(db) +} + +fn initial_db_setup(db: &mut Connection) -> error::Result<()> { + // tables already exist? + if db + .prepare_cached("select null from sqlite_master where type = 'table' and name = 'media'")? + .exists([])? + { + return Ok(()); + } + + db.execute("begin", [])?; + db.execute_batch(include_str!("schema.sql"))?; + db.execute_batch("commit; vacuum; analyze;")?; + + Ok(()) +} + #[cfg(test)] mod test { + use tempfile::TempDir; + use crate::{ error::Result, io::new_tempfile, - media::{database::MediaEntry, files::sha1_of_data, MediaManager}, + media::{files::sha1_of_data, MediaManager}, + sync::media::database::client::MediaEntry, }; #[test] fn database() -> Result<()> { + let folder = TempDir::new()?; let db_file = new_tempfile()?; let db_file_path = db_file.path().to_str().unwrap(); - let mut mgr = MediaManager::new("/dummy", db_file_path)?; - let mut ctx = mgr.dbctx(); - - ctx.transact(|ctx| { + let mut mgr = MediaManager::new(folder.path(), db_file_path)?; + mgr.db.transact(|ctx| { // no entry exists yet assert_eq!(ctx.get_entry("test.mp3")?, None); @@ -308,26 +357,24 @@ mod test { let mut meta = ctx.get_meta()?; assert_eq!(meta.folder_mtime, 0); - assert_eq!(meta.last_sync_usn, 0); + assert_eq!(meta.last_sync_usn.0, 0); meta.folder_mtime = 123; - meta.last_sync_usn = 321; + meta.last_sync_usn.0 = 321; ctx.set_meta(&meta)?; meta = ctx.get_meta()?; assert_eq!(meta.folder_mtime, 123); - assert_eq!(meta.last_sync_usn, 321); + assert_eq!(meta.last_sync_usn.0, 321); Ok(()) })?; // reopen database and ensure data was committed - drop(ctx); drop(mgr); - mgr = MediaManager::new("/dummy", db_file_path)?; - let mut ctx = mgr.dbctx(); - let meta = ctx.get_meta()?; + mgr = MediaManager::new(folder.path(), db_file_path)?; + let meta = mgr.db.get_meta()?; assert_eq!(meta.folder_mtime, 123); Ok(()) diff --git a/rslib/src/media/schema.sql b/rslib/src/sync/media/database/client/schema.sql similarity index 100% rename from rslib/src/media/schema.sql rename to rslib/src/sync/media/database/client/schema.sql diff --git a/rslib/src/sync/media/database/mod.rs b/rslib/src/sync/media/database/mod.rs new file mode 100644 index 000000000..da14a1d67 --- /dev/null +++ b/rslib/src/sync/media/database/mod.rs @@ -0,0 +1,5 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +pub mod client; +pub mod server; diff --git a/rslib/src/sync/media/database/server/entry/changes.rs b/rslib/src/sync/media/database/server/entry/changes.rs new file mode 100644 index 000000000..4edb6ec50 --- /dev/null +++ b/rslib/src/sync/media/database/server/entry/changes.rs @@ -0,0 +1,28 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use rusqlite::Row; + +use crate::{ + prelude::*, + sync::media::{changes::MediaChange, database::server::ServerMediaDatabase}, +}; + +impl MediaChange { + fn from_row(row: &Row) -> Result { + Ok(Self { + fname: row.get(0)?, + usn: row.get(1)?, + sha1: row.get(2)?, + }) + } +} +impl ServerMediaDatabase { + pub fn media_changes_chunk(&self, after_usn: Usn) -> Result> { + Ok(self + .db + .prepare_cached(include_str!("changes.sql"))? + .query_map([after_usn], MediaChange::from_row)? + .collect::>()?) + } +} diff --git a/rslib/src/sync/media/database/server/entry/changes.sql b/rslib/src/sync/media/database/server/entry/changes.sql new file mode 100644 index 000000000..853dbcf08 --- /dev/null +++ b/rslib/src/sync/media/database/server/entry/changes.sql @@ -0,0 +1,11 @@ +SELECT fname, + usn, + ( + CASE + WHEN size > 0 THEN lower(hex(csum)) + ELSE '' + END + ) +FROM media +WHERE usn > ? +LIMIT 1000 \ No newline at end of file diff --git a/rslib/src/sync/media/database/server/entry/download.rs b/rslib/src/sync/media/database/server/entry/download.rs new file mode 100644 index 000000000..6b3b4e91c --- /dev/null +++ b/rslib/src/sync/media/database/server/entry/download.rs @@ -0,0 +1,59 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use rusqlite::params; + +use crate::{ + error, + sync::{ + error::{HttpResult, OrHttpErr}, + media::{ + database::server::{entry::MediaEntry, ServerMediaDatabase}, + MAX_MEDIA_FILES_IN_ZIP, MEDIA_SYNC_TARGET_ZIP_BYTES, + }, + }, +}; + +impl ServerMediaDatabase { + /// Return a list of entries in the same order as the provided files, truncating + /// the list if the configured total bytes is exceeded. + /// + /// If any file entries were missing or deleted, we don't have any way in the current + /// sync protocol to signal that they should be skipped, so we abort with a conflict. + pub fn get_entries_for_download(&self, files: &[String]) -> HttpResult> { + if files.len() > MAX_MEDIA_FILES_IN_ZIP { + None.or_bad_request("too many files requested")?; + } + + let mut entries = vec![]; + let mut accumulated_size = 0; + for filename in files { + let Some(entry) = self.get_nonempty_entry(filename).or_internal_err("fetching entry")? else { + return None.or_conflict(format!("missing/empty file entry: {filename}")); + }; + + accumulated_size += entry.size; + entries.push(entry); + if accumulated_size > MEDIA_SYNC_TARGET_ZIP_BYTES as u64 { + break; + } + } + Ok(entries) + } + + /// Delete provided file from media DB, leaving no record of deletion. It was probably + /// missing due to an interrupted deletion, but removing the entry errs on the side of + /// caution, ensuring the deletion doesn't propagate to other clients. + pub fn forget_missing_file(&mut self, entry: &MediaEntry) -> error::Result<()> { + assert!(entry.size > 0); + self.with_transaction(|db, meta| { + meta.total_bytes = meta.total_bytes.saturating_sub(entry.size); + meta.total_nonempty_files = meta.total_nonempty_files.saturating_sub(1); + db.db + .prepare_cached("delete from media where fname = ?")? + .execute(params![&entry.nfc_filename,])?; + Ok(()) + })?; + Ok(()) + } +} diff --git a/rslib/src/sync/media/database/server/entry/get_entry.sql b/rslib/src/sync/media/database/server/entry/get_entry.sql new file mode 100644 index 000000000..824632983 --- /dev/null +++ b/rslib/src/sync/media/database/server/entry/get_entry.sql @@ -0,0 +1,7 @@ +SELECT fname, + csum, + size, + usn, + mtime +FROM media +WHERE fname = ?; \ No newline at end of file diff --git a/rslib/src/sync/media/database/server/entry/mod.rs b/rslib/src/sync/media/database/server/entry/mod.rs new file mode 100644 index 000000000..d2b882f11 --- /dev/null +++ b/rslib/src/sync/media/database/server/entry/mod.rs @@ -0,0 +1,119 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::mem; + +use rusqlite::{params, OptionalExtension, Row}; + +use crate::{ + error, + prelude::{TimestampSecs, Usn}, + sync::media::database::server::{meta::StoreMetadata, ServerMediaDatabase}, +}; + +pub mod changes; +mod download; +pub mod upload; + +impl ServerMediaDatabase { + /// Does not return a deletion entry. + pub fn get_nonempty_entry(&self, nfc_filename: &str) -> error::Result> { + self.get_entry(nfc_filename) + .map(|e| e.filter(|e| !e.is_deleted())) + } + + pub fn get_entry(&self, nfc_filename: &str) -> error::Result> { + self.db + .prepare_cached(include_str!("get_entry.sql"))? + .query_row([nfc_filename], MediaEntry::from_row) + .optional() + .map_err(Into::into) + } + + /// Saves entry to the DB, overwriting any existing entry. Does no validation on its own; + /// caller is responsible for mutating meta (which will update mtime as well). + pub fn set_entry(&mut self, entry: &mut MediaEntry) -> error::Result<()> { + self.db + .prepare_cached(include_str!("set_entry.sql"))? + .execute(params![ + &entry.nfc_filename, + &entry.sha1, + &entry.size, + &entry.usn, + &entry.mtime.0, + ])?; + Ok(()) + } + + fn add_entry( + &mut self, + meta: &mut StoreMetadata, + filename: String, + total_bytes: usize, + sha1: Vec, + ) -> error::Result { + assert!(total_bytes > 0); + let mut new_entry = MediaEntry { + nfc_filename: filename, + sha1, + size: total_bytes as u64, + // set by following call + usn: Default::default(), + mtime: Default::default(), + }; + meta.add_entry(&mut new_entry); + self.set_entry(&mut new_entry)?; + Ok(new_entry) + } + + /// Returns the old sha1 + fn replace_entry( + &mut self, + meta: &mut StoreMetadata, + existing_nonempty: &mut MediaEntry, + total_bytes: usize, + sha1: Vec, + ) -> error::Result> { + assert!(total_bytes > 0); + meta.replace_entry(existing_nonempty, total_bytes as u64); + let old_sha1 = mem::replace(&mut existing_nonempty.sha1, sha1); + self.set_entry(existing_nonempty)?; + Ok(old_sha1) + } + + fn remove_entry( + &mut self, + meta: &mut StoreMetadata, + existing_nonempty: &mut MediaEntry, + ) -> error::Result<()> { + meta.remove_entry(existing_nonempty); + self.set_entry(existing_nonempty)?; + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MediaEntry { + pub nfc_filename: String, + pub sha1: Vec, + /// Set to 0 to indicate deletion. + pub size: u64, + pub usn: Usn, + pub mtime: TimestampSecs, +} + +impl MediaEntry { + pub fn from_row(row: &Row) -> std::result::Result { + Ok(Self { + nfc_filename: row.get(0)?, + sha1: row.get(1)?, + size: row.get(2)?, + usn: row.get(3)?, + mtime: TimestampSecs(row.get(4)?), + }) + } + + fn is_deleted(&self) -> bool { + self.size == 0 + } +} diff --git a/rslib/src/sync/media/database/server/entry/set_entry.sql b/rslib/src/sync/media/database/server/entry/set_entry.sql new file mode 100644 index 000000000..76070f2a3 --- /dev/null +++ b/rslib/src/sync/media/database/server/entry/set_entry.sql @@ -0,0 +1,3 @@ +INSERT + OR REPLACE INTO media (fname, csum, size, usn, mtime) +VALUES (?, ?, ?, ?, ?); \ No newline at end of file diff --git a/rslib/src/sync/media/database/server/entry/upload.rs b/rslib/src/sync/media/database/server/entry/upload.rs new file mode 100644 index 000000000..74d3a0221 --- /dev/null +++ b/rslib/src/sync/media/database/server/entry/upload.rs @@ -0,0 +1,102 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use crate::{ + error, + sync::media::{ + database::server::{meta::StoreMetadata, ServerMediaDatabase}, + zip::{UploadedChange, UploadedChangeKind}, + }, +}; + +pub enum UploadedChangeResult { + FileAlreadyDeleted { + filename: String, + }, + FileIdentical { + filename: String, + sha1: Vec, + }, + Added { + filename: String, + data: Vec, + sha1: Vec, + }, + Removed { + filename: String, + sha1: Vec, + }, + Replaced { + filename: String, + data: Vec, + old_sha1: Vec, + new_sha1: Vec, + }, +} + +impl ServerMediaDatabase { + /// Add/modify/remove a single file. + pub fn register_uploaded_change( + &mut self, + meta: &mut StoreMetadata, + update: UploadedChange, + ) -> error::Result { + let existing_file = self.get_nonempty_entry(&update.nfc_filename)?; + match (existing_file, update.kind) { + // deletion + (None, UploadedChangeKind::Delete) => Ok(UploadedChangeResult::FileAlreadyDeleted { + filename: update.nfc_filename, + }), + (Some(mut existing_nonempty), UploadedChangeKind::Delete) => { + self.remove_entry(meta, &mut existing_nonempty)?; + Ok(UploadedChangeResult::Removed { + filename: existing_nonempty.nfc_filename, + sha1: existing_nonempty.sha1, + }) + } + // addition + ( + None, + UploadedChangeKind::AddOrReplace { + nonempty_data, + sha1, + }, + ) => { + let entry = self.add_entry(meta, update.nfc_filename, nonempty_data.len(), sha1)?; + Ok(UploadedChangeResult::Added { + filename: entry.nfc_filename, + data: nonempty_data, + sha1: entry.sha1, + }) + } + // replacement + ( + Some(mut existing_nonempty), + UploadedChangeKind::AddOrReplace { + nonempty_data, + sha1, + }, + ) => { + if existing_nonempty.sha1 == sha1 { + Ok(UploadedChangeResult::FileIdentical { + filename: existing_nonempty.nfc_filename, + sha1, + }) + } else { + let old_sha1 = self.replace_entry( + meta, + &mut existing_nonempty, + nonempty_data.len(), + sha1, + )?; + Ok(UploadedChangeResult::Replaced { + filename: existing_nonempty.nfc_filename, + data: nonempty_data, + old_sha1, + new_sha1: existing_nonempty.sha1, + }) + } + } + } + } +} diff --git a/rslib/src/sync/media/database/server/meta/get_meta.sql b/rslib/src/sync/media/database/server/meta/get_meta.sql new file mode 100644 index 000000000..9e0437d3e --- /dev/null +++ b/rslib/src/sync/media/database/server/meta/get_meta.sql @@ -0,0 +1,4 @@ +SELECT last_usn, + total_bytes, + total_nonempty_files +FROM meta; \ No newline at end of file diff --git a/rslib/src/sync/media/database/server/meta/mod.rs b/rslib/src/sync/media/database/server/meta/mod.rs new file mode 100644 index 000000000..2fc4f5011 --- /dev/null +++ b/rslib/src/sync/media/database/server/meta/mod.rs @@ -0,0 +1,113 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use rusqlite::{params, Row}; + +use crate::{ + error, + prelude::{TimestampSecs, Usn}, + sync::media::database::server::{entry::MediaEntry, ServerMediaDatabase}, +}; + +#[derive(Debug, PartialEq, Eq)] +pub struct StoreMetadata { + pub last_usn: Usn, + pub total_bytes: u64, + pub total_nonempty_files: u32, +} + +impl StoreMetadata { + pub(crate) fn add_entry(&mut self, entry: &mut MediaEntry) { + assert!(entry.size > 0); + self.total_bytes += entry.size; + self.total_nonempty_files += 1; + entry.usn = self.next_usn(); + entry.mtime = TimestampSecs::now(); + } + + /// Expects entry to have its old size; the new size will be set. + pub(crate) fn replace_entry(&mut self, entry: &mut MediaEntry, new_size: u64) { + assert!(entry.size > 0); + assert!(new_size > 0); + self.total_bytes = self.total_bytes.saturating_sub(entry.size) + new_size; + entry.size = new_size; + entry.usn = self.next_usn(); + entry.mtime = TimestampSecs::now(); + } + + pub(crate) fn remove_entry(&mut self, entry: &mut MediaEntry) { + assert!(entry.size > 0); + self.total_bytes = self.total_bytes.saturating_sub(entry.size); + self.total_nonempty_files = self.total_nonempty_files.saturating_sub(1); + entry.size = 0; + entry.usn = self.next_usn(); + entry.mtime = TimestampSecs::now(); + } +} + +impl StoreMetadata { + fn from_row(row: &Row) -> error::Result { + Ok(Self { + last_usn: row.get(0)?, + total_bytes: row.get(1)?, + total_nonempty_files: row.get(2)?, + }) + } + + fn next_usn(&mut self) -> Usn { + self.last_usn.0 += 1; + self.last_usn + } +} + +impl ServerMediaDatabase { + /// Perform an exclusive transaction. Will implicitly commit if no error returned, + /// after flushing the updated metadata. Returns the latest usn. + pub fn with_transaction(&mut self, op: F) -> error::Result + where + F: FnOnce(&mut Self, &mut StoreMetadata) -> error::Result<()>, + { + self.db.execute("begin exclusive", [])?; + let mut meta = self.get_meta()?; + op(self, &mut meta) + .and_then(|_| { + self.set_meta(&meta)?; + self.db.execute("commit", [])?; + Ok(meta.last_usn) + }) + .map_err(|e| { + let _ = self.db.execute("rollback", []); + e + }) + } + + pub fn last_usn(&self) -> error::Result { + Ok(self.get_meta()?.last_usn) + } + + fn get_meta(&self) -> error::Result { + self.db + .prepare_cached(include_str!("get_meta.sql"))? + .query_row([], StoreMetadata::from_row) + .map_err(Into::into) + } + + fn set_meta(&mut self, meta: &StoreMetadata) -> error::Result<()> { + self.db + .prepare_cached(include_str!("set_meta.sql"))? + .execute(params![ + meta.last_usn, + meta.total_bytes, + meta.total_nonempty_files + ])?; + Ok(()) + } + + pub fn nonempty_file_count(&self) -> error::Result { + Ok(self.get_meta()?.total_nonempty_files) + } + + pub fn total_bytes(&self) -> error::Result { + Ok(self.get_meta()?.total_bytes) + } +} diff --git a/rslib/src/sync/media/database/server/meta/set_meta.sql b/rslib/src/sync/media/database/server/meta/set_meta.sql new file mode 100644 index 000000000..80e6752a1 --- /dev/null +++ b/rslib/src/sync/media/database/server/meta/set_meta.sql @@ -0,0 +1,4 @@ +UPDATE meta +SET last_usn = ?, + total_bytes = ?, + total_nonempty_files = ?; \ No newline at end of file diff --git a/rslib/src/sync/media/database/server/mod.rs b/rslib/src/sync/media/database/server/mod.rs new file mode 100644 index 000000000..0c2ef648a --- /dev/null +++ b/rslib/src/sync/media/database/server/mod.rs @@ -0,0 +1,40 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +pub mod entry; +pub mod meta; + +use std::path::Path; + +use rusqlite::Connection; + +use crate::prelude::*; + +pub struct ServerMediaDatabase { + pub db: Connection, +} + +impl ServerMediaDatabase { + pub fn new(path: &Path) -> Result { + Ok(Self { + db: open_or_create_db(path)?, + }) + } +} + +fn open_or_create_db(path: &Path) -> Result { + let db = Connection::open(path)?; + db.busy_timeout(std::time::Duration::from_secs(0))?; + db.pragma_update(None, "locking_mode", "exclusive")?; + db.pragma_update(None, "journal_mode", "wal")?; + let ver: u32 = db.query_row("select user_version from pragma_user_version", [], |r| { + r.get(0) + })?; + if ver < 3 { + db.execute_batch(include_str!("schema_v3.sql"))?; + } + if ver < 4 { + db.execute_batch(include_str!("schema_v4.sql"))?; + } + Ok(db) +} diff --git a/rslib/src/sync/media/database/server/schema_v3.sql b/rslib/src/sync/media/database/server/schema_v3.sql new file mode 100644 index 000000000..3517562fd --- /dev/null +++ b/rslib/src/sync/media/database/server/schema_v3.sql @@ -0,0 +1,14 @@ +BEGIN exclusive; +CREATE TABLE IF NOT EXISTS media ( + fname text NOT NULL PRIMARY KEY, + csum blob, + sz int NOT NULL, + usn int NOT NULL, + deleted int NOT NULL +); +CREATE INDEX IF NOT EXISTS ix_usn ON media (usn); +CREATE TABLE IF NOT EXISTS meta (usn int NOT NULL, sz int NOT NULL); +INSERT INTO meta (usn, sz) +VALUES (0, 0); +pragma user_version = 3; +COMMIT; \ No newline at end of file diff --git a/rslib/src/sync/media/database/server/schema_v4.sql b/rslib/src/sync/media/database/server/schema_v4.sql new file mode 100644 index 000000000..e04d25343 --- /dev/null +++ b/rslib/src/sync/media/database/server/schema_v4.sql @@ -0,0 +1,46 @@ +-- csum is no longer nulled on deletion +-- sz renamed to size +-- deleted renamed to mtime +BEGIN exclusive; +ALTER TABLE media + RENAME TO media_tmp; +DROP INDEX ix_usn; +CREATE TABLE media ( + fname text NOT NULL PRIMARY KEY, + csum blob NOT NULL, + -- if zero, file has been deleted + size int NOT NULL, + usn int NOT NULL, + mtime int NOT NULL +); +INSERT INTO media (fname, csum, size, usn, mtime) +SELECT fname, + csum, + sz, + usn, + deleted +FROM media_tmp +WHERE csum IS NOT NULL; +DROP TABLE media_tmp; +CREATE INDEX ix_usn ON media (usn); +DROP TABLE meta; +-- columns renamed; file count added +CREATE TABLE meta ( + last_usn int NOT NULL, + total_bytes int NOT NULL, + total_nonempty_files int NOT NULL +); +INSERT INTO meta (last_usn, total_bytes, total_nonempty_files) +SELECT coalesce(max(usn), 0), + coalesce(sum(size), 0), + 0 +FROM media; +UPDATE meta +SET total_nonempty_files = ( + SELECT COUNT(*) + FROM media + WHERE size > 0 + ); +pragma user_version = 4; +COMMIT; +vacuum; \ No newline at end of file diff --git a/rslib/src/sync/media/download.rs b/rslib/src/sync/media/download.rs new file mode 100644 index 000000000..20d57c654 --- /dev/null +++ b/rslib/src/sync/media/download.rs @@ -0,0 +1,50 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::{collections::HashMap, io, io::Read, path::Path}; + +use serde::{Deserialize, Serialize}; + +use crate::{ + error, + error::{AnkiError, SyncErrorKind}, + media::files::{add_file_from_ankiweb, AddedFile}, +}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct DownloadFilesRequest { + pub files: Vec, +} + +pub(crate) fn extract_into_media_folder( + media_folder: &Path, + zip: Vec, +) -> error::Result> { + let reader = io::Cursor::new(zip); + let mut zip = zip::ZipArchive::new(reader)?; + + let meta_file = zip.by_name("_meta")?; + let fmap: HashMap = serde_json::from_reader(meta_file)?; + let mut output = Vec::with_capacity(fmap.len()); + + for i in 0..zip.len() { + let mut file = zip.by_index(i)?; + let name = file.name(); + if name == "_meta" { + continue; + } + + let real_name = fmap + .get(name) + .ok_or_else(|| AnkiError::sync_error("malformed zip", SyncErrorKind::Other))?; + + let mut data = Vec::with_capacity(file.size() as usize); + file.read_to_end(&mut data)?; + + let added = add_file_from_ankiweb(media_folder, real_name, &data)?; + + output.push(added); + } + + Ok(output) +} diff --git a/rslib/src/sync/media/mod.rs b/rslib/src/sync/media/mod.rs new file mode 100644 index 000000000..69616aa79 --- /dev/null +++ b/rslib/src/sync/media/mod.rs @@ -0,0 +1,32 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +pub mod begin; +pub mod changes; +pub mod database; +pub mod download; +pub mod progress; +pub mod protocol; +pub mod sanity; +pub mod syncer; +mod tests; +pub mod upload; +pub mod zip; + +/// The maximum length we allow a filename to be. When combined +/// with the rest of the path, the full path needs to be under ~240 chars +/// on some platforms, and some filesystems like eCryptFS will increase +/// the length of the filename. +pub static MAX_MEDIA_FILENAME_LENGTH: usize = 120; + +// We can't enforce the 120 limit until all clients have shifted over to the +// Rust codebase. +pub const MAX_MEDIA_FILENAME_LENGTH_SERVER: usize = 255; + +/// Media syncing does not support files over 100MiB. +pub static MAX_INDIVIDUAL_MEDIA_FILE_SIZE: usize = 100 * 1024 * 1024; + +pub static MAX_MEDIA_FILES_IN_ZIP: usize = 25; + +/// If reached, no further files are placed into the zip. +pub static MEDIA_SYNC_TARGET_ZIP_BYTES: usize = (2.5 * 1024.0 * 1024.0) as usize; diff --git a/rslib/src/sync/media/progress.rs b/rslib/src/sync/media/progress.rs new file mode 100644 index 000000000..ea295c0eb --- /dev/null +++ b/rslib/src/sync/media/progress.rs @@ -0,0 +1,11 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +#[derive(Debug, Default, Clone, Copy)] +pub struct MediaSyncProgress { + pub checked: usize, + pub downloaded_files: usize, + pub downloaded_deletions: usize, + pub uploaded_files: usize, + pub uploaded_deletions: usize, +} diff --git a/rslib/src/sync/media/protocol.rs b/rslib/src/sync/media/protocol.rs new file mode 100644 index 000000000..70806d260 --- /dev/null +++ b/rslib/src/sync/media/protocol.rs @@ -0,0 +1,103 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use async_trait::async_trait; +use reqwest::Url; +use serde::de::DeserializeOwned; +use serde_derive::{Deserialize, Serialize}; +use strum::IntoStaticStr; + +use crate::{ + error, + error::AnkiError, + sync::{ + collection::protocol::AsSyncEndpoint, + error::HttpResult, + media::{ + begin::{SyncBeginRequest, SyncBeginResponse}, + changes::{MediaChangesRequest, MediaChangesResponse}, + download::DownloadFilesRequest, + sanity::{MediaSanityCheckResponse, SanityCheckRequest}, + upload::MediaUploadResponse, + }, + request::SyncRequest, + response::SyncResponse, + }, +}; + +#[derive(IntoStaticStr, Deserialize, PartialEq, Eq, Debug)] +#[serde(rename_all = "camelCase")] +#[strum(serialize_all = "camelCase")] +pub enum MediaSyncMethod { + Begin, + MediaChanges, + UploadChanges, + DownloadFiles, + MediaSanity, +} + +impl AsSyncEndpoint for MediaSyncMethod { + fn as_sync_endpoint(&self, base: &Url) -> Url { + base.join("msync/").unwrap().join(self.into()).unwrap() + } +} + +#[async_trait] +pub trait MediaSyncProtocol: Send + Sync + 'static { + async fn begin( + &self, + req: SyncRequest, + ) -> HttpResult>>; + async fn media_changes( + &self, + req: SyncRequest, + ) -> HttpResult>>; + async fn upload_changes( + &self, + req: SyncRequest>, + ) -> HttpResult>>; + async fn download_files( + &self, + req: SyncRequest, + ) -> HttpResult>>; + async fn media_sanity_check( + &self, + req: SyncRequest, + ) -> HttpResult>>; +} + +/// Media endpoints wrap their returns in a JSON result, and legacy +/// clients expect it to always have an err field, even if it's empty. +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum JsonResult { + Ok { + data: T, + #[serde(default)] + err: String, + }, + Err { + err: String, + }, +} + +impl JsonResult { + pub fn ok(inner: T) -> Self { + Self::Ok { + data: inner, + err: String::new(), + } + } +} + +impl SyncResponse> +where + T: DeserializeOwned, +{ + pub fn json_result(&self) -> error::Result { + match serde_json::from_slice(&self.data)? { + JsonResult::Ok { data, .. } => Ok(data), + JsonResult::Err { err } => Err(AnkiError::server_message(err)), + } + } +} diff --git a/rslib/src/sync/media/sanity.rs b/rslib/src/sync/media/sanity.rs new file mode 100644 index 000000000..dda940750 --- /dev/null +++ b/rslib/src/sync/media/sanity.rs @@ -0,0 +1,17 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use serde_derive::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub struct SanityCheckRequest { + pub local: u32, +} + +#[derive(Serialize, Deserialize, Eq, PartialEq, Debug)] +pub enum MediaSanityCheckResponse { + #[serde(rename = "OK")] + Ok, + #[serde(rename = "mediaSanity")] + SanityCheckFailed, +} diff --git a/rslib/src/sync/media/syncer.rs b/rslib/src/sync/media/syncer.rs new file mode 100644 index 000000000..26d860688 --- /dev/null +++ b/rslib/src/sync/media/syncer.rs @@ -0,0 +1,291 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use tracing::debug; +use version::sync_client_version; + +use crate::{ + error::{AnkiError, Result, SyncErrorKind}, + media::{files::mtime_as_i64, MediaManager}, + prelude::*, + sync::{ + http_client::HttpSyncClient, + media::{ + begin::{SyncBeginRequest, SyncBeginResponse}, + changes, + changes::MediaChangesRequest, + database::client::{changetracker::ChangeTracker, MediaDatabaseMetadata, MediaEntry}, + download, + download::DownloadFilesRequest, + progress::MediaSyncProgress, + protocol::MediaSyncProtocol, + sanity::{MediaSanityCheckResponse, SanityCheckRequest}, + upload::gather_zip_data_for_upload, + zip::zip_files_for_upload, + MAX_MEDIA_FILES_IN_ZIP, + }, + request::IntoSyncRequest, + }, + version, +}; + +pub struct MediaSyncer

+where + P: FnMut(MediaSyncProgress) -> bool, +{ + mgr: MediaManager, + client: HttpSyncClient, + progress_cb: P, + progress: MediaSyncProgress, +} + +impl

MediaSyncer

+where + P: FnMut(MediaSyncProgress) -> bool, +{ + pub fn new( + mgr: MediaManager, + progress_cb: P, + client: HttpSyncClient, + ) -> Result> { + Ok(MediaSyncer { + mgr, + client, + progress_cb, + progress: Default::default(), + }) + } + + fn fire_progress_cb(&mut self) -> Result<()> { + if (self.progress_cb)(self.progress) { + Ok(()) + } else { + Err(AnkiError::Interrupted) + } + } + + pub async fn sync(&mut self) -> Result<()> { + self.sync_inner().await.map_err(|e| { + debug!("sync error: {:?}", e); + e + }) + } + + #[allow(clippy::useless_let_if_seq)] + async fn sync_inner(&mut self) -> Result<()> { + self.register_changes()?; + + let meta = self.mgr.db.get_meta()?; + let client_usn = meta.last_sync_usn; + let server_usn = self.begin_sync().await?; + + let mut actions_performed = false; + + // need to fetch changes from server? + if client_usn != server_usn { + debug!("differs from local usn {}, fetching changes", client_usn); + self.fetch_changes(meta).await?; + actions_performed = true; + } + + // need to send changes to server? + let changes_pending = !self.mgr.db.get_pending_uploads(1)?.is_empty(); + if changes_pending { + self.send_changes().await?; + actions_performed = true; + } + + if actions_performed { + self.finalize_sync().await?; + } + + self.fire_progress_cb()?; + + debug!("media sync complete"); + + Ok(()) + } + + async fn begin_sync(&mut self) -> Result { + debug!("begin media sync"); + let SyncBeginResponse { + host_key: _, + usn: server_usn, + } = self + .client + .begin( + SyncBeginRequest { + client_version: sync_client_version().into(), + } + .try_into_sync_request()?, + ) + .await? + .json_result()?; + + debug!("server usn was {}", server_usn); + Ok(server_usn) + } + + /// Make sure media DB is up to date. + fn register_changes(&mut self) -> Result<()> { + // make borrow checker happy + let progress = &mut self.progress; + let progress_cb = &mut self.progress_cb; + + let progress = |checked| { + progress.checked = checked; + (progress_cb)(*progress) + }; + + ChangeTracker::new(self.mgr.media_folder.as_path(), progress).register_changes(&self.mgr.db) + } + + async fn fetch_changes(&mut self, mut meta: MediaDatabaseMetadata) -> Result<()> { + let mut last_usn = meta.last_sync_usn; + loop { + debug!(start_usn = ?last_usn, "fetching record batch"); + + let batch = self + .client + .media_changes(MediaChangesRequest { last_usn }.try_into_sync_request()?) + .await? + .json_result()?; + if batch.is_empty() { + debug!("empty batch, done"); + break; + } + last_usn = batch.last().unwrap().usn; + + self.progress.checked += batch.len(); + self.fire_progress_cb()?; + + let (to_download, to_delete, to_remove_pending) = + changes::determine_required_changes(&self.mgr.db, batch)?; + + // file removal + self.mgr.remove_files(to_delete.as_slice())?; + self.progress.downloaded_deletions += to_delete.len(); + self.fire_progress_cb()?; + + // file download + let mut downloaded = vec![]; + let mut dl_fnames = to_download.as_slice(); + while !dl_fnames.is_empty() { + let batch: Vec<_> = dl_fnames + .iter() + .take(MAX_MEDIA_FILES_IN_ZIP) + .map(ToOwned::to_owned) + .collect(); + let zip_data = self + .client + .download_files(DownloadFilesRequest { files: batch }.try_into_sync_request()?) + .await? + .data; + let download_batch = + download::extract_into_media_folder(self.mgr.media_folder.as_path(), zip_data)? + .into_iter(); + let len = download_batch.len(); + dl_fnames = &dl_fnames[len..]; + downloaded.extend(download_batch); + + self.progress.downloaded_files += len; + self.fire_progress_cb()?; + } + + // then update the DB + let dirmod = mtime_as_i64(&self.mgr.media_folder)?; + self.mgr.db.transact(|ctx| { + ctx.record_clean(&to_remove_pending)?; + ctx.record_removals(&to_delete)?; + ctx.record_additions(downloaded)?; + + // update usn + meta.last_sync_usn = last_usn; + meta.folder_mtime = dirmod; + ctx.set_meta(&meta)?; + + Ok(()) + })?; + } + Ok(()) + } + + async fn send_changes(&mut self) -> Result<()> { + loop { + let pending: Vec = self + .mgr + .db + .get_pending_uploads(MAX_MEDIA_FILES_IN_ZIP as u32)?; + if pending.is_empty() { + break; + } + + let data_for_zip = + gather_zip_data_for_upload(&self.mgr.db, &self.mgr.media_folder, &pending)?; + let zip_bytes = match data_for_zip { + None => { + // discard zip info and retry batch - not particularly efficient, + // but this is a corner case + self.progress.checked += pending.len(); + self.fire_progress_cb()?; + continue; + } + Some(data) => zip_files_for_upload(data)?, + }; + + let reply = self + .client + .upload_changes(zip_bytes.try_into_sync_request()?) + .await? + .json_result()?; + + let (processed_files, processed_deletions): (Vec<_>, Vec<_>) = pending + .into_iter() + .take(reply.processed) + .partition(|e| e.sha1.is_some()); + + self.progress.uploaded_files += processed_files.len(); + self.progress.uploaded_deletions += processed_deletions.len(); + self.fire_progress_cb()?; + + let fnames: Vec<_> = processed_files + .into_iter() + .chain(processed_deletions.into_iter()) + .map(|e| e.fname) + .collect(); + let fname_cnt = fnames.len() as i32; + self.mgr.db.transact(|ctx| { + ctx.record_clean(fnames.as_slice())?; + let mut meta = ctx.get_meta()?; + if meta.last_sync_usn.0 + fname_cnt == reply.current_usn.0 { + meta.last_sync_usn = reply.current_usn; + ctx.set_meta(&meta)?; + } else { + debug!( + "server usn {} is not {}, skipping usn update", + reply.current_usn, + meta.last_sync_usn.0 + fname_cnt + ); + } + Ok(()) + })?; + } + + Ok(()) + } + + async fn finalize_sync(&mut self) -> Result<()> { + let local = self.mgr.db.count()?; + let msg = self + .client + .media_sanity_check(SanityCheckRequest { local }.try_into_sync_request()?) + .await? + .json_result()?; + if msg == MediaSanityCheckResponse::Ok { + Ok(()) + } else { + self.mgr.db.transact(|ctx| ctx.force_resync())?; + Err(AnkiError::sync_error("", SyncErrorKind::ResyncRequired)) + } + } +} diff --git a/rslib/src/sync/media/tests.rs b/rslib/src/sync/media/tests.rs new file mode 100644 index 000000000..c68bbecd8 --- /dev/null +++ b/rslib/src/sync/media/tests.rs @@ -0,0 +1,292 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +#![cfg(test)] + +use std::{fs, net::IpAddr, thread::sleep, time::Duration}; + +use nom::AsBytes; +use reqwest::{multipart, Client}; + +use crate::{ + error::Result, + media::MediaManager, + prelude::AnkiError, + sync::{ + collection::{ + protocol::AsSyncEndpoint, + tests::{with_active_server, SyncTestContext}, + }, + media::{ + begin::{SyncBeginQuery, SyncBeginRequest}, + progress::MediaSyncProgress, + protocol::{MediaSyncMethod, MediaSyncProtocol}, + sanity::{MediaSanityCheckResponse, SanityCheckRequest}, + syncer::MediaSyncer, + zip::zip_files_for_upload, + }, + request::{IntoSyncRequest, SyncRequest}, + version::SyncVersion, + }, + version::sync_client_version, +}; + +/// Older Rust versions sent hkey/version in GET query string. +#[tokio::test] +async fn begin_supports_get() -> Result<()> { + with_active_server(|client_| async move { + let url = client_.endpoint().join("msync/begin").unwrap(); + let client = Client::new(); + client + .get(url) + .query(&SyncBeginQuery { + host_key: client_.sync_key.clone(), + client_version: sync_client_version().into(), + }) + .send() + .await? + .error_for_status()?; + Ok(()) + }) + .await +} + +/// Older clients used a `v` variable in the begin multipart instead of placing the +/// version in the JSON payload. +#[tokio::test] +async fn begin_supports_version_in_form() -> Result<()> { + with_active_server(|client_| async move { + let url = MediaSyncMethod::Begin.as_sync_endpoint(client_.endpoint()); + let client = Client::new(); + + let form = multipart::Form::new() + .text("c", "0") + .text("v", "client") + .text("k", client_.sync_key.clone()); + client + .post(url) + .multipart(form) + .send() + .await? + .error_for_status()?; + Ok(()) + }) + .await +} + +/// Older clients sent key in `sk` multipart variable for non-begin requests. +#[tokio::test] +async fn legacy_session_key_works() -> Result<()> { + with_active_server(|client_| async move { + let url = MediaSyncMethod::MediaChanges.as_sync_endpoint(client_.endpoint()); + let client = Client::new(); + + let form = multipart::Form::new() + .text("c", "0") + .text("v", "client") + .text("sk", client_.sync_key.clone()) + .part( + "data", + multipart::Part::bytes(b"{\"lastUsn\": 0}".as_bytes()), + ); + client + .post(url) + .multipart(form) + .send() + .await? + .error_for_status()?; + Ok(()) + }) + .await +} + +#[tokio::test] +async fn sanity_check() -> Result<()> { + with_active_server(|client| async move { + let ctx = SyncTestContext::new(client.partial_clone()); + let media1 = ctx.media1(); + ctx.sync_media1().await?; + // may be non-zero when testing on external endpoint + let starting_file_count = fs::read_dir(&media1.media_folder).unwrap().count() as u32; + let resp = client + .media_sanity_check( + SanityCheckRequest { + local: starting_file_count, + } + .try_into_sync_request()?, + ) + .await? + .json_result()?; + assert_eq!(resp, MediaSanityCheckResponse::Ok); + let resp = client + .media_sanity_check( + SanityCheckRequest { + local: starting_file_count + 1, + } + .try_into_sync_request()?, + ) + .await? + .json_result()?; + assert_eq!(resp, MediaSanityCheckResponse::SanityCheckFailed); + Ok(()) + }) + .await +} + +fn ignore_progress(_progress: MediaSyncProgress) -> bool { + true +} + +impl SyncTestContext { + fn media1(&self) -> MediaManager { + self.col1().media().unwrap() + } + + fn media2(&self) -> MediaManager { + self.col2().media().unwrap() + } + + async fn sync_media1(&self) -> Result<()> { + let mut syncer = + MediaSyncer::new(self.media1(), ignore_progress, self.client.partial_clone()).unwrap(); + syncer.sync().await + } + + async fn sync_media2(&self) -> Result<()> { + let mut syncer = + MediaSyncer::new(self.media2(), ignore_progress, self.client.partial_clone()).unwrap(); + syncer.sync().await + } + + /// As local change detection depends on a millisecond timestamp, + /// we need to wait a little while between steps to ensure changes are observed. Theoretically + /// 1ms should suffice, but I was seeing flaky tests on a ZFS system with the delay set to a + /// few milliseconds. + fn sleep(&self) { + sleep(Duration::from_millis(10)) + } +} + +#[tokio::test] +async fn media_roundtrip() -> Result<()> { + with_active_server(|client| async move { + let ctx = SyncTestContext::new(client.partial_clone()); + let media1 = ctx.media1(); + let media2 = ctx.media2(); + ctx.sync_media1().await?; + ctx.sync_media2().await?; + ctx.sleep(); + // may be non-zero when testing on external endpoint + let starting_file_count = fs::read_dir(&media1.media_folder).unwrap().count(); + // add some files + fs::write(media1.media_folder.join("manual1"), "manual1").unwrap(); + media1.add_file("auto1", b"auto1").unwrap(); + fs::write(media1.media_folder.join("manual2"), "manual2").unwrap(); + // sync to server and then other client + ctx.sync_media1().await?; + ctx.sync_media2().await?; + // modify a file and remove the other + ctx.sleep(); + fs::write(media2.media_folder.join("manual1"), "changed1").unwrap(); + fs::remove_file(media2.media_folder.join("manual2")).unwrap(); + ctx.sync_media2().await?; + ctx.sync_media1().await?; + assert_eq!( + fs::read_to_string(media1.media_folder.join("manual1")).unwrap(), + "changed1" + ); + // remove remaining files + ctx.sleep(); + fs::remove_file(media1.media_folder.join("manual1")).unwrap(); + fs::remove_file(media2.media_folder.join("auto1")).unwrap(); + ctx.sync_media1().await?; + ctx.sync_media2().await?; + ctx.sync_media1().await?; + assert_eq!( + fs::read_dir(media1.media_folder).unwrap().count(), + starting_file_count + ); + assert_eq!( + fs::read_dir(media2.media_folder).unwrap().count(), + starting_file_count + ); + Ok(()) + }) + .await +} + +#[tokio::test] +async fn parallel_requests() -> Result<()> { + with_active_server(|client| async move { + let ctx = SyncTestContext::new(client.partial_clone()); + let media1 = ctx.media1(); + let media2 = ctx.media2(); + ctx.sleep(); + // multiple clients should be able to add the same file + media1.add_file("auto", b"auto").unwrap(); + media2.add_file("auto", b"auto").unwrap(); + ctx.sync_media1().await?; + // Normally the second client would notice the addition of the file when + // fetching changes from the server; here we manually upload the change to simulate + // two parallel syncs going on. + let get_usn = || async { + Ok::<_, AnkiError>( + ctx.client + .begin( + SyncBeginRequest { + client_version: "x".into(), + } + .try_into_sync_request()?, + ) + .await? + .json_result()? + .usn, + ) + }; + let start_usn = get_usn().await?; + let zip_data = zip_files_for_upload(vec![("auto".into(), Some(b"auto".to_vec()))])?; + client + .upload_changes(SyncRequest::from_data( + zip_data, + ctx.client.sync_key.clone(), + String::new(), + IpAddr::from([0, 0, 0, 0]), + SyncVersion::latest(), + )) + .await?; + let end_usn = get_usn().await?; + assert_eq!(start_usn, end_usn); + // Parallel deletions should work too + media1.remove_files(&["auto"])?; + media2.remove_files(&["auto"])?; + ctx.sync_media1().await?; + let start_usn = get_usn().await?; + let zip_data = zip_files_for_upload(vec![("auto".into(), None)])?; + client + .upload_changes(SyncRequest::from_data( + zip_data, + ctx.client.sync_key.clone(), + String::new(), + IpAddr::from([0, 0, 0, 0]), + SyncVersion::latest(), + )) + .await?; + let end_usn = get_usn().await?; + assert_eq!(start_usn, end_usn); + // In the case of differing content, server (first sync) content wins + media1.add_file("diff", b"1").unwrap(); + media2.add_file("diff", b"2").unwrap(); + ctx.sync_media1().await?; + ctx.sync_media2().await?; + assert_eq!( + fs::read_to_string(media1.media_folder.join("diff")).unwrap(), + "1" + ); + assert_eq!( + fs::read_to_string(media2.media_folder.join("diff")).unwrap(), + "1" + ); + Ok(()) + }) + .await +} diff --git a/rslib/src/sync/media/upload.rs b/rslib/src/sync/media/upload.rs new file mode 100644 index 000000000..f00ec502e --- /dev/null +++ b/rslib/src/sync/media/upload.rs @@ -0,0 +1,111 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::{borrow::Cow, path::Path}; + +use serde_derive::Deserialize; +use serde_tuple::Serialize_tuple; +use tracing::debug; + +use crate::{ + media::files::{data_for_file, normalize_filename}, + prelude::*, + sync::media::{ + database::client::{MediaDatabase, MediaEntry}, + MAX_INDIVIDUAL_MEDIA_FILE_SIZE, MEDIA_SYNC_TARGET_ZIP_BYTES, + }, +}; + +#[derive(Serialize_tuple, Deserialize, Debug)] +pub struct MediaUploadResponse { + /// Always equal to number of uploaded files now. Old AnkiWeb versions used to + /// terminate processing early if too much time had elapsed, so older clients + /// will upload the same material again if this is less than the count they + /// uploaded. + pub processed: usize, + pub current_usn: Usn, +} + +/// Filename -> Some(Data), or None in the deleted case. +type ZipDataForUpload = Vec<(String, Option>)>; + +/// Gather [(filename, data)] for provided entries, up to configured limit. +/// Data is None if file is deleted. +/// Returns None if one or more of the entries were inaccessible or in the wrong format. +pub fn gather_zip_data_for_upload( + ctx: &MediaDatabase, + media_folder: &Path, + files: &[MediaEntry], +) -> Result> { + let mut invalid_entries = vec![]; + let mut accumulated_size = 0; + let mut entries = vec![]; + + for file in files { + if accumulated_size > MEDIA_SYNC_TARGET_ZIP_BYTES { + break; + } + + #[cfg(target_vendor = "apple")] + { + use unicode_normalization::is_nfc; + if !is_nfc(&file.fname) { + // older Anki versions stored non-normalized filenames in the DB; clean them up + debug!(fname = file.fname, "clean up non-nfc entry"); + invalid_entries.push(&file.fname); + continue; + } + } + + let file_data = if file.sha1.is_some() { + match data_for_file(media_folder, &file.fname) { + Ok(data) => data, + Err(e) => { + debug!("error accessing {}: {}", &file.fname, e); + invalid_entries.push(&file.fname); + continue; + } + } + } else { + // uploading deletion + None + }; + + if let Some(data) = file_data { + let normalized = normalize_filename(&file.fname); + if let Cow::Owned(o) = normalized { + debug!("media check required: {} should be {}", &file.fname, o); + invalid_entries.push(&file.fname); + continue; + } + + if data.is_empty() { + invalid_entries.push(&file.fname); + continue; + } + if data.len() > MAX_INDIVIDUAL_MEDIA_FILE_SIZE { + invalid_entries.push(&file.fname); + continue; + } + accumulated_size += data.len(); + entries.push((file.fname.clone(), Some(data))); + debug!(file.fname, kind = "addition", "will upload"); + } else { + entries.push((file.fname.clone(), None)); + debug!(file.fname, kind = "removal", "will upload"); + } + } + + if !invalid_entries.is_empty() { + // clean up invalid entries; we'll build a new zip + ctx.transact(|ctx| { + for fname in invalid_entries { + ctx.remove_entry(fname)?; + } + Ok(()) + })?; + return Ok(None); + } + + Ok(Some(entries)) +} diff --git a/rslib/src/sync/media/zip.rs b/rslib/src/sync/media/zip.rs new file mode 100644 index 000000000..064137bf4 --- /dev/null +++ b/rslib/src/sync/media/zip.rs @@ -0,0 +1,156 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::{ + collections::HashMap, + io, + io::{Read, Write}, +}; + +use serde::Deserialize; +use serde_tuple::Serialize_tuple; +use unicode_normalization::is_nfc; +use zip::{self, write::FileOptions, ZipWriter}; + +use crate::{ + media::files::sha1_of_data, + prelude::*, + sync::media::{MAX_INDIVIDUAL_MEDIA_FILE_SIZE, MAX_MEDIA_FILENAME_LENGTH_SERVER}, +}; + +pub struct ZipFileMetadata { + pub filename: String, + pub total_bytes: u32, + pub sha1: String, +} + +/// Write provided `[(filename, data)]` into a zip file, returning its data. +/// The metadata is in a different format to the upload case, since deletions don't need +/// to be represented. +pub fn zip_files_for_download(files: Vec<(String, Vec)>) -> Result> { + let options = FileOptions::default().compression_method(zip::CompressionMethod::Stored); + let mut zip = ZipWriter::new(io::Cursor::new(vec![])); + let mut entries = HashMap::new(); + + for (idx, (filename, data)) in files.into_iter().enumerate() { + assert!(!data.is_empty()); + let idx_str = idx.to_string(); + entries.insert(idx_str.clone(), filename); + zip.start_file(idx_str, options)?; + zip.write_all(&data)?; + } + + let meta = serde_json::to_vec(&entries)?; + zip.start_file("_meta", options)?; + zip.write_all(&meta)?; + + Ok(zip.finish()?.into_inner()) +} + +pub fn zip_files_for_upload(entries_: Vec<(String, Option>)>) -> Result> { + let options = FileOptions::default().compression_method(zip::CompressionMethod::Stored); + let mut zip = ZipWriter::new(io::Cursor::new(vec![])); + let mut entries = vec![]; + + for (idx, (filename, data)) in entries_.into_iter().enumerate() { + match data { + None => { + entries.push(UploadEntry { + actual_filename: filename, + filename_in_zip: None, + }); + } + Some(data) => { + let idx_str = idx.to_string(); + zip.start_file(&idx_str, options)?; + zip.write_all(&data)?; + entries.push(UploadEntry { + actual_filename: filename, + filename_in_zip: Some(idx_str), + }); + } + } + } + + let meta = serde_json::to_vec(&entries)?; + zip.start_file("_meta", options)?; + zip.write_all(&meta)?; + + Ok(zip.finish()?.into_inner()) +} + +pub struct UploadedChange { + pub nfc_filename: String, + pub kind: UploadedChangeKind, +} + +pub enum UploadedChangeKind { + AddOrReplace { + nonempty_data: Vec, + sha1: Vec, + }, + Delete, +} + +pub fn unzip_and_validate_files(zip_data: &[u8]) -> Result> { + let mut zip = zip::ZipArchive::new(io::Cursor::new(zip_data))?; + + // meta map first, limited to a reasonable size + let meta_file = zip.by_name("_meta")?; + let entries: Vec = serde_json::from_reader(meta_file.take(50 * 1024))?; + if entries.len() > 25 { + invalid_input!("too many files in zip"); + } + + // extract files/deletions from zip + entries + .into_iter() + .map(|entry| { + if entry.actual_filename.len() > MAX_MEDIA_FILENAME_LENGTH_SERVER { + invalid_input!("filename too long: {}", entry.actual_filename.len()); + } + if !is_nfc(&entry.actual_filename) { + invalid_input!("filename was not not in nfc: {}", entry.actual_filename); + } + if entry.actual_filename.contains(std::path::is_separator) { + invalid_input!("filename contained separator: {}", entry.actual_filename); + } + let data = if let Some(filename_in_zip) = entry.filename_in_zip.as_ref() { + if filename_in_zip.is_empty() { + // older clients/AnkiDroid use an empty string instead of null + UploadedChangeKind::Delete + } else { + let file = zip.by_name(filename_in_zip)?; + if file.size() > MAX_INDIVIDUAL_MEDIA_FILE_SIZE as u64 { + invalid_input!("file too large"); + } + let mut data = vec![]; + // the .take() is because we don't trust the header to be correct + let bytes_read = file + .take(MAX_INDIVIDUAL_MEDIA_FILE_SIZE as u64) + .read_to_end(&mut data)?; + if bytes_read == 0 { + invalid_input!("file entry was zero bytes"); + } + let sha1 = sha1_of_data(&data).to_vec(); + UploadedChangeKind::AddOrReplace { + nonempty_data: data, + sha1, + } + } + } else { + UploadedChangeKind::Delete + }; + Ok(UploadedChange { + nfc_filename: entry.actual_filename, + kind: data, + }) + }) + .collect() +} + +#[derive(Serialize_tuple, Deserialize)] +struct UploadEntry { + actual_filename: String, + filename_in_zip: Option, +} diff --git a/rslib/src/sync/mod.rs b/rslib/src/sync/mod.rs index 613cd82f2..447c6f351 100644 --- a/rslib/src/sync/mod.rs +++ b/rslib/src/sync/mod.rs @@ -1,1611 +1,12 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -pub mod http; -mod http_client; -mod server; - -use std::collections::HashMap; - -use http_client::HttpSyncClient; -pub use http_client::{FullSyncProgressFn, Timeouts}; -use itertools::Itertools; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use serde_tuple::Serialize_tuple; -pub(crate) use server::{LocalServer, SyncServer}; -use tracing::debug; - -use crate::{ - card::{Card, CardQueue, CardType}, - deckconfig::DeckConfSchema11, - decks::DeckSchema11, - error::{SyncError, SyncErrorKind}, - io::atomic_rename, - notes::Note, - notetype::{Notetype, NotetypeSchema11}, - pb::sync::{sync_status_response, SyncStatusResponse}, - prelude::*, - revlog::RevlogEntry, - serde::{default_on_invalid, deserialize_int_from_number}, - storage::{ - card::data::{card_data_string, CardData}, - open_and_check_sqlite_file, SchemaVersion, - }, - tags::{join_tags, split_tags, Tag}, -}; - -pub static SYNC_VERSION_MIN: u8 = 7; -pub static SYNC_VERSION_MAX: u8 = 10; - -#[derive(Default, Debug, Clone, Copy)] -pub struct NormalSyncProgress { - pub stage: SyncStage, - pub local_update: usize, - pub local_remove: usize, - pub remote_update: usize, - pub remote_remove: usize, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SyncStage { - Connecting, - Syncing, - Finalizing, -} - -impl Default for SyncStage { - fn default() -> Self { - SyncStage::Connecting - } -} - -#[derive(Serialize, Deserialize, Debug, Default)] -pub struct SyncMeta { - #[serde(rename = "mod")] - pub modified: TimestampMillis, - #[serde(rename = "scm")] - pub schema: TimestampMillis, - pub usn: Usn, - #[serde(rename = "ts")] - pub current_time: TimestampSecs, - #[serde(rename = "msg")] - pub server_message: String, - #[serde(rename = "cont")] - pub should_continue: bool, - #[serde(rename = "hostNum")] - pub host_number: u32, - #[serde(default)] - pub empty: bool, -} - -#[derive(Serialize, Deserialize, Debug, Default)] -pub struct Graves { - pub(crate) cards: Vec, - pub(crate) decks: Vec, - pub(crate) notes: Vec, -} - -#[derive(Serialize_tuple, Deserialize, Debug, Default)] -pub struct DecksAndConfig { - decks: Vec, - config: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Default)] -pub struct UnchunkedChanges { - #[serde(rename = "models")] - notetypes: Vec, - #[serde(rename = "decks")] - decks_and_config: DecksAndConfig, - tags: Vec, - - // the following are only sent if local is newer - #[serde(skip_serializing_if = "Option::is_none", rename = "conf")] - config: Option>, - #[serde(skip_serializing_if = "Option::is_none", rename = "crt")] - creation_stamp: Option, -} - -#[derive(Serialize, Deserialize, Debug, Default)] -pub struct Chunk { - #[serde(default)] - done: bool, - #[serde(skip_serializing_if = "Vec::is_empty", default)] - revlog: Vec, - #[serde(skip_serializing_if = "Vec::is_empty", default)] - cards: Vec, - #[serde(skip_serializing_if = "Vec::is_empty", default)] - notes: Vec, -} - -struct ChunkableIds { - revlog: Vec, - cards: Vec, - notes: Vec, -} - -#[derive(Serialize_tuple, Deserialize, Debug)] -pub struct NoteEntry { - pub id: NoteId, - pub guid: String, - #[serde(rename = "mid")] - pub ntid: NotetypeId, - #[serde(rename = "mod")] - pub mtime: TimestampSecs, - pub usn: Usn, - pub tags: String, - pub fields: String, - pub sfld: String, // always empty - pub csum: String, // always empty - pub flags: u32, - pub data: String, -} - -#[derive(Serialize_tuple, Deserialize, Debug)] -pub struct CardEntry { - pub id: CardId, - pub nid: NoteId, - pub did: DeckId, - pub ord: u16, - #[serde(deserialize_with = "deserialize_int_from_number")] - pub mtime: TimestampSecs, - pub usn: Usn, - pub ctype: CardType, - pub queue: CardQueue, - #[serde(deserialize_with = "deserialize_int_from_number")] - pub due: i32, - #[serde(deserialize_with = "deserialize_int_from_number")] - pub ivl: u32, - pub factor: u16, - pub reps: u32, - pub lapses: u32, - pub left: u32, - #[serde(deserialize_with = "deserialize_int_from_number")] - pub odue: i32, - pub odid: DeckId, - pub flags: u8, - pub data: String, -} - -#[derive(Serialize, Deserialize, Debug)] -pub struct SanityCheckResponse { - pub status: SanityCheckStatus, - #[serde(rename = "c", default, deserialize_with = "default_on_invalid")] - pub client: Option, - #[serde(rename = "s", default, deserialize_with = "default_on_invalid")] - pub server: Option, -} - -#[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] -#[serde(rename_all = "lowercase")] -pub enum SanityCheckStatus { - Ok, - Bad, -} - -#[derive(Serialize_tuple, Deserialize, Debug, PartialEq, Eq)] -pub struct SanityCheckCounts { - pub counts: SanityCheckDueCounts, - pub cards: u32, - pub notes: u32, - pub revlog: u32, - pub graves: u32, - #[serde(rename = "models")] - pub notetypes: u32, - pub decks: u32, - pub deck_config: u32, -} - -#[derive(Serialize_tuple, Deserialize, Debug, Default, PartialEq, Eq)] -pub struct SanityCheckDueCounts { - pub new: u32, - pub learn: u32, - pub review: u32, -} - -#[derive(Debug, Default, Clone, Copy)] -pub struct FullSyncProgress { - pub transferred_bytes: usize, - pub total_bytes: usize, -} - -#[derive(PartialEq, Eq, Debug, Clone, Copy)] -pub enum SyncActionRequired { - NoChanges, - FullSyncRequired { upload_ok: bool, download_ok: bool }, - NormalSyncRequired, -} -#[derive(Debug)] -struct SyncState { - required: SyncActionRequired, - local_is_newer: bool, - usn_at_last_sync: Usn, - // latest usn, used for adding new items - latest_usn: Usn, - // usn to use when locating pending objects - pending_usn: Usn, - // usn to replace pending items with - the same as latest_usn in the client case - new_usn: Option, - server_message: String, - host_number: u32, -} - -#[derive(Debug)] -pub struct SyncOutput { - pub required: SyncActionRequired, - pub server_message: String, - pub host_number: u32, -} - -#[derive(Clone)] -pub struct SyncAuth { - pub hkey: String, - pub host_number: u32, -} - -struct NormalSyncer<'a, F> { - col: &'a mut Collection, - remote: Box, - progress: NormalSyncProgress, - progress_fn: F, -} - -impl Usn { - /// Used when gathering pending objects during sync. - pub(crate) fn pending_object_clause(self) -> &'static str { - if self.0 == -1 { - "usn = ?" - } else { - "usn >= ?" - } - } - - pub(crate) fn is_pending_sync(self, pending_usn: Usn) -> bool { - if pending_usn.0 == -1 { - self.0 == -1 - } else { - self.0 >= pending_usn.0 - } - } -} - -impl SyncMeta { - fn compared_to_remote(&self, remote: SyncMeta) -> SyncState { - let local = self; - let required = if remote.modified == local.modified { - SyncActionRequired::NoChanges - } else if remote.schema != local.schema { - let upload_ok = !local.empty || remote.empty; - let download_ok = !remote.empty || local.empty; - SyncActionRequired::FullSyncRequired { - upload_ok, - download_ok, - } - } else { - SyncActionRequired::NormalSyncRequired - }; - - SyncState { - required, - local_is_newer: local.modified > remote.modified, - usn_at_last_sync: local.usn, - latest_usn: remote.usn, - pending_usn: Usn(-1), - new_usn: Some(remote.usn), - server_message: remote.server_message, - host_number: remote.host_number, - } - } -} - -impl NormalSyncer<'_, F> -where - F: FnMut(NormalSyncProgress, bool), -{ - pub fn new( - col: &mut Collection, - server: Box, - progress_fn: F, - ) -> NormalSyncer<'_, F> - where - F: FnMut(NormalSyncProgress, bool), - { - NormalSyncer { - col, - remote: server, - progress: NormalSyncProgress::default(), - progress_fn, - } - } - - fn fire_progress_cb(&mut self, throttle: bool) { - (self.progress_fn)(self.progress, throttle) - } - - pub async fn sync(&mut self) -> Result { - debug!("fetching meta..."); - self.fire_progress_cb(false); - let state: SyncState = self.get_sync_state().await?; - debug!(?state, "fetched"); - match state.required { - SyncActionRequired::NoChanges => Ok(state.into()), - SyncActionRequired::FullSyncRequired { .. } => Ok(state.into()), - SyncActionRequired::NormalSyncRequired => { - self.col.discard_undo_and_study_queues(); - self.col.storage.begin_trx()?; - let timing = self.col.timing_today()?; - self.col.unbury_if_day_rolled_over(timing)?; - match self.normal_sync_inner(state).await { - Ok(success) => { - self.col.storage.commit_trx()?; - Ok(success) - } - Err(e) => { - self.col.storage.rollback_trx()?; - let _ = self.remote.abort().await; - - if let AnkiError::SyncError { - source: - SyncError { - info, - kind: SyncErrorKind::DatabaseCheckRequired, - }, - } = &e - { - debug!("sanity check failed:\n{}", info); - } - - Err(e) - } - } - } - } - } - - async fn get_sync_state(&self) -> Result { - let remote: SyncMeta = self.remote.meta().await?; - debug!("remote {:?}", &remote); - if !remote.should_continue { - debug!(remote.server_message, "server says abort"); - return Err(AnkiError::sync_error( - remote.server_message, - SyncErrorKind::ServerMessage, - )); - } - - let local = self.col.sync_meta()?; - debug!("local {:?}", &local); - let delta = remote.current_time.0 - local.current_time.0; - if delta.abs() > 300 { - debug!(delta, "clock off"); - return Err(AnkiError::sync_error("", SyncErrorKind::ClockIncorrect)); - } - - Ok(local.compared_to_remote(remote)) - } - - /// Sync. Caller must have created a transaction, and should call - /// abort on failure. - async fn normal_sync_inner(&mut self, mut state: SyncState) -> Result { - self.progress.stage = SyncStage::Syncing; - self.fire_progress_cb(false); - - debug!("start"); - self.start_and_process_deletions(&state).await?; - debug!("unchunked changes"); - self.process_unchunked_changes(&state).await?; - debug!("begin stream from server"); - self.process_chunks_from_server(&state).await?; - debug!("begin stream to server"); - self.send_chunks_to_server(&state).await?; - - self.progress.stage = SyncStage::Finalizing; - self.fire_progress_cb(false); - - debug!("sanity check"); - self.sanity_check().await?; - debug!("finalize"); - self.finalize(&state).await?; - state.required = SyncActionRequired::NoChanges; - Ok(state.into()) - } - - // The following operations assume a transaction has been set up. - - async fn start_and_process_deletions(&mut self, state: &SyncState) -> Result<()> { - let remote: Graves = self - .remote - .start(state.usn_at_last_sync, state.local_is_newer, None) - .await?; - - debug!( - cards = remote.cards.len(), - notes = remote.notes.len(), - decks = remote.decks.len(), - "removed on remote" - ); - - let mut local = self.col.storage.pending_graves(state.pending_usn)?; - if let Some(new_usn) = state.new_usn { - self.col.storage.update_pending_grave_usns(new_usn)?; - } - - debug!( - cards = local.cards.len(), - notes = local.notes.len(), - decks = local.decks.len(), - "locally removed " - ); - - while let Some(chunk) = local.take_chunk() { - debug!("sending graves chunk"); - self.progress.local_remove += chunk.cards.len() + chunk.notes.len() + chunk.decks.len(); - self.remote.apply_graves(chunk).await?; - self.fire_progress_cb(true); - } - - self.progress.remote_remove = remote.cards.len() + remote.notes.len() + remote.decks.len(); - self.col.apply_graves(remote, state.latest_usn)?; - self.fire_progress_cb(true); - debug!("applied server graves"); - - Ok(()) - } - - // This was assumed to a cheap operation when originally written - it didn't anticipate - // the large deck trees and note types some users would create. They should be chunked - // in the future, like other objects. Syncing tags explicitly is also probably of limited - // usefulness. - async fn process_unchunked_changes(&mut self, state: &SyncState) -> Result<()> { - debug!("gathering local changes"); - let local = self.col.local_unchunked_changes( - state.pending_usn, - state.new_usn, - state.local_is_newer, - )?; - - debug!( - notetypes = local.notetypes.len(), - decks = local.decks_and_config.decks.len(), - deck_config = local.decks_and_config.config.len(), - tags = local.tags.len(), - "sending" - ); - - self.progress.local_update += local.notetypes.len() - + local.decks_and_config.decks.len() - + local.decks_and_config.config.len() - + local.tags.len(); - let remote = self.remote.apply_changes(local).await?; - self.fire_progress_cb(true); - - debug!( - notetypes = remote.notetypes.len(), - decks = remote.decks_and_config.decks.len(), - deck_config = remote.decks_and_config.config.len(), - tags = remote.tags.len(), - "received" - ); - - self.progress.remote_update += remote.notetypes.len() - + remote.decks_and_config.decks.len() - + remote.decks_and_config.config.len() - + remote.tags.len(); - - self.col.apply_changes(remote, state.latest_usn)?; - self.fire_progress_cb(true); - Ok(()) - } - - async fn process_chunks_from_server(&mut self, state: &SyncState) -> Result<()> { - loop { - let chunk: Chunk = self.remote.chunk().await?; - - debug!( - done = chunk.done, - cards = chunk.cards.len(), - notes = chunk.notes.len(), - revlog = chunk.revlog.len(), - "received" - ); - - self.progress.remote_update += - chunk.cards.len() + chunk.notes.len() + chunk.revlog.len(); - - let done = chunk.done; - self.col.apply_chunk(chunk, state.pending_usn)?; - - self.fire_progress_cb(true); - - if done { - return Ok(()); - } - } - } - - async fn send_chunks_to_server(&mut self, state: &SyncState) -> Result<()> { - let mut ids = self.col.get_chunkable_ids(state.pending_usn)?; - - loop { - let chunk: Chunk = self.col.get_chunk(&mut ids, state.new_usn)?; - let done = chunk.done; - - debug!( - done = chunk.done, - cards = chunk.cards.len(), - notes = chunk.notes.len(), - revlog = chunk.revlog.len(), - "sending" - ); - - self.progress.local_update += - chunk.cards.len() + chunk.notes.len() + chunk.revlog.len(); - - self.remote.apply_chunk(chunk).await?; - - self.fire_progress_cb(true); - - if done { - return Ok(()); - } - } - } - - /// Caller should force full sync after rolling back. - async fn sanity_check(&mut self) -> Result<()> { - let mut local_counts = self.col.storage.sanity_check_info()?; - self.col.add_due_counts(&mut local_counts.counts)?; - - debug!("gathered local counts; waiting for server reply"); - let out: SanityCheckResponse = self.remote.sanity_check(local_counts).await?; - debug!("got server reply"); - if out.status != SanityCheckStatus::Ok { - Err(AnkiError::sync_error( - format!("local {:?}\nremote {:?}", out.client, out.server), - SyncErrorKind::DatabaseCheckRequired, - )) - } else { - Ok(()) - } - } - - async fn finalize(&mut self, state: &SyncState) -> Result<()> { - let new_server_mtime = self.remote.finish().await?; - self.col.finalize_sync(state, new_server_mtime) - } -} - -const CHUNK_SIZE: usize = 250; - -impl Graves { - fn take_chunk(&mut self) -> Option { - let mut limit = CHUNK_SIZE; - let mut out = Graves::default(); - while limit > 0 && !self.cards.is_empty() { - out.cards.push(self.cards.pop().unwrap()); - limit -= 1; - } - while limit > 0 && !self.notes.is_empty() { - out.notes.push(self.notes.pop().unwrap()); - limit -= 1; - } - while limit > 0 && !self.decks.is_empty() { - out.decks.push(self.decks.pop().unwrap()); - limit -= 1; - } - if limit == CHUNK_SIZE { - None - } else { - Some(out) - } - } -} - -pub async fn sync_login(username: &str, password: &str) -> Result { - let mut remote = HttpSyncClient::new(None, 0); - remote.login(username, password).await?; - Ok(SyncAuth { - hkey: remote.hkey().to_string(), - host_number: 0, - }) -} - -pub async fn sync_abort(hkey: String, host_number: u32) -> Result<()> { - let mut remote = HttpSyncClient::new(Some(hkey), host_number); - remote.abort().await -} - -pub(crate) async fn get_remote_sync_meta(auth: SyncAuth) -> Result { - let remote = HttpSyncClient::new(Some(auth.hkey), auth.host_number); - remote.meta().await -} - -impl Collection { - pub fn get_local_sync_status(&mut self) -> Result { - let stamps = self.storage.get_collection_timestamps()?; - let required = if stamps.schema_changed_since_sync() { - sync_status_response::Required::FullSync - } else if stamps.collection_changed_since_sync() { - sync_status_response::Required::NormalSync - } else { - sync_status_response::Required::NoChanges - }; - - Ok(required) - } - - pub fn get_sync_status(&self, remote: SyncMeta) -> Result { - Ok(self.sync_meta()?.compared_to_remote(remote).required.into()) - } - - /// Create a new syncing instance. If host_number is unavailable, use 0. - pub async fn normal_sync(&mut self, auth: SyncAuth, progress_fn: F) -> Result - where - F: FnMut(NormalSyncProgress, bool), - { - NormalSyncer::new( - self, - Box::new(HttpSyncClient::new(Some(auth.hkey), auth.host_number)), - progress_fn, - ) - .sync() - .await - } - - /// Upload collection to AnkiWeb. Caller must re-open afterwards. - pub async fn full_upload(self, auth: SyncAuth, progress_fn: FullSyncProgressFn) -> Result<()> { - let mut server = HttpSyncClient::new(Some(auth.hkey), auth.host_number); - server.set_full_sync_progress_fn(Some(progress_fn)); - self.full_upload_inner(Box::new(server)).await - } - - pub(crate) async fn full_upload_inner(mut self, server: Box) -> Result<()> { - self.before_upload()?; - let col_path = self.col_path.clone(); - self.close(Some(SchemaVersion::V11))?; - server.full_upload(&col_path, false).await - } - - /// Download collection from AnkiWeb. Caller must re-open afterwards. - pub async fn full_download( - self, - auth: SyncAuth, - progress_fn: FullSyncProgressFn, - ) -> Result<()> { - let mut server = HttpSyncClient::new(Some(auth.hkey), auth.host_number); - server.set_full_sync_progress_fn(Some(progress_fn)); - self.full_download_inner(Box::new(server)).await - } - - pub(crate) async fn full_download_inner(self, server: Box) -> Result<()> { - let col_path = self.col_path.clone(); - let col_folder = col_path.parent().or_invalid("couldn't get col_folder")?; - self.close(None)?; - let out_file = server.full_download(Some(col_folder)).await?; - // check file ok - let db = open_and_check_sqlite_file(out_file.path())?; - db.execute_batch("update col set ls=mod")?; - drop(db); - atomic_rename(out_file, &col_path, true)?; - - Ok(()) - } - - fn sync_meta(&self) -> Result { - let stamps = self.storage.get_collection_timestamps()?; - Ok(SyncMeta { - modified: stamps.collection_change, - schema: stamps.schema_change, - usn: self.storage.usn(true)?, - current_time: TimestampSecs::now(), - server_message: "".into(), - should_continue: true, - host_number: 0, - empty: !self.storage.have_at_least_one_card()?, - }) - } - - pub fn apply_graves(&self, graves: Graves, latest_usn: Usn) -> Result<()> { - for nid in graves.notes { - self.storage.remove_note(nid)?; - self.storage.add_note_grave(nid, latest_usn)?; - } - for cid in graves.cards { - self.storage.remove_card(cid)?; - self.storage.add_card_grave(cid, latest_usn)?; - } - for did in graves.decks { - self.storage.remove_deck(did)?; - self.storage.add_deck_grave(did, latest_usn)?; - } - Ok(()) - } - - // Local->remote unchunked changes - //---------------------------------------------------------------- - - fn local_unchunked_changes( - &mut self, - pending_usn: Usn, - new_usn: Option, - local_is_newer: bool, - ) -> Result { - let mut changes = UnchunkedChanges { - notetypes: self.changed_notetypes(pending_usn, new_usn)?, - decks_and_config: DecksAndConfig { - decks: self.changed_decks(pending_usn, new_usn)?, - config: self.changed_deck_config(pending_usn, new_usn)?, - }, - tags: self.changed_tags(pending_usn, new_usn)?, - ..Default::default() - }; - if local_is_newer { - changes.config = Some(self.changed_config()?); - changes.creation_stamp = Some(self.storage.creation_stamp()?); - } - - Ok(changes) - } - - fn changed_notetypes( - &mut self, - pending_usn: Usn, - new_usn: Option, - ) -> Result> { - let ids = self - .storage - .objects_pending_sync("notetypes", pending_usn)?; - self.storage - .maybe_update_object_usns("notetypes", &ids, new_usn)?; - self.state.notetype_cache.clear(); - ids.into_iter() - .map(|id| { - self.storage.get_notetype(id).map(|opt| { - let mut nt: NotetypeSchema11 = opt.unwrap().into(); - nt.usn = new_usn.unwrap_or(nt.usn); - nt - }) - }) - .collect() - } - - fn changed_decks( - &mut self, - pending_usn: Usn, - new_usn: Option, - ) -> Result> { - let ids = self.storage.objects_pending_sync("decks", pending_usn)?; - self.storage - .maybe_update_object_usns("decks", &ids, new_usn)?; - self.state.deck_cache.clear(); - ids.into_iter() - .map(|id| { - self.storage.get_deck(id).map(|opt| { - let mut deck = opt.unwrap(); - deck.usn = new_usn.unwrap_or(deck.usn); - deck.into() - }) - }) - .collect() - } - - fn changed_deck_config( - &self, - pending_usn: Usn, - new_usn: Option, - ) -> Result> { - let ids = self - .storage - .objects_pending_sync("deck_config", pending_usn)?; - self.storage - .maybe_update_object_usns("deck_config", &ids, new_usn)?; - ids.into_iter() - .map(|id| { - self.storage.get_deck_config(id).map(|opt| { - let mut conf: DeckConfSchema11 = opt.unwrap().into(); - conf.usn = new_usn.unwrap_or(conf.usn); - conf - }) - }) - .collect() - } - - fn changed_tags(&self, pending_usn: Usn, new_usn: Option) -> Result> { - let changed = self.storage.tags_pending_sync(pending_usn)?; - if let Some(usn) = new_usn { - self.storage.update_tag_usns(&changed, usn)?; - } - Ok(changed) - } - - /// Currently this is all config, as legacy clients overwrite the local items - /// with the provided value. - fn changed_config(&self) -> Result> { - let conf = self.storage.get_all_config()?; - self.storage.clear_config_usns()?; - Ok(conf) - } - - // Remote->local unchunked changes - //---------------------------------------------------------------- - - fn apply_changes(&mut self, remote: UnchunkedChanges, latest_usn: Usn) -> Result<()> { - self.merge_notetypes(remote.notetypes, latest_usn)?; - self.merge_decks(remote.decks_and_config.decks, latest_usn)?; - self.merge_deck_config(remote.decks_and_config.config)?; - self.merge_tags(remote.tags, latest_usn)?; - if let Some(crt) = remote.creation_stamp { - self.set_creation_stamp(crt)?; - } - if let Some(config) = remote.config { - self.storage - .set_all_config(config, latest_usn, TimestampSecs::now())?; - } - - Ok(()) - } - - fn merge_notetypes(&mut self, notetypes: Vec, latest_usn: Usn) -> Result<()> { - for nt in notetypes { - let mut nt: Notetype = nt.into(); - let proceed = if let Some(existing_nt) = self.storage.get_notetype(nt.id)? { - if existing_nt.mtime_secs <= nt.mtime_secs { - if (existing_nt.fields.len() != nt.fields.len()) - || (existing_nt.templates.len() != nt.templates.len()) - { - return Err(AnkiError::sync_error( - "notetype schema changed", - SyncErrorKind::ResyncRequired, - )); - } - true - } else { - false - } - } else { - true - }; - if proceed { - self.ensure_notetype_name_unique(&mut nt, latest_usn)?; - self.storage.add_or_update_notetype_with_existing_id(&nt)?; - self.state.notetype_cache.remove(&nt.id); - } - } - Ok(()) - } - - fn merge_decks(&mut self, decks: Vec, latest_usn: Usn) -> Result<()> { - for deck in decks { - let proceed = if let Some(existing_deck) = self.storage.get_deck(deck.id())? { - existing_deck.mtime_secs <= deck.common().mtime - } else { - true - }; - if proceed { - let mut deck = deck.into(); - self.ensure_deck_name_unique(&mut deck, latest_usn)?; - self.storage.add_or_update_deck_with_existing_id(&deck)?; - self.state.deck_cache.remove(&deck.id); - } - } - Ok(()) - } - - fn merge_deck_config(&self, dconf: Vec) -> Result<()> { - for conf in dconf { - let proceed = if let Some(existing_conf) = self.storage.get_deck_config(conf.id)? { - existing_conf.mtime_secs <= conf.mtime - } else { - true - }; - if proceed { - let conf = conf.into(); - self.storage - .add_or_update_deck_config_with_existing_id(&conf)?; - } - } - Ok(()) - } - - fn merge_tags(&mut self, tags: Vec, latest_usn: Usn) -> Result<()> { - for tag in tags { - self.register_tag(&mut Tag::new(tag, latest_usn))?; - } - Ok(()) - } - - // Remote->local chunks - //---------------------------------------------------------------- - - /// pending_usn is used to decide whether the local objects are newer. - /// If the provided objects are not modified locally, the USN inside - /// the individual objects is used. - fn apply_chunk(&mut self, chunk: Chunk, pending_usn: Usn) -> Result<()> { - self.merge_revlog(chunk.revlog)?; - self.merge_cards(chunk.cards, pending_usn)?; - self.merge_notes(chunk.notes, pending_usn) - } - - fn merge_revlog(&self, entries: Vec) -> Result<()> { - for entry in entries { - self.storage.add_revlog_entry(&entry, false)?; - } - Ok(()) - } - - fn merge_cards(&self, entries: Vec, pending_usn: Usn) -> Result<()> { - for entry in entries { - self.add_or_update_card_if_newer(entry, pending_usn)?; - } - Ok(()) - } - - fn add_or_update_card_if_newer(&self, entry: CardEntry, pending_usn: Usn) -> Result<()> { - let proceed = if let Some(existing_card) = self.storage.get_card(entry.id)? { - !existing_card.usn.is_pending_sync(pending_usn) || existing_card.mtime < entry.mtime - } else { - true - }; - if proceed { - let card = entry.into(); - self.storage.add_or_update_card(&card)?; - } - Ok(()) - } - - fn merge_notes(&mut self, entries: Vec, pending_usn: Usn) -> Result<()> { - for entry in entries { - self.add_or_update_note_if_newer(entry, pending_usn)?; - } - Ok(()) - } - - fn add_or_update_note_if_newer(&mut self, entry: NoteEntry, pending_usn: Usn) -> Result<()> { - let proceed = if let Some(existing_note) = self.storage.get_note(entry.id)? { - !existing_note.usn.is_pending_sync(pending_usn) || existing_note.mtime < entry.mtime - } else { - true - }; - if proceed { - let mut note: Note = entry.into(); - let nt = self - .get_notetype(note.notetype_id)? - .or_invalid("note missing notetype")?; - note.prepare_for_update(&nt, false)?; - self.storage.add_or_update_note(¬e)?; - } - Ok(()) - } - - // Local->remote chunks - //---------------------------------------------------------------- - - fn get_chunkable_ids(&self, pending_usn: Usn) -> Result { - Ok(ChunkableIds { - revlog: self.storage.objects_pending_sync("revlog", pending_usn)?, - cards: self.storage.objects_pending_sync("cards", pending_usn)?, - notes: self.storage.objects_pending_sync("notes", pending_usn)?, - }) - } - - /// Fetch a chunk of ids from `ids`, returning the referenced objects. - fn get_chunk(&self, ids: &mut ChunkableIds, new_usn: Option) -> Result { - // get a bunch of IDs - let mut limit = CHUNK_SIZE as i32; - let mut revlog_ids = vec![]; - let mut card_ids = vec![]; - let mut note_ids = vec![]; - let mut chunk = Chunk::default(); - while limit > 0 { - let last_limit = limit; - if let Some(id) = ids.revlog.pop() { - revlog_ids.push(id); - limit -= 1; - } - if let Some(id) = ids.notes.pop() { - note_ids.push(id); - limit -= 1; - } - if let Some(id) = ids.cards.pop() { - card_ids.push(id); - limit -= 1; - } - if limit == last_limit { - // all empty - break; - } - } - if limit > 0 { - chunk.done = true; - } - - // remove pending status - if !self.server { - self.storage - .maybe_update_object_usns("revlog", &revlog_ids, new_usn)?; - self.storage - .maybe_update_object_usns("cards", &card_ids, new_usn)?; - self.storage - .maybe_update_object_usns("notes", ¬e_ids, new_usn)?; - } - - // the fetch associated objects, and return - chunk.revlog = revlog_ids - .into_iter() - .map(|id| { - self.storage.get_revlog_entry(id).map(|e| { - let mut e = e.unwrap(); - e.usn = new_usn.unwrap_or(e.usn); - e - }) - }) - .collect::>()?; - chunk.cards = card_ids - .into_iter() - .map(|id| { - self.storage.get_card(id).map(|e| { - let mut e: CardEntry = e.unwrap().into(); - e.usn = new_usn.unwrap_or(e.usn); - e - }) - }) - .collect::>()?; - chunk.notes = note_ids - .into_iter() - .map(|id| { - self.storage.get_note(id).map(|e| { - let mut e: NoteEntry = e.unwrap().into(); - e.usn = new_usn.unwrap_or(e.usn); - e - }) - }) - .collect::>()?; - - Ok(chunk) - } - - // Final steps - //---------------------------------------------------------------- - - fn add_due_counts(&mut self, counts: &mut SanityCheckDueCounts) -> Result<()> { - if let Some(tree) = self.current_deck_tree()? { - counts.new = tree.new_count; - counts.review = tree.review_count; - counts.learn = tree.learn_count; - } - Ok(()) - } - - fn finalize_sync(&self, state: &SyncState, new_server_mtime: TimestampMillis) -> Result<()> { - self.storage.set_last_sync(new_server_mtime)?; - let mut usn = state.latest_usn; - usn.0 += 1; - self.storage.set_usn(usn)?; - self.storage.set_modified_time(new_server_mtime) - } -} - -impl From for Card { - fn from(e: CardEntry) -> Self { - let CardData { - original_position, - custom_data, - } = CardData::from_str(&e.data); - Card { - id: e.id, - note_id: e.nid, - deck_id: e.did, - template_idx: e.ord, - mtime: e.mtime, - usn: e.usn, - ctype: e.ctype, - queue: e.queue, - due: e.due, - interval: e.ivl, - ease_factor: e.factor, - reps: e.reps, - lapses: e.lapses, - remaining_steps: e.left, - original_due: e.odue, - original_deck_id: e.odid, - flags: e.flags, - original_position, - custom_data, - } - } -} - -impl From for CardEntry { - fn from(e: Card) -> Self { - CardEntry { - id: e.id, - nid: e.note_id, - did: e.deck_id, - ord: e.template_idx, - mtime: e.mtime, - usn: e.usn, - ctype: e.ctype, - queue: e.queue, - due: e.due, - ivl: e.interval, - factor: e.ease_factor, - reps: e.reps, - lapses: e.lapses, - left: e.remaining_steps, - odue: e.original_due, - odid: e.original_deck_id, - flags: e.flags, - data: card_data_string(&e), - } - } -} - -impl From for Note { - fn from(e: NoteEntry) -> Self { - let fields = e.fields.split('\x1f').map(ToString::to_string).collect(); - Note::new_from_storage( - e.id, - e.guid, - e.ntid, - e.mtime, - e.usn, - split_tags(&e.tags).map(ToString::to_string).collect(), - fields, - None, - None, - ) - } -} - -impl From for NoteEntry { - fn from(e: Note) -> Self { - NoteEntry { - id: e.id, - fields: e.fields().iter().join("\x1f"), - guid: e.guid, - ntid: e.notetype_id, - mtime: e.mtime, - usn: e.usn, - tags: join_tags(&e.tags), - sfld: String::new(), - csum: String::new(), - flags: 0, - data: String::new(), - } - } -} - -impl From for SyncOutput { - fn from(s: SyncState) -> Self { - SyncOutput { - required: s.required, - server_message: s.server_message, - host_number: s.host_number, - } - } -} - -impl From for SyncStatusResponse { - fn from(r: sync_status_response::Required) -> Self { - SyncStatusResponse { required: r.into() } - } -} - -impl From for sync_status_response::Required { - fn from(r: SyncActionRequired) -> Self { - match r { - SyncActionRequired::NoChanges => sync_status_response::Required::NoChanges, - SyncActionRequired::FullSyncRequired { .. } => sync_status_response::Required::FullSync, - SyncActionRequired::NormalSyncRequired => sync_status_response::Required::NormalSync, - } - } -} - -#[cfg(test)] -mod test { - use std::path::Path; - - use async_trait::async_trait; - use lazy_static::lazy_static; - use tempfile::{tempdir, TempDir}; - use tokio::runtime::Runtime; - - use super::{server::LocalServer, *}; - use crate::{ - collection::CollectionBuilder, deckconfig::DeckConfig, decks::DeckKind, - notetype::all_stock_notetypes, search::SortMode, - }; - - fn norm_progress(_: NormalSyncProgress, _: bool) {} - - fn full_progress(_: FullSyncProgress, _: bool) {} - - #[test] - /// Run remote tests if hkey provided in environment; otherwise local. - fn syncing() -> Result<()> { - let ctx: Box = if let Ok(hkey) = std::env::var("TEST_HKEY") { - Box::new(RemoteTestContext { - auth: SyncAuth { - hkey, - host_number: 0, - }, - }) - } else { - Box::new(LocalTestContext {}) - }; - let rt = Runtime::new().unwrap(); - rt.block_on(upload_download(&ctx))?; - rt.block_on(regular_sync(&ctx)) - } - - #[async_trait(?Send)] - trait TestContext { - fn server(&self) -> Box; - - fn col1(&self) -> Collection { - CollectionBuilder::new(self.dir().join("col1.anki2")) - .build() - .unwrap() - } - - fn col2(&self) -> Collection { - CollectionBuilder::new(self.dir().join("col2.anki2")) - .build() - .unwrap() - } - - fn dir(&self) -> &Path { - lazy_static! { - static ref DIR: TempDir = tempdir().unwrap(); - } - DIR.path() - } - - async fn normal_sync(&self, col: &mut Collection) -> SyncOutput { - NormalSyncer::new(col, self.server(), norm_progress) - .sync() - .await - .unwrap() - } - - async fn full_upload(&self, col: Collection) { - col.full_upload_inner(self.server()).await.unwrap() - } - - async fn full_download(&self, col: Collection) { - col.full_download_inner(self.server()).await.unwrap() - } - } - - // Local specifics - ///////////////////// - - struct LocalTestContext {} - - #[async_trait(?Send)] - impl TestContext for LocalTestContext { - fn server(&self) -> Box { - let col_path = self.dir().join("server.anki2"); - let col = CollectionBuilder::new(col_path) - .set_server(true) - .build() - .unwrap(); - Box::new(LocalServer::new(col)) - } - } - - // Remote specifics - ///////////////////// - - struct RemoteTestContext { - auth: SyncAuth, - } - - impl RemoteTestContext { - fn server_inner(&self) -> HttpSyncClient { - let auth = self.auth.clone(); - HttpSyncClient::new(Some(auth.hkey), auth.host_number) - } - } - - #[async_trait(?Send)] - impl TestContext for RemoteTestContext { - fn server(&self) -> Box { - Box::new(self.server_inner()) - } - - async fn full_upload(&self, col: Collection) { - let mut server = self.server_inner(); - server.set_full_sync_progress_fn(Some(Box::new(full_progress))); - col.full_upload_inner(Box::new(server)).await.unwrap() - } - - async fn full_download(&self, col: Collection) { - let mut server = self.server_inner(); - server.set_full_sync_progress_fn(Some(Box::new(full_progress))); - col.full_download_inner(Box::new(server)).await.unwrap() - } - } - - // Setup + full syncs - ///////////////////// - - fn col1_setup(col: &mut Collection) { - let nt = col.get_notetype_by_name("Basic").unwrap().unwrap(); - let mut note = nt.new_note(); - note.set_field(0, "1").unwrap(); - col.add_note(&mut note, DeckId(1)).unwrap(); - - // // set our schema time back, so when initial server - // // col is created, it's not identical - // col.storage - // .db - // .execute_batch("update col set scm = 123") - // .unwrap() - } - - #[allow(clippy::borrowed_box)] - async fn upload_download(ctx: &Box) -> Result<()> { - let mut col1 = ctx.col1(); - col1_setup(&mut col1); - - let out = ctx.normal_sync(&mut col1).await; - assert!(matches!( - out.required, - SyncActionRequired::FullSyncRequired { .. } - )); - - ctx.full_upload(col1).await; - - // another collection - let mut col2 = ctx.col2(); - - // won't allow ankiweb clobber - let out = ctx.normal_sync(&mut col2).await; - assert_eq!( - out.required, - SyncActionRequired::FullSyncRequired { - upload_ok: false, - download_ok: true - } - ); - - // fetch so we're in sync - ctx.full_download(col2).await; - - Ok(()) - } - - // Regular syncs - ///////////////////// - - #[allow(clippy::borrowed_box)] - async fn regular_sync(ctx: &Box) -> Result<()> { - // add a deck - let mut col1 = ctx.col1(); - let mut col2 = ctx.col2(); - - let mut deck = col1.get_or_create_normal_deck("new deck")?; - - // give it a new option group - let mut dconf = DeckConfig { - name: "new dconf".into(), - ..Default::default() - }; - col1.add_or_update_deck_config(&mut dconf)?; - if let DeckKind::Normal(deck) = &mut deck.kind { - deck.config_id = dconf.id.0; - } - col1.add_or_update_deck(&mut deck)?; - - // and a new notetype - let mut nt = all_stock_notetypes(&col1.tr).remove(0); - nt.name = "new".into(); - col1.add_notetype(&mut nt, false)?; - - // add another note+card+tag - let mut note = nt.new_note(); - note.set_field(0, "2")?; - note.tags.push("tag".into()); - col1.add_note(&mut note, deck.id)?; - - // mock revlog entry - col1.storage.add_revlog_entry( - &RevlogEntry { - id: RevlogId(123), - cid: CardId(456), - usn: Usn(-1), - interval: 10, - ..Default::default() - }, - true, - )?; - - // config + creation - col1.set_config("test", &"test1")?; - // bumping this will affect 'last studied at' on decks at the moment - // col1.storage.set_creation_stamp(TimestampSecs(12345))?; - - // and sync our changes - let remote_meta = ctx.server().meta().await.unwrap(); - let out = col1.get_sync_status(remote_meta)?; - assert_eq!(out, sync_status_response::Required::NormalSync); - - let out = ctx.normal_sync(&mut col1).await; - assert_eq!(out.required, SyncActionRequired::NoChanges); - - // sync the other collection - let out = ctx.normal_sync(&mut col2).await; - assert_eq!(out.required, SyncActionRequired::NoChanges); - - let ntid = nt.id; - let deckid = deck.id; - let dconfid = dconf.id; - let noteid = note.id; - let cardid = col1.search_cards(note.id, SortMode::NoOrder)?[0]; - let revlogid = RevlogId(123); - - let compare_sides = |col1: &mut Collection, col2: &mut Collection| -> Result<()> { - assert_eq!( - col1.get_notetype(ntid)?.unwrap(), - col2.get_notetype(ntid)?.unwrap() - ); - assert_eq!( - col1.get_deck(deckid)?.unwrap(), - col2.get_deck(deckid)?.unwrap() - ); - assert_eq!( - col1.get_deck_config(dconfid, false)?.unwrap(), - col2.get_deck_config(dconfid, false)?.unwrap() - ); - assert_eq!( - col1.storage.get_note(noteid)?.unwrap(), - col2.storage.get_note(noteid)?.unwrap() - ); - assert_eq!( - col1.storage.get_card(cardid)?.unwrap(), - col2.storage.get_card(cardid)?.unwrap() - ); - assert_eq!( - col1.storage.get_revlog_entry(revlogid)?, - col2.storage.get_revlog_entry(revlogid)?, - ); - assert_eq!( - col1.storage.get_all_config()?, - col2.storage.get_all_config()? - ); - assert_eq!( - col1.storage.creation_stamp()?, - col2.storage.creation_stamp()? - ); - - // server doesn't send tag usns, so we can only compare tags, not usns, - // as the usns may not match - assert_eq!( - col1.storage - .all_tags()? - .into_iter() - .map(|t| t.name) - .collect::>(), - col2.storage - .all_tags()? - .into_iter() - .map(|t| t.name) - .collect::>() - ); - std::thread::sleep(std::time::Duration::from_millis(1)); - Ok(()) - }; - - // make sure everything has been transferred across - compare_sides(&mut col1, &mut col2)?; - - // make some modifications - let mut note = col2.storage.get_note(note.id)?.unwrap(); - note.set_field(1, "new")?; - note.tags.push("tag2".into()); - col2.update_note(&mut note)?; - - col2.get_and_update_card(cardid, |card| { - card.queue = CardQueue::Review; - Ok(()) - })?; - - let mut deck = col2.storage.get_deck(deck.id)?.unwrap(); - deck.name = NativeDeckName::from_native_str("newer"); - col2.add_or_update_deck(&mut deck)?; - - let mut nt = col2.storage.get_notetype(nt.id)?.unwrap(); - nt.name = "newer".into(); - col2.update_notetype(&mut nt, false)?; - - // sync the changes back - let out = ctx.normal_sync(&mut col2).await; - assert_eq!(out.required, SyncActionRequired::NoChanges); - let out = ctx.normal_sync(&mut col1).await; - assert_eq!(out.required, SyncActionRequired::NoChanges); - - // should still match - compare_sides(&mut col1, &mut col2)?; - - // deletions should sync too - for table in &["cards", "notes", "decks"] { - assert_eq!( - col1.storage - .db_scalar::(&format!("select count() from {}", table))?, - 2 - ); - } - - // fixme: inconsistent usn arg - std::thread::sleep(std::time::Duration::from_millis(1)); - col1.remove_cards_and_orphaned_notes(&[cardid])?; - let usn = col1.usn()?; - col1.remove_note_only_undoable(noteid, usn)?; - col1.remove_decks_and_child_decks(&[deckid])?; - - let out = ctx.normal_sync(&mut col1).await; - assert_eq!(out.required, SyncActionRequired::NoChanges); - let out = ctx.normal_sync(&mut col2).await; - assert_eq!(out.required, SyncActionRequired::NoChanges); - - for table in &["cards", "notes", "decks"] { - assert_eq!( - col2.storage - .db_scalar::(&format!("select count() from {}", table))?, - 1 - ); - } - - // removing things like a notetype forces a full sync - std::thread::sleep(std::time::Duration::from_millis(1)); - col2.remove_notetype(ntid)?; - let out = ctx.normal_sync(&mut col2).await; - assert!(matches!( - out.required, - SyncActionRequired::FullSyncRequired { .. } - )); - Ok(()) - } - - // Helper to reproduce issues with a copy of the client and server collections. - // #[test] - // fn repro_test() { - // let rt = Runtime::new().unwrap(); - // rt.block_on(repro_test_inner()).unwrap(); - // } - - // async fn repro_test_inner() -> Result<()> { - // let client_fname = "/path/to/collection1.anki2"; - // let server_fname = "/path/to/collection2.anki2"; - - // use std::env::temp_dir; - // use std::fs; - // use tempfile::NamedTempFile; - - // let client_col_file = new_named_tempfile()?; - // let client_col_name = client_col_file - // .path() - // .file_name() - // .unwrap() - // .to_string_lossy(); - // fs::copy(client_fname, client_col_file.path())?; - // let server_col_file = new_named_tempfile()?; - // let server_col_name = server_col_file - // .path() - // .file_name() - // .unwrap() - // .to_string_lossy(); - // fs::copy(server_fname, server_col_file.path())?; - // let dir = temp_dir(); - // let server = Box::new(LocalServer::new(open_col(&dir, true, &server_col_name)?)); - // let mut client_col = open_col(&dir, false, &client_col_name)?; - // NormalSyncer::new(&mut client_col, server, norm_progress) - // .sync() - // .await - // .unwrap(); - - // Ok(()) - // } -} +pub mod collection; +pub mod error; +pub mod http_client; +pub mod http_server; +pub mod login; +pub mod media; +pub mod request; +pub mod response; +pub mod version; diff --git a/rslib/src/sync/request/header_and_stream.rs b/rslib/src/sync/request/header_and_stream.rs new file mode 100644 index 000000000..2de54cb30 --- /dev/null +++ b/rslib/src/sync/request/header_and_stream.rs @@ -0,0 +1,155 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::{ + fmt::Display, + io::{Cursor, ErrorKind}, + marker::PhantomData, + net::IpAddr, +}; + +use axum::{ + extract::BodyStream, + headers::{Header, HeaderName, HeaderValue}, + http::StatusCode, +}; +use bytes::Bytes; +use futures::{Stream, TryStreamExt}; +use serde::de::DeserializeOwned; +use serde_derive::{Deserialize, Serialize}; +use tokio::io::AsyncReadExt; +use tokio_util::io::ReaderStream; + +use crate::sync::{ + error::{HttpResult, HttpSnafu, OrHttpErr}, + request::{SyncRequest, MAXIMUM_SYNC_PAYLOAD_BYTES_UNCOMPRESSED}, + version::SyncVersion, +}; + +impl SyncRequest { + pub(super) async fn from_header_and_stream( + sync_header: SyncHeader, + body_stream: BodyStream, + ip: IpAddr, + ) -> HttpResult> + where + T: DeserializeOwned, + { + sync_header.sync_version.ensure_supported()?; + let data = decode_zstd_body(body_stream).await?; + Ok(Self { + sync_key: sync_header.sync_key, + session_key: sync_header.session_key, + media_client_version: None, + data, + ip, + json_output_type: PhantomData, + sync_version: sync_header.sync_version, + client_version: sync_header.client_ver, + }) + } +} + +pub async fn decode_zstd_body(data: S) -> HttpResult> +where + S: Stream> + Unpin, + E: Display, +{ + let reader = tokio_util::io::StreamReader::new( + data.map_err(|e| std::io::Error::new(ErrorKind::ConnectionAborted, format!("{e}"))), + ); + let reader = async_compression::tokio::bufread::ZstdDecoder::new(reader); + let mut buf: Vec = vec![]; + reader + .take(*MAXIMUM_SYNC_PAYLOAD_BYTES_UNCOMPRESSED) + .read_to_end(&mut buf) + .await + .or_bad_request("decoding zstd body")?; + Ok(buf) +} + +pub fn decode_zstd_body_stream(data: S) -> impl Stream> +where + S: Stream> + Unpin, + E: Display, +{ + let reader = tokio_util::io::StreamReader::new( + data.map_err(|e| std::io::Error::new(ErrorKind::ConnectionAborted, format!("{e}"))), + ); + let reader = async_compression::tokio::bufread::ZstdDecoder::new(reader); + ReaderStream::new(reader.take(*MAXIMUM_SYNC_PAYLOAD_BYTES_UNCOMPRESSED)).map_err(|err| { + HttpSnafu { + code: StatusCode::BAD_REQUEST, + context: "decode zstd body", + source: Some(Box::new(err) as _), + } + .build() + }) +} + +pub fn encode_zstd_body(data: Vec) -> impl Stream> + Unpin { + let enc = async_compression::tokio::bufread::ZstdEncoder::new(Cursor::new(data)); + ReaderStream::new(enc).map_err(|err| { + HttpSnafu { + code: StatusCode::INTERNAL_SERVER_ERROR, + context: "encode zstd body", + source: Some(Box::new(err) as _), + } + .build() + }) +} + +pub fn encode_zstd_body_stream(data: S) -> impl Stream> +where + S: Stream> + Unpin, + E: Display, +{ + let reader = tokio_util::io::StreamReader::new( + data.map_err(|e| std::io::Error::new(ErrorKind::ConnectionAborted, format!("{e}"))), + ); + let reader = async_compression::tokio::bufread::ZstdEncoder::new(reader); + ReaderStream::new(reader).map_err(|err| { + HttpSnafu { + code: StatusCode::BAD_REQUEST, + context: "encode zstd body", + source: Some(Box::new(err) as _), + } + .build() + }) +} + +#[derive(Serialize, Deserialize)] +pub struct SyncHeader { + #[serde(rename = "v")] + pub sync_version: SyncVersion, + #[serde(rename = "k")] + pub sync_key: String, + #[serde(rename = "c")] + pub client_ver: String, + #[serde(rename = "s")] + pub session_key: String, +} + +pub static SYNC_HEADER_NAME: HeaderName = HeaderName::from_static("anki-sync"); + +impl Header for SyncHeader { + fn name() -> &'static HeaderName { + &SYNC_HEADER_NAME + } + + fn decode<'i, I>(values: &mut I) -> Result + where + Self: Sized, + I: Iterator, + { + values + .next() + .and_then(|value| value.to_str().ok()) + .and_then(|s| serde_json::from_str(s).ok()) + .ok_or_else(axum::headers::Error::invalid) + } + + fn encode>(&self, _values: &mut E) { + todo!() + } +} diff --git a/rslib/src/sync/request/mod.rs b/rslib/src/sync/request/mod.rs new file mode 100644 index 000000000..b6f0c7a37 --- /dev/null +++ b/rslib/src/sync/request/mod.rs @@ -0,0 +1,187 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +pub mod header_and_stream; +mod multipart; + +use std::{any::Any, env, marker::PhantomData, net::IpAddr}; + +use async_trait::async_trait; +use axum::{ + extract::{BodyStream, FromRequest, Multipart}, + http::Request, + RequestPartsExt, TypedHeader, +}; +use axum_client_ip::ClientIp; +use header_and_stream::SyncHeader; +use hyper::Body; +use once_cell::sync::Lazy; +use serde::{de::DeserializeOwned, Serialize}; +use serde_json::Error; +use tracing::Span; + +use crate::{ + sync::{ + error::{HttpError, HttpResult, OrHttpErr}, + version::SyncVersion, + }, + version::sync_client_version_short, +}; + +/// Stores the bytes of a sync request, the associated type they +/// represent, and authentication info provided in headers/multipart +/// forms. For a SyncRequest, you can call .json() to get a Foo +/// struct from the bytes. +#[derive(Clone)] +pub struct SyncRequest { + pub data: Vec, + json_output_type: PhantomData, + pub sync_version: SyncVersion, + /// empty with older clients + pub client_version: String, + pub ip: IpAddr, + /// Non-empty on every non-login request. + pub sync_key: String, + /// May not be set on some requests by legacy clients. Used by stateful sync methods to check + /// for concurrent access. + pub session_key: String, + /// Set by legacy clients when posting to msync/begin + pub media_client_version: Option, +} + +impl SyncRequest +where + T: DeserializeOwned, +{ + pub fn from_data( + data: Vec, + host_key: String, + session_key: String, + ip: IpAddr, + sync_version: SyncVersion, + ) -> SyncRequest { + SyncRequest { + data, + json_output_type: Default::default(), + ip, + sync_key: host_key, + session_key, + media_client_version: None, + sync_version, + client_version: String::new(), + } + } + + /// Given a generic Self>, infer the actual type based on context. + pub fn into_output_type(self) -> SyncRequest { + SyncRequest { + data: self.data, + json_output_type: PhantomData, + ip: self.ip, + sync_key: self.sync_key, + session_key: self.session_key, + media_client_version: self.media_client_version, + sync_version: self.sync_version, + client_version: self.client_version, + } + } + + pub fn json(&self) -> HttpResult { + serde_json::from_slice(&self.data).or_bad_request("invalid json") + } + + pub fn skey(&self) -> HttpResult<&str> { + if self.session_key.is_empty() { + None.or_bad_request("missing skey")?; + } + Ok(&self.session_key) + } +} + +#[async_trait] +impl FromRequest for SyncRequest +where + S: Send + Sync, + T: DeserializeOwned, +{ + type Rejection = HttpError; + + async fn from_request(req: Request, state: &S) -> HttpResult { + let (mut parts, body) = req.into_parts(); + + let ClientIp(ip) = parts + .extract() + .await + .ok() + .or_internal_err("unable to get ip")?; + Span::current().record("ip", ip.to_string()); + + let sync_header: Option> = + parts.extract().await.or_bad_request("bad sync header")?; + let req = Request::from_parts(parts, body); + + if let Some(TypedHeader(sync_header)) = sync_header { + let stream = BodyStream::from_request(req, state) + .await + .expect("infallible"); + SyncRequest::from_header_and_stream(sync_header, stream, ip).await + } else { + let multi = Multipart::from_request(req, state) + .await + .or_bad_request("multipart")?; + SyncRequest::from_multipart(multi, ip).await + } + } +} + +pub trait IntoSyncRequest { + fn try_into_sync_request(self) -> Result, serde_json::Error> + where + Self: Sized + 'static; +} + +impl IntoSyncRequest for T +where + T: Serialize, +{ + fn try_into_sync_request(self) -> Result, Error> + where + Self: Sized + 'static, + { + // A not-very-elegant workaround for the fact that a separate impl for vec would + // conflict with this generic one. + let is_data = (&self as &dyn Any).is::>(); + let data = if is_data { + let boxed_self = (Box::new(self) as Box) + .downcast::>() + .unwrap(); + *boxed_self + } else { + serde_json::to_vec(&self)? + }; + Ok(SyncRequest { + data, + json_output_type: PhantomData, + ip: IpAddr::from([0, 0, 0, 0]), + media_client_version: None, + sync_version: SyncVersion::latest(), + client_version: sync_client_version_short().to_string(), + // injected by client.request() + sync_key: String::new(), + session_key: String::new(), + }) + } +} + +pub static MAXIMUM_SYNC_PAYLOAD_BYTES: Lazy = Lazy::new(|| { + env::var("MAX_SYNC_PAYLOAD_MEGS") + .map(|v| v.parse().expect("invalid upload limit")) + .unwrap_or(100) + * 1024 + * 1024 +}); +/// Client ignores this when a non-AnkiWeb endpoint is configured. Controls the maximum +/// size of a payload after decompression, which effectively limits the how large a collection +/// file can be uploaded. +pub static MAXIMUM_SYNC_PAYLOAD_BYTES_UNCOMPRESSED: Lazy = + Lazy::new(|| (*MAXIMUM_SYNC_PAYLOAD_BYTES * 3) as u64); diff --git a/rslib/src/sync/request/multipart.rs b/rslib/src/sync/request/multipart.rs new file mode 100644 index 000000000..85babee8a --- /dev/null +++ b/rslib/src/sync/request/multipart.rs @@ -0,0 +1,98 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::{io::Read, marker::PhantomData, net::IpAddr}; + +use axum::extract::Multipart; +use bytes::{Buf, Bytes}; +use flate2::read::GzDecoder; +use tokio::task::spawn_blocking; + +use crate::sync::{ + error::{HttpResult, OrHttpErr}, + request::{SyncRequest, MAXIMUM_SYNC_PAYLOAD_BYTES_UNCOMPRESSED}, + version::{SyncVersion, SYNC_VERSION_10_V2_TIMEZONE}, +}; + +impl SyncRequest { + pub(super) async fn from_multipart( + mut multi: Multipart, + ip: IpAddr, + ) -> HttpResult> { + let mut host_key = String::new(); + let mut session_key = String::new(); + let mut media_client_version = None; + let mut compressed = false; + let mut data = None; + while let Some(field) = multi + .next_field() + .await + .or_bad_request("invalid multipart")? + { + match field.name() { + Some("c") => { + // normal syncs should always be compressed, but media syncs may compress the + // zip instead + let c = field.text().await.or_bad_request("malformed c")?; + compressed = c != "0"; + } + Some("k") | Some("sk") => { + host_key = field.text().await.or_bad_request("malformed (s)k")?; + } + Some("s") => session_key = field.text().await.or_bad_request("malformed s")?, + Some("v") => { + media_client_version = Some(field.text().await.or_bad_request("malformed v")?) + } + Some("data") => { + data = Some( + field + .bytes() + .await + .or_bad_request("missing data for multi")?, + ) + } + _ => {} + } + } + let data = { + let data = data.unwrap_or_default(); + if data.is_empty() { + // AnkiDroid omits 'data' when downloading + b"{}".to_vec() + } else if compressed { + decode_gzipped_data(data).await? + } else { + data.to_vec() + } + }; + Ok(Self { + ip, + sync_key: host_key, + session_key, + media_client_version, + data, + json_output_type: PhantomData, + // may be lower - the old protocol didn't provide the version on every request + sync_version: SyncVersion(SYNC_VERSION_10_V2_TIMEZONE), + client_version: String::new(), + }) + } +} + +pub async fn decode_gzipped_data(data: Bytes) -> HttpResult> { + // actix uses this threshold, so presumably they've measured + if data.len() < 2049 { + decode_gzipped_data_inner(data) + } else { + spawn_blocking(move || decode_gzipped_data_inner(data)) + .await + .or_internal_err("decode gzip join")? + } +} + +fn decode_gzipped_data_inner(data: Bytes) -> HttpResult> { + let mut gz = GzDecoder::new(data.reader()).take(*MAXIMUM_SYNC_PAYLOAD_BYTES_UNCOMPRESSED); + let mut data = Vec::new(); + gz.read_to_end(&mut data).or_bad_request("invalid gzip")?; + Ok(data) +} diff --git a/rslib/src/sync/response.rs b/rslib/src/sync/response.rs new file mode 100644 index 000000000..d00f3e2a9 --- /dev/null +++ b/rslib/src/sync/response.rs @@ -0,0 +1,90 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use std::marker::PhantomData; + +use axum::{ + body::StreamBody, + headers::HeaderName, + response::{IntoResponse, Response}, +}; +use serde::{de::DeserializeOwned, Serialize}; + +use crate::{ + prelude::*, + sync::{ + collection::upload::UploadResponse, + error::{HttpResult, OrHttpErr}, + request::header_and_stream::encode_zstd_body, + version::SyncVersion, + }, +}; + +pub static ORIGINAL_SIZE: HeaderName = HeaderName::from_static("anki-original-size"); + +/// Stores the data returned from a sync request, and the type +/// it represents. Given a SyncResponse, you can get a Foo +/// struct via .json(), except for uploads/downloads. +#[derive(Debug)] +pub struct SyncResponse { + pub data: Vec, + json_output_type: PhantomData, +} + +impl SyncResponse { + pub fn from_vec(data: Vec) -> SyncResponse { + SyncResponse { + data, + json_output_type: Default::default(), + } + } + + pub fn make_response(self, sync_version: SyncVersion) -> Response { + if sync_version.is_zstd() { + let header = (&ORIGINAL_SIZE, self.data.len().to_string()); + let body = StreamBody::new(encode_zstd_body(self.data)); + ([header], body).into_response() + } else { + self.data.into_response() + } + } +} + +impl SyncResponse { + // Unfortunately the sync protocol sends this as a bare string + // instead of JSON. + pub fn upload_response(&self) -> UploadResponse { + let resp = String::from_utf8_lossy(&self.data); + match resp.as_ref() { + "OK" => UploadResponse::Ok, + other => UploadResponse::Err(other.into()), + } + } + + pub fn from_upload_response(resp: UploadResponse) -> Self { + let text = match resp { + UploadResponse::Ok => "OK".into(), + UploadResponse::Err(other) => other, + }; + SyncResponse::from_vec(text.into_bytes()) + } +} + +impl SyncResponse +where + T: Serialize, +{ + pub fn try_from_obj(obj: T) -> HttpResult> { + let data = serde_json::to_vec(&obj).or_internal_err("couldn't serialize object")?; + Ok(SyncResponse::from_vec(data)) + } +} + +impl SyncResponse +where + T: DeserializeOwned, +{ + pub fn json(&self) -> Result { + serde_json::from_slice(&self.data).map_err(Into::into) + } +} diff --git a/rslib/src/sync/server.rs b/rslib/src/sync/server.rs deleted file mode 100644 index 377f7515a..000000000 --- a/rslib/src/sync/server.rs +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright: Ankitects Pty Ltd and contributors -// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html - -use std::{fs, path::Path}; - -use async_trait::async_trait; -use tempfile::NamedTempFile; - -use super::ChunkableIds; -use crate::{ - collection::CollectionBuilder, - io::new_tempfile, - prelude::*, - storage::{open_and_check_sqlite_file, SchemaVersion}, - sync::{ - Chunk, Graves, SanityCheckCounts, SanityCheckResponse, SanityCheckStatus, SyncMeta, - UnchunkedChanges, Usn, - }, -}; -#[async_trait(?Send)] -pub trait SyncServer { - async fn meta(&self) -> Result; - async fn start( - &mut self, - client_usn: Usn, - local_is_newer: bool, - deprecated_client_graves: Option, - ) -> Result; - async fn apply_graves(&mut self, client_chunk: Graves) -> Result<()>; - async fn apply_changes(&mut self, client_changes: UnchunkedChanges) - -> Result; - async fn chunk(&mut self) -> Result; - async fn apply_chunk(&mut self, client_chunk: Chunk) -> Result<()>; - async fn sanity_check(&mut self, client: SanityCheckCounts) -> Result; - async fn finish(&mut self) -> Result; - async fn abort(&mut self) -> Result<()>; - - /// If `can_consume` is true, the local server will move or remove the file, instead - /// creating a copy. The remote server ignores this argument. - async fn full_upload(self: Box, col_path: &Path, can_consume: bool) -> Result<()>; - /// If the calling code intends to .persist() the named temp file to - /// atomically update the collection, it should pass in the collection's - /// folder, as .persist() can't work across filesystems. - async fn full_download(self: Box, temp_folder: Option<&Path>) -> Result; -} - -pub struct LocalServer { - col: Collection, - - // The current sync protocol is stateful, so unfortunately we need to - // retain a bunch of information across requests. These are set either - // on start, or on subsequent methods. - server_usn: Usn, - client_usn: Usn, - /// Only used to determine whether we should send our - /// config to client. - client_is_newer: bool, - /// Set on the first call to chunk() - server_chunk_ids: Option, -} - -impl LocalServer { - #[allow(dead_code)] - pub fn new(col: Collection) -> LocalServer { - assert!(col.server); - LocalServer { - col, - server_usn: Usn(0), - client_usn: Usn(0), - client_is_newer: false, - server_chunk_ids: None, - } - } - - /// Consumes self and returns the stored collection. If a sync has begun, caller must ensure they - /// call .finish() or .abort() before calling this. - pub fn into_col(self) -> Collection { - self.col - } -} - -#[async_trait(?Send)] -impl SyncServer for LocalServer { - async fn meta(&self) -> Result { - let stamps = self.col.storage.get_collection_timestamps()?; - Ok(SyncMeta { - modified: stamps.collection_change, - schema: stamps.schema_change, - usn: self.col.storage.usn(true)?, - current_time: TimestampSecs::now(), - server_message: String::new(), - should_continue: true, - host_number: 0, - empty: !self.col.storage.have_at_least_one_card()?, - }) - } - - async fn start( - &mut self, - client_usn: Usn, - client_is_newer: bool, - deprecated_client_graves: Option, - ) -> Result { - self.server_usn = self.col.usn()?; - self.client_usn = client_usn; - self.client_is_newer = client_is_newer; - - self.col.discard_undo_and_study_queues(); - self.col.storage.begin_rust_trx()?; - - // make sure any pending cards have been unburied first if necessary - let timing = self.col.timing_today()?; - self.col.unbury_if_day_rolled_over(timing)?; - - // fetch local graves - let server_graves = self.col.storage.pending_graves(client_usn)?; - // handle AnkiDroid using old protocol - if let Some(graves) = deprecated_client_graves { - self.col.apply_graves(graves, self.server_usn)?; - } - - Ok(server_graves) - } - - async fn apply_graves(&mut self, client_chunk: Graves) -> Result<()> { - self.col.apply_graves(client_chunk, self.server_usn) - } - - async fn apply_changes( - &mut self, - client_changes: UnchunkedChanges, - ) -> Result { - let server_changes = - self.col - .local_unchunked_changes(self.client_usn, None, !self.client_is_newer)?; - self.col.apply_changes(client_changes, self.server_usn)?; - Ok(server_changes) - } - - async fn chunk(&mut self) -> Result { - if self.server_chunk_ids.is_none() { - self.server_chunk_ids = Some(self.col.get_chunkable_ids(self.client_usn)?); - } - - self.col - .get_chunk(self.server_chunk_ids.as_mut().unwrap(), None) - } - - async fn apply_chunk(&mut self, client_chunk: Chunk) -> Result<()> { - self.col.apply_chunk(client_chunk, self.client_usn) - } - - async fn sanity_check(&mut self, mut client: SanityCheckCounts) -> Result { - let mut server = self.col.storage.sanity_check_info()?; - client.counts = Default::default(); - // clients on schema 17 and below may send duplicate - // deletion markers, so we can't compare graves until - // the minimum syncing version is schema 18. - client.graves = 0; - server.graves = 0; - Ok(SanityCheckResponse { - status: if client == server { - SanityCheckStatus::Ok - } else { - SanityCheckStatus::Bad - }, - client: Some(client), - server: Some(server), - }) - } - - async fn finish(&mut self) -> Result { - let now = TimestampMillis::now(); - self.col.storage.set_modified_time(now)?; - self.col.storage.set_last_sync(now)?; - self.col.storage.increment_usn()?; - self.col.storage.commit_rust_trx()?; - Ok(now) - } - - async fn abort(&mut self) -> Result<()> { - self.col.storage.rollback_rust_trx() - } - - /// `col_path` should point to the uploaded file, and the caller is - /// responsible for imposing limits on its size if it wishes. - /// If `can_consume` is true, the provided file will be moved into place, - /// or removed on failure. If false, the original will be left alone. - async fn full_upload( - mut self: Box, - mut col_path: &Path, - can_consume: bool, - ) -> Result<()> { - // create a copy if necessary - let new_file: NamedTempFile; - if !can_consume { - new_file = new_tempfile()?; - fs::copy(col_path, new_file.path())?; - col_path = new_file.path(); - } - - // ensure it's a valid sqlite file, and a valid collection - open_and_check_sqlite_file(col_path) - .and_then(|_| CollectionBuilder::new(col_path).build()) - .map_err(|check_err| match fs::remove_file(col_path) { - Ok(_) => check_err, - Err(remove_err) => remove_err.into(), - })?; - - let target_col_path = self.col.col_path.clone(); - self.col.close(None)?; - fs::rename(col_path, &target_col_path).map_err(Into::into) - } - - /// The provided folder is ignored, as in the server case the local data - /// will be sent over the network, instead of written into a local file. - async fn full_download( - mut self: Box, - _col_folder: Option<&Path>, - ) -> Result { - // bump usn/mod & close - self.col - .transact_no_undo(|col| col.storage.increment_usn())?; - let col_path = self.col.col_path.clone(); - self.col.close(Some(SchemaVersion::V11))?; - - // copy file and return path - let temp_file = new_tempfile()?; - fs::copy(&col_path, temp_file.path())?; - - Ok(temp_file) - } -} diff --git a/rslib/src/sync/version.rs b/rslib/src/sync/version.rs new file mode 100644 index 000000000..a7c6ad5b7 --- /dev/null +++ b/rslib/src/sync/version.rs @@ -0,0 +1,80 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +use serde_derive::{Deserialize, Serialize}; + +use crate::{ + storage::SchemaVersion, + sync::error::{HttpResult, OrHttpErr}, +}; + +pub const SYNC_VERSION_MIN: u8 = SYNC_VERSION_08_SESSIONKEY; +pub const SYNC_VERSION_MAX: u8 = SYNC_VERSION_11_DIRECT_POST; + +/// Added in 2013. Introduced a session key to identify parallel attempts at syncing. +/// At the end of 2022, only used by 0.045% of syncers. Half are AnkiUniversal users, +/// as it never added support for the V2 scheduler. +pub const SYNC_VERSION_08_SESSIONKEY: u8 = 8; + +/// Added Jan 2018. No functional changes to protocol, but marks that the client +/// supports the V2 scheduler. +/// +/// In July 2018 a separate chunked graves method was added, but was optional. At +/// the end of 2022, AnkiDroid is still using the old approach of passing all +/// graves to the start method in the legacy schema path. +pub const SYNC_VERSION_09_V2_SCHEDULER: u8 = 9; + +/// Added Mar 2020. No functional changes to protocol, but marks that the client +/// supports the V2 timezone changes. +pub const SYNC_VERSION_10_V2_TIMEZONE: u8 = 10; + +/// Added Jan 2023. Switches from packaging messages in a multipart request with +/// gzip to using headers and zstd, and stops using a separate session key for +/// media syncs. Schema 18 uploads/downloads are now supported, and hostNum has +/// been deprecated in favour of a redirect. +pub const SYNC_VERSION_11_DIRECT_POST: u8 = 11; + +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[repr(transparent)] +pub struct SyncVersion(pub u8); + +impl SyncVersion { + pub fn is_too_old(&self) -> bool { + self.0 < SYNC_VERSION_MIN + } + + pub fn is_too_new(&self) -> bool { + self.0 > SYNC_VERSION_MAX + } + + pub fn ensure_supported(&self) -> HttpResult<()> { + if self.is_too_old() || self.is_too_new() { + None.or_bad_request(&format!("unsupported sync version: {}", self.0))?; + } + Ok(()) + } + + pub fn latest() -> Self { + SyncVersion(SYNC_VERSION_MAX) + } + + pub fn multipart() -> Self { + Self(SYNC_VERSION_10_V2_TIMEZONE) + } + + pub fn is_multipart(&self) -> bool { + self.0 < SYNC_VERSION_11_DIRECT_POST + } + + pub fn is_zstd(&self) -> bool { + self.0 >= SYNC_VERSION_11_DIRECT_POST + } + + pub fn collection_schema(&self) -> SchemaVersion { + if self.is_multipart() { + SchemaVersion::V11 + } else { + SchemaVersion::V18 + } + } +} diff --git a/rslib/src/tests.rs b/rslib/src/tests.rs index 8033dba5d..81a38d448 100644 --- a/rslib/src/tests.rs +++ b/rslib/src/tests.rs @@ -49,9 +49,8 @@ pub(crate) fn open_test_collection_with_relearning_card() -> Collection { impl Collection { pub(crate) fn add_media(&self, media: &[(&str, &[u8])]) { let mgr = MediaManager::new(&self.media_folder, &self.media_db).unwrap(); - let mut ctx = mgr.dbctx(); for (name, data) in media { - mgr.add_file(&mut ctx, name, data).unwrap(); + mgr.add_file(name, data).unwrap(); } } diff --git a/rslib/src/version.rs b/rslib/src/version.rs index 6cda89c10..dfe7cab4f 100644 --- a/rslib/src/version.rs +++ b/rslib/src/version.rs @@ -24,3 +24,15 @@ pub(crate) fn sync_client_version() -> &'static str { } &VER } + +pub(crate) fn sync_client_version_short() -> &'static str { + lazy_static! { + static ref VER: String = format!( + "{version},{buildhash},{platform}", + version = version(), + buildhash = buildhash(), + platform = env::consts::OS + ); + } + &VER +} diff --git a/tools/workspace-hack/Cargo.toml b/tools/workspace-hack/Cargo.toml index 9bc791fc2..65874e06d 100644 --- a/tools/workspace-hack/Cargo.toml +++ b/tools/workspace-hack/Cargo.toml @@ -19,28 +19,42 @@ bytes = { version = "1", features = ["std"] } crossbeam-utils = { version = "0.8", features = ["std"] } either = { version = "1", features = ["use_std"] } flate2 = { version = "1", features = ["miniz_oxide", "rust_backend"] } +futures-channel = { version = "0.3", features = ["alloc", "futures-sink", "sink", "std"] } +futures-io = { version = "0.3", features = ["std"] } +futures-task = { version = "0.3", default-features = false, features = ["alloc", "std"] } +futures-util = { version = "0.3", default-features = false, features = ["alloc", "async-await", "async-await-macro", "channel", "futures-channel", "futures-io", "futures-macro", "futures-sink", "io", "memchr", "sink", "slab", "std"] } getrandom = { version = "0.2", default-features = false, features = ["std"] } hashbrown = { version = "0.12", features = ["ahash", "inline-more", "raw"] } hmac = { version = "0.12", default-features = false, features = ["reset"] } +hyper = { version = "0.14", features = ["client", "full", "h2", "http1", "http2", "runtime", "server", "socket2", "stream", "tcp"] } indexmap = { version = "1", default-features = false, features = ["std"] } log = { version = "0.4", default-features = false, features = ["std"] } num-traits = { version = "0.2", features = ["std"] } phf_shared = { version = "0.11", default-features = false, features = ["std"] } prost = { version = "0.11", features = ["prost-derive", "std"] } -rand = { version = "0.8", features = ["alloc", "getrandom", "libc", "rand_chacha", "small_rng", "std", "std_rng"] } +rand-c38e5c1d305a1b54 = { package = "rand", version = "0.8", features = ["alloc", "getrandom", "libc", "rand_chacha", "small_rng", "std", "std_rng"] } +rand-ca01ad9e24f5d932 = { package = "rand", version = "0.7", features = ["alloc", "getrandom", "getrandom_package", "libc", "rand_pcg", "small_rng", "std"] } rand_core = { version = "0.6", default-features = false, features = ["alloc", "getrandom", "std"] } regex = { version = "1", features = ["aho-corasick", "memchr", "perf", "perf-cache", "perf-dfa", "perf-inline", "perf-literal", "std", "unicode", "unicode-age", "unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script", "unicode-segment"] } +regex-automata = { version = "0.1", features = ["regex-syntax", "std"] } regex-syntax = { version = "0.6", features = ["unicode", "unicode-age", "unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script", "unicode-segment"] } +reqwest = { version = "0.11", features = ["__rustls", "__tls", "blocking", "default-tls", "hyper-rustls", "hyper-tls", "json", "mime_guess", "multipart", "native-tls", "native-tls-crate", "rustls", "rustls-native-certs", "rustls-pemfile", "rustls-tls", "rustls-tls-native-roots", "rustls-tls-webpki-roots", "serde_json", "socks", "stream", "tokio-native-tls", "tokio-rustls", "tokio-socks", "tokio-util", "webpki-roots"] } scopeguard = { version = "1", features = ["use_std"] } serde = { version = "1", features = ["alloc", "derive", "rc", "serde_derive", "std"] } +serde_json = { version = "1", features = ["raw_value", "std"] } sha2 = { version = "0.10", features = ["std"] } snafu = { version = "0.7", features = ["backtrace", "backtraces", "rust_1_39", "rust_1_46", "std"] } -time = { version = "0.3", features = ["alloc", "formatting", "local-offset", "macros", "parsing", "std"] } +time = { version = "0.3", features = ["alloc", "formatting", "macros", "parsing", "std"] } tokio = { version = "1", features = ["bytes", "fs", "full", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "parking_lot", "process", "rt", "rt-multi-thread", "signal", "signal-hook-registry", "socket2", "sync", "time", "tokio-macros"] } +tokio-util = { version = "0.7", features = ["codec", "io", "tracing"] } +tracing = { version = "0.1", features = ["attributes", "log", "max_level_trace", "release_max_level_debug", "std", "tracing-attributes"] } +tracing-core = { version = "0.1", features = ["once_cell", "std"] } url = { version = "2", features = ["serde"] } zip = { version = "0.6", features = ["aes", "aes-crypto", "bzip2", "constant_time_eq", "deflate", "flate2", "hmac", "pbkdf2", "sha1", "time", "zstd"] } -zstd = { version = "0.12", features = ["arrays", "legacy", "zdict_builder", "zstdmt"] } -zstd-safe = { version = "6", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder", "zstdmt"] } +zstd-5ef9efb8ec2df382 = { package = "zstd", version = "0.12", features = ["arrays", "legacy", "zdict_builder", "zstdmt"] } +zstd-a6292c17cd707f01 = { package = "zstd", version = "0.11", features = ["arrays", "legacy", "zdict_builder"] } +zstd-safe-a490c3000a992113 = { package = "zstd-safe", version = "6", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder", "zstdmt"] } +zstd-safe-cdf1610d3e1514e9 = { package = "zstd-safe", version = "5", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder"] } zstd-sys = { version = "2", default-features = false, features = ["legacy", "std", "zdict_builder", "zstdmt"] } [build-dependencies] @@ -53,12 +67,14 @@ indexmap = { version = "1", default-features = false, features = ["std"] } log = { version = "0.4", default-features = false, features = ["std"] } phf_shared = { version = "0.11", default-features = false, features = ["std"] } prost = { version = "0.11", features = ["prost-derive", "std"] } -rand = { version = "0.8", features = ["alloc", "getrandom", "libc", "rand_chacha", "small_rng", "std", "std_rng"] } +rand-ca01ad9e24f5d932 = { package = "rand", version = "0.7", features = ["alloc", "getrandom", "getrandom_package", "libc", "rand_pcg", "small_rng", "std"] } +rand-c38e5c1d305a1b54 = { package = "rand", version = "0.8", features = ["alloc", "getrandom", "libc", "rand_chacha", "small_rng", "std", "std_rng"] } rand_core = { version = "0.6", default-features = false, features = ["alloc", "getrandom", "std"] } regex = { version = "1", features = ["aho-corasick", "memchr", "perf", "perf-cache", "perf-dfa", "perf-inline", "perf-literal", "std", "unicode", "unicode-age", "unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script", "unicode-segment"] } regex-syntax = { version = "0.6", features = ["unicode", "unicode-age", "unicode-bool", "unicode-case", "unicode-gencat", "unicode-perl", "unicode-script", "unicode-segment"] } serde = { version = "1", features = ["alloc", "derive", "rc", "serde_derive", "std"] } -syn = { version = "1", features = ["clone-impls", "derive", "extra-traits", "fold", "full", "parsing", "printing", "proc-macro", "quote", "visit-mut"] } +serde_json = { version = "1", features = ["raw_value", "std"] } +syn = { version = "1", features = ["clone-impls", "derive", "extra-traits", "fold", "full", "parsing", "printing", "proc-macro", "quote", "visit", "visit-mut"] } time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing"] } ### END HAKARI SECTION