From b37063e20a5ada1047d98be3d8ca4852d613897a Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Thu, 22 Jun 2023 09:38:05 +1000 Subject: [PATCH] More service generation refactoring - Dropped the protobuf extensions in favor of explicitly listing out methods in both services if we want to implement both, as it's clearer. - Move Service/Method wrappers into a separate crate that the various clients can import, to easily get at the list of backend services and their correct indices and comments. --- Cargo.lock | 14 +- cargo/licenses.json | 9 + proto/anki/ankidroid.proto | 2 + proto/anki/card_rendering.proto | 12 +- proto/anki/cards.proto | 4 + proto/anki/codegen.proto | 31 -- proto/anki/collection.proto | 2 + proto/anki/config.proto | 4 + proto/anki/deckconfig.proto | 4 + proto/anki/decks.proto | 4 + proto/anki/i18n.proto | 21 +- proto/anki/image_occlusion.proto | 4 + proto/anki/import_export.proto | 3 +- proto/anki/links.proto | 4 + proto/anki/media.proto | 4 + proto/anki/notes.proto | 4 + proto/anki/notetypes.proto | 4 + proto/anki/scheduler.proto | 4 + proto/anki/search.proto | 4 + proto/anki/stats.proto | 4 + proto/anki/sync.proto | 3 + proto/anki/tags.proto | 4 + rslib/Cargo.toml | 3 +- rslib/proto/Cargo.toml | 1 + rslib/proto/build.rs | 11 +- rslib/proto/python.rs | 39 +-- rslib/proto/{rust_protos.rs => rust.rs} | 0 rslib/proto/src/lib.rs | 1 - rslib/proto/ts.rs | 56 ++-- rslib/proto/utils.rs | 36 --- rslib/proto_gen/Cargo.toml | 16 + rslib/proto_gen/src/lib.rs | 201 +++++++++++++ rslib/rust_interface.rs | 384 +++++++++++------------- tools/workspace-hack/Cargo.toml | 2 + 34 files changed, 545 insertions(+), 354 deletions(-) delete mode 100644 proto/anki/codegen.proto rename rslib/proto/{rust_protos.rs => rust.rs} (100%) delete mode 100644 rslib/proto/utils.rs create mode 100644 rslib/proto_gen/Cargo.toml create mode 100644 rslib/proto_gen/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 169c99538..a66f632ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,7 @@ dependencies = [ "anki_i18n", "anki_io", "anki_proto", + "anki_proto_gen", "anyhow", "async-compression", "async-stream", @@ -127,7 +128,6 @@ dependencies = [ "prettyplease 0.2.7", "prost", "prost-reflect", - "prost-types", "pulldown-cmark 0.9.2", "rand 0.8.5", "regex", @@ -213,6 +213,7 @@ name = "anki_proto" version = "0.0.0" dependencies = [ "anki_io", + "anki_proto_gen", "anyhow", "inflections", "itertools", @@ -226,6 +227,16 @@ dependencies = [ "strum", ] +[[package]] +name = "anki_proto_gen" +version = "0.0.0" +dependencies = [ + "inflections", + "itertools", + "prost-reflect", + "prost-types", +] + [[package]] name = "anstream" version = "0.2.6" @@ -5067,6 +5078,7 @@ dependencies = [ "hmac", "hyper", "indexmap", + "itertools", "log", "num-traits", "phf_shared 0.11.1", diff --git a/cargo/licenses.json b/cargo/licenses.json index 26c1b77de..d409f97e0 100644 --- a/cargo/licenses.json +++ b/cargo/licenses.json @@ -107,6 +107,15 @@ "license_file": null, "description": "Anki's Rust library protobuf code" }, + { + "name": "anki_proto_gen", + "version": "0.0.0", + "authors": "Ankitects Pty Ltd and contributors ", + "repository": null, + "license": "AGPL-3.0-or-later", + "license_file": null, + "description": "Helpers for interface code generation" + }, { "name": "anstream", "version": "0.2.6", diff --git a/proto/anki/ankidroid.proto b/proto/anki/ankidroid.proto index cb9db1ecd..49fb7d191 100644 --- a/proto/anki/ankidroid.proto +++ b/proto/anki/ankidroid.proto @@ -20,6 +20,8 @@ service AnkidroidService { returns (GetActiveSequenceNumbersResponse); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. service BackendAnkidroidService { rpc SchedTimingTodayLegacy(SchedTimingTodayLegacyRequest) returns (scheduler.SchedTimingTodayResponse); diff --git a/proto/anki/card_rendering.proto b/proto/anki/card_rendering.proto index 866e2e9ae..a0f13af6b 100644 --- a/proto/anki/card_rendering.proto +++ b/proto/anki/card_rendering.proto @@ -10,7 +10,6 @@ package anki.card_rendering; import "anki/generic.proto"; import "anki/notes.proto"; import "anki/notetypes.proto"; -import "anki/codegen.proto"; service CardRenderingService { rpc ExtractAvTags(ExtractAvTagsRequest) returns (ExtractAvTagsResponse); @@ -26,10 +25,7 @@ service CardRenderingService { rpc RenderMarkdown(RenderMarkdownRequest) returns (generic.String); rpc EncodeIriPaths(generic.String) returns (generic.String); rpc DecodeIriPaths(generic.String) returns (generic.String); - rpc StripHtml(StripHtmlRequest) returns (generic.String) { - // a bunch of our unit tests access this without a collection - option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT; - } + rpc StripHtml(StripHtmlRequest) returns (generic.String); rpc CompareAnswer(CompareAnswerRequest) returns (generic.String); rpc ExtractClozeForTyping(ExtractClozeForTypingRequest) returns (generic.String); @@ -37,6 +33,12 @@ service CardRenderingService { rpc WriteTtsStream(WriteTtsStreamRequest) returns (generic.Empty); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendCardRenderingService { + rpc StripHtml(StripHtmlRequest) returns (generic.String); +} + message ExtractAvTagsRequest { string text = 1; bool question_side = 2; diff --git a/proto/anki/cards.proto b/proto/anki/cards.proto index a7c6400ea..0b4b9354e 100644 --- a/proto/anki/cards.proto +++ b/proto/anki/cards.proto @@ -18,6 +18,10 @@ service CardsService { rpc SetFlag(SetFlagRequest) returns (collection.OpChangesWithCount); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendCardsService {} + message CardId { int64 cid = 1; } diff --git a/proto/anki/codegen.proto b/proto/anki/codegen.proto deleted file mode 100644 index 51feed224..000000000 --- a/proto/anki/codegen.proto +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright: Ankitects Pty Ltd and contributors -// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html - -syntax = "proto3"; - -package anki.codegen; - -import "google/protobuf/descriptor.proto"; - -extend google.protobuf.MethodOptions { - BackendMethod backend_method = 50000; -} - -message MethodOptions { - BackendMethod backend_method = 50000; -} - -enum BackendMethod { - /// Used for typical collection-based operations. We must implement the - // method on Collection. The same method is automatically implemented on - // Backend, which forwards to Collection. - BACKEND_METHOD_DELEGATE = 0; - /// Both the backend and collection need to implement the method; there - /// is no auto-delegation. Can be used to provide a method on both, but - /// skip the Collection mutex lock when a backend handle is available. - /// In practice we only do this for the i18n methods; for the occasional - /// method in other services that doesn't happen to need the collection, - /// we just delegate to the collection method for convenience, and to make - /// sure it's available even if the consumer is not using Backend. - BACKEND_METHOD_IMPLEMENT = 1; -} diff --git a/proto/anki/collection.proto b/proto/anki/collection.proto index 08c618223..e82ca9f04 100644 --- a/proto/anki/collection.proto +++ b/proto/anki/collection.proto @@ -20,6 +20,8 @@ service CollectionService { rpc SetWantsAbort(generic.Empty) returns (generic.Empty); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. service BackendCollectionService { rpc OpenCollection(OpenCollectionRequest) returns (generic.Empty); rpc CloseCollection(CloseCollectionRequest) returns (generic.Empty); diff --git a/proto/anki/config.proto b/proto/anki/config.proto index 867c62f80..9cbd9f3b5 100644 --- a/proto/anki/config.proto +++ b/proto/anki/config.proto @@ -24,6 +24,10 @@ service ConfigService { rpc SetPreferences(Preferences) returns (collection.OpChanges); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendConfigService {} + message ConfigKey { enum Bool { BROWSER_TABLE_SHOW_NOTES_MODE = 0; diff --git a/proto/anki/deckconfig.proto b/proto/anki/deckconfig.proto index 1e571b135..e12efaa28 100644 --- a/proto/anki/deckconfig.proto +++ b/proto/anki/deckconfig.proto @@ -25,6 +25,10 @@ service DeckConfigService { returns (collection.OpChanges); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendDeckConfigService {} + message DeckConfigId { int64 dcid = 1; } diff --git a/proto/anki/decks.proto b/proto/anki/decks.proto index 3a38f3a8b..6ecd55ad8 100644 --- a/proto/anki/decks.proto +++ b/proto/anki/decks.proto @@ -39,6 +39,10 @@ service DecksService { rpc GetCurrentDeck(generic.Empty) returns (Deck); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendDecksService {} + message DeckId { int64 did = 1; } diff --git a/proto/anki/i18n.proto b/proto/anki/i18n.proto index c00898eb9..9414e5fce 100644 --- a/proto/anki/i18n.proto +++ b/proto/anki/i18n.proto @@ -8,18 +8,19 @@ option java_multiple_files = true; package anki.i18n; import "anki/generic.proto"; -import "anki/codegen.proto"; service I18nService { - rpc TranslateString(TranslateStringRequest) returns (generic.String) { - option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT; - } - rpc FormatTimespan(FormatTimespanRequest) returns (generic.String) { - option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT; - } - rpc I18nResources(I18nResourcesRequest) returns (generic.Json) { - option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT; - } + rpc TranslateString(TranslateStringRequest) returns (generic.String); + rpc FormatTimespan(FormatTimespanRequest) returns (generic.String); + rpc I18nResources(I18nResourcesRequest) returns (generic.Json); +} + +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendI18nService { + rpc TranslateString(TranslateStringRequest) returns (generic.String); + rpc FormatTimespan(FormatTimespanRequest) returns (generic.String); + rpc I18nResources(I18nResourcesRequest) returns (generic.Json); } message TranslateStringRequest { diff --git a/proto/anki/image_occlusion.proto b/proto/anki/image_occlusion.proto index 2e4032780..6bea48569 100644 --- a/proto/anki/image_occlusion.proto +++ b/proto/anki/image_occlusion.proto @@ -23,6 +23,10 @@ service ImageOcclusionService { rpc AddImageOcclusionNotetype(generic.Empty) returns (collection.OpChanges); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendImageOcclusionService {} + message GetImageForOcclusionRequest { string path = 1; } diff --git a/proto/anki/import_export.proto b/proto/anki/import_export.proto index f490aee3a..bad5230e1 100644 --- a/proto/anki/import_export.proto +++ b/proto/anki/import_export.proto @@ -11,7 +11,6 @@ import "anki/cards.proto"; import "anki/collection.proto"; import "anki/notes.proto"; import "anki/generic.proto"; -import "anki/codegen.proto"; service ImportExportService { rpc ImportAnkiPackage(ImportAnkiPackageRequest) returns (ImportResponse); @@ -24,6 +23,8 @@ service ImportExportService { rpc ImportJsonString(generic.String) returns (ImportResponse); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. service BackendImportExportService { rpc ImportCollectionPackage(ImportCollectionPackageRequest) returns (generic.Empty); diff --git a/proto/anki/links.proto b/proto/anki/links.proto index d66d10bc9..4d862adb9 100644 --- a/proto/anki/links.proto +++ b/proto/anki/links.proto @@ -13,6 +13,10 @@ service LinksService { rpc HelpPageLink(HelpPageLinkRequest) returns (generic.String); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendLinksService {} + message HelpPageLinkRequest { enum HelpPage { NOTE_TYPE = 0; diff --git a/proto/anki/media.proto b/proto/anki/media.proto index 47b0ffcea..1455bdeb5 100644 --- a/proto/anki/media.proto +++ b/proto/anki/media.proto @@ -17,6 +17,10 @@ service MediaService { rpc RestoreTrash(generic.Empty) returns (generic.Empty); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendMediaService {} + message CheckMediaResponse { repeated string unused = 1; repeated string missing = 2; diff --git a/proto/anki/notes.proto b/proto/anki/notes.proto index eb48042d2..26c36bff9 100644 --- a/proto/anki/notes.proto +++ b/proto/anki/notes.proto @@ -30,6 +30,10 @@ service NotesService { rpc GetSingleNotetypeOfNotes(notes.NoteIds) returns (notetypes.NotetypeId); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendNotesService {} + message NoteId { int64 nid = 1; } diff --git a/proto/anki/notetypes.proto b/proto/anki/notetypes.proto index 68816efea..87d7a2eb8 100644 --- a/proto/anki/notetypes.proto +++ b/proto/anki/notetypes.proto @@ -34,6 +34,10 @@ service NotetypesService { returns (collection.OpChanges); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendNotetypesService {} + message NotetypeId { int64 ntid = 1; } diff --git a/proto/anki/scheduler.proto b/proto/anki/scheduler.proto index ac78654ff..cc6fe1fb2 100644 --- a/proto/anki/scheduler.proto +++ b/proto/anki/scheduler.proto @@ -54,6 +54,10 @@ service SchedulerService { rpc RepositionDefaults(generic.Empty) returns (RepositionDefaultsResponse); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendSchedulerService {} + message SchedulingState { message New { uint32 position = 1; diff --git a/proto/anki/search.proto b/proto/anki/search.proto index 3a253ed72..01c460579 100644 --- a/proto/anki/search.proto +++ b/proto/anki/search.proto @@ -23,6 +23,10 @@ service SearchService { rpc SetActiveBrowserColumns(generic.StringList) returns (generic.Empty); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendSearchService {} + message SearchNode { message Dupe { int64 notetype_id = 1; diff --git a/proto/anki/stats.proto b/proto/anki/stats.proto index 8e1ca8566..8131eba1c 100644 --- a/proto/anki/stats.proto +++ b/proto/anki/stats.proto @@ -17,6 +17,10 @@ service StatsService { rpc SetGraphPreferences(GraphPreferences) returns (generic.Empty); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendStatsService {} + message CardStatsResponse { message StatsRevlogEntry { int64 time = 1; diff --git a/proto/anki/sync.proto b/proto/anki/sync.proto index 43e3ac198..368287195 100644 --- a/proto/anki/sync.proto +++ b/proto/anki/sync.proto @@ -9,6 +9,9 @@ package anki.sync; import "anki/generic.proto"; +/// Syncing methods are only available with a Backend handle. +service SyncService {} + service BackendSyncService { rpc SyncMedia(SyncAuth) returns (generic.Empty); rpc AbortMediaSync(generic.Empty) returns (generic.Empty); diff --git a/proto/anki/tags.proto b/proto/anki/tags.proto index ab55f92bb..a31591be6 100644 --- a/proto/anki/tags.proto +++ b/proto/anki/tags.proto @@ -27,6 +27,10 @@ service TagsService { rpc CompleteTag(CompleteTagRequest) returns (CompleteTagResponse); } +// Implicitly includes any of the above methods that are not listed in the +// backend service. +service BackendTagsService {} + message SetTagCollapsedRequest { string name = 1; bool collapsed = 2; diff --git a/rslib/Cargo.toml b/rslib/Cargo.toml index e562cc6cd..8242007dc 100644 --- a/rslib/Cargo.toml +++ b/rslib/Cargo.toml @@ -22,12 +22,13 @@ required-features = ["bench"] [build-dependencies] anki_io = { version = "0.0.0", path = "io" } anki_proto = { version = "0.0.0", path = "proto" } +anki_proto_gen = { version = "0.0.0", path = "proto_gen" } anyhow = "1.0.71" inflections = "1.1.1" +itertools = "0.10.5" prettyplease = "0.2.7" prost = "0.11.8" prost-reflect = "0.11.4" -prost-types = "0.11.9" syn = { version = "2.0.18", features = ["parsing", "printing"] } [dev-dependencies] diff --git a/rslib/proto/Cargo.toml b/rslib/proto/Cargo.toml index c3fdafb31..f96283a38 100644 --- a/rslib/proto/Cargo.toml +++ b/rslib/proto/Cargo.toml @@ -11,6 +11,7 @@ rust-version.workspace = true [build-dependencies] anki_io = { version = "0.0.0", path = "../io" } +anki_proto_gen = { version = "0.0.0", path = "../proto_gen" } anyhow = "1.0.71" inflections = "1.1.1" itertools = "0.10.5" diff --git a/rslib/proto/build.rs b/rslib/proto/build.rs index cd339a03b..5e5e1359a 100644 --- a/rslib/proto/build.rs +++ b/rslib/proto/build.rs @@ -2,21 +2,22 @@ // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html pub mod python; -pub mod rust_protos; +pub mod rust; pub mod ts; -pub mod utils; use std::env; use std::path::PathBuf; +use anki_proto_gen::get_services; use anyhow::Result; fn main() -> Result<()> { let descriptors_path = env::var("DESCRIPTORS_BIN").ok().map(PathBuf::from); - let pool = rust_protos::write_rust_protos(descriptors_path)?; - python::write_python_interface(&pool)?; - ts::write_ts_interface(&pool)?; + let pool = rust::write_rust_protos(descriptors_path)?; + let (_, services) = get_services(&pool); + python::write_python_interface(&services)?; + ts::write_ts_interface(&services)?; Ok(()) } diff --git a/rslib/proto/python.rs b/rslib/proto/python.rs index 1f7f6f357..01850158d 100644 --- a/rslib/proto/python.rs +++ b/rslib/proto/python.rs @@ -7,30 +7,26 @@ use std::path::Path; use anki_io::create_dir_all; use anki_io::create_file; +use anki_proto_gen::BackendService; +use anki_proto_gen::Method; use anyhow::Result; use inflections::Inflect; -use prost_reflect::DescriptorPool; use prost_reflect::FieldDescriptor; use prost_reflect::Kind; use prost_reflect::MessageDescriptor; -use prost_reflect::MethodDescriptor; -use prost_reflect::ServiceDescriptor; -use crate::utils::Comments; - -pub(crate) fn write_python_interface(pool: &DescriptorPool) -> Result<()> { +pub(crate) fn write_python_interface(services: &[BackendService]) -> Result<()> { let output_path = Path::new("../../out/pylib/anki/_backend_generated.py"); create_dir_all(output_path.parent().unwrap())?; let mut out = BufWriter::new(create_file(output_path)?); write_header(&mut out)?; - for service in pool.services() { - if service.name() == "AnkidroidService" { + for service in services { + if service.name == "BackendAnkidroidService" { continue; } - let comments = Comments::from_file(service.parent_file().file_descriptor_proto()); - for method in service.methods() { - render_method(&service, &method, &comments, &mut out); + for method in service.all_methods() { + render_method(service, method, &mut out); } } @@ -48,18 +44,13 @@ pub(crate) fn write_python_interface(pool: &DescriptorPool) -> Result<()> { /// output = anki.generic_pb2.StringList() /// output.ParseFromString(raw_bytes) /// return output.vals -fn render_method( - service: &ServiceDescriptor, - method: &MethodDescriptor, - comments: &Comments, - out: &mut impl Write, -) { - let method_name = method.name().to_snake_case(); - let input = method.input(); - let output = method.output(); - let service_idx = service.index(); - let method_idx = method.index(); - let comments = format_comments(comments.get_for_path(method.path())); +fn render_method(service: &BackendService, method: &Method, out: &mut impl Write) { + let method_name = method.name.to_snake_case(); + let input = method.proto.input(); + let output = method.proto.output(); + let service_idx = service.index; + let method_idx = method.index; + let comments = format_comments(&method.comments); // raw bytes write!( @@ -89,7 +80,7 @@ fn render_method( .unwrap(); } -fn format_comments(comments: Option<&str>) -> String { +fn format_comments(comments: &Option) -> String { comments .as_ref() .map(|c| { diff --git a/rslib/proto/rust_protos.rs b/rslib/proto/rust.rs similarity index 100% rename from rslib/proto/rust_protos.rs rename to rslib/proto/rust.rs diff --git a/rslib/proto/src/lib.rs b/rslib/proto/src/lib.rs index ec3c47053..a995c57a4 100644 --- a/rslib/proto/src/lib.rs +++ b/rslib/proto/src/lib.rs @@ -15,7 +15,6 @@ protobuf!(ankidroid, "ankidroid"); protobuf!(backend, "backend"); protobuf!(card_rendering, "card_rendering"); protobuf!(cards, "cards"); -protobuf!(codegen, "codegen"); protobuf!(collection, "collection"); protobuf!(config, "config"); protobuf!(deckconfig, "deckconfig"); diff --git a/rslib/proto/ts.rs b/rslib/proto/ts.rs index 46540c27f..de7405445 100644 --- a/rslib/proto/ts.rs +++ b/rslib/proto/ts.rs @@ -9,46 +9,39 @@ use std::path::Path; use anki_io::create_dir_all; use anki_io::create_file; +use anki_proto_gen::BackendService; +use anki_proto_gen::Method; use anyhow::Result; use inflections::Inflect; -use prost_reflect::DescriptorPool; -use prost_reflect::MethodDescriptor; -use prost_reflect::ServiceDescriptor; -use crate::utils::Comments; - -pub(crate) fn write_ts_interface(pool: &DescriptorPool) -> Result<()> { +pub(crate) fn write_ts_interface(services: &[BackendService]) -> Result<()> { let root = Path::new("../../out/ts/lib/anki"); create_dir_all(root)?; - for service in pool.services() { - if service.name() == "AnkidroidService" { + for service in services { + if service.name == "BackendAnkidroidService" { continue; } - let service_name = service.name().replace("Service", "").to_snake_case(); - let comments = Comments::from_file(service.parent_file().file_descriptor_proto()); - write_dts_file(root, &service_name, &service, &comments)?; - write_js_file(root, &service_name, &service, &comments)?; + let service_name = service.name.replace("Service", "").to_snake_case(); + + write_dts_file(root, &service_name, service)?; + write_js_file(root, &service_name, service)?; } Ok(()) } -fn write_dts_file( - root: &Path, - service_name: &str, - service: &ServiceDescriptor, - comments: &Comments, -) -> Result<()> { +fn write_dts_file(root: &Path, service_name: &str, service: &BackendService) -> Result<()> { let output_path = root.join(format!("{service_name}_service.d.ts")); let mut out = BufWriter::new(create_file(output_path)?); write_dts_header(&mut out)?; let mut referenced_packages = HashSet::new(); let mut method_text = String::new(); - for method in service.methods() { - let method = MethodDetails::from_descriptor(&method, comments); + + for method in service.all_methods() { + let method = MethodDetails::from_method(method); record_referenced_type(&mut referenced_packages, &method.input_type)?; record_referenced_type(&mut referenced_packages, &method.output_type)?; write_dts_method(&method, &mut method_text)?; @@ -100,20 +93,15 @@ fn write_dts_method( Ok(()) } -fn write_js_file( - root: &Path, - service_name: &str, - service: &ServiceDescriptor, - comments: &Comments, -) -> Result<()> { +fn write_js_file(root: &Path, service_name: &str, service: &BackendService) -> Result<()> { let output_path = root.join(format!("{service_name}_service.js")); let mut out = BufWriter::new(create_file(output_path)?); write_js_header(&mut out)?; let mut referenced_packages = HashSet::new(); let mut method_text = String::new(); - for method in service.methods() { - let method = MethodDetails::from_descriptor(&method, comments); + for method in service.all_methods() { + let method = MethodDetails::from_method(method); record_referenced_type(&mut referenced_packages, &method.input_type)?; record_referenced_type(&mut referenced_packages, &method.output_type)?; write_js_method(&method, &mut method_text)?; @@ -169,16 +157,16 @@ struct MethodDetails { } impl MethodDetails { - fn from_descriptor(method: &MethodDescriptor, comments: &Comments) -> MethodDetails { - let name = method.name().to_camel_case(); - let input_type = full_name_to_imported_reference(method.input().full_name()); - let output_type = full_name_to_imported_reference(method.output().full_name()); - let comments = comments.get_for_path(method.path()); + fn from_method(method: &Method) -> MethodDetails { + let name = method.name.to_camel_case(); + let input_type = full_name_to_imported_reference(method.proto.input().full_name()); + let output_type = full_name_to_imported_reference(method.proto.output().full_name()); + let comments = method.comments.clone(); Self { method_name: name, input_type, output_type, - comments: comments.map(ToString::to_string), + comments, } } } diff --git a/rslib/proto/utils.rs b/rslib/proto/utils.rs deleted file mode 100644 index 9dca9ec8d..000000000 --- a/rslib/proto/utils.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright: Ankitects Pty Ltd and contributors -// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html - -use std::collections::HashMap; - -use prost_types::FileDescriptorProto; - -#[derive(Debug)] -pub struct Comments { - path_map: HashMap, String>, -} - -impl Comments { - pub fn from_file(file: &FileDescriptorProto) -> Self { - Self { - path_map: file - .source_code_info - .as_ref() - .unwrap() - .location - .iter() - .map(|l| (l.path.clone(), l.leading_comments().trim().to_string())) - .collect(), - } - } - - pub fn get_for_path(&self, path: &[i32]) -> Option<&str> { - self.path_map.get(path).map(|s| s.as_str()).and_then(|s| { - if s.is_empty() { - None - } else { - Some(s) - } - }) - } -} diff --git a/rslib/proto_gen/Cargo.toml b/rslib/proto_gen/Cargo.toml new file mode 100644 index 000000000..f5b057f9a --- /dev/null +++ b/rslib/proto_gen/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "anki_proto_gen" +publish = false +description = "Helpers for interface code generation" + +version.workspace = true +authors.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[dependencies] +inflections = "1.1.1" +itertools = "0.10.5" +prost-reflect = "0.11.4" +prost-types = "0.11.9" diff --git a/rslib/proto_gen/src/lib.rs b/rslib/proto_gen/src/lib.rs new file mode 100644 index 000000000..d600b7fc8 --- /dev/null +++ b/rslib/proto_gen/src/lib.rs @@ -0,0 +1,201 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +//! Some helpers for code generation in external crates, that ensure indexes +//! match. + +use std::collections::HashMap; + +use inflections::Inflect; +use itertools::Either; +use itertools::Itertools; +use prost_reflect::DescriptorPool; +use prost_reflect::MessageDescriptor; +use prost_reflect::MethodDescriptor; +use prost_reflect::ServiceDescriptor; + +/// We look for ExampleService and BackedExampleService, both of which are +/// expected to exist (but may be empty). +/// +/// - If a method is listed in BackendExampleService and not in ExampleService, +/// that method is only available with a Backend. +/// - If a method is listed in both services, you can provide separate +/// implementations for each of the traits. +/// - If a method is listed only in ExampleService, a forwarding method on +/// Backend is automatically implemented. This bypasses the trait and implements +/// directly on Backend. +/// +/// It's important that service and method indices are the same for +/// client-generated code, so the client code should use the .index fields +/// of Service and Method provided by get_services(), and not +/// .enumerate() or .proto.index() +/// +/// Client code will want to ignore CollectionServices, and focus on +/// BackendServices. +pub fn get_services(pool: &DescriptorPool) -> (Vec, Vec) { + // split services into backend and collection + let (mut col_services, mut backend_services): (Vec<_>, Vec<_>) = + pool.services().partition_map(|service| { + if service.name().starts_with("Backend") { + Either::Right(BackendService::from_proto(service)) + } else { + Either::Left(CollectionService::from_proto(service)) + } + }); + assert!(col_services.len() == backend_services.len()); + // copy collection methods into backend services if they don't have one with + // a matching name + for service in &mut backend_services { + // locate associated collection service + let Some(col_service) = col_services + .iter() + .find(|cs| cs.name == service.name.trim_start_matches("Backend")) else { panic!("missing associated service: {}", service.name) }; + + // add any methods that don't exist in backend trait methods to the delegating + // methods + service.delegating_methods = col_service + .trait_methods + .iter() + .filter(|m| service.trait_methods.iter().all(|bm| bm.name != m.name)) + .map(|method| Method { + index: method.index + service.trait_methods.len(), + ..method.clone() + }) + .collect(); + } + // fill comments in + let comments = MethodComments::from_pool(pool); + for service in &mut col_services { + for method in &mut service.trait_methods { + method.comments = comments.get_for_method(&method.proto); + } + } + for service in &mut backend_services { + for method in &mut service.trait_methods { + method.comments = comments.get_for_method(&method.proto); + } + for method in &mut service.delegating_methods { + method.comments = comments.get_for_method(&method.proto); + } + } + + (col_services, backend_services) +} + +#[derive(Debug)] +pub struct CollectionService { + pub name: String, + pub index: usize, + pub trait_methods: Vec, + pub proto: ServiceDescriptor, +} + +#[derive(Debug)] +pub struct BackendService { + pub name: String, + pub index: usize, + pub trait_methods: Vec, + pub delegating_methods: Vec, + pub proto: ServiceDescriptor, +} + +#[derive(Debug, Clone)] +pub struct Method { + pub name: String, + pub index: usize, + pub comments: Option, + pub proto: MethodDescriptor, +} + +impl CollectionService { + pub fn from_proto(service: prost_reflect::ServiceDescriptor) -> Self { + CollectionService { + name: service.name().to_string(), + index: service.index(), + trait_methods: service.methods().map(Method::from_proto).collect(), + proto: service, + } + } +} + +impl BackendService { + pub fn from_proto(service: prost_reflect::ServiceDescriptor) -> Self { + BackendService { + name: service.name().to_string(), + index: service.index(), + trait_methods: service.methods().map(Method::from_proto).collect(), + proto: service, + // filled in later + delegating_methods: vec![], + } + } + + pub fn all_methods(&self) -> impl Iterator { + self.trait_methods + .iter() + .chain(self.delegating_methods.iter()) + } +} + +impl Method { + pub fn from_proto(method: prost_reflect::MethodDescriptor) -> Self { + Method { + name: method.name().to_snake_case(), + index: method.index(), + proto: method, + // filled in later + comments: None, + } + } + + /// The input type, if not empty. + pub fn input(&self) -> Option { + msg_if_empty(self.proto.input()) + } + + /// The output type, if not empty. + pub fn output(&self) -> Option { + msg_if_empty(self.proto.output()) + } +} + +fn msg_if_empty(msg: MessageDescriptor) -> Option { + if msg.full_name() == "anki.generic.Empty" { + None + } else { + Some(msg) + } +} + +#[derive(Debug)] +struct MethodComments<'a> { + // package name -> method path -> comment + by_package_and_path: HashMap<&'a str, HashMap, String>>, +} + +impl<'a> MethodComments<'a> { + pub fn from_pool(pool: &'a DescriptorPool) -> MethodComments<'a> { + let mut by_package_and_path = HashMap::new(); + for file in pool.file_descriptor_protos() { + let path_map = file + .source_code_info + .as_ref() + .unwrap() + .location + .iter() + .map(|l| (l.path.clone(), l.leading_comments().trim().to_string())) + .collect(); + by_package_and_path.insert(file.package(), path_map); + } + Self { + by_package_and_path, + } + } + + pub fn get_for_method(&self, method: &MethodDescriptor) -> Option { + self.by_package_and_path + .get(method.parent_file().package_name()) + .and_then(|by_path| by_path.get(method.path())) + .and_then(|s| if s.is_empty() { None } else { Some(s.into()) }) + } +} diff --git a/rslib/rust_interface.rs b/rslib/rust_interface.rs index 3ac9adaa8..0e9b8ef44 100644 --- a/rslib/rust_interface.rs +++ b/rslib/rust_interface.rs @@ -6,7 +6,10 @@ use std::fmt::Write; use std::path::PathBuf; use anki_io::write_file_if_changed; -use anki_proto::codegen::BackendMethod; +use anki_proto_gen::get_services; +use anki_proto_gen::BackendService; +use anki_proto_gen::CollectionService; +use anki_proto_gen::Method; use anyhow::Context; use anyhow::Result; use inflections::Inflect; @@ -15,105 +18,50 @@ use prost_reflect::DescriptorPool; pub fn write_rust_interface(pool: &DescriptorPool) -> Result<()> { let mut buf = String::new(); buf.push_str("use crate::error::Result; use prost::Message;"); - let services = pool - .services() - .map(RustService::from_proto) - .collect::>(); - for service in &services { - render_service(service, &mut buf); - } - render_top_level_run_method(&mut buf, &services, true); - render_top_level_run_method(&mut buf, &services, false); + let (col_services, backend_services) = get_services(pool); + + render_collection_services(&col_services, &mut buf)?; + render_backend_services(&backend_services, &mut buf)?; - // println!("{}", &buf); let buf = format_code(buf)?; - // write into OUT_DIR so we can use it in build.rs + // println!("{}", &buf); + // panic!(); let out_dir = env::var("OUT_DIR").unwrap(); let path = PathBuf::from(out_dir).join("backend.rs"); write_file_if_changed(path, buf).context("write file")?; Ok(()) } -#[derive(Debug)] -struct RustService { - name: String, - methods: Vec, +fn render_collection_services(col_services: &[CollectionService], buf: &mut String) -> Result<()> { + for service in col_services { + render_collection_trait(service, buf); + render_individual_service_run_method_for_collection(buf, service); + } + render_top_level_run_method( + col_services.iter().map(|s| (s.index, s.name.as_str())), + "&mut self", + "crate::collection::Collection", + buf, + ); + + Ok(()) } -#[derive(Debug)] -struct RustMethod { - name: String, - input_type: Option, - output_type: Option, - options: anki_proto::codegen::MethodOptions, - service_name: String, -} - -impl RustMethod { - /// No text if generic::Empty - fn text_if_input_not_empty(&self, text: impl Fn(&String) -> String) -> String { - self.input_type.as_ref().map(text).unwrap_or_default() +fn render_backend_services(backend_services: &[BackendService], buf: &mut String) -> Result<()> { + for service in backend_services { + render_backend_trait(service, buf); + render_delegating_backend_methods(service, buf); + render_individual_service_run_method_for_backend(buf, service); } + render_top_level_run_method( + backend_services.iter().map(|s| (s.index, s.name.as_str())), + "&self", + "crate::backend::Backend", + buf, + ); - /// No text if generic::Empty - fn get_input_arg_with_label(&self) -> String { - self.input_type - .as_ref() - .map(|t| format!("input: {}", t)) - .unwrap_or_default() - } - - /// () if generic::Empty - fn get_output_type(&self) -> String { - self.output_type.as_deref().unwrap_or("()").into() - } - - fn text_if_output_not_empty(&self, text: impl Fn(&String) -> String) -> String { - self.output_type.as_ref().map(text).unwrap_or_default() - } - - fn wants_abstract_backend_method(&self) -> bool { - self.service_name.starts_with("Backend") - || self.options.backend_method() == BackendMethod::Implement - } - - fn wants_abstract_collection_method(&self) -> bool { - !self.service_name.starts_with("Backend") - } - - fn from_proto(method: prost_reflect::MethodDescriptor) -> Self { - RustMethod { - name: method.name().to_snake_case(), - input_type: rust_type(method.input().full_name()), - output_type: rust_type(method.output().full_name()), - options: method.options().transcode_to().unwrap(), - service_name: method.parent_service().name().to_string(), - } - } -} - -impl RustService { - fn from_proto(service: prost_reflect::ServiceDescriptor) -> Self { - RustService { - name: service.name().to_string(), - methods: service.methods().map(RustMethod::from_proto).collect(), - } - } -} - -fn rust_type(name: &str) -> Option { - if name == "anki.generic.Empty" { - return None; - } - let Some((head, tail)) = name.rsplit_once( '.') else { panic!() }; - Some(format!( - "{}::{}", - head.to_snake_case() - .replace('.', "::") - .replace("anki::", "anki_proto::"), - tail - )) + Ok(()) } fn format_code(code: String) -> Result { @@ -121,118 +69,75 @@ fn format_code(code: String) -> Result { Ok(prettyplease::unparse(&syntax_tree)) } -fn render_abstract_collection_method(method: &RustMethod, buf: &mut String) { +fn render_collection_trait(service: &CollectionService, buf: &mut String) { + let name = &service.name; + writeln!(buf, "pub trait {name} {{").unwrap(); + for method in &service.trait_methods { + render_trait_method(method, "&mut self", buf); + } + buf.push('}'); +} + +fn render_trait_method(method: &Method, self_kind: &str, buf: &mut String) { let method_name = &method.name; let input_with_label = method.get_input_arg_with_label(); let output_type = method.get_output_type(); writeln!( buf, - "fn {method_name}(&mut self, {input_with_label}) -> Result<{output_type}>;" + "fn {method_name}({self_kind}, {input_with_label}) -> Result<{output_type}>;" ) .unwrap(); } -fn render_abstract_backend_method(method: &RustMethod, buf: &mut String, _service: &RustService) { - let method_name = &method.name; - let input_with_label = method.get_input_arg_with_label(); - let output_type = method.get_output_type(); - writeln!( - buf, - "fn {method_name}(&self, {input_with_label}) -> Result<{output_type}>;" - ) - .unwrap(); +fn render_backend_trait(service: &BackendService, buf: &mut String) { + let name = &service.name; + writeln!(buf, "pub trait {name} {{").unwrap(); + for method in &service.trait_methods { + render_trait_method(method, "&self", buf); + } + buf.push('}'); } -fn render_delegating_backend_method(method: &RustMethod, buf: &mut String, service: &RustService) { +fn render_delegating_backend_methods(service: &BackendService, buf: &mut String) { + buf.push_str("impl crate::backend::Backend {"); + for method in &service.delegating_methods { + render_delegating_backend_method(method, service.name.trim_start_matches("Backend"), buf); + } + buf.push('}'); +} + +fn render_delegating_backend_method(method: &Method, method_qualifier: &str, buf: &mut String) { let method_name = &method.name; let input_with_label = method.get_input_arg_with_label(); let input = method.text_if_input_not_empty(|_| "input".into()); let output_type = method.get_output_type(); - let col_service = &service.name; writeln!( buf, "fn {method_name}(&self, {input_with_label}) -> Result<{output_type}> {{ - self.with_col(|col| {col_service}::{method_name}(col, {input})) }}", + self.with_col(|col| {method_qualifier}::{method_name}(col, {input})) }}", ) .unwrap(); } -fn render_service(service: &RustService, buf: &mut String) { - let have_collection = service - .methods - .iter() - .any(|m| m.wants_abstract_collection_method()); - if have_collection { - render_collection_trait(service, buf); - } - if service - .methods - .iter() - .any(|m| m.wants_abstract_backend_method()) - { - render_backend_trait(service, buf); - } - render_delegating_backend_methods(service, buf); - render_individual_service_run_method(buf, service, true); - render_individual_service_run_method(buf, service, false); -} - -fn render_collection_trait(service: &RustService, buf: &mut String) { - let name = &service.name; - writeln!(buf, "pub trait {name} {{").unwrap(); - for method in &service.methods { - if method.wants_abstract_collection_method() { - render_abstract_collection_method(method, buf); - } - } - buf.push('}'); -} - -fn render_backend_trait(service: &RustService, buf: &mut String) { - let name = if !service.name.starts_with("Backend") { - format!("Backend{}", service.name) - } else { - service.name.clone() - }; - writeln!(buf, "pub trait {name} {{").unwrap(); - for method in &service.methods { - if method.wants_abstract_backend_method() { - render_abstract_backend_method(method, buf, service); - } - } - buf.push('}'); -} - -fn render_delegating_backend_methods(service: &RustService, buf: &mut String) { - buf.push_str("impl crate::backend::Backend {"); - for method in &service.methods { - if method.wants_abstract_backend_method() { - continue; - } - render_delegating_backend_method(method, buf, service); - } - buf.push('}'); -} - // Matches all service types and delegates to the revelant self.run_foo_method() -fn render_top_level_run_method(buf: &mut String, services: &[RustService], backend: bool) { - let self_kind = if backend { "&self" } else { "&mut self" }; - let struct_to_impl = if backend { - "crate::backend::Backend" - } else { - "crate::collection::Collection" - }; +fn render_top_level_run_method<'a>( + // (index, name) + services: impl Iterator, + self_kind: &str, + struct_name: &str, + buf: &mut String, +) { writeln!(buf, - r#" impl {struct_to_impl} {{ + r#" impl {struct_name} {{ pub fn run_service_method({self_kind}, service: u32, method: u32, input: &[u8]) -> Result, Vec> {{ match service {{ "#, ).unwrap(); - for (idx, service) in services.iter().enumerate() { + for (idx, service) in services { writeln!( buf, "{idx} => self.run_{service}_method(method, input),", - service = service.name.to_snake_case() + service = service.to_snake_case() ) .unwrap(); } @@ -250,51 +155,21 @@ fn render_top_level_run_method(buf: &mut String, services: &[RustService], backe ); } -fn render_individual_service_run_method(buf: &mut String, service: &RustService, backend: bool) { - let self_kind = if backend { "&self" } else { "&mut self" }; - let struct_to_impl = if backend { - "crate::backend::Backend" - } else { - "crate::collection::Collection" - }; - let method_qualifier = if backend { - struct_to_impl - } else { - &service.name - }; - +fn render_individual_service_run_method_for_collection( + buf: &mut String, + service: &CollectionService, +) { let service_name = &service.name.to_snake_case(); writeln!( buf, "#[allow(unused_variables, clippy::match_single_binding)] - impl {struct_to_impl} {{ pub(crate) fn run_{service_name}_method({self_kind}, + impl crate::collection::Collection {{ pub(crate) fn run_{service_name}_method(&mut self, method: u32, input: &[u8]) -> Result> {{ match method {{", ) .unwrap(); - for (idx, method) in service.methods.iter().enumerate() { - if !backend && !method.wants_abstract_collection_method() { - continue; - } - let decode_input = - method.text_if_input_not_empty(|kind| format!("let input = {kind}::decode(input)?;")); - let rust_method = &method.name; - let input = method.text_if_input_not_empty(|_| "input".into()); - let output_assign = method.text_if_output_not_empty(|_| "let output = ".into()); - let output = if method.output_type.is_none() { - "Vec::new()" - } else { - "{ let mut out_bytes = Vec::new(); - output.encode(&mut out_bytes)?; - out_bytes }" - }; - writeln!( - buf, - "{idx} => {{ {decode_input} - {output_assign} {method_qualifier}::{rust_method}(self, {input})?; - Ok({output}) }},", - ) - .unwrap(); + for method in &service.trait_methods { + render_method_in_match_expression(method, &service.name, buf); } buf.push_str( r#" @@ -304,3 +179,102 @@ fn render_individual_service_run_method(buf: &mut String, service: &RustService, "#, ); } + +fn render_individual_service_run_method_for_backend(buf: &mut String, service: &BackendService) { + let service_name = &service.name.to_snake_case(); + writeln!( + buf, + "#[allow(unused_variables, clippy::match_single_binding)] + impl crate::backend::Backend {{ pub(crate) fn run_{service_name}_method(&self, + method: u32, input: &[u8]) -> Result> {{ + match method {{", + ) + .unwrap(); + for method in &service.trait_methods { + render_method_in_match_expression(method, &service.name, buf); + } + for method in &service.delegating_methods { + render_method_in_match_expression(method, "crate::backend::Backend", buf); + } + buf.push_str( + r#" + _ => Err(crate::error::AnkiError::InvalidMethodIndex), + } +} } +"#, + ); +} + +fn render_method_in_match_expression(method: &Method, method_qualifier: &str, buf: &mut String) { + let decode_input = + method.text_if_input_not_empty(|kind| format!("let input = {kind}::decode(input)?;")); + let rust_method = &method.name; + let input = method.text_if_input_not_empty(|_| "input".into()); + let output_assign = method.text_if_output_not_empty(|_| "let output = ".into()); + let idx = method.index; + let output = if method.output().is_none() { + "Vec::new()" + } else { + "{ let mut out_bytes = Vec::new(); + output.encode(&mut out_bytes)?; + out_bytes }" + }; + writeln!( + buf, + "{idx} => {{ {decode_input} + {output_assign} {method_qualifier}::{rust_method}(self, {input})?; + Ok({output}) }},", + ) + .unwrap(); +} + +trait MethodHelpers { + fn input_type(&self) -> Option; + fn output_type(&self) -> Option; + fn text_if_input_not_empty(&self, text: impl Fn(&String) -> String) -> String; + fn get_input_arg_with_label(&self) -> String; + fn get_output_type(&self) -> String; + fn text_if_output_not_empty(&self, text: impl Fn(&String) -> String) -> String; +} + +impl MethodHelpers for Method { + fn input_type(&self) -> Option { + self.input().map(|t| rust_type(t.full_name())) + } + + fn output_type(&self) -> Option { + self.output().map(|t| rust_type(t.full_name())) + } + /// No text if generic::Empty + fn text_if_input_not_empty(&self, text: impl Fn(&String) -> String) -> String { + self.input_type().as_ref().map(text).unwrap_or_default() + } + + /// No text if generic::Empty + fn get_input_arg_with_label(&self) -> String { + self.input_type() + .as_ref() + .map(|t| format!("input: {}", t)) + .unwrap_or_default() + } + + /// () if generic::Empty + fn get_output_type(&self) -> String { + self.output_type().as_deref().unwrap_or("()").into() + } + + fn text_if_output_not_empty(&self, text: impl Fn(&String) -> String) -> String { + self.output_type().as_ref().map(text).unwrap_or_default() + } +} + +fn rust_type(name: &str) -> String { + let Some((head, tail)) = name.rsplit_once( '.') else { panic!() }; + format!( + "{}::{}", + head.to_snake_case() + .replace('.', "::") + .replace("anki::", "anki_proto::"), + tail + ) +} diff --git a/tools/workspace-hack/Cargo.toml b/tools/workspace-hack/Cargo.toml index 3d9f7bb2d..9f7eed4fe 100644 --- a/tools/workspace-hack/Cargo.toml +++ b/tools/workspace-hack/Cargo.toml @@ -28,6 +28,7 @@ hashbrown = { version = "0.12", features = ["raw"] } hmac = { version = "0.12", default-features = false, features = ["reset"] } hyper = { version = "0.14", features = ["full"] } indexmap = { version = "1", default-features = false, features = ["std"] } +itertools = { version = "0.10" } log = { version = "0.4", default-features = false, features = ["std"] } num-traits = { version = "0.2" } phf_shared = { version = "0.11", default-features = false, features = ["std"] } @@ -69,6 +70,7 @@ hashbrown = { version = "0.12", features = ["raw"] } hmac = { version = "0.12", default-features = false, features = ["reset"] } hyper = { version = "0.14", features = ["full"] } indexmap = { version = "1", default-features = false, features = ["std"] } +itertools = { version = "0.10" } log = { version = "0.4", default-features = false, features = ["std"] } num-traits = { version = "0.2" } phf_shared = { version = "0.11", default-features = false, features = ["std"] }