More service generation refactoring

- Dropped the protobuf extensions in favor of explicitly listing out
methods in both services if we want to implement both, as it's clearer.
- Move Service/Method wrappers into a separate crate that the various
clients can import, to easily get at the list of backend services and
their correct indices and comments.

Cargo.lock

@@ -91,6 +91,7 @@ dependencies = [
  "anki_i18n",
  "anki_io",
  "anki_proto",
+ "anki_proto_gen",
  "anyhow",
  "async-compression",
  "async-stream",
@@ -127,7 +128,6 @@ dependencies = [
  "prettyplease 0.2.7",
  "prost",
  "prost-reflect",
- "prost-types",
  "pulldown-cmark 0.9.2",
  "rand 0.8.5",
  "regex",
@@ -213,6 +213,7 @@ name = "anki_proto"
 version = "0.0.0"
 dependencies = [
  "anki_io",
+ "anki_proto_gen",
  "anyhow",
  "inflections",
  "itertools",
@@ -226,6 +227,16 @@ dependencies = [
  "strum",
 ]
 
+[[package]]
+name = "anki_proto_gen"
+version = "0.0.0"
+dependencies = [
+ "inflections",
+ "itertools",
+ "prost-reflect",
+ "prost-types",
+]
+
 [[package]]
 name = "anstream"
 version = "0.2.6"
@@ -5067,6 +5078,7 @@ dependencies = [
  "hmac",
  "hyper",
  "indexmap",
+ "itertools",
  "log",
  "num-traits",
  "phf_shared 0.11.1",

cargo/licenses.json

@@ -107,6 +107,15 @@
     "license_file": null,
     "description": "Anki's Rust library protobuf code"
   },
+  {
+    "name": "anki_proto_gen",
+    "version": "0.0.0",
+    "authors": "Ankitects Pty Ltd and contributors <https://help.ankiweb.net>",
+    "repository": null,
+    "license": "AGPL-3.0-or-later",
+    "license_file": null,
+    "description": "Helpers for interface code generation"
+  },
   {
     "name": "anstream",
     "version": "0.2.6",

proto/anki/ankidroid.proto

@@ -20,6 +20,8 @@ service AnkidroidService {
       returns (GetActiveSequenceNumbersResponse);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
 service BackendAnkidroidService {
   rpc SchedTimingTodayLegacy(SchedTimingTodayLegacyRequest)
       returns (scheduler.SchedTimingTodayResponse);

proto/anki/card_rendering.proto

@@ -10,7 +10,6 @@ package anki.card_rendering;
 import "anki/generic.proto";
 import "anki/notes.proto";
 import "anki/notetypes.proto";
-import "anki/codegen.proto";
 
 service CardRenderingService {
   rpc ExtractAvTags(ExtractAvTagsRequest) returns (ExtractAvTagsResponse);
@@ -26,10 +25,7 @@ service CardRenderingService {
   rpc RenderMarkdown(RenderMarkdownRequest) returns (generic.String);
   rpc EncodeIriPaths(generic.String) returns (generic.String);
   rpc DecodeIriPaths(generic.String) returns (generic.String);
-  rpc StripHtml(StripHtmlRequest) returns (generic.String) {
+  rpc StripHtml(StripHtmlRequest) returns (generic.String);
-    // a bunch of our unit tests access this without a collection
-    option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT;
-  }
   rpc CompareAnswer(CompareAnswerRequest) returns (generic.String);
   rpc ExtractClozeForTyping(ExtractClozeForTypingRequest)
       returns (generic.String);
@@ -37,6 +33,12 @@ service CardRenderingService {
   rpc WriteTtsStream(WriteTtsStreamRequest) returns (generic.Empty);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendCardRenderingService {
+  rpc StripHtml(StripHtmlRequest) returns (generic.String);
+}
+
 message ExtractAvTagsRequest {
   string text = 1;
   bool question_side = 2;

proto/anki/cards.proto

@@ -18,6 +18,10 @@ service CardsService {
   rpc SetFlag(SetFlagRequest) returns (collection.OpChangesWithCount);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendCardsService {}
+
 message CardId {
   int64 cid = 1;
 }

proto/anki/codegen.proto

@@ -1,31 +0,0 @@
-// Copyright: Ankitects Pty Ltd and contributors
-// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
-
-syntax = "proto3";
-
-package anki.codegen;
-
-import "google/protobuf/descriptor.proto";
-
-extend google.protobuf.MethodOptions {
-  BackendMethod backend_method = 50000;
-}
-
-message MethodOptions {
-  BackendMethod backend_method = 50000;
-}
-
-enum BackendMethod {
-  /// Used for typical collection-based operations. We must implement the
-  //  method on Collection. The same method is automatically implemented on
-  //  Backend, which forwards to Collection.
-  BACKEND_METHOD_DELEGATE = 0;
-  /// Both the backend and collection need to implement the method; there
-  /// is no auto-delegation. Can be used to provide a method on both, but
-  /// skip the Collection mutex lock when a backend handle is available.
-  /// In practice we only do this for the i18n methods; for the occasional
-  /// method in other services that doesn't happen to need the collection,
-  /// we just delegate to the collection method for convenience, and to make
-  /// sure it's available even if the consumer is not using Backend.
-  BACKEND_METHOD_IMPLEMENT = 1;
-}

proto/anki/collection.proto

@@ -20,6 +20,8 @@ service CollectionService {
   rpc SetWantsAbort(generic.Empty) returns (generic.Empty);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
 service BackendCollectionService {
   rpc OpenCollection(OpenCollectionRequest) returns (generic.Empty);
   rpc CloseCollection(CloseCollectionRequest) returns (generic.Empty);

proto/anki/config.proto

@@ -24,6 +24,10 @@ service ConfigService {
   rpc SetPreferences(Preferences) returns (collection.OpChanges);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendConfigService {}
+
 message ConfigKey {
   enum Bool {
     BROWSER_TABLE_SHOW_NOTES_MODE = 0;

proto/anki/deckconfig.proto

@@ -25,6 +25,10 @@ service DeckConfigService {
       returns (collection.OpChanges);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendDeckConfigService {}
+
 message DeckConfigId {
   int64 dcid = 1;
 }

proto/anki/decks.proto

@@ -39,6 +39,10 @@ service DecksService {
   rpc GetCurrentDeck(generic.Empty) returns (Deck);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendDecksService {}
+
 message DeckId {
   int64 did = 1;
 }

proto/anki/i18n.proto

@@ -8,18 +8,19 @@ option java_multiple_files = true;
 package anki.i18n;
 
 import "anki/generic.proto";
-import "anki/codegen.proto";
 
 service I18nService {
-  rpc TranslateString(TranslateStringRequest) returns (generic.String) {
+  rpc TranslateString(TranslateStringRequest) returns (generic.String);
-    option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT;
+  rpc FormatTimespan(FormatTimespanRequest) returns (generic.String);
-  }
+  rpc I18nResources(I18nResourcesRequest) returns (generic.Json);
-  rpc FormatTimespan(FormatTimespanRequest) returns (generic.String) {
+}
-    option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT;
+
-  }
+// Implicitly includes any of the above methods that are not listed in the
-  rpc I18nResources(I18nResourcesRequest) returns (generic.Json) {
+// backend service.
-    option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT;
+service BackendI18nService {
-  }
+  rpc TranslateString(TranslateStringRequest) returns (generic.String);
+  rpc FormatTimespan(FormatTimespanRequest) returns (generic.String);
+  rpc I18nResources(I18nResourcesRequest) returns (generic.Json);
 }
 
 message TranslateStringRequest {

proto/anki/image_occlusion.proto

@@ -23,6 +23,10 @@ service ImageOcclusionService {
   rpc AddImageOcclusionNotetype(generic.Empty) returns (collection.OpChanges);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendImageOcclusionService {}
+
 message GetImageForOcclusionRequest {
   string path = 1;
 }

proto/anki/import_export.proto

@@ -11,7 +11,6 @@ import "anki/cards.proto";
 import "anki/collection.proto";
 import "anki/notes.proto";
 import "anki/generic.proto";
-import "anki/codegen.proto";
 
 service ImportExportService {
   rpc ImportAnkiPackage(ImportAnkiPackageRequest) returns (ImportResponse);
@@ -24,6 +23,8 @@ service ImportExportService {
   rpc ImportJsonString(generic.String) returns (ImportResponse);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
 service BackendImportExportService {
   rpc ImportCollectionPackage(ImportCollectionPackageRequest)
       returns (generic.Empty);

proto/anki/links.proto

@@ -13,6 +13,10 @@ service LinksService {
   rpc HelpPageLink(HelpPageLinkRequest) returns (generic.String);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendLinksService {}
+
 message HelpPageLinkRequest {
   enum HelpPage {
     NOTE_TYPE = 0;

proto/anki/media.proto

@@ -17,6 +17,10 @@ service MediaService {
   rpc RestoreTrash(generic.Empty) returns (generic.Empty);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendMediaService {}
+
 message CheckMediaResponse {
   repeated string unused = 1;
   repeated string missing = 2;

proto/anki/notes.proto

@@ -30,6 +30,10 @@ service NotesService {
   rpc GetSingleNotetypeOfNotes(notes.NoteIds) returns (notetypes.NotetypeId);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendNotesService {}
+
 message NoteId {
   int64 nid = 1;
 }

proto/anki/notetypes.proto

@@ -34,6 +34,10 @@ service NotetypesService {
       returns (collection.OpChanges);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendNotetypesService {}
+
 message NotetypeId {
   int64 ntid = 1;
 }

proto/anki/scheduler.proto

@@ -54,6 +54,10 @@ service SchedulerService {
   rpc RepositionDefaults(generic.Empty) returns (RepositionDefaultsResponse);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendSchedulerService {}
+
 message SchedulingState {
   message New {
     uint32 position = 1;

proto/anki/search.proto

@@ -23,6 +23,10 @@ service SearchService {
   rpc SetActiveBrowserColumns(generic.StringList) returns (generic.Empty);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendSearchService {}
+
 message SearchNode {
   message Dupe {
     int64 notetype_id = 1;

proto/anki/stats.proto

@@ -17,6 +17,10 @@ service StatsService {
   rpc SetGraphPreferences(GraphPreferences) returns (generic.Empty);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendStatsService {}
+
 message CardStatsResponse {
   message StatsRevlogEntry {
     int64 time = 1;

proto/anki/sync.proto

@@ -9,6 +9,9 @@ package anki.sync;
 
 import "anki/generic.proto";
 
+/// Syncing methods are only available with a Backend handle.
+service SyncService {}
+
 service BackendSyncService {
   rpc SyncMedia(SyncAuth) returns (generic.Empty);
   rpc AbortMediaSync(generic.Empty) returns (generic.Empty);

proto/anki/tags.proto

@@ -27,6 +27,10 @@ service TagsService {
   rpc CompleteTag(CompleteTagRequest) returns (CompleteTagResponse);
 }
 
+// Implicitly includes any of the above methods that are not listed in the
+// backend service.
+service BackendTagsService {}
+
 message SetTagCollapsedRequest {
   string name = 1;
   bool collapsed = 2;

rslib/Cargo.toml

@@ -22,12 +22,13 @@ required-features = ["bench"]
 [build-dependencies]
 anki_io = { version = "0.0.0", path = "io" }
 anki_proto = { version = "0.0.0", path = "proto" }
+anki_proto_gen = { version = "0.0.0", path = "proto_gen" }
 anyhow = "1.0.71"
 inflections = "1.1.1"
+itertools = "0.10.5"
 prettyplease = "0.2.7"
 prost = "0.11.8"
 prost-reflect = "0.11.4"
-prost-types = "0.11.9"
 syn = { version = "2.0.18", features = ["parsing", "printing"] }
 
 [dev-dependencies]

rslib/proto/Cargo.toml

@@ -11,6 +11,7 @@ rust-version.workspace = true
 
 [build-dependencies]
 anki_io = { version = "0.0.0", path = "../io" }
+anki_proto_gen = { version = "0.0.0", path = "../proto_gen" }
 anyhow = "1.0.71"
 inflections = "1.1.1"
 itertools = "0.10.5"

rslib/proto/build.rs

@@ -2,21 +2,22 @@
 // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
 pub mod python;
-pub mod rust_protos;
+pub mod rust;
 pub mod ts;
-pub mod utils;
 
 use std::env;
 use std::path::PathBuf;
 
+use anki_proto_gen::get_services;
 use anyhow::Result;
 
 fn main() -> Result<()> {
     let descriptors_path = env::var("DESCRIPTORS_BIN").ok().map(PathBuf::from);
 
-    let pool = rust_protos::write_rust_protos(descriptors_path)?;
+    let pool = rust::write_rust_protos(descriptors_path)?;
-    python::write_python_interface(&pool)?;
+    let (_, services) = get_services(&pool);
-    ts::write_ts_interface(&pool)?;
+    python::write_python_interface(&services)?;
+    ts::write_ts_interface(&services)?;
 
     Ok(())
 }

rslib/proto/python.rs

@@ -7,30 +7,26 @@ use std::path::Path;
 
 use anki_io::create_dir_all;
 use anki_io::create_file;
+use anki_proto_gen::BackendService;
+use anki_proto_gen::Method;
 use anyhow::Result;
 use inflections::Inflect;
-use prost_reflect::DescriptorPool;
 use prost_reflect::FieldDescriptor;
 use prost_reflect::Kind;
 use prost_reflect::MessageDescriptor;
-use prost_reflect::MethodDescriptor;
-use prost_reflect::ServiceDescriptor;
 
-use crate::utils::Comments;
+pub(crate) fn write_python_interface(services: &[BackendService]) -> Result<()> {
-
-pub(crate) fn write_python_interface(pool: &DescriptorPool) -> Result<()> {
     let output_path = Path::new("../../out/pylib/anki/_backend_generated.py");
     create_dir_all(output_path.parent().unwrap())?;
     let mut out = BufWriter::new(create_file(output_path)?);
     write_header(&mut out)?;
 
-    for service in pool.services() {
+    for service in services {
-        if service.name() == "AnkidroidService" {
+        if service.name == "BackendAnkidroidService" {
             continue;
         }
-        let comments = Comments::from_file(service.parent_file().file_descriptor_proto());
+        for method in service.all_methods() {
-        for method in service.methods() {
+            render_method(service, method, &mut out);
-            render_method(&service, &method, &comments, &mut out);
         }
     }
 
@@ -48,18 +44,13 @@ pub(crate) fn write_python_interface(pool: &DescriptorPool) -> Result<()> {
 ///     output = anki.generic_pb2.StringList()
 ///     output.ParseFromString(raw_bytes)
 ///     return output.vals
-fn render_method(
+fn render_method(service: &BackendService, method: &Method, out: &mut impl Write) {
-    service: &ServiceDescriptor,
+    let method_name = method.name.to_snake_case();
-    method: &MethodDescriptor,
+    let input = method.proto.input();
-    comments: &Comments,
+    let output = method.proto.output();
-    out: &mut impl Write,
+    let service_idx = service.index;
-) {
+    let method_idx = method.index;
-    let method_name = method.name().to_snake_case();
+    let comments = format_comments(&method.comments);
-    let input = method.input();
-    let output = method.output();
-    let service_idx = service.index();
-    let method_idx = method.index();
-    let comments = format_comments(comments.get_for_path(method.path()));
 
     // raw bytes
     write!(
@@ -89,7 +80,7 @@ fn render_method(
     .unwrap();
 }
 
-fn format_comments(comments: Option<&str>) -> String {
+fn format_comments(comments: &Option<String>) -> String {
     comments
         .as_ref()
         .map(|c| {

rslib/proto/src/lib.rs

@@ -15,7 +15,6 @@ protobuf!(ankidroid, "ankidroid");
 protobuf!(backend, "backend");
 protobuf!(card_rendering, "card_rendering");
 protobuf!(cards, "cards");
-protobuf!(codegen, "codegen");
 protobuf!(collection, "collection");
 protobuf!(config, "config");
 protobuf!(deckconfig, "deckconfig");

rslib/proto/ts.rs

@@ -9,46 +9,39 @@ use std::path::Path;
 
 use anki_io::create_dir_all;
 use anki_io::create_file;
+use anki_proto_gen::BackendService;
+use anki_proto_gen::Method;
 use anyhow::Result;
 use inflections::Inflect;
-use prost_reflect::DescriptorPool;
-use prost_reflect::MethodDescriptor;
-use prost_reflect::ServiceDescriptor;
 
-use crate::utils::Comments;
+pub(crate) fn write_ts_interface(services: &[BackendService]) -> Result<()> {
-
-pub(crate) fn write_ts_interface(pool: &DescriptorPool) -> Result<()> {
     let root = Path::new("../../out/ts/lib/anki");
     create_dir_all(root)?;
 
-    for service in pool.services() {
+    for service in services {
-        if service.name() == "AnkidroidService" {
+        if service.name == "BackendAnkidroidService" {
             continue;
         }
-        let service_name = service.name().replace("Service", "").to_snake_case();
-        let comments = Comments::from_file(service.parent_file().file_descriptor_proto());
 
-        write_dts_file(root, &service_name, &service, &comments)?;
+        let service_name = service.name.replace("Service", "").to_snake_case();
-        write_js_file(root, &service_name, &service, &comments)?;
+
+        write_dts_file(root, &service_name, service)?;
+        write_js_file(root, &service_name, service)?;
     }
 
     Ok(())
 }
 
-fn write_dts_file(
+fn write_dts_file(root: &Path, service_name: &str, service: &BackendService) -> Result<()> {
-    root: &Path,
-    service_name: &str,
-    service: &ServiceDescriptor,
-    comments: &Comments,
-) -> Result<()> {
     let output_path = root.join(format!("{service_name}_service.d.ts"));
     let mut out = BufWriter::new(create_file(output_path)?);
     write_dts_header(&mut out)?;
 
     let mut referenced_packages = HashSet::new();
     let mut method_text = String::new();
-    for method in service.methods() {
+
-        let method = MethodDetails::from_descriptor(&method, comments);
+    for method in service.all_methods() {
+        let method = MethodDetails::from_method(method);
         record_referenced_type(&mut referenced_packages, &method.input_type)?;
         record_referenced_type(&mut referenced_packages, &method.output_type)?;
         write_dts_method(&method, &mut method_text)?;
@@ -100,20 +93,15 @@ fn write_dts_method(
     Ok(())
 }
 
-fn write_js_file(
+fn write_js_file(root: &Path, service_name: &str, service: &BackendService) -> Result<()> {
-    root: &Path,
-    service_name: &str,
-    service: &ServiceDescriptor,
-    comments: &Comments,
-) -> Result<()> {
     let output_path = root.join(format!("{service_name}_service.js"));
     let mut out = BufWriter::new(create_file(output_path)?);
     write_js_header(&mut out)?;
 
     let mut referenced_packages = HashSet::new();
     let mut method_text = String::new();
-    for method in service.methods() {
+    for method in service.all_methods() {
-        let method = MethodDetails::from_descriptor(&method, comments);
+        let method = MethodDetails::from_method(method);
         record_referenced_type(&mut referenced_packages, &method.input_type)?;
         record_referenced_type(&mut referenced_packages, &method.output_type)?;
         write_js_method(&method, &mut method_text)?;
@@ -169,16 +157,16 @@ struct MethodDetails {
 }
 
 impl MethodDetails {
-    fn from_descriptor(method: &MethodDescriptor, comments: &Comments) -> MethodDetails {
+    fn from_method(method: &Method) -> MethodDetails {
-        let name = method.name().to_camel_case();
+        let name = method.name.to_camel_case();
-        let input_type = full_name_to_imported_reference(method.input().full_name());
+        let input_type = full_name_to_imported_reference(method.proto.input().full_name());
-        let output_type = full_name_to_imported_reference(method.output().full_name());
+        let output_type = full_name_to_imported_reference(method.proto.output().full_name());
-        let comments = comments.get_for_path(method.path());
+        let comments = method.comments.clone();
         Self {
             method_name: name,
             input_type,
             output_type,
-            comments: comments.map(ToString::to_string),
+            comments,
         }
     }
 }

rslib/proto/utils.rs

@@ -1,36 +0,0 @@
-// Copyright: Ankitects Pty Ltd and contributors
-// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
-
-use std::collections::HashMap;
-
-use prost_types::FileDescriptorProto;
-
-#[derive(Debug)]
-pub struct Comments {
-    path_map: HashMap<Vec<i32>, String>,
-}
-
-impl Comments {
-    pub fn from_file(file: &FileDescriptorProto) -> Self {
-        Self {
-            path_map: file
-                .source_code_info
-                .as_ref()
-                .unwrap()
-                .location
-                .iter()
-                .map(|l| (l.path.clone(), l.leading_comments().trim().to_string()))
-                .collect(),
-        }
-    }
-
-    pub fn get_for_path(&self, path: &[i32]) -> Option<&str> {
-        self.path_map.get(path).map(|s| s.as_str()).and_then(|s| {
-            if s.is_empty() {
-                None
-            } else {
-                Some(s)
-            }
-        })
-    }
-}

rslib/proto_gen/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "anki_proto_gen"
+publish = false
+description = "Helpers for interface code generation"
+
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+rust-version.workspace = true
+
+[dependencies]
+inflections = "1.1.1"
+itertools = "0.10.5"
+prost-reflect = "0.11.4"
+prost-types = "0.11.9"

rslib/proto_gen/src/lib.rs

@@ -0,0 +1,201 @@
+// Copyright: Ankitects Pty Ltd and contributors
+// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+//! Some helpers for code generation in external crates, that ensure indexes
+//! match.
+
+use std::collections::HashMap;
+
+use inflections::Inflect;
+use itertools::Either;
+use itertools::Itertools;
+use prost_reflect::DescriptorPool;
+use prost_reflect::MessageDescriptor;
+use prost_reflect::MethodDescriptor;
+use prost_reflect::ServiceDescriptor;
+
+/// We look for ExampleService and BackedExampleService, both of which are
+/// expected to exist (but may be empty).
+///
+/// - If a method is listed in BackendExampleService and not in ExampleService,
+/// that method is only available with a Backend.
+/// - If a method is listed in both services, you can provide separate
+/// implementations for each of the traits.
+/// - If a method is listed only in ExampleService, a forwarding method on
+/// Backend is automatically implemented. This bypasses the trait and implements
+/// directly on Backend.
+///
+/// It's important that service and method indices are the same for
+/// client-generated code, so the client code should use the .index fields
+/// of Service and Method provided by get_services(), and not
+/// .enumerate() or .proto.index()
+///
+/// Client code will want to ignore CollectionServices, and focus on
+/// BackendServices.
+pub fn get_services(pool: &DescriptorPool) -> (Vec<CollectionService>, Vec<BackendService>) {
+    // split services into backend and collection
+    let (mut col_services, mut backend_services): (Vec<_>, Vec<_>) =
+        pool.services().partition_map(|service| {
+            if service.name().starts_with("Backend") {
+                Either::Right(BackendService::from_proto(service))
+            } else {
+                Either::Left(CollectionService::from_proto(service))
+            }
+        });
+    assert!(col_services.len() == backend_services.len());
+    // copy collection methods into backend services if they don't have one with
+    // a matching name
+    for service in &mut backend_services {
+        // locate associated collection service
+        let Some(col_service) = col_services
+            .iter()
+            .find(|cs| cs.name == service.name.trim_start_matches("Backend")) else { panic!("missing associated service: {}", service.name) };
+
+        // add any methods that don't exist in backend trait methods to the delegating
+        // methods
+        service.delegating_methods = col_service
+            .trait_methods
+            .iter()
+            .filter(|m| service.trait_methods.iter().all(|bm| bm.name != m.name))
+            .map(|method| Method {
+                index: method.index + service.trait_methods.len(),
+                ..method.clone()
+            })
+            .collect();
+    }
+    // fill comments in
+    let comments = MethodComments::from_pool(pool);
+    for service in &mut col_services {
+        for method in &mut service.trait_methods {
+            method.comments = comments.get_for_method(&method.proto);
+        }
+    }
+    for service in &mut backend_services {
+        for method in &mut service.trait_methods {
+            method.comments = comments.get_for_method(&method.proto);
+        }
+        for method in &mut service.delegating_methods {
+            method.comments = comments.get_for_method(&method.proto);
+        }
+    }
+
+    (col_services, backend_services)
+}
+
+#[derive(Debug)]
+pub struct CollectionService {
+    pub name: String,
+    pub index: usize,
+    pub trait_methods: Vec<Method>,
+    pub proto: ServiceDescriptor,
+}
+
+#[derive(Debug)]
+pub struct BackendService {
+    pub name: String,
+    pub index: usize,
+    pub trait_methods: Vec<Method>,
+    pub delegating_methods: Vec<Method>,
+    pub proto: ServiceDescriptor,
+}
+
+#[derive(Debug, Clone)]
+pub struct Method {
+    pub name: String,
+    pub index: usize,
+    pub comments: Option<String>,
+    pub proto: MethodDescriptor,
+}
+
+impl CollectionService {
+    pub fn from_proto(service: prost_reflect::ServiceDescriptor) -> Self {
+        CollectionService {
+            name: service.name().to_string(),
+            index: service.index(),
+            trait_methods: service.methods().map(Method::from_proto).collect(),
+            proto: service,
+        }
+    }
+}
+
+impl BackendService {
+    pub fn from_proto(service: prost_reflect::ServiceDescriptor) -> Self {
+        BackendService {
+            name: service.name().to_string(),
+            index: service.index(),
+            trait_methods: service.methods().map(Method::from_proto).collect(),
+            proto: service,
+            // filled in later
+            delegating_methods: vec![],
+        }
+    }
+
+    pub fn all_methods(&self) -> impl Iterator<Item = &Method> {
+        self.trait_methods
+            .iter()
+            .chain(self.delegating_methods.iter())
+    }
+}
+
+impl Method {
+    pub fn from_proto(method: prost_reflect::MethodDescriptor) -> Self {
+        Method {
+            name: method.name().to_snake_case(),
+            index: method.index(),
+            proto: method,
+            // filled in later
+            comments: None,
+        }
+    }
+
+    /// The input type, if not empty.
+    pub fn input(&self) -> Option<MessageDescriptor> {
+        msg_if_empty(self.proto.input())
+    }
+
+    /// The output type, if not empty.
+    pub fn output(&self) -> Option<MessageDescriptor> {
+        msg_if_empty(self.proto.output())
+    }
+}
+
+fn msg_if_empty(msg: MessageDescriptor) -> Option<MessageDescriptor> {
+    if msg.full_name() == "anki.generic.Empty" {
+        None
+    } else {
+        Some(msg)
+    }
+}
+
+#[derive(Debug)]
+struct MethodComments<'a> {
+    // package name -> method path -> comment
+    by_package_and_path: HashMap<&'a str, HashMap<Vec<i32>, String>>,
+}
+
+impl<'a> MethodComments<'a> {
+    pub fn from_pool(pool: &'a DescriptorPool) -> MethodComments<'a> {
+        let mut by_package_and_path = HashMap::new();
+        for file in pool.file_descriptor_protos() {
+            let path_map = file
+                .source_code_info
+                .as_ref()
+                .unwrap()
+                .location
+                .iter()
+                .map(|l| (l.path.clone(), l.leading_comments().trim().to_string()))
+                .collect();
+            by_package_and_path.insert(file.package(), path_map);
+        }
+        Self {
+            by_package_and_path,
+        }
+    }
+
+    pub fn get_for_method(&self, method: &MethodDescriptor) -> Option<String> {
+        self.by_package_and_path
+            .get(method.parent_file().package_name())
+            .and_then(|by_path| by_path.get(method.path()))
+            .and_then(|s| if s.is_empty() { None } else { Some(s.into()) })
+    }
+}

rslib/rust_interface.rs

@@ -6,7 +6,10 @@ use std::fmt::Write;
 use std::path::PathBuf;
 
 use anki_io::write_file_if_changed;
-use anki_proto::codegen::BackendMethod;
+use anki_proto_gen::get_services;
+use anki_proto_gen::BackendService;
+use anki_proto_gen::CollectionService;
+use anki_proto_gen::Method;
 use anyhow::Context;
 use anyhow::Result;
 use inflections::Inflect;
@@ -15,105 +18,50 @@ use prost_reflect::DescriptorPool;
 pub fn write_rust_interface(pool: &DescriptorPool) -> Result<()> {
     let mut buf = String::new();
     buf.push_str("use crate::error::Result; use prost::Message;");
-    let services = pool
-        .services()
-        .map(RustService::from_proto)
-        .collect::<Vec<_>>();
-    for service in &services {
-        render_service(service, &mut buf);
-    }
 
-    render_top_level_run_method(&mut buf, &services, true);
+    let (col_services, backend_services) = get_services(pool);
-    render_top_level_run_method(&mut buf, &services, false);
+
+    render_collection_services(&col_services, &mut buf)?;
+    render_backend_services(&backend_services, &mut buf)?;
 
-    // println!("{}", &buf);
     let buf = format_code(buf)?;
-    // write into OUT_DIR so we can use it in build.rs
+    // println!("{}", &buf);
+    // panic!();
     let out_dir = env::var("OUT_DIR").unwrap();
     let path = PathBuf::from(out_dir).join("backend.rs");
     write_file_if_changed(path, buf).context("write file")?;
     Ok(())
 }
 
-#[derive(Debug)]
+fn render_collection_services(col_services: &[CollectionService], buf: &mut String) -> Result<()> {
-struct RustService {
+    for service in col_services {
-    name: String,
+        render_collection_trait(service, buf);
-    methods: Vec<RustMethod>,
+        render_individual_service_run_method_for_collection(buf, service);
+    }
+    render_top_level_run_method(
+        col_services.iter().map(|s| (s.index, s.name.as_str())),
+        "&mut self",
+        "crate::collection::Collection",
+        buf,
+    );
+
+    Ok(())
 }
 
-#[derive(Debug)]
+fn render_backend_services(backend_services: &[BackendService], buf: &mut String) -> Result<()> {
-struct RustMethod {
+    for service in backend_services {
-    name: String,
+        render_backend_trait(service, buf);
-    input_type: Option<String>,
+        render_delegating_backend_methods(service, buf);
-    output_type: Option<String>,
+        render_individual_service_run_method_for_backend(buf, service);
-    options: anki_proto::codegen::MethodOptions,
-    service_name: String,
-}
-
-impl RustMethod {
-    /// No text if generic::Empty
-    fn text_if_input_not_empty(&self, text: impl Fn(&String) -> String) -> String {
-        self.input_type.as_ref().map(text).unwrap_or_default()
     }
+    render_top_level_run_method(
+        backend_services.iter().map(|s| (s.index, s.name.as_str())),
+        "&self",
+        "crate::backend::Backend",
+        buf,
+    );
 
-    /// No text if generic::Empty
+    Ok(())
-    fn get_input_arg_with_label(&self) -> String {
-        self.input_type
-            .as_ref()
-            .map(|t| format!("input: {}", t))
-            .unwrap_or_default()
-    }
-
-    /// () if generic::Empty
-    fn get_output_type(&self) -> String {
-        self.output_type.as_deref().unwrap_or("()").into()
-    }
-
-    fn text_if_output_not_empty(&self, text: impl Fn(&String) -> String) -> String {
-        self.output_type.as_ref().map(text).unwrap_or_default()
-    }
-
-    fn wants_abstract_backend_method(&self) -> bool {
-        self.service_name.starts_with("Backend")
-            || self.options.backend_method() == BackendMethod::Implement
-    }
-
-    fn wants_abstract_collection_method(&self) -> bool {
-        !self.service_name.starts_with("Backend")
-    }
-
-    fn from_proto(method: prost_reflect::MethodDescriptor) -> Self {
-        RustMethod {
-            name: method.name().to_snake_case(),
-            input_type: rust_type(method.input().full_name()),
-            output_type: rust_type(method.output().full_name()),
-            options: method.options().transcode_to().unwrap(),
-            service_name: method.parent_service().name().to_string(),
-        }
-    }
-}
-
-impl RustService {
-    fn from_proto(service: prost_reflect::ServiceDescriptor) -> Self {
-        RustService {
-            name: service.name().to_string(),
-            methods: service.methods().map(RustMethod::from_proto).collect(),
-        }
-    }
-}
-
-fn rust_type(name: &str) -> Option<String> {
-    if name == "anki.generic.Empty" {
-        return None;
-    }
-    let Some((head, tail)) = name.rsplit_once( '.') else { panic!() };
-    Some(format!(
-        "{}::{}",
-        head.to_snake_case()
-            .replace('.', "::")
-            .replace("anki::", "anki_proto::"),
-        tail
-    ))
 }
 
 fn format_code(code: String) -> Result<String> {
@@ -121,118 +69,75 @@ fn format_code(code: String) -> Result<String> {
     Ok(prettyplease::unparse(&syntax_tree))
 }
 
-fn render_abstract_collection_method(method: &RustMethod, buf: &mut String) {
+fn render_collection_trait(service: &CollectionService, buf: &mut String) {
+    let name = &service.name;
+    writeln!(buf, "pub trait {name} {{").unwrap();
+    for method in &service.trait_methods {
+        render_trait_method(method, "&mut self", buf);
+    }
+    buf.push('}');
+}
+
+fn render_trait_method(method: &Method, self_kind: &str, buf: &mut String) {
     let method_name = &method.name;
     let input_with_label = method.get_input_arg_with_label();
     let output_type = method.get_output_type();
     writeln!(
         buf,
-        "fn {method_name}(&mut self, {input_with_label}) -> Result<{output_type}>;"
+        "fn {method_name}({self_kind}, {input_with_label}) -> Result<{output_type}>;"
     )
     .unwrap();
 }
 
-fn render_abstract_backend_method(method: &RustMethod, buf: &mut String, _service: &RustService) {
+fn render_backend_trait(service: &BackendService, buf: &mut String) {
-    let method_name = &method.name;
+    let name = &service.name;
-    let input_with_label = method.get_input_arg_with_label();
+    writeln!(buf, "pub trait {name} {{").unwrap();
-    let output_type = method.get_output_type();
+    for method in &service.trait_methods {
-    writeln!(
+        render_trait_method(method, "&self", buf);
-        buf,
+    }
-        "fn {method_name}(&self, {input_with_label}) -> Result<{output_type}>;"
+    buf.push('}');
-    )
-    .unwrap();
 }
 
-fn render_delegating_backend_method(method: &RustMethod, buf: &mut String, service: &RustService) {
+fn render_delegating_backend_methods(service: &BackendService, buf: &mut String) {
+    buf.push_str("impl crate::backend::Backend {");
+    for method in &service.delegating_methods {
+        render_delegating_backend_method(method, service.name.trim_start_matches("Backend"), buf);
+    }
+    buf.push('}');
+}
+
+fn render_delegating_backend_method(method: &Method, method_qualifier: &str, buf: &mut String) {
     let method_name = &method.name;
     let input_with_label = method.get_input_arg_with_label();
     let input = method.text_if_input_not_empty(|_| "input".into());
     let output_type = method.get_output_type();
-    let col_service = &service.name;
     writeln!(
         buf,
         "fn {method_name}(&self, {input_with_label}) -> Result<{output_type}> {{
-        self.with_col(|col| {col_service}::{method_name}(col, {input})) }}",
+        self.with_col(|col| {method_qualifier}::{method_name}(col, {input})) }}",
     )
     .unwrap();
 }
 
-fn render_service(service: &RustService, buf: &mut String) {
-    let have_collection = service
-        .methods
-        .iter()
-        .any(|m| m.wants_abstract_collection_method());
-    if have_collection {
-        render_collection_trait(service, buf);
-    }
-    if service
-        .methods
-        .iter()
-        .any(|m| m.wants_abstract_backend_method())
-    {
-        render_backend_trait(service, buf);
-    }
-    render_delegating_backend_methods(service, buf);
-    render_individual_service_run_method(buf, service, true);
-    render_individual_service_run_method(buf, service, false);
-}
-
-fn render_collection_trait(service: &RustService, buf: &mut String) {
-    let name = &service.name;
-    writeln!(buf, "pub trait {name} {{").unwrap();
-    for method in &service.methods {
-        if method.wants_abstract_collection_method() {
-            render_abstract_collection_method(method, buf);
-        }
-    }
-    buf.push('}');
-}
-
-fn render_backend_trait(service: &RustService, buf: &mut String) {
-    let name = if !service.name.starts_with("Backend") {
-        format!("Backend{}", service.name)
-    } else {
-        service.name.clone()
-    };
-    writeln!(buf, "pub trait {name} {{").unwrap();
-    for method in &service.methods {
-        if method.wants_abstract_backend_method() {
-            render_abstract_backend_method(method, buf, service);
-        }
-    }
-    buf.push('}');
-}
-
-fn render_delegating_backend_methods(service: &RustService, buf: &mut String) {
-    buf.push_str("impl crate::backend::Backend {");
-    for method in &service.methods {
-        if method.wants_abstract_backend_method() {
-            continue;
-        }
-        render_delegating_backend_method(method, buf, service);
-    }
-    buf.push('}');
-}
-
 // Matches all service types and delegates to the revelant self.run_foo_method()
-fn render_top_level_run_method(buf: &mut String, services: &[RustService], backend: bool) {
+fn render_top_level_run_method<'a>(
-    let self_kind = if backend { "&self" } else { "&mut self" };
+    // (index, name)
-    let struct_to_impl = if backend {
+    services: impl Iterator<Item = (usize, &'a str)>,
-        "crate::backend::Backend"
+    self_kind: &str,
-    } else {
+    struct_name: &str,
-        "crate::collection::Collection"
+    buf: &mut String,
-    };
+) {
     writeln!(buf,
-        r#" impl {struct_to_impl} {{
+        r#" impl {struct_name} {{
     pub fn run_service_method({self_kind}, service: u32, method: u32, input: &[u8]) -> Result<Vec<u8>, Vec<u8>> {{
         match service {{
 "#,
     ).unwrap();
-    for (idx, service) in services.iter().enumerate() {
+    for (idx, service) in services {
         writeln!(
             buf,
             "{idx} => self.run_{service}_method(method, input),",
-            service = service.name.to_snake_case()
+            service = service.to_snake_case()
         )
         .unwrap();
     }
@@ -250,51 +155,21 @@ fn render_top_level_run_method(buf: &mut String, services: &[RustService], backe
     );
 }
 
-fn render_individual_service_run_method(buf: &mut String, service: &RustService, backend: bool) {
+fn render_individual_service_run_method_for_collection(
-    let self_kind = if backend { "&self" } else { "&mut self" };
+    buf: &mut String,
-    let struct_to_impl = if backend {
+    service: &CollectionService,
-        "crate::backend::Backend"
+) {
-    } else {
-        "crate::collection::Collection"
-    };
-    let method_qualifier = if backend {
-        struct_to_impl
-    } else {
-        &service.name
-    };
-
     let service_name = &service.name.to_snake_case();
     writeln!(
         buf,
         "#[allow(unused_variables, clippy::match_single_binding)]
-        impl {struct_to_impl} {{ pub(crate) fn run_{service_name}_method({self_kind},
+        impl crate::collection::Collection {{ pub(crate) fn run_{service_name}_method(&mut self,
         method: u32, input: &[u8]) -> Result<Vec<u8>> {{
         match method {{",
     )
     .unwrap();
-    for (idx, method) in service.methods.iter().enumerate() {
+    for method in &service.trait_methods {
-        if !backend && !method.wants_abstract_collection_method() {
+        render_method_in_match_expression(method, &service.name, buf);
-            continue;
-        }
-        let decode_input =
-            method.text_if_input_not_empty(|kind| format!("let input = {kind}::decode(input)?;"));
-        let rust_method = &method.name;
-        let input = method.text_if_input_not_empty(|_| "input".into());
-        let output_assign = method.text_if_output_not_empty(|_| "let output = ".into());
-        let output = if method.output_type.is_none() {
-            "Vec::new()"
-        } else {
-            "{ let mut out_bytes = Vec::new();
-            output.encode(&mut out_bytes)?;
-            out_bytes }"
-        };
-        writeln!(
-            buf,
-            "{idx} => {{ {decode_input}
-                {output_assign} {method_qualifier}::{rust_method}(self, {input})?;
-                Ok({output}) }},",
-        )
-        .unwrap();
     }
     buf.push_str(
         r#"
@@ -304,3 +179,102 @@ fn render_individual_service_run_method(buf: &mut String, service: &RustService,
 "#,
     );
 }
+
+fn render_individual_service_run_method_for_backend(buf: &mut String, service: &BackendService) {
+    let service_name = &service.name.to_snake_case();
+    writeln!(
+        buf,
+        "#[allow(unused_variables, clippy::match_single_binding)]
+        impl crate::backend::Backend {{ pub(crate) fn run_{service_name}_method(&self,
+        method: u32, input: &[u8]) -> Result<Vec<u8>> {{
+        match method {{",
+    )
+    .unwrap();
+    for method in &service.trait_methods {
+        render_method_in_match_expression(method, &service.name, buf);
+    }
+    for method in &service.delegating_methods {
+        render_method_in_match_expression(method, "crate::backend::Backend", buf);
+    }
+    buf.push_str(
+        r#"
+            _ => Err(crate::error::AnkiError::InvalidMethodIndex),
+        }
+} }
+"#,
+    );
+}
+
+fn render_method_in_match_expression(method: &Method, method_qualifier: &str, buf: &mut String) {
+    let decode_input =
+        method.text_if_input_not_empty(|kind| format!("let input = {kind}::decode(input)?;"));
+    let rust_method = &method.name;
+    let input = method.text_if_input_not_empty(|_| "input".into());
+    let output_assign = method.text_if_output_not_empty(|_| "let output = ".into());
+    let idx = method.index;
+    let output = if method.output().is_none() {
+        "Vec::new()"
+    } else {
+        "{ let mut out_bytes = Vec::new();
+            output.encode(&mut out_bytes)?;
+            out_bytes }"
+    };
+    writeln!(
+        buf,
+        "{idx} => {{ {decode_input}
+                {output_assign} {method_qualifier}::{rust_method}(self, {input})?;
+                Ok({output}) }},",
+    )
+    .unwrap();
+}
+
+trait MethodHelpers {
+    fn input_type(&self) -> Option<String>;
+    fn output_type(&self) -> Option<String>;
+    fn text_if_input_not_empty(&self, text: impl Fn(&String) -> String) -> String;
+    fn get_input_arg_with_label(&self) -> String;
+    fn get_output_type(&self) -> String;
+    fn text_if_output_not_empty(&self, text: impl Fn(&String) -> String) -> String;
+}
+
+impl MethodHelpers for Method {
+    fn input_type(&self) -> Option<String> {
+        self.input().map(|t| rust_type(t.full_name()))
+    }
+
+    fn output_type(&self) -> Option<String> {
+        self.output().map(|t| rust_type(t.full_name()))
+    }
+    /// No text if generic::Empty
+    fn text_if_input_not_empty(&self, text: impl Fn(&String) -> String) -> String {
+        self.input_type().as_ref().map(text).unwrap_or_default()
+    }
+
+    /// No text if generic::Empty
+    fn get_input_arg_with_label(&self) -> String {
+        self.input_type()
+            .as_ref()
+            .map(|t| format!("input: {}", t))
+            .unwrap_or_default()
+    }
+
+    /// () if generic::Empty
+    fn get_output_type(&self) -> String {
+        self.output_type().as_deref().unwrap_or("()").into()
+    }
+
+    fn text_if_output_not_empty(&self, text: impl Fn(&String) -> String) -> String {
+        self.output_type().as_ref().map(text).unwrap_or_default()
+    }
+}
+
+fn rust_type(name: &str) -> String {
+    let Some((head, tail)) = name.rsplit_once( '.') else { panic!() };
+    format!(
+        "{}::{}",
+        head.to_snake_case()
+            .replace('.', "::")
+            .replace("anki::", "anki_proto::"),
+        tail
+    )
+}

tools/workspace-hack/Cargo.toml

@@ -28,6 +28,7 @@ hashbrown = { version = "0.12", features = ["raw"] }
 hmac = { version = "0.12", default-features = false, features = ["reset"] }
 hyper = { version = "0.14", features = ["full"] }
 indexmap = { version = "1", default-features = false, features = ["std"] }
+itertools = { version = "0.10" }
 log = { version = "0.4", default-features = false, features = ["std"] }
 num-traits = { version = "0.2" }
 phf_shared = { version = "0.11", default-features = false, features = ["std"] }
@@ -69,6 +70,7 @@ hashbrown = { version = "0.12", features = ["raw"] }
 hmac = { version = "0.12", default-features = false, features = ["reset"] }
 hyper = { version = "0.14", features = ["full"] }
 indexmap = { version = "1", default-features = false, features = ["std"] }
+itertools = { version = "0.10" }
 log = { version = "0.4", default-features = false, features = ["std"] }
 num-traits = { version = "0.2" }
 phf_shared = { version = "0.11", default-features = false, features = ["std"] }