GitHub/Anki
1
0
Fork 0
mirror of https://github.com/ankitects/anki.git synced 2025-09-18 14:02:21 -04:00
Code Issues Projects Releases Packages 1 Wiki Activity Actions

More service generation refactoring

Browse source 
- Dropped the protobuf extensions in favor of explicitly listing out
methods in both services if we want to implement both, as it's clearer.
- Move Service/Method wrappers into a separate crate that the various
clients can import, to easily get at the list of backend services and
their correct indices and comments.
This commit is contained in:
Damien Elmes 2023-06-22 09:38:05 +10:00
parent dee7860f08
commit b37063e20a
34 changed files with 545 additions and 354 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
Cargo.lock generated
View file

@ -91,6 +91,7 @@ dependencies = [
"anki_i18n",
"anki_io",
"anki_proto",
"anki_proto_gen",
"anyhow",
"async-compression",
"async-stream",
@ -127,7 +128,6 @@ dependencies = [
"prettyplease 0.2.7",
"prost",
"prost-reflect",
"prost-types",
"pulldown-cmark 0.9.2",
"rand 0.8.5",
"regex",
@ -213,6 +213,7 @@ name = "anki_proto"
version = "0.0.0"
dependencies = [
"anki_io",
"anki_proto_gen",
"anyhow",
"inflections",
"itertools",
@ -226,6 +227,16 @@ dependencies = [
"strum",
]
[[package]]
name = "anki_proto_gen"
version = "0.0.0"
dependencies = [
"inflections",
"itertools",
"prost-reflect",
"prost-types",
]
[[package]]
name = "anstream"
version = "0.2.6"
@ -5067,6 +5078,7 @@ dependencies = [
"hmac",
"hyper",
"indexmap",
"itertools",
"log",
"num-traits",
"phf_shared 0.11.1",

9
cargo/licenses.json
View file

@ -107,6 +107,15 @@
"license_file": null,
"description": "Anki's Rust library protobuf code"
},
{
"name": "anki_proto_gen",
"version": "0.0.0",
"authors": "Ankitects Pty Ltd and contributors <https://help.ankiweb.net>",
"repository": null,
"license": "AGPL-3.0-or-later",
"license_file": null,
"description": "Helpers for interface code generation"
},
{
"name": "anstream",
"version": "0.2.6",

2
proto/anki/ankidroid.proto
View file

@ -20,6 +20,8 @@ service AnkidroidService {
returns (GetActiveSequenceNumbersResponse);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendAnkidroidService {
rpc SchedTimingTodayLegacy(SchedTimingTodayLegacyRequest)
returns (scheduler.SchedTimingTodayResponse);

12
proto/anki/card_rendering.proto
View file

@ -10,7 +10,6 @@ package anki.card_rendering;
import "anki/generic.proto";
import "anki/notes.proto";
import "anki/notetypes.proto";
import "anki/codegen.proto";
service CardRenderingService {
rpc ExtractAvTags(ExtractAvTagsRequest) returns (ExtractAvTagsResponse);
@ -26,10 +25,7 @@ service CardRenderingService {
rpc RenderMarkdown(RenderMarkdownRequest) returns (generic.String);
rpc EncodeIriPaths(generic.String) returns (generic.String);
rpc DecodeIriPaths(generic.String) returns (generic.String);
rpc StripHtml(StripHtmlRequest) returns (generic.String) {
// a bunch of our unit tests access this without a collection
option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT;
}
rpc StripHtml(StripHtmlRequest) returns (generic.String);
rpc CompareAnswer(CompareAnswerRequest) returns (generic.String);
rpc ExtractClozeForTyping(ExtractClozeForTypingRequest)
returns (generic.String);
@ -37,6 +33,12 @@ service CardRenderingService {
rpc WriteTtsStream(WriteTtsStreamRequest) returns (generic.Empty);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendCardRenderingService {
rpc StripHtml(StripHtmlRequest) returns (generic.String);
}
message ExtractAvTagsRequest {
string text = 1;
bool question_side = 2;

4
proto/anki/cards.proto
View file

@ -18,6 +18,10 @@ service CardsService {
rpc SetFlag(SetFlagRequest) returns (collection.OpChangesWithCount);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendCardsService {}
message CardId {
int64 cid = 1;
}

31
proto/anki/codegen.proto
View file

@ -1,31 +0,0 @@
// Copyright: Ankitects Pty Ltd and contributors
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
syntax = "proto3";
package anki.codegen;
import "google/protobuf/descriptor.proto";
extend google.protobuf.MethodOptions {
BackendMethod backend_method = 50000;
}
message MethodOptions {
BackendMethod backend_method = 50000;
}
enum BackendMethod {
/// Used for typical collection-based operations. We must implement the
// method on Collection. The same method is automatically implemented on
// Backend, which forwards to Collection.
BACKEND_METHOD_DELEGATE = 0;
/// Both the backend and collection need to implement the method; there
/// is no auto-delegation. Can be used to provide a method on both, but
/// skip the Collection mutex lock when a backend handle is available.
/// In practice we only do this for the i18n methods; for the occasional
/// method in other services that doesn't happen to need the collection,
/// we just delegate to the collection method for convenience, and to make
/// sure it's available even if the consumer is not using Backend.
BACKEND_METHOD_IMPLEMENT = 1;
}

2
proto/anki/collection.proto
View file

@ -20,6 +20,8 @@ service CollectionService {
rpc SetWantsAbort(generic.Empty) returns (generic.Empty);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendCollectionService {
rpc OpenCollection(OpenCollectionRequest) returns (generic.Empty);
rpc CloseCollection(CloseCollectionRequest) returns (generic.Empty);

4
proto/anki/config.proto
View file

@ -24,6 +24,10 @@ service ConfigService {
rpc SetPreferences(Preferences) returns (collection.OpChanges);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendConfigService {}
message ConfigKey {
enum Bool {
BROWSER_TABLE_SHOW_NOTES_MODE = 0;

4
proto/anki/deckconfig.proto
View file

@ -25,6 +25,10 @@ service DeckConfigService {
returns (collection.OpChanges);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendDeckConfigService {}
message DeckConfigId {
int64 dcid = 1;
}

4
proto/anki/decks.proto
View file

@ -39,6 +39,10 @@ service DecksService {
rpc GetCurrentDeck(generic.Empty) returns (Deck);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendDecksService {}
message DeckId {
int64 did = 1;
}

21
proto/anki/i18n.proto
View file

@ -8,18 +8,19 @@ option java_multiple_files = true;
package anki.i18n;
import "anki/generic.proto";
import "anki/codegen.proto";
service I18nService {
rpc TranslateString(TranslateStringRequest) returns (generic.String) {
option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT;
}
rpc FormatTimespan(FormatTimespanRequest) returns (generic.String) {
option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT;
}
rpc I18nResources(I18nResourcesRequest) returns (generic.Json) {
option (codegen.backend_method) = BACKEND_METHOD_IMPLEMENT;
}
rpc TranslateString(TranslateStringRequest) returns (generic.String);
rpc FormatTimespan(FormatTimespanRequest) returns (generic.String);
rpc I18nResources(I18nResourcesRequest) returns (generic.Json);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendI18nService {
rpc TranslateString(TranslateStringRequest) returns (generic.String);
rpc FormatTimespan(FormatTimespanRequest) returns (generic.String);
rpc I18nResources(I18nResourcesRequest) returns (generic.Json);
}
message TranslateStringRequest {

4
proto/anki/image_occlusion.proto
View file

@ -23,6 +23,10 @@ service ImageOcclusionService {
rpc AddImageOcclusionNotetype(generic.Empty) returns (collection.OpChanges);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendImageOcclusionService {}
message GetImageForOcclusionRequest {
string path = 1;
}

3
proto/anki/import_export.proto
View file

@ -11,7 +11,6 @@ import "anki/cards.proto";
import "anki/collection.proto";
import "anki/notes.proto";
import "anki/generic.proto";
import "anki/codegen.proto";
service ImportExportService {
rpc ImportAnkiPackage(ImportAnkiPackageRequest) returns (ImportResponse);
@ -24,6 +23,8 @@ service ImportExportService {
rpc ImportJsonString(generic.String) returns (ImportResponse);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendImportExportService {
rpc ImportCollectionPackage(ImportCollectionPackageRequest)
returns (generic.Empty);

4
proto/anki/links.proto
View file

@ -13,6 +13,10 @@ service LinksService {
rpc HelpPageLink(HelpPageLinkRequest) returns (generic.String);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendLinksService {}
message HelpPageLinkRequest {
enum HelpPage {
NOTE_TYPE = 0;

4
proto/anki/media.proto
View file

@ -17,6 +17,10 @@ service MediaService {
rpc RestoreTrash(generic.Empty) returns (generic.Empty);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendMediaService {}
message CheckMediaResponse {
repeated string unused = 1;
repeated string missing = 2;

4
proto/anki/notes.proto
View file

@ -30,6 +30,10 @@ service NotesService {
rpc GetSingleNotetypeOfNotes(notes.NoteIds) returns (notetypes.NotetypeId);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendNotesService {}
message NoteId {
int64 nid = 1;
}

4
proto/anki/notetypes.proto
View file

@ -34,6 +34,10 @@ service NotetypesService {
returns (collection.OpChanges);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendNotetypesService {}
message NotetypeId {
int64 ntid = 1;
}

4
proto/anki/scheduler.proto
View file

@ -54,6 +54,10 @@ service SchedulerService {
rpc RepositionDefaults(generic.Empty) returns (RepositionDefaultsResponse);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendSchedulerService {}
message SchedulingState {
message New {
uint32 position = 1;

4
proto/anki/search.proto
View file

@ -23,6 +23,10 @@ service SearchService {
rpc SetActiveBrowserColumns(generic.StringList) returns (generic.Empty);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendSearchService {}
message SearchNode {
message Dupe {
int64 notetype_id = 1;

4
proto/anki/stats.proto
View file

@ -17,6 +17,10 @@ service StatsService {
rpc SetGraphPreferences(GraphPreferences) returns (generic.Empty);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendStatsService {}
message CardStatsResponse {
message StatsRevlogEntry {
int64 time = 1;

3
proto/anki/sync.proto
View file

@ -9,6 +9,9 @@ package anki.sync;
import "anki/generic.proto";
/// Syncing methods are only available with a Backend handle.
service SyncService {}
service BackendSyncService {
rpc SyncMedia(SyncAuth) returns (generic.Empty);
rpc AbortMediaSync(generic.Empty) returns (generic.Empty);

4
proto/anki/tags.proto
View file

@ -27,6 +27,10 @@ service TagsService {
rpc CompleteTag(CompleteTagRequest) returns (CompleteTagResponse);
}
// Implicitly includes any of the above methods that are not listed in the
// backend service.
service BackendTagsService {}
message SetTagCollapsedRequest {
string name = 1;
bool collapsed = 2;

3
rslib/Cargo.toml
View file

@ -22,12 +22,13 @@ required-features = ["bench"]
[build-dependencies]
anki_io = { version = "0.0.0", path = "io" }
anki_proto = { version = "0.0.0", path = "proto" }
anki_proto_gen = { version = "0.0.0", path = "proto_gen" }
anyhow = "1.0.71"
inflections = "1.1.1"
itertools = "0.10.5"
prettyplease = "0.2.7"
prost = "0.11.8"
prost-reflect = "0.11.4"
prost-types = "0.11.9"
syn = { version = "2.0.18", features = ["parsing", "printing"] }
[dev-dependencies]

1
rslib/proto/Cargo.toml
View file

@ -11,6 +11,7 @@ rust-version.workspace = true
[build-dependencies]
anki_io = { version = "0.0.0", path = "../io" }
anki_proto_gen = { version = "0.0.0", path = "../proto_gen" }
anyhow = "1.0.71"
inflections = "1.1.1"
itertools = "0.10.5"

11
rslib/proto/build.rs
View file

@ -2,21 +2,22 @@
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
pub mod python;
pub mod rust_protos;
pub mod rust;
pub mod ts;
pub mod utils;
use std::env;
use std::path::PathBuf;
use anki_proto_gen::get_services;
use anyhow::Result;
fn main() -> Result<()> {
let descriptors_path = env::var("DESCRIPTORS_BIN").ok().map(PathBuf::from);
let pool = rust_protos::write_rust_protos(descriptors_path)?;
python::write_python_interface(&pool)?;
ts::write_ts_interface(&pool)?;
let pool = rust::write_rust_protos(descriptors_path)?;
let (_, services) = get_services(&pool);
python::write_python_interface(&services)?;
ts::write_ts_interface(&services)?;
Ok(())
}

39
rslib/proto/python.rs
View file

@ -7,30 +7,26 @@ use std::path::Path;
use anki_io::create_dir_all;
use anki_io::create_file;
use anki_proto_gen::BackendService;
use anki_proto_gen::Method;
use anyhow::Result;
use inflections::Inflect;
use prost_reflect::DescriptorPool;
use prost_reflect::FieldDescriptor;
use prost_reflect::Kind;
use prost_reflect::MessageDescriptor;
use prost_reflect::MethodDescriptor;
use prost_reflect::ServiceDescriptor;
use crate::utils::Comments;
pub(crate) fn write_python_interface(pool: &DescriptorPool) -> Result<()> {
pub(crate) fn write_python_interface(services: &[BackendService]) -> Result<()> {
let output_path = Path::new("../../out/pylib/anki/_backend_generated.py");
create_dir_all(output_path.parent().unwrap())?;
let mut out = BufWriter::new(create_file(output_path)?);
write_header(&mut out)?;
for service in pool.services() {
if service.name() == "AnkidroidService" {
for service in services {
if service.name == "BackendAnkidroidService" {
continue;
}
let comments = Comments::from_file(service.parent_file().file_descriptor_proto());
for method in service.methods() {
render_method(&service, &method, &comments, &mut out);
for method in service.all_methods() {
render_method(service, method, &mut out);
}
}
@ -48,18 +44,13 @@ pub(crate) fn write_python_interface(pool: &DescriptorPool) -> Result<()> {
/// output = anki.generic_pb2.StringList()
/// output.ParseFromString(raw_bytes)
/// return output.vals
fn render_method(
service: &ServiceDescriptor,
method: &MethodDescriptor,
comments: &Comments,
out: &mut impl Write,
) {
let method_name = method.name().to_snake_case();
let input = method.input();
let output = method.output();
let service_idx = service.index();
let method_idx = method.index();
let comments = format_comments(comments.get_for_path(method.path()));
fn render_method(service: &BackendService, method: &Method, out: &mut impl Write) {
let method_name = method.name.to_snake_case();
let input = method.proto.input();
let output = method.proto.output();
let service_idx = service.index;
let method_idx = method.index;
let comments = format_comments(&method.comments);
// raw bytes
write!(
@ -89,7 +80,7 @@ fn render_method(
.unwrap();
}
fn format_comments(comments: Option<&str>) -> String {
fn format_comments(comments: &Option<String>) -> String {
comments
.as_ref()
.map(|c| {

0
rslib/proto/rust_protos.rs → rslib/proto/rust.rs
View file

1
rslib/proto/src/lib.rs
View file

@ -15,7 +15,6 @@ protobuf!(ankidroid, "ankidroid");
protobuf!(backend, "backend");
protobuf!(card_rendering, "card_rendering");
protobuf!(cards, "cards");
protobuf!(codegen, "codegen");
protobuf!(collection, "collection");
protobuf!(config, "config");
protobuf!(deckconfig, "deckconfig");

56
rslib/proto/ts.rs
View file

@ -9,46 +9,39 @@ use std::path::Path;
use anki_io::create_dir_all;
use anki_io::create_file;
use anki_proto_gen::BackendService;
use anki_proto_gen::Method;
use anyhow::Result;
use inflections::Inflect;
use prost_reflect::DescriptorPool;
use prost_reflect::MethodDescriptor;
use prost_reflect::ServiceDescriptor;
use crate::utils::Comments;
pub(crate) fn write_ts_interface(pool: &DescriptorPool) -> Result<()> {
pub(crate) fn write_ts_interface(services: &[BackendService]) -> Result<()> {
let root = Path::new("../../out/ts/lib/anki");
create_dir_all(root)?;
for service in pool.services() {
if service.name() == "AnkidroidService" {
for service in services {
if service.name == "BackendAnkidroidService" {
continue;
}
let service_name = service.name().replace("Service", "").to_snake_case();
let comments = Comments::from_file(service.parent_file().file_descriptor_proto());
write_dts_file(root, &service_name, &service, &comments)?;
write_js_file(root, &service_name, &service, &comments)?;
let service_name = service.name.replace("Service", "").to_snake_case();
write_dts_file(root, &service_name, service)?;
write_js_file(root, &service_name, service)?;
}
Ok(())
}
fn write_dts_file(
root: &Path,
service_name: &str,
service: &ServiceDescriptor,
comments: &Comments,
) -> Result<()> {
fn write_dts_file(root: &Path, service_name: &str, service: &BackendService) -> Result<()> {
let output_path = root.join(format!("{service_name}_service.d.ts"));
let mut out = BufWriter::new(create_file(output_path)?);
write_dts_header(&mut out)?;
let mut referenced_packages = HashSet::new();
let mut method_text = String::new();
for method in service.methods() {
let method = MethodDetails::from_descriptor(&method, comments);
for method in service.all_methods() {
let method = MethodDetails::from_method(method);
record_referenced_type(&mut referenced_packages, &method.input_type)?;
record_referenced_type(&mut referenced_packages, &method.output_type)?;
write_dts_method(&method, &mut method_text)?;
@ -100,20 +93,15 @@ fn write_dts_method(
Ok(())
}
fn write_js_file(
root: &Path,
service_name: &str,
service: &ServiceDescriptor,
comments: &Comments,
) -> Result<()> {
fn write_js_file(root: &Path, service_name: &str, service: &BackendService) -> Result<()> {
let output_path = root.join(format!("{service_name}_service.js"));
let mut out = BufWriter::new(create_file(output_path)?);
write_js_header(&mut out)?;
let mut referenced_packages = HashSet::new();
let mut method_text = String::new();
for method in service.methods() {
let method = MethodDetails::from_descriptor(&method, comments);
for method in service.all_methods() {
let method = MethodDetails::from_method(method);
record_referenced_type(&mut referenced_packages, &method.input_type)?;
record_referenced_type(&mut referenced_packages, &method.output_type)?;
write_js_method(&method, &mut method_text)?;
@ -169,16 +157,16 @@ struct MethodDetails {
}
impl MethodDetails {
fn from_descriptor(method: &MethodDescriptor, comments: &Comments) -> MethodDetails {
let name = method.name().to_camel_case();
let input_type = full_name_to_imported_reference(method.input().full_name());
let output_type = full_name_to_imported_reference(method.output().full_name());
let comments = comments.get_for_path(method.path());
fn from_method(method: &Method) -> MethodDetails {
let name = method.name.to_camel_case();
let input_type = full_name_to_imported_reference(method.proto.input().full_name());
let output_type = full_name_to_imported_reference(method.proto.output().full_name());
let comments = method.comments.clone();
Self {
method_name: name,
input_type,
output_type,
comments: comments.map(ToString::to_string),
comments,
}
}
}

36
rslib/proto/utils.rs
View file

@ -1,36 +0,0 @@
// Copyright: Ankitects Pty Ltd and contributors
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
use std::collections::HashMap;
use prost_types::FileDescriptorProto;
#[derive(Debug)]
pub struct Comments {
path_map: HashMap<Vec<i32>, String>,
}
impl Comments {
pub fn from_file(file: &FileDescriptorProto) -> Self {
Self {
path_map: file
.source_code_info
.as_ref()
.unwrap()
.location
.iter()
.map(|l| (l.path.clone(), l.leading_comments().trim().to_string()))
.collect(),
}
}
pub fn get_for_path(&self, path: &[i32]) -> Option<&str> {
self.path_map.get(path).map(|s| s.as_str()).and_then(|s| {
if s.is_empty() {
None
} else {
Some(s)
}
})
}
}

16
rslib/proto_gen/Cargo.toml Normal file
View file

@ -0,0 +1,16 @@
[package]
name = "anki_proto_gen"
publish = false
description = "Helpers for interface code generation"
version.workspace = true
authors.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
[dependencies]
inflections = "1.1.1"
itertools = "0.10.5"
prost-reflect = "0.11.4"
prost-types = "0.11.9"

201
rslib/proto_gen/src/lib.rs Normal file
View file

@ -0,0 +1,201 @@
// Copyright: Ankitects Pty Ltd and contributors
// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
//! Some helpers for code generation in external crates, that ensure indexes
//! match.
use std::collections::HashMap;
use inflections::Inflect;
use itertools::Either;
use itertools::Itertools;
use prost_reflect::DescriptorPool;
use prost_reflect::MessageDescriptor;
use prost_reflect::MethodDescriptor;
use prost_reflect::ServiceDescriptor;
/// We look for ExampleService and BackedExampleService, both of which are
/// expected to exist (but may be empty).
///
/// - If a method is listed in BackendExampleService and not in ExampleService,
/// that method is only available with a Backend.
/// - If a method is listed in both services, you can provide separate
/// implementations for each of the traits.
/// - If a method is listed only in ExampleService, a forwarding method on
/// Backend is automatically implemented. This bypasses the trait and implements
/// directly on Backend.
///
/// It's important that service and method indices are the same for
/// client-generated code, so the client code should use the .index fields
/// of Service and Method provided by get_services(), and not
/// .enumerate() or .proto.index()
///
/// Client code will want to ignore CollectionServices, and focus on
/// BackendServices.
pub fn get_services(pool: &DescriptorPool) -> (Vec<CollectionService>, Vec<BackendService>) {
// split services into backend and collection
let (mut col_services, mut backend_services): (Vec<_>, Vec<_>) =
pool.services().partition_map(|service| {
if service.name().starts_with("Backend") {
Either::Right(BackendService::from_proto(service))
} else {
Either::Left(CollectionService::from_proto(service))
}
});
assert!(col_services.len() == backend_services.len());
// copy collection methods into backend services if they don't have one with
// a matching name
for service in &mut backend_services {
// locate associated collection service
let Some(col_service) = col_services
.iter()
.find(|cs| cs.name == service.name.trim_start_matches("Backend")) else { panic!("missing associated service: {}", service.name) };
// add any methods that don't exist in backend trait methods to the delegating
// methods
service.delegating_methods = col_service
.trait_methods
.iter()
.filter(|m| service.trait_methods.iter().all(|bm| bm.name != m.name))
.map(|method| Method {
index: method.index + service.trait_methods.len(),
..method.clone()
})
.collect();
}
// fill comments in
let comments = MethodComments::from_pool(pool);
for service in &mut col_services {
for method in &mut service.trait_methods {
method.comments = comments.get_for_method(&method.proto);
}
}
for service in &mut backend_services {
for method in &mut service.trait_methods {
method.comments = comments.get_for_method(&method.proto);
}
for method in &mut service.delegating_methods {
method.comments = comments.get_for_method(&method.proto);
}
}
(col_services, backend_services)
}
#[derive(Debug)]
pub struct CollectionService {
pub name: String,
pub index: usize,
pub trait_methods: Vec<Method>,
pub proto: ServiceDescriptor,
}
#[derive(Debug)]
pub struct BackendService {
pub name: String,
pub index: usize,
pub trait_methods: Vec<Method>,
pub delegating_methods: Vec<Method>,
pub proto: ServiceDescriptor,
}
#[derive(Debug, Clone)]
pub struct Method {
pub name: String,
pub index: usize,
pub comments: Option<String>,
pub proto: MethodDescriptor,
}
impl CollectionService {
pub fn from_proto(service: prost_reflect::ServiceDescriptor) -> Self {
CollectionService {
name: service.name().to_string(),
index: service.index(),
trait_methods: service.methods().map(Method::from_proto).collect(),
proto: service,
}
}
}
impl BackendService {
pub fn from_proto(service: prost_reflect::ServiceDescriptor) -> Self {
BackendService {
name: service.name().to_string(),
index: service.index(),
trait_methods: service.methods().map(Method::from_proto).collect(),
proto: service,
// filled in later
delegating_methods: vec![],
}
}
pub fn all_methods(&self) -> impl Iterator<Item = &Method> {
self.trait_methods
.iter()
.chain(self.delegating_methods.iter())
}
}
impl Method {
pub fn from_proto(method: prost_reflect::MethodDescriptor) -> Self {
Method {
name: method.name().to_snake_case(),
index: method.index(),
proto: method,
// filled in later
comments: None,
}
}
/// The input type, if not empty.
pub fn input(&self) -> Option<MessageDescriptor> {
msg_if_empty(self.proto.input())
}
/// The output type, if not empty.
pub fn output(&self) -> Option<MessageDescriptor> {
msg_if_empty(self.proto.output())
}
}
fn msg_if_empty(msg: MessageDescriptor) -> Option<MessageDescriptor> {
if msg.full_name() == "anki.generic.Empty" {
None
} else {
Some(msg)
}
}
#[derive(Debug)]
struct MethodComments<'a> {
// package name -> method path -> comment
by_package_and_path: HashMap<&'a str, HashMap<Vec<i32>, String>>,
}
impl<'a> MethodComments<'a> {
pub fn from_pool(pool: &'a DescriptorPool) -> MethodComments<'a> {
let mut by_package_and_path = HashMap::new();
for file in pool.file_descriptor_protos() {
let path_map = file
.source_code_info
.as_ref()
.unwrap()
.location
.iter()
.map(|l| (l.path.clone(), l.leading_comments().trim().to_string()))
.collect();
by_package_and_path.insert(file.package(), path_map);
}
Self {
by_package_and_path,
}
}
pub fn get_for_method(&self, method: &MethodDescriptor) -> Option<String> {
self.by_package_and_path
.get(method.parent_file().package_name())
.and_then(|by_path| by_path.get(method.path()))
.and_then(|s| if s.is_empty() { None } else { Some(s.into()) })
}
}

384
rslib/rust_interface.rs
View file

@ -6,7 +6,10 @@ use std::fmt::Write;
use std::path::PathBuf;
use anki_io::write_file_if_changed;
use anki_proto::codegen::BackendMethod;
use anki_proto_gen::get_services;
use anki_proto_gen::BackendService;
use anki_proto_gen::CollectionService;
use anki_proto_gen::Method;
use anyhow::Context;
use anyhow::Result;
use inflections::Inflect;
@ -15,105 +18,50 @@ use prost_reflect::DescriptorPool;
pub fn write_rust_interface(pool: &DescriptorPool) -> Result<()> {
let mut buf = String::new();
buf.push_str("use crate::error::Result; use prost::Message;");
let services = pool
.services()
.map(RustService::from_proto)
.collect::<Vec<_>>();
for service in &services {
render_service(service, &mut buf);
}
render_top_level_run_method(&mut buf, &services, true);
render_top_level_run_method(&mut buf, &services, false);
let (col_services, backend_services) = get_services(pool);
render_collection_services(&col_services, &mut buf)?;
render_backend_services(&backend_services, &mut buf)?;
// println!("{}", &buf);
let buf = format_code(buf)?;
// write into OUT_DIR so we can use it in build.rs
// println!("{}", &buf);
// panic!();
let out_dir = env::var("OUT_DIR").unwrap();
let path = PathBuf::from(out_dir).join("backend.rs");
write_file_if_changed(path, buf).context("write file")?;
Ok(())
}
#[derive(Debug)]
struct RustService {
name: String,
methods: Vec<RustMethod>,
fn render_collection_services(col_services: &[CollectionService], buf: &mut String) -> Result<()> {
for service in col_services {
render_collection_trait(service, buf);
render_individual_service_run_method_for_collection(buf, service);
}
render_top_level_run_method(
col_services.iter().map(|s| (s.index, s.name.as_str())),
"&mut self",
"crate::collection::Collection",
buf,
);
Ok(())
}
#[derive(Debug)]
struct RustMethod {
name: String,
input_type: Option<String>,
output_type: Option<String>,
options: anki_proto::codegen::MethodOptions,
service_name: String,
}
impl RustMethod {
/// No text if generic::Empty
fn text_if_input_not_empty(&self, text: impl Fn(&String) -> String) -> String {
self.input_type.as_ref().map(text).unwrap_or_default()
fn render_backend_services(backend_services: &[BackendService], buf: &mut String) -> Result<()> {
for service in backend_services {
render_backend_trait(service, buf);
render_delegating_backend_methods(service, buf);
render_individual_service_run_method_for_backend(buf, service);
}
render_top_level_run_method(
backend_services.iter().map(|s| (s.index, s.name.as_str())),
"&self",
"crate::backend::Backend",
buf,
);
/// No text if generic::Empty
fn get_input_arg_with_label(&self) -> String {
self.input_type
.as_ref()
.map(|t| format!("input: {}", t))
.unwrap_or_default()
}
/// () if generic::Empty
fn get_output_type(&self) -> String {
self.output_type.as_deref().unwrap_or("()").into()
}
fn text_if_output_not_empty(&self, text: impl Fn(&String) -> String) -> String {
self.output_type.as_ref().map(text).unwrap_or_default()
}
fn wants_abstract_backend_method(&self) -> bool {
self.service_name.starts_with("Backend")
|| self.options.backend_method() == BackendMethod::Implement
}
fn wants_abstract_collection_method(&self) -> bool {
!self.service_name.starts_with("Backend")
}
fn from_proto(method: prost_reflect::MethodDescriptor) -> Self {
RustMethod {
name: method.name().to_snake_case(),
input_type: rust_type(method.input().full_name()),
output_type: rust_type(method.output().full_name()),
options: method.options().transcode_to().unwrap(),
service_name: method.parent_service().name().to_string(),
}
}
}
impl RustService {
fn from_proto(service: prost_reflect::ServiceDescriptor) -> Self {
RustService {
name: service.name().to_string(),
methods: service.methods().map(RustMethod::from_proto).collect(),
}
}
}
fn rust_type(name: &str) -> Option<String> {
if name == "anki.generic.Empty" {
return None;
}
let Some((head, tail)) = name.rsplit_once( '.') else { panic!() };
Some(format!(
"{}::{}",
head.to_snake_case()
.replace('.', "::")
.replace("anki::", "anki_proto::"),
tail
))
Ok(())
}
fn format_code(code: String) -> Result<String> {
@ -121,118 +69,75 @@ fn format_code(code: String) -> Result<String> {
Ok(prettyplease::unparse(&syntax_tree))
}
fn render_abstract_collection_method(method: &RustMethod, buf: &mut String) {
fn render_collection_trait(service: &CollectionService, buf: &mut String) {
let name = &service.name;
writeln!(buf, "pub trait {name} {{").unwrap();
for method in &service.trait_methods {
render_trait_method(method, "&mut self", buf);
}
buf.push('}');
}
fn render_trait_method(method: &Method, self_kind: &str, buf: &mut String) {
let method_name = &method.name;
let input_with_label = method.get_input_arg_with_label();
let output_type = method.get_output_type();
writeln!(
buf,
"fn {method_name}(&mut self, {input_with_label}) -> Result<{output_type}>;"
"fn {method_name}({self_kind}, {input_with_label}) -> Result<{output_type}>;"
)
.unwrap();
}
fn render_abstract_backend_method(method: &RustMethod, buf: &mut String, _service: &RustService) {
let method_name = &method.name;
let input_with_label = method.get_input_arg_with_label();
let output_type = method.get_output_type();
writeln!(
buf,
"fn {method_name}(&self, {input_with_label}) -> Result<{output_type}>;"
)
.unwrap();
fn render_backend_trait(service: &BackendService, buf: &mut String) {
let name = &service.name;
writeln!(buf, "pub trait {name} {{").unwrap();
for method in &service.trait_methods {
render_trait_method(method, "&self", buf);
}
buf.push('}');
}
fn render_delegating_backend_method(method: &RustMethod, buf: &mut String, service: &RustService) {
fn render_delegating_backend_methods(service: &BackendService, buf: &mut String) {
buf.push_str("impl crate::backend::Backend {");
for method in &service.delegating_methods {
render_delegating_backend_method(method, service.name.trim_start_matches("Backend"), buf);
}
buf.push('}');
}
fn render_delegating_backend_method(method: &Method, method_qualifier: &str, buf: &mut String) {
let method_name = &method.name;
let input_with_label = method.get_input_arg_with_label();
let input = method.text_if_input_not_empty(|_| "input".into());
let output_type = method.get_output_type();
let col_service = &service.name;
writeln!(
buf,
"fn {method_name}(&self, {input_with_label}) -> Result<{output_type}> {{
self.with_col(|col| {col_service}::{method_name}(col, {input})) }}",
self.with_col(|col| {method_qualifier}::{method_name}(col, {input})) }}",
)
.unwrap();
}
fn render_service(service: &RustService, buf: &mut String) {
let have_collection = service
.methods
.iter()
.any(|m| m.wants_abstract_collection_method());
if have_collection {
render_collection_trait(service, buf);
}
if service
.methods
.iter()
.any(|m| m.wants_abstract_backend_method())
{
render_backend_trait(service, buf);
}
render_delegating_backend_methods(service, buf);
render_individual_service_run_method(buf, service, true);
render_individual_service_run_method(buf, service, false);
}
fn render_collection_trait(service: &RustService, buf: &mut String) {
let name = &service.name;
writeln!(buf, "pub trait {name} {{").unwrap();
for method in &service.methods {
if method.wants_abstract_collection_method() {
render_abstract_collection_method(method, buf);
}
}
buf.push('}');
}
fn render_backend_trait(service: &RustService, buf: &mut String) {
let name = if !service.name.starts_with("Backend") {
format!("Backend{}", service.name)
} else {
service.name.clone()
};
writeln!(buf, "pub trait {name} {{").unwrap();
for method in &service.methods {
if method.wants_abstract_backend_method() {
render_abstract_backend_method(method, buf, service);
}
}
buf.push('}');
}
fn render_delegating_backend_methods(service: &RustService, buf: &mut String) {
buf.push_str("impl crate::backend::Backend {");
for method in &service.methods {
if method.wants_abstract_backend_method() {
continue;
}
render_delegating_backend_method(method, buf, service);
}
buf.push('}');
}
// Matches all service types and delegates to the revelant self.run_foo_method()
fn render_top_level_run_method(buf: &mut String, services: &[RustService], backend: bool) {
let self_kind = if backend { "&self" } else { "&mut self" };
let struct_to_impl = if backend {
"crate::backend::Backend"
} else {
"crate::collection::Collection"
};
fn render_top_level_run_method<'a>(
// (index, name)
services: impl Iterator<Item = (usize, &'a str)>,
self_kind: &str,
struct_name: &str,
buf: &mut String,
) {
writeln!(buf,
r#" impl {struct_to_impl} {{
r#" impl {struct_name} {{
pub fn run_service_method({self_kind}, service: u32, method: u32, input: &[u8]) -> Result<Vec<u8>, Vec<u8>> {{
match service {{
"#,
).unwrap();
for (idx, service) in services.iter().enumerate() {
for (idx, service) in services {
writeln!(
buf,
"{idx} => self.run_{service}_method(method, input),",
service = service.name.to_snake_case()
service = service.to_snake_case()
)
.unwrap();
}
@ -250,51 +155,21 @@ fn render_top_level_run_method(buf: &mut String, services: &[RustService], backe
);
}
fn render_individual_service_run_method(buf: &mut String, service: &RustService, backend: bool) {
let self_kind = if backend { "&self" } else { "&mut self" };
let struct_to_impl = if backend {
"crate::backend::Backend"
} else {
"crate::collection::Collection"
};
let method_qualifier = if backend {
struct_to_impl
} else {
&service.name
};
fn render_individual_service_run_method_for_collection(
buf: &mut String,
service: &CollectionService,
) {
let service_name = &service.name.to_snake_case();
writeln!(
buf,
"#[allow(unused_variables, clippy::match_single_binding)]
impl {struct_to_impl} {{ pub(crate) fn run_{service_name}_method({self_kind},
impl crate::collection::Collection {{ pub(crate) fn run_{service_name}_method(&mut self,
method: u32, input: &[u8]) -> Result<Vec<u8>> {{
match method {{",
)
.unwrap();
for (idx, method) in service.methods.iter().enumerate() {
if !backend && !method.wants_abstract_collection_method() {
continue;
}
let decode_input =
method.text_if_input_not_empty(|kind| format!("let input = {kind}::decode(input)?;"));
let rust_method = &method.name;
let input = method.text_if_input_not_empty(|_| "input".into());
let output_assign = method.text_if_output_not_empty(|_| "let output = ".into());
let output = if method.output_type.is_none() {
"Vec::new()"
} else {
"{ let mut out_bytes = Vec::new();
output.encode(&mut out_bytes)?;
out_bytes }"
};
writeln!(
buf,
"{idx} => {{ {decode_input}
{output_assign} {method_qualifier}::{rust_method}(self, {input})?;
Ok({output}) }},",
)
.unwrap();
for method in &service.trait_methods {
render_method_in_match_expression(method, &service.name, buf);
}
buf.push_str(
r#"
@ -304,3 +179,102 @@ fn render_individual_service_run_method(buf: &mut String, service: &RustService,
"#,
);
}
fn render_individual_service_run_method_for_backend(buf: &mut String, service: &BackendService) {
let service_name = &service.name.to_snake_case();
writeln!(
buf,
"#[allow(unused_variables, clippy::match_single_binding)]
impl crate::backend::Backend {{ pub(crate) fn run_{service_name}_method(&self,
method: u32, input: &[u8]) -> Result<Vec<u8>> {{
match method {{",
)
.unwrap();
for method in &service.trait_methods {
render_method_in_match_expression(method, &service.name, buf);
}
for method in &service.delegating_methods {
render_method_in_match_expression(method, "crate::backend::Backend", buf);
}
buf.push_str(
r#"
_ => Err(crate::error::AnkiError::InvalidMethodIndex),
}
} }
"#,
);
}
fn render_method_in_match_expression(method: &Method, method_qualifier: &str, buf: &mut String) {
let decode_input =
method.text_if_input_not_empty(|kind| format!("let input = {kind}::decode(input)?;"));
let rust_method = &method.name;
let input = method.text_if_input_not_empty(|_| "input".into());
let output_assign = method.text_if_output_not_empty(|_| "let output = ".into());
let idx = method.index;
let output = if method.output().is_none() {
"Vec::new()"
} else {
"{ let mut out_bytes = Vec::new();
output.encode(&mut out_bytes)?;
out_bytes }"
};
writeln!(
buf,
"{idx} => {{ {decode_input}
{output_assign} {method_qualifier}::{rust_method}(self, {input})?;
Ok({output}) }},",
)
.unwrap();
}
trait MethodHelpers {
fn input_type(&self) -> Option<String>;
fn output_type(&self) -> Option<String>;
fn text_if_input_not_empty(&self, text: impl Fn(&String) -> String) -> String;
fn get_input_arg_with_label(&self) -> String;
fn get_output_type(&self) -> String;
fn text_if_output_not_empty(&self, text: impl Fn(&String) -> String) -> String;
}
impl MethodHelpers for Method {
fn input_type(&self) -> Option<String> {
self.input().map(|t| rust_type(t.full_name()))
}
fn output_type(&self) -> Option<String> {
self.output().map(|t| rust_type(t.full_name()))
}
/// No text if generic::Empty
fn text_if_input_not_empty(&self, text: impl Fn(&String) -> String) -> String {
self.input_type().as_ref().map(text).unwrap_or_default()
}
/// No text if generic::Empty
fn get_input_arg_with_label(&self) -> String {
self.input_type()
.as_ref()
.map(|t| format!("input: {}", t))
.unwrap_or_default()
}
/// () if generic::Empty
fn get_output_type(&self) -> String {
self.output_type().as_deref().unwrap_or("()").into()
}
fn text_if_output_not_empty(&self, text: impl Fn(&String) -> String) -> String {
self.output_type().as_ref().map(text).unwrap_or_default()
}
}
fn rust_type(name: &str) -> String {
let Some((head, tail)) = name.rsplit_once( '.') else { panic!() };
format!(
"{}::{}",
head.to_snake_case()
.replace('.', "::")
.replace("anki::", "anki_proto::"),
tail
)
}

2
tools/workspace-hack/Cargo.toml
View file

@ -28,6 +28,7 @@ hashbrown = { version = "0.12", features = ["raw"] }
hmac = { version = "0.12", default-features = false, features = ["reset"] }
hyper = { version = "0.14", features = ["full"] }
indexmap = { version = "1", default-features = false, features = ["std"] }
itertools = { version = "0.10" }
log = { version = "0.4", default-features = false, features = ["std"] }
num-traits = { version = "0.2" }
phf_shared = { version = "0.11", default-features = false, features = ["std"] }
@ -69,6 +70,7 @@ hashbrown = { version = "0.12", features = ["raw"] }
hmac = { version = "0.12", default-features = false, features = ["reset"] }
hyper = { version = "0.14", features = ["full"] }
indexmap = { version = "1", default-features = false, features = ["std"] }
itertools = { version = "0.10" }
log = { version = "0.4", default-features = false, features = ["std"] }
num-traits = { version = "0.2" }
phf_shared = { version = "0.11", default-features = false, features = ["std"] }