From 616db33c0ea90fb9ac70f96465e6b47848fe51f5 Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 10 Jul 2021 17:50:18 +1000 Subject: [PATCH] refactor protobuf handling for split/import In order to split backend.proto into a more manageable size, the protobuf handling needed to be updated. This took more time than I would have liked, as each language handles protobuf differently: - The Python Protobuf code ignores "package" directives, and relies solely on how the files are laid out on disk. While it would have been nice to keep the generated files in a private subpackage, Protobuf gets confused if the files are located in a location that does not match their original .proto layout, so the old approach of storing them in _backend/ will not work. They now clutter up pylib/anki instead. I'm rather annoyed by that, but alternatives seem to be having to add an extra level to the Protobuf path, making the other languages suffer, or trying to hack around the issue by munging sys.modules. - Protobufjs fails to expose packages if they don't start with a capital letter, despite the fact that lowercase packages are the norm in most languages :-( This required a patch to fix. - Rust was the easiest, as Prost is relatively straightforward compared to Google's tools. The Protobuf files are now stored in /proto/anki, with a separate package for each file. I've split backend.proto into a few files as a test, but the majority of that work is still to come. The Python Protobuf building is a bit of a hack at the moment, hard-coding "proto" as the top level folder, but it seems to get the job done for now. Also changed the workspace name, as there seems to be a number of Bazel repos moving away from the more awkward reverse DNS naming style. --- WORKSPACE | 2 +- defs.bzl | 10 +- docs/new-platform.md | 4 +- proto/.top_level | 0 proto/BUILD.bazel | 20 ++- proto/{ => anki}/backend.proto | 263 ++++++++++------------------ proto/anki/generic.proto | 44 +++++ proto/anki/i18n.proto | 42 +++++ proto/clang_format.bzl | 8 +- proto/format.bzl | 8 +- protobuf.bzl | 8 +- pylib/anki/BUILD.bazel | 28 +++ pylib/anki/_backend/BUILD.bazel | 12 +- pylib/anki/_backend/__init__.py | 20 +-- pylib/anki/_backend/backend_pb2.pyi | 1 - pylib/anki/_backend/genbackend.py | 33 ++-- pylib/anki/backend_pb2.pyi | 1 + pylib/anki/cards.py | 2 +- pylib/anki/collection.py | 8 +- pylib/anki/config.py | 2 +- pylib/anki/decks.py | 2 +- pylib/anki/generic_pb2.pyi | 1 + pylib/anki/i18n_pb2.pyi | 1 + pylib/anki/lang.py | 3 +- pylib/anki/latex.py | 2 +- pylib/anki/media.py | 2 +- pylib/anki/models.py | 2 +- pylib/anki/notes.py | 2 +- pylib/anki/scheduler/base.py | 2 +- pylib/anki/scheduler/v2.py | 2 +- pylib/anki/scheduler/v3.py | 2 +- pylib/anki/stdmodels.py | 2 +- pylib/anki/sync.py | 2 +- pylib/anki/syncserver/__init__.py | 2 +- pylib/anki/tags.py | 2 +- pylib/anki/template.py | 2 +- pylib/protobuf.bzl | 24 +-- pylib/tools/protoc_wrapper.py | 23 ++- qt/dmypy.py | 4 +- rslib/BUILD.bazel | 5 +- rslib/build/protobuf.rs | 41 +++-- rslib/src/backend_proto.rs | 14 +- ts/change-notetype/BUILD.bazel | 4 +- ts/change-notetype/lib.test.ts | 10 +- ts/change-notetype/lib.ts | 38 ++-- ts/compile_sass.bzl | 2 +- ts/components/BUILD.bazel | 5 +- ts/congrats/BUILD.bazel | 6 +- ts/congrats/CongratsPage.svelte | 4 +- ts/congrats/lib.ts | 8 +- ts/deck-options/BUILD.bazel | 4 +- ts/deck-options/lib.test.ts | 4 +- ts/deck-options/lib.ts | 35 ++-- ts/graphs/AddedGraph.svelte | 6 +- ts/graphs/BUILD.bazel | 2 - ts/graphs/ButtonsGraph.svelte | 4 +- ts/graphs/CalendarGraph.svelte | 6 +- ts/graphs/CardCounts.svelte | 6 +- ts/graphs/EaseGraph.svelte | 6 +- ts/graphs/FutureDue.svelte | 6 +- ts/graphs/HourGraph.svelte | 4 +- ts/graphs/IntervalsGraph.svelte | 6 +- ts/graphs/ReviewsGraph.svelte | 4 +- ts/graphs/TodayStats.svelte | 4 +- ts/graphs/WithGraphData.svelte | 22 ++- ts/graphs/added.ts | 6 +- ts/graphs/buttons.ts | 13 +- ts/graphs/calendar.ts | 10 +- ts/graphs/card-counts.ts | 11 +- ts/graphs/ease.ts | 6 +- ts/graphs/future-due.ts | 12 +- ts/graphs/graph-helpers.ts | 6 +- ts/graphs/hours.ts | 10 +- ts/graphs/intervals.ts | 8 +- ts/graphs/reviews.ts | 8 +- ts/graphs/today.ts | 8 +- ts/lib/proto.ts | 6 + ts/reviewer/BUILD.bazel | 6 +- ts/reviewer/answering.ts | 12 +- ts/svelte/svelte.bzl | 1 - ts/svelte/svelte.ts | 6 +- 81 files changed, 516 insertions(+), 467 deletions(-) create mode 100644 proto/.top_level rename proto/{ => anki}/backend.proto (86%) create mode 100644 proto/anki/generic.proto create mode 100644 proto/anki/i18n.proto delete mode 120000 pylib/anki/_backend/backend_pb2.pyi create mode 120000 pylib/anki/backend_pb2.pyi create mode 120000 pylib/anki/generic_pb2.pyi create mode 120000 pylib/anki/i18n_pb2.pyi create mode 100644 ts/lib/proto.ts diff --git a/WORKSPACE b/WORKSPACE index 3d3ecc885..a33c39caa 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,5 +1,5 @@ workspace( - name = "net_ankiweb_anki", + name = "anki", managed_directories = {"@npm": [ "ts/node_modules", ]}, diff --git a/defs.bzl b/defs.bzl index 0002bdd2a..eef38cd66 100644 --- a/defs.bzl +++ b/defs.bzl @@ -1,7 +1,7 @@ load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") load("@bazel_skylib//lib:versions.bzl", "versions") load("@rules_rust//rust:repositories.bzl", "rust_repositories") -load("@net_ankiweb_anki//cargo:crates.bzl", "raze_fetch_remote_crates") +load("@anki//cargo:crates.bzl", "raze_fetch_remote_crates") load(":python.bzl", "setup_local_python") load(":protobuf.bzl", "setup_protobuf_binary") load("//proto:format.bzl", "setup_clang_format") @@ -35,7 +35,7 @@ def setup_deps(): pip_import( name = "py_deps", - requirements = "@net_ankiweb_anki//pip:requirements.txt", + requirements = "@anki//pip:requirements.txt", python_runtime = "@python//:python", ) @@ -44,12 +44,12 @@ def setup_deps(): python_runtime = "@python//:python", ) - node_repositories(package_json = ["@net_ankiweb_anki//ts:package.json"]) + node_repositories(package_json = ["@anki//ts:package.json"]) yarn_install( name = "npm", - package_json = "@net_ankiweb_anki//ts:package.json", - yarn_lock = "@net_ankiweb_anki//ts:yarn.lock", + package_json = "@anki//ts:package.json", + yarn_lock = "@anki//ts:yarn.lock", ) sass_repositories() diff --git a/docs/new-platform.md b/docs/new-platform.md index 20767cc84..3c276a114 100644 --- a/docs/new-platform.md +++ b/docs/new-platform.md @@ -74,14 +74,14 @@ index eff3d9df2..fb2e9f7fe 100644 python_runtime = "@python//:python", ) -- node_repositories(package_json = ["@net_ankiweb_anki//ts:package.json"]) +- node_repositories(package_json = ["@anki//ts:package.json"]) + native.local_repository( + name = "local_node", + path = "local_node", + ) + + node_repositories( -+ package_json = ["@net_ankiweb_anki//ts:package.json"], ++ package_json = ["@anki//ts:package.json"], + vendored_node = "@local_node//:node", + ) diff --git a/proto/.top_level b/proto/.top_level new file mode 100644 index 000000000..e69de29bb diff --git a/proto/BUILD.bazel b/proto/BUILD.bazel index da881c352..ad81b1cad 100644 --- a/proto/BUILD.bazel +++ b/proto/BUILD.bazel @@ -6,13 +6,27 @@ load("//proto:clang_format.bzl", "proto_format") proto_format( name = "format", - srcs = ["backend.proto"], + srcs = glob(["**/*.proto"]), + visibility = ["//visibility:public"], ) proto_library( name = "backend_proto_lib", - srcs = ["backend.proto"], + srcs = glob(["**/*.proto"]), + # "" removes the "proto/" prefix + strip_import_prefix = "", visibility = ["//visibility:public"], ) -exports_files(["backend.proto"]) +filegroup( + name = "proto", + srcs = glob(["**/*.proto"]), + visibility = ["//visibility:public"], +) + +exports_files([ + # for external workspace use + "format.py", + # an empty file we use to get the root proto/ path in the Rust build + ".top_level", +]) diff --git a/proto/backend.proto b/proto/anki/backend.proto similarity index 86% rename from proto/backend.proto rename to proto/anki/backend.proto index 978c9d353..847f314b5 100644 --- a/proto/backend.proto +++ b/proto/anki/backend.proto @@ -1,59 +1,11 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + syntax = "proto3"; -package BackendProto; +package anki.backend; -// Generic containers -/////////////////////////////////////////////////////////// - -message Empty {} - -message OptionalInt32 { - sint32 val = 1; -} - -message OptionalUInt32 { - uint32 val = 1; -} - -message Int32 { - sint32 val = 1; -} - -message UInt32 { - uint32 val = 1; -} - -message Int64 { - int64 val = 1; -} - -message String { - string val = 1; -} - -message Json { - bytes json = 1; -} - -message Bool { - bool val = 1; -} - -message StringList { - repeated string vals = 1; -} - -message OpChangesWithCount { - uint32 count = 1; - OpChanges changes = 2; -} - -message OpChangesWithId { - int64 id = 1; - OpChanges changes = 2; -} +import "anki/generic.proto"; // IDs used in RPC calls /////////////////////////////////////////////////////////// @@ -115,13 +67,13 @@ enum ServiceIndex { } service SchedulingService { - rpc SchedTimingToday(Empty) returns (SchedTimingTodayResponse); - rpc StudiedToday(Empty) returns (String); - rpc StudiedTodayMessage(StudiedTodayMessageRequest) returns (String); - rpc UpdateStats(UpdateStatsRequest) returns (Empty); - rpc ExtendLimits(ExtendLimitsRequest) returns (Empty); + rpc SchedTimingToday(generic.Empty) returns (SchedTimingTodayResponse); + rpc StudiedToday(generic.Empty) returns (generic.String); + rpc StudiedTodayMessage(StudiedTodayMessageRequest) returns (generic.String); + rpc UpdateStats(UpdateStatsRequest) returns (generic.Empty); + rpc ExtendLimits(ExtendLimitsRequest) returns (generic.Empty); rpc CountsForDeckToday(DeckId) returns (CountsForDeckTodayResponse); - rpc CongratsInfo(Empty) returns (CongratsInfoResponse); + rpc CongratsInfo(generic.Empty) returns (CongratsInfoResponse); rpc RestoreBuriedAndSuspendedCards(CardIds) returns (OpChanges); rpc UnburyDeck(UnburyDeckRequest) returns (OpChanges); rpc BuryOrSuspendCards(BuryOrSuspendCardsRequest) @@ -133,35 +85,35 @@ service SchedulingService { rpc SortCards(SortCardsRequest) returns (OpChangesWithCount); rpc SortDeck(SortDeckRequest) returns (OpChangesWithCount); rpc GetNextCardStates(CardId) returns (NextCardStates); - rpc DescribeNextStates(NextCardStates) returns (StringList); - rpc StateIsLeech(SchedulingState) returns (Bool); + rpc DescribeNextStates(NextCardStates) returns (generic.StringList); + rpc StateIsLeech(SchedulingState) returns (generic.Bool); rpc AnswerCard(CardAnswer) returns (OpChanges); - rpc UpgradeScheduler(Empty) returns (Empty); + rpc UpgradeScheduler(generic.Empty) returns (generic.Empty); rpc GetQueuedCards(GetQueuedCardsRequest) returns (QueuedCards); } service DecksService { - rpc AddDeckLegacy(Json) returns (OpChangesWithId); + rpc AddDeckLegacy(generic.Json) returns (OpChangesWithId); rpc AddOrUpdateDeckLegacy(AddOrUpdateDeckLegacyRequest) returns (DeckId); rpc DeckTree(DeckTreeRequest) returns (DeckTreeNode); - rpc DeckTreeLegacy(Empty) returns (Json); - rpc GetAllDecksLegacy(Empty) returns (Json); - rpc GetDeckIdByName(String) returns (DeckId); + rpc DeckTreeLegacy(generic.Empty) returns (generic.Json); + rpc GetAllDecksLegacy(generic.Empty) returns (generic.Json); + rpc GetDeckIdByName(generic.String) returns (DeckId); rpc GetDeck(DeckId) returns (Deck); rpc UpdateDeck(Deck) returns (OpChanges); - rpc UpdateDeckLegacy(Json) returns (OpChanges); + rpc UpdateDeckLegacy(generic.Json) returns (OpChanges); rpc SetDeckCollapsed(SetDeckCollapsedRequest) returns (OpChanges); - rpc GetDeckLegacy(DeckId) returns (Json); + rpc GetDeckLegacy(DeckId) returns (generic.Json); rpc GetDeckNames(GetDeckNamesRequest) returns (DeckNames); - rpc NewDeckLegacy(Bool) returns (Json); + rpc NewDeckLegacy(generic.Bool) returns (generic.Json); rpc RemoveDecks(DeckIds) returns (OpChangesWithCount); rpc ReparentDecks(ReparentDecksRequest) returns (OpChangesWithCount); rpc RenameDeck(RenameDeckRequest) returns (OpChanges); rpc GetOrCreateFilteredDeck(DeckId) returns (FilteredDeckForUpdate); rpc AddOrUpdateFilteredDeck(FilteredDeckForUpdate) returns (OpChangesWithId); - rpc FilteredDeckOrderLabels(Empty) returns (StringList); + rpc FilteredDeckOrderLabels(generic.Empty) returns (generic.StringList); rpc SetCurrentDeck(DeckId) returns (OpChanges); - rpc GetCurrentDeck(Empty) returns (Deck); + rpc GetCurrentDeck(generic.Empty) returns (Deck); } service NotesService { @@ -181,47 +133,48 @@ service NotesService { } service SyncService { - rpc SyncMedia(SyncAuth) returns (Empty); - rpc AbortSync(Empty) returns (Empty); - rpc AbortMediaSync(Empty) returns (Empty); - rpc BeforeUpload(Empty) returns (Empty); + rpc SyncMedia(SyncAuth) returns (generic.Empty); + rpc AbortSync(generic.Empty) returns (generic.Empty); + rpc AbortMediaSync(generic.Empty) returns (generic.Empty); + rpc BeforeUpload(generic.Empty) returns (generic.Empty); rpc SyncLogin(SyncLoginRequest) returns (SyncAuth); rpc SyncStatus(SyncAuth) returns (SyncStatusResponse); rpc SyncCollection(SyncAuth) returns (SyncCollectionResponse); - rpc FullUpload(SyncAuth) returns (Empty); - rpc FullDownload(SyncAuth) returns (Empty); - rpc SyncServerMethod(SyncServerMethodRequest) returns (Json); + rpc FullUpload(SyncAuth) returns (generic.Empty); + rpc FullDownload(SyncAuth) returns (generic.Empty); + rpc SyncServerMethod(SyncServerMethodRequest) returns (generic.Json); } service ConfigService { - rpc GetConfigJson(String) returns (Json); + rpc GetConfigJson(generic.String) returns (generic.Json); rpc SetConfigJson(SetConfigJsonRequest) returns (OpChanges); - rpc SetConfigJsonNoUndo(SetConfigJsonRequest) returns (Empty); - rpc RemoveConfig(String) returns (OpChanges); - rpc GetAllConfig(Empty) returns (Json); - rpc GetConfigBool(Config.Bool) returns (Bool); + rpc SetConfigJsonNoUndo(SetConfigJsonRequest) returns (generic.Empty); + rpc RemoveConfig(generic.String) returns (OpChanges); + rpc GetAllConfig(generic.Empty) returns (generic.Json); + rpc GetConfigBool(Config.Bool) returns (generic.Bool); rpc SetConfigBool(SetConfigBoolRequest) returns (OpChanges); - rpc GetConfigString(Config.String) returns (String); + rpc GetConfigString(Config.String) returns (generic.String); rpc SetConfigString(SetConfigStringRequest) returns (OpChanges); - rpc GetPreferences(Empty) returns (Preferences); + rpc GetPreferences(generic.Empty) returns (Preferences); rpc SetPreferences(Preferences) returns (OpChanges); } service NotetypesService { rpc AddNotetype(Notetype) returns (OpChangesWithId); rpc UpdateNotetype(Notetype) returns (OpChanges); - rpc AddNotetypeLegacy(Json) returns (OpChangesWithId); - rpc UpdateNotetypeLegacy(Json) returns (OpChanges); + rpc AddNotetypeLegacy(generic.Json) returns (OpChangesWithId); + rpc UpdateNotetypeLegacy(generic.Json) returns (OpChanges); rpc AddOrUpdateNotetype(AddOrUpdateNotetypeRequest) returns (NotetypeId); - rpc GetStockNotetypeLegacy(StockNotetype) returns (Json); + rpc GetStockNotetypeLegacy(StockNotetype) returns (generic.Json); rpc GetNotetype(NotetypeId) returns (Notetype); - rpc GetNotetypeLegacy(NotetypeId) returns (Json); - rpc GetNotetypeNames(Empty) returns (NotetypeNames); - rpc GetNotetypeNamesAndCounts(Empty) returns (NotetypeUseCounts); - rpc GetNotetypeIdByName(String) returns (NotetypeId); + rpc GetNotetypeLegacy(NotetypeId) returns (generic.Json); + rpc GetNotetypeNames(generic.Empty) returns (NotetypeNames); + rpc GetNotetypeNamesAndCounts(generic.Empty) returns (NotetypeUseCounts); + rpc GetNotetypeIdByName(generic.String) returns (NotetypeId); rpc RemoveNotetype(NotetypeId) returns (OpChanges); - rpc GetAuxNotetypeConfigKey(GetAuxConfigKeyRequest) returns (String); - rpc GetAuxTemplateConfigKey(GetAuxTemplateConfigKeyRequest) returns (String); + rpc GetAuxNotetypeConfigKey(GetAuxConfigKeyRequest) returns (generic.String); + rpc GetAuxTemplateConfigKey(GetAuxTemplateConfigKeyRequest) + returns (generic.String); rpc GetSingleNotetypeOfNotes(NoteIds) returns (NotetypeId); rpc GetChangeNotetypeInfo(GetChangeNotetypeInfoRequest) returns (ChangeNotetypeInfo); @@ -231,34 +184,34 @@ service NotetypesService { service CardRenderingService { rpc ExtractAVTags(ExtractAVTagsRequest) returns (ExtractAVTagsResponse); rpc ExtractLatex(ExtractLatexRequest) returns (ExtractLatexResponse); - rpc GetEmptyCards(Empty) returns (EmptyCardsReport); + rpc GetEmptyCards(generic.Empty) returns (EmptyCardsReport); rpc RenderExistingCard(RenderExistingCardRequest) returns (RenderCardResponse); rpc RenderUncommittedCard(RenderUncommittedCardRequest) returns (RenderCardResponse); rpc RenderUncommittedCardLegacy(RenderUncommittedCardLegacyRequest) returns (RenderCardResponse); - rpc StripAVTags(String) returns (String); - rpc RenderMarkdown(RenderMarkdownRequest) returns (String); + rpc StripAVTags(generic.String) returns (generic.String); + rpc RenderMarkdown(RenderMarkdownRequest) returns (generic.String); } service DeckConfigService { - rpc AddOrUpdateDeckConfigLegacy(Json) returns (DeckConfigId); + rpc AddOrUpdateDeckConfigLegacy(generic.Json) returns (DeckConfigId); rpc GetDeckConfig(DeckConfigId) returns (DeckConfig); - rpc AllDeckConfigLegacy(Empty) returns (Json); - rpc GetDeckConfigLegacy(DeckConfigId) returns (Json); - rpc NewDeckConfigLegacy(Empty) returns (Json); - rpc RemoveDeckConfig(DeckConfigId) returns (Empty); + rpc AllDeckConfigLegacy(generic.Empty) returns (generic.Json); + rpc GetDeckConfigLegacy(DeckConfigId) returns (generic.Json); + rpc NewDeckConfigLegacy(generic.Empty) returns (generic.Json); + rpc RemoveDeckConfig(DeckConfigId) returns (generic.Empty); rpc GetDeckConfigsForUpdate(DeckId) returns (DeckConfigsForUpdate); rpc UpdateDeckConfigs(UpdateDeckConfigsRequest) returns (OpChanges); } service TagsService { - rpc ClearUnusedTags(Empty) returns (OpChangesWithCount); - rpc AllTags(Empty) returns (StringList); - rpc RemoveTags(String) returns (OpChangesWithCount); + rpc ClearUnusedTags(generic.Empty) returns (OpChangesWithCount); + rpc AllTags(generic.Empty) returns (generic.StringList); + rpc RemoveTags(generic.String) returns (OpChangesWithCount); rpc SetTagCollapsed(SetTagCollapsedRequest) returns (OpChanges); - rpc TagTree(Empty) returns (TagTreeNode); + rpc TagTree(generic.Empty) returns (TagTreeNode); rpc ReparentTags(ReparentTagsRequest) returns (OpChangesWithCount); rpc RenameTags(RenameTagsRequest) returns (OpChangesWithCount); rpc AddNoteTags(NoteIdsAndTagsRequest) returns (OpChangesWithCount); @@ -267,55 +220,49 @@ service TagsService { } service SearchService { - rpc BuildSearchString(SearchNode) returns (String); + rpc BuildSearchString(SearchNode) returns (generic.String); rpc SearchCards(SearchRequest) returns (SearchResponse); rpc SearchNotes(SearchRequest) returns (SearchResponse); - rpc JoinSearchNodes(JoinSearchNodesRequest) returns (String); - rpc ReplaceSearchNode(ReplaceSearchNodeRequest) returns (String); + rpc JoinSearchNodes(JoinSearchNodesRequest) returns (generic.String); + rpc ReplaceSearchNode(ReplaceSearchNodeRequest) returns (generic.String); rpc FindAndReplace(FindAndReplaceRequest) returns (OpChangesWithCount); - rpc AllBrowserColumns(Empty) returns (BrowserColumns); - rpc BrowserRowForId(Int64) returns (BrowserRow); - rpc SetActiveBrowserColumns(StringList) returns (Empty); + rpc AllBrowserColumns(generic.Empty) returns (BrowserColumns); + rpc BrowserRowForId(generic.Int64) returns (BrowserRow); + rpc SetActiveBrowserColumns(generic.StringList) returns (generic.Empty); } service StatsService { - rpc CardStats(CardId) returns (String); + rpc CardStats(CardId) returns (generic.String); rpc Graphs(GraphsRequest) returns (GraphsResponse); - rpc GetGraphPreferences(Empty) returns (GraphPreferences); - rpc SetGraphPreferences(GraphPreferences) returns (Empty); + rpc GetGraphPreferences(generic.Empty) returns (GraphPreferences); + rpc SetGraphPreferences(GraphPreferences) returns (generic.Empty); } service MediaService { - rpc CheckMedia(Empty) returns (CheckMediaResponse); - rpc TrashMediaFiles(TrashMediaFilesRequest) returns (Empty); - rpc AddMediaFile(AddMediaFileRequest) returns (String); - rpc EmptyTrash(Empty) returns (Empty); - rpc RestoreTrash(Empty) returns (Empty); -} - -service I18nService { - rpc TranslateString(TranslateStringRequest) returns (String); - rpc FormatTimespan(FormatTimespanRequest) returns (String); - rpc I18nResources(I18nResourcesRequest) returns (Json); + rpc CheckMedia(generic.Empty) returns (CheckMediaResponse); + rpc TrashMediaFiles(TrashMediaFilesRequest) returns (generic.Empty); + rpc AddMediaFile(AddMediaFileRequest) returns (generic.String); + rpc EmptyTrash(generic.Empty) returns (generic.Empty); + rpc RestoreTrash(generic.Empty) returns (generic.Empty); } service CollectionService { - rpc OpenCollection(OpenCollectionRequest) returns (Empty); - rpc CloseCollection(CloseCollectionRequest) returns (Empty); - rpc CheckDatabase(Empty) returns (CheckDatabaseResponse); - rpc GetUndoStatus(Empty) returns (UndoStatus); - rpc Undo(Empty) returns (OpChangesAfterUndo); - rpc Redo(Empty) returns (OpChangesAfterUndo); - rpc AddCustomUndoEntry(String) returns (UInt32); - rpc MergeUndoEntries(UInt32) returns (OpChanges); - rpc LatestProgress(Empty) returns (Progress); - rpc SetWantsAbort(Empty) returns (Empty); + rpc OpenCollection(OpenCollectionRequest) returns (generic.Empty); + rpc CloseCollection(CloseCollectionRequest) returns (generic.Empty); + rpc CheckDatabase(generic.Empty) returns (CheckDatabaseResponse); + rpc GetUndoStatus(generic.Empty) returns (UndoStatus); + rpc Undo(generic.Empty) returns (OpChangesAfterUndo); + rpc Redo(generic.Empty) returns (OpChangesAfterUndo); + rpc AddCustomUndoEntry(generic.String) returns (generic.UInt32); + rpc MergeUndoEntries(generic.UInt32) returns (OpChanges); + rpc LatestProgress(generic.Empty) returns (Progress); + rpc SetWantsAbort(generic.Empty) returns (generic.Empty); } service CardsService { rpc GetCard(CardId) returns (Card); rpc UpdateCard(UpdateCardRequest) returns (OpChanges); - rpc RemoveCards(RemoveCardsRequest) returns (Empty); + rpc RemoveCards(RemoveCardsRequest) returns (generic.Empty); rpc SetDeck(SetDeckRequest) returns (OpChangesWithCount); rpc SetFlag(SetFlagRequest) returns (OpChangesWithCount); } @@ -525,7 +472,7 @@ message Notetype { bytes other = 255; } - OptionalUInt32 ord = 1; + generic.OptionalUInt32 ord = 1; string name = 2; Config config = 5; } @@ -542,7 +489,7 @@ message Notetype { bytes other = 255; } - OptionalUInt32 ord = 1; + generic.OptionalUInt32 ord = 1; string name = 2; int64 mtime_secs = 3; sint32 usn = 4; @@ -661,7 +608,7 @@ message Progress { uint32 stage_current = 3; } oneof value { - Empty none = 1; + generic.Empty none = 1; MediaSync media_sync = 2; string media_check = 3; FullSync full_sync = 4; @@ -797,34 +744,6 @@ message TrashMediaFilesRequest { repeated string fnames = 1; } -message TranslateStringRequest { - uint32 module_index = 1; - uint32 message_index = 2; - map args = 3; -} - -message TranslateArgValue { - oneof value { - string str = 1; - double number = 2; - } -} - -message FormatTimespanRequest { - enum Context { - PRECISE = 0; - ANSWER_BUTTONS = 1; - INTERVALS = 2; - } - - float seconds = 1; - Context context = 2; -} - -message I18nResourcesRequest { - repeated string modules = 1; -} - message StudiedTodayMessageRequest { uint32 cards = 1; double seconds = 2; @@ -857,7 +776,7 @@ message SortOrder { bool reverse = 2; } oneof value { - Empty none = 1; + generic.Empty none = 1; string custom = 2; Builtin builtin = 3; } @@ -1606,6 +1525,16 @@ message OpChanges { bool study_queues = 10; } +message OpChangesWithCount { + uint32 count = 1; + OpChanges changes = 2; +} + +message OpChangesWithId { + int64 id = 1; + OpChanges changes = 2; +} + message UndoStatus { string undo = 1; string redo = 2; diff --git a/proto/anki/generic.proto b/proto/anki/generic.proto new file mode 100644 index 000000000..5156e48d6 --- /dev/null +++ b/proto/anki/generic.proto @@ -0,0 +1,44 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +syntax = "proto3"; + +package anki.generic; + +message Empty {} + +message OptionalInt32 { + sint32 val = 1; +} + +message OptionalUInt32 { + uint32 val = 1; +} + +message Int32 { + sint32 val = 1; +} + +message UInt32 { + uint32 val = 1; +} + +message Int64 { + int64 val = 1; +} + +message String { + string val = 1; +} + +message Json { + bytes json = 1; +} + +message Bool { + bool val = 1; +} + +message StringList { + repeated string vals = 1; +} diff --git a/proto/anki/i18n.proto b/proto/anki/i18n.proto new file mode 100644 index 000000000..1f135287b --- /dev/null +++ b/proto/anki/i18n.proto @@ -0,0 +1,42 @@ +// Copyright: Ankitects Pty Ltd and contributors +// License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +syntax = "proto3"; + +package anki.i18n; + +import "anki/generic.proto"; + +service I18nService { + rpc TranslateString(TranslateStringRequest) returns (generic.String); + rpc FormatTimespan(FormatTimespanRequest) returns (generic.String); + rpc I18nResources(I18nResourcesRequest) returns (generic.Json); +} + +message TranslateStringRequest { + uint32 module_index = 1; + uint32 message_index = 2; + map args = 3; +} + +message TranslateArgValue { + oneof value { + string str = 1; + double number = 2; + } +} + +message FormatTimespanRequest { + enum Context { + PRECISE = 0; + ANSWER_BUTTONS = 1; + INTERVALS = 2; + } + + float seconds = 1; + Context context = 2; +} + +message I18nResourcesRequest { + repeated string modules = 1; +} diff --git a/proto/clang_format.bzl b/proto/clang_format.bzl index 165e82bd2..4af4ad56e 100644 --- a/proto/clang_format.bzl +++ b/proto/clang_format.bzl @@ -14,9 +14,9 @@ def _impl(rctx): alias( name = "clang_format", actual = select({ - "@net_ankiweb_anki//platforms:windows_x86_64": "@clang_format_windows_x86_64//:clang-format.exe", - "@net_ankiweb_anki//platforms:macos_x86_64": "@clang_format_macos_x86_64//:clang-format", - "@net_ankiweb_anki//platforms:linux_x86_64": "@clang_format_linux_x86_64//:clang-format", + "@anki//platforms:windows_x86_64": "@clang_format_windows_x86_64//:clang-format.exe", + "@anki//platforms:macos_x86_64": "@clang_format_macos_x86_64//:clang-format", + "@anki//platforms:linux_x86_64": "@clang_format_linux_x86_64//:clang-format", }), visibility = ["//visibility:public"] ) @@ -68,7 +68,7 @@ def proto_format(name, srcs, **kwargs): py_test( name = name, srcs = [ - "format.py", + "@anki//proto:format.py", ], data = ["@clang_format//:clang_format"] + srcs, args = ["$(location @clang_format//:clang_format)"] + [native.package_name() + "/" + f for f in srcs], diff --git a/proto/format.bzl b/proto/format.bzl index 165e82bd2..08e92a099 100644 --- a/proto/format.bzl +++ b/proto/format.bzl @@ -14,9 +14,9 @@ def _impl(rctx): alias( name = "clang_format", actual = select({ - "@net_ankiweb_anki//platforms:windows_x86_64": "@clang_format_windows_x86_64//:clang-format.exe", - "@net_ankiweb_anki//platforms:macos_x86_64": "@clang_format_macos_x86_64//:clang-format", - "@net_ankiweb_anki//platforms:linux_x86_64": "@clang_format_linux_x86_64//:clang-format", + "@anki//platforms:windows_x86_64": "@clang_format_windows_x86_64//:clang-format.exe", + "@anki//platforms:macos_x86_64": "@clang_format_macos_x86_64//:clang-format", + "@anki//platforms:linux_x86_64": "@clang_format_linux_x86_64//:clang-format", }), visibility = ["//visibility:public"] ) @@ -68,7 +68,7 @@ def proto_format(name, srcs, **kwargs): py_test( name = name, srcs = [ - "format.py", + "@anki//format.py", ], data = ["@clang_format//:clang_format"] + srcs, args = ["$(location @clang_format//:clang_format)"] + [native.package_name() + "/" + f for f in srcs], diff --git a/protobuf.bzl b/protobuf.bzl index b46af971b..79a4690c0 100644 --- a/protobuf.bzl +++ b/protobuf.bzl @@ -12,10 +12,10 @@ def _impl(rctx): alias( name = "protoc", actual = select({ - "@net_ankiweb_anki//platforms:windows_x86_64": "@protoc_bin_windows//:bin/protoc.exe", - "@net_ankiweb_anki//platforms:macos_x86_64": "@protoc_bin_macos//:bin/protoc", - "@net_ankiweb_anki//platforms:linux_x86_64": "@protoc_bin_linux_x86_64//:bin/protoc", - "@net_ankiweb_anki//platforms:linux_arm64": "@protoc_bin_linux_arm64//:bin/protoc" + "@anki//platforms:windows_x86_64": "@protoc_bin_windows//:bin/protoc.exe", + "@anki//platforms:macos_x86_64": "@protoc_bin_macos//:bin/protoc", + "@anki//platforms:linux_x86_64": "@protoc_bin_linux_x86_64//:bin/protoc", + "@anki//platforms:linux_arm64": "@protoc_bin_linux_arm64//:bin/protoc" }), visibility = ["//visibility:public"] ) diff --git a/pylib/anki/BUILD.bazel b/pylib/anki/BUILD.bazel index 056449dbc..f586402b5 100644 --- a/pylib/anki/BUILD.bazel +++ b/pylib/anki/BUILD.bazel @@ -32,6 +32,7 @@ py_library( "py.typed", ":buildinfo", ":hooks_gen", + ":proto", "//pylib/anki/_backend", ], imports = [ @@ -105,3 +106,30 @@ filegroup( "//pylib:__subpackages__", ], ) + +load("//pylib:protobuf.bzl", "py_proto") + +py_proto( + name = "proto_files", + srcs = ["//proto"], + visibility = [ + "//visibility:public", + ], +) + +filegroup( + name = "proto", + srcs = [ + # "__init__.py", + ":proto_files", + ], + visibility = ["//pylib:__subpackages__"], +) + +# only used for genbackend.py +py_library( + name = "proto_lib", + srcs = [":proto"], + imports = [".."], + visibility = ["//pylib:__subpackages__"], +) diff --git a/pylib/anki/_backend/BUILD.bazel b/pylib/anki/_backend/BUILD.bazel index 8a2fdf4ee..4707ad1ff 100644 --- a/pylib/anki/_backend/BUILD.bazel +++ b/pylib/anki/_backend/BUILD.bazel @@ -1,27 +1,18 @@ load("@rules_python//python:defs.bzl", "py_binary") load("@py_deps//:requirements.bzl", "requirement") -load("//pylib:protobuf.bzl", "py_proto_library_typed") load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@bazel_skylib//lib:selects.bzl", "selects") -py_proto_library_typed( - name = "backend_pb2", - src = "//proto:backend.proto", - visibility = [ - "//visibility:public", - ], -) - py_binary( name = "genbackend", srcs = [ - "backend_pb2", "genbackend.py", ], deps = [ requirement("black"), requirement("stringcase"), requirement("protobuf"), + "//pylib/anki:proto_lib", ], ) @@ -94,7 +85,6 @@ filegroup( srcs = [ "__init__.py", "rsbridge.pyi", - ":backend_pb2", ":fluent_gen", ":rsbackend_gen", ":rsbridge", diff --git a/pylib/anki/_backend/__init__.py b/pylib/anki/_backend/__init__.py index 83cd3a2d7..3decc0312 100644 --- a/pylib/anki/_backend/__init__.py +++ b/pylib/anki/_backend/__init__.py @@ -11,6 +11,7 @@ from weakref import ref from markdown import markdown import anki.buildinfo +from anki import backend_pb2, i18n_pb2 from anki._backend.generated import RustBackendGenerated from anki.dbproxy import Row as DBRow from anki.dbproxy import ValueForDB @@ -32,7 +33,6 @@ from ..errors import ( TemplateError, UndoEmpty, ) -from . import backend_pb2 as pb from . import rsbridge from .fluent import GeneratedTranslations, LegacyTranslationEnum @@ -65,7 +65,7 @@ class RustBackend(RustBackendGenerated): if langs is None: langs = [anki.lang.currentLang] - init_msg = pb.BackendInit( + init_msg = backend_pb2.BackendInit( preferred_langs=langs, server=server, ) @@ -95,7 +95,7 @@ class RustBackend(RustBackendGenerated): return from_json_bytes(self._backend.db_command(to_json_bytes(input))) except Exception as e: err_bytes = bytes(e.args[0]) - err = pb.BackendError() + err = backend_pb2.BackendError() err.ParseFromString(err_bytes) raise backend_exception_to_pylib(err) @@ -125,21 +125,21 @@ class RustBackend(RustBackendGenerated): return self._backend.command(service, method, input_bytes) except Exception as e: err_bytes = bytes(e.args[0]) - err = pb.BackendError() + err = backend_pb2.BackendError() err.ParseFromString(err_bytes) raise backend_exception_to_pylib(err) def translate_string_in( module_index: int, message_index: int, **kwargs: Union[str, int, float] -) -> pb.TranslateStringRequest: +) -> i18n_pb2.TranslateStringRequest: args = {} for (k, v) in kwargs.items(): if isinstance(v, str): - args[k] = pb.TranslateArgValue(str=v) + args[k] = i18n_pb2.TranslateArgValue(str=v) else: - args[k] = pb.TranslateArgValue(number=v) - return pb.TranslateStringRequest( + args[k] = i18n_pb2.TranslateArgValue(number=v) + return i18n_pb2.TranslateStringRequest( module_index=module_index, message_index=message_index, args=args ) @@ -167,8 +167,8 @@ class Translations(GeneratedTranslations): ) -def backend_exception_to_pylib(err: pb.BackendError) -> Exception: - kind = pb.BackendError +def backend_exception_to_pylib(err: backend_pb2.BackendError) -> Exception: + kind = backend_pb2.BackendError val = err.kind if val == kind.INTERRUPTED: return Interrupted() diff --git a/pylib/anki/_backend/backend_pb2.pyi b/pylib/anki/_backend/backend_pb2.pyi deleted file mode 120000 index fb507905b..000000000 --- a/pylib/anki/_backend/backend_pb2.pyi +++ /dev/null @@ -1 +0,0 @@ -../../../bazel-bin/pylib/anki/_backend/backend_pb2.pyi \ No newline at end of file diff --git a/pylib/anki/_backend/genbackend.py b/pylib/anki/_backend/genbackend.py index d86e6ceb9..3d4031c1b 100755 --- a/pylib/anki/_backend/genbackend.py +++ b/pylib/anki/_backend/genbackend.py @@ -7,7 +7,10 @@ import re import sys import google.protobuf.descriptor -import pylib.anki._backend.backend_pb2 as pb + +import anki.backend_pb2 +import anki.i18n_pb2 + import stringcase TYPE_DOUBLE = 1 @@ -73,11 +76,11 @@ def python_type_inner(field): raise Exception(f"unknown type: {type}") -def fullname(fullname): - if "FluentString" in fullname: - return fullname.replace("BackendProto", "anki.fluent_pb2") - else: - return fullname.replace("BackendProto", "pb") +def fullname(fullname: str) -> str: + # eg anki.generic.Empty -> anki.generic_pb2.Empty + components = fullname.split(".") + components[1] += "_pb2" + return ".".join(components) # get_deck_i_d -> get_deck_id etc @@ -131,7 +134,7 @@ def render_method(service_idx, method_idx, method): return_type = python_type(f) else: single_field = "" - return_type = f"pb.{method.output_type.name}" + return_type = fullname(method.output_type.full_name) if method.name in SKIP_DECODE: return_type = "bytes" @@ -144,7 +147,7 @@ def render_method(service_idx, method_idx, method): buf += f"""return self._run_command({service_idx}, {method_idx}, input) """ else: - buf += f"""output = pb.{method.output_type.name}() + buf += f"""output = {fullname(method.output_type.full_name)}() output.ParseFromString(self._run_command({service_idx}, {method_idx}, input)) return output{single_field} """ @@ -162,12 +165,14 @@ def render_service( out.append(render_method(service_index, method_index, method)) -for service in pb.ServiceIndex.DESCRIPTOR.values: +service_modules = dict(I18N="i18n") + +for service in anki.backend_pb2.ServiceIndex.DESCRIPTOR.values: # SERVICE_INDEX_TEST -> _TESTSERVICE - service_var = ( - "_" + service.name.replace("SERVICE_INDEX", "").replace("_", "") + "SERVICE" - ) - service_obj = getattr(pb, service_var) + base = service.name.replace("SERVICE_INDEX_", "") + service_pkg = (service_modules.get(base) or "backend") + "" + service_var = "_" + base.replace("_", "") + "SERVICE" + service_obj = getattr(getattr(anki, service_pkg + "_pb2"), service_var) service_index = service.number render_service(service_obj, service_index) @@ -194,7 +199,7 @@ col.decks.all_config() from typing import * -import anki._backend.backend_pb2 as pb +import anki class RustBackendGenerated: def _run_command(self, service: int, method: int, input: Any) -> bytes: diff --git a/pylib/anki/backend_pb2.pyi b/pylib/anki/backend_pb2.pyi new file mode 120000 index 000000000..d3937d76d --- /dev/null +++ b/pylib/anki/backend_pb2.pyi @@ -0,0 +1 @@ +../../bazel-bin/pylib/anki/backend_pb2.pyi \ No newline at end of file diff --git a/pylib/anki/cards.py b/pylib/anki/cards.py index 60f15c42b..933f044ff 100644 --- a/pylib/anki/cards.py +++ b/pylib/anki/cards.py @@ -10,7 +10,7 @@ import time from typing import List, NewType, Optional import anki # pylint: disable=unused-import -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb from anki import hooks from anki._legacy import DeprecatedNamesMixin, deprecated from anki.consts import * diff --git a/pylib/anki/collection.py b/pylib/anki/collection.py index dc7cfaeb2..83e406927 100644 --- a/pylib/anki/collection.py +++ b/pylib/anki/collection.py @@ -7,7 +7,7 @@ from __future__ import annotations from typing import Any, Generator, List, Literal, Optional, Sequence, Tuple, Union, cast -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb # protobuf we publicly export - listed first to avoid circular imports from anki._legacy import DeprecatedNamesMixin, deprecated @@ -35,7 +35,7 @@ import weakref from dataclasses import dataclass, field import anki.latex -from anki import hooks +from anki import generic_pb2, hooks from anki._backend import RustBackend, Translations from anki.browser import BrowserConfig, BrowserDefaults from anki.cards import Card, CardId @@ -492,7 +492,7 @@ class Collection(DeprecatedNamesMixin): return _pb.SortOrder(custom=order) if isinstance(order, bool): if order is False: - return _pb.SortOrder(none=_pb.Empty()) + return _pb.SortOrder(none=generic_pb2.Empty()) # order=True: set args to sort column and reverse from config sort_key = BrowserConfig.sort_column_key(finding_notes) order = self.get_browser_column(self.get_config(sort_key)) @@ -506,7 +506,7 @@ class Collection(DeprecatedNamesMixin): # eg, user is ordering on an add-on field with the add-on not installed print(f"{order} is not a valid sort order.") - return _pb.SortOrder(none=_pb.Empty()) + return _pb.SortOrder(none=generic_pb2.Empty()) def find_and_replace( self, diff --git a/pylib/anki/config.py b/pylib/anki/config.py index af71eeaf5..0ed79f085 100644 --- a/pylib/anki/config.py +++ b/pylib/anki/config.py @@ -25,7 +25,7 @@ from typing import Any from weakref import ref import anki -from anki._backend import backend_pb2 as _pb +from anki import backend_pb2 as _pb from anki.collection import OpChanges from anki.errors import NotFoundError from anki.utils import from_json_bytes, to_json_bytes diff --git a/pylib/anki/decks.py b/pylib/anki/decks.py index bda0c1e2e..d2291caf8 100644 --- a/pylib/anki/decks.py +++ b/pylib/anki/decks.py @@ -23,7 +23,7 @@ from typing import ( if TYPE_CHECKING: import anki -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb from anki._legacy import DeprecatedNamesMixin, deprecated, print_deprecation_warning from anki.cards import CardId from anki.collection import OpChanges, OpChangesWithCount, OpChangesWithId diff --git a/pylib/anki/generic_pb2.pyi b/pylib/anki/generic_pb2.pyi new file mode 120000 index 000000000..77017f778 --- /dev/null +++ b/pylib/anki/generic_pb2.pyi @@ -0,0 +1 @@ +../../bazel-bin/pylib/anki/generic_pb2.pyi \ No newline at end of file diff --git a/pylib/anki/i18n_pb2.pyi b/pylib/anki/i18n_pb2.pyi new file mode 120000 index 000000000..d496aec75 --- /dev/null +++ b/pylib/anki/i18n_pb2.pyi @@ -0,0 +1 @@ +../../bazel-bin/pylib/anki/i18n_pb2.pyi \ No newline at end of file diff --git a/pylib/anki/lang.py b/pylib/anki/lang.py index 020712444..87ccee947 100644 --- a/pylib/anki/lang.py +++ b/pylib/anki/lang.py @@ -9,7 +9,8 @@ import weakref from typing import Optional, Tuple import anki -import anki._backend.backend_pb2 as _pb +import anki._backend +import anki.i18n_pb2 as _pb # public exports TR = anki._backend.LegacyTranslationEnum diff --git a/pylib/anki/latex.py b/pylib/anki/latex.py index 2132d5972..b3f19385f 100644 --- a/pylib/anki/latex.py +++ b/pylib/anki/latex.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from typing import Any, List, Optional, Tuple import anki -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb from anki import hooks from anki.models import NotetypeDict from anki.template import TemplateRenderContext, TemplateRenderOutput diff --git a/pylib/anki/media.py b/pylib/anki/media.py index e54f43485..f452134e7 100644 --- a/pylib/anki/media.py +++ b/pylib/anki/media.py @@ -11,7 +11,7 @@ import time from typing import Any, Callable, List, Optional, Tuple import anki -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb from anki._legacy import deprecated from anki.consts import * from anki.latex import render_latex, render_latex_returning_errors diff --git a/pylib/anki/models.py b/pylib/anki/models.py index 8a3bd13f6..50066e711 100644 --- a/pylib/anki/models.py +++ b/pylib/anki/models.py @@ -12,7 +12,7 @@ import time from typing import Any, Dict, List, NewType, Optional, Sequence, Tuple, Union import anki # pylint: disable=unused-import -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb from anki._legacy import DeprecatedNamesMixin, deprecated, print_deprecation_warning from anki.collection import OpChanges, OpChangesWithId from anki.consts import * diff --git a/pylib/anki/notes.py b/pylib/anki/notes.py index 356c34ff5..d6ad1f5de 100644 --- a/pylib/anki/notes.py +++ b/pylib/anki/notes.py @@ -9,7 +9,7 @@ import copy from typing import Any, List, NewType, Optional, Sequence, Tuple, Union import anki # pylint: disable=unused-import -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb from anki import hooks from anki._legacy import DeprecatedNamesMixin from anki.consts import MODEL_STD diff --git a/pylib/anki/scheduler/base.py b/pylib/anki/scheduler/base.py index a4b082623..c3f3534d8 100644 --- a/pylib/anki/scheduler/base.py +++ b/pylib/anki/scheduler/base.py @@ -4,7 +4,7 @@ from __future__ import annotations import anki -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb from anki.collection import OpChanges, OpChangesWithCount, OpChangesWithId from anki.config import Config diff --git a/pylib/anki/scheduler/v2.py b/pylib/anki/scheduler/v2.py index 713a9e15d..9c32366eb 100644 --- a/pylib/anki/scheduler/v2.py +++ b/pylib/anki/scheduler/v2.py @@ -11,7 +11,7 @@ from heapq import * from typing import Any, Callable, Dict, List, Optional, Tuple, Union import anki # pylint: disable=unused-import -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb from anki import hooks from anki.cards import Card, CardId from anki.consts import * diff --git a/pylib/anki/scheduler/v3.py b/pylib/anki/scheduler/v3.py index 4986e279a..da84b6118 100644 --- a/pylib/anki/scheduler/v3.py +++ b/pylib/anki/scheduler/v3.py @@ -14,7 +14,7 @@ from __future__ import annotations from typing import List, Literal, Sequence, Tuple -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb from anki.cards import Card from anki.collection import OpChanges from anki.consts import * diff --git a/pylib/anki/stdmodels.py b/pylib/anki/stdmodels.py index a3da07a74..01ac09b45 100644 --- a/pylib/anki/stdmodels.py +++ b/pylib/anki/stdmodels.py @@ -6,7 +6,7 @@ from __future__ import annotations from typing import Any, Callable, List, Tuple import anki -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb from anki.utils import from_json_bytes # pylint: disable=no-member diff --git a/pylib/anki/sync.py b/pylib/anki/sync.py index f81b7d972..54c2196d6 100644 --- a/pylib/anki/sync.py +++ b/pylib/anki/sync.py @@ -1,7 +1,7 @@ # Copyright: Ankitects Pty Ltd and contributors # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb # public exports SyncAuth = _pb.SyncAuth diff --git a/pylib/anki/syncserver/__init__.py b/pylib/anki/syncserver/__init__.py index c1d60d1e8..48da5dd1c 100644 --- a/pylib/anki/syncserver/__init__.py +++ b/pylib/anki/syncserver/__init__.py @@ -27,7 +27,7 @@ except ImportError as e: from flask import Response from anki import Collection -from anki._backend.backend_pb2 import SyncServerMethodRequest +from anki.backend_pb2 import SyncServerMethodRequest Method = SyncServerMethodRequest.Method # pylint: disable=no-member diff --git a/pylib/anki/tags.py b/pylib/anki/tags.py index 89bb5e158..7f3382fbd 100644 --- a/pylib/anki/tags.py +++ b/pylib/anki/tags.py @@ -16,7 +16,7 @@ import re from typing import Collection, List, Match, Optional, Sequence import anki # pylint: disable=unused-import -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb import anki.collection from anki.collection import OpChanges, OpChangesWithCount from anki.decks import DeckId diff --git a/pylib/anki/template.py b/pylib/anki/template.py index 332420f7d..588212665 100644 --- a/pylib/anki/template.py +++ b/pylib/anki/template.py @@ -32,7 +32,7 @@ from dataclasses import dataclass from typing import Any, Dict, List, Optional, Sequence, Tuple, Union import anki -import anki._backend.backend_pb2 as _pb +import anki.backend_pb2 as _pb from anki import hooks from anki.cards import Card from anki.decks import DeckManager diff --git a/pylib/protobuf.bzl b/pylib/protobuf.bzl index 7dc321a12..4ba3b65f4 100644 --- a/pylib/protobuf.bzl +++ b/pylib/protobuf.bzl @@ -1,21 +1,21 @@ load("@bazel_skylib//lib:paths.bzl", "paths") -def _py_proto_library_impl(ctx): - basename = ctx.file.src.basename - outs = [ - ctx.actions.declare_file(paths.replace_extension(basename, "_pb2.py")), - ctx.actions.declare_file(paths.replace_extension(basename, "_pb2.pyi")), - ] +def _py_proto_impl(ctx): + outs = [] + for src in ctx.files.srcs: + base = paths.basename(src.path) + outs.append(ctx.actions.declare_file(paths.replace_extension(base, "_pb2.py"))) + outs.append(ctx.actions.declare_file(paths.replace_extension(base, "_pb2.pyi"))) + ctx.actions.run( outputs = outs, - inputs = [ctx.file.src], + inputs = ctx.files.srcs, executable = ctx.executable.protoc_wrapper, arguments = [ ctx.executable.protoc.path, ctx.executable.mypy_protobuf.path, - ctx.file.src.path, paths.dirname(outs[0].path), - ], + ] + [file.path for file in ctx.files.srcs], tools = [ ctx.executable.protoc, ctx.executable.mypy_protobuf, @@ -26,10 +26,10 @@ def _py_proto_library_impl(ctx): DefaultInfo(files = depset(direct = outs), data_runfiles = ctx.runfiles(files = outs)), ] -py_proto_library_typed = rule( - implementation = _py_proto_library_impl, +py_proto = rule( + implementation = _py_proto_impl, attrs = { - "src": attr.label(allow_single_file = [".proto"]), + "srcs": attr.label_list(allow_files = [".proto"]), "protoc_wrapper": attr.label( executable = True, cfg = "exec", diff --git a/pylib/tools/protoc_wrapper.py b/pylib/tools/protoc_wrapper.py index a7a86fcfd..14165c514 100644 --- a/pylib/tools/protoc_wrapper.py +++ b/pylib/tools/protoc_wrapper.py @@ -10,16 +10,9 @@ import shutil import subprocess import sys -(protoc, mypy_protobuf, proto, outdir) = sys.argv[1:] +(protoc, mypy_protobuf, outdir, *protos) = sys.argv[1:] -# copy to current dir -basename = os.path.basename(proto) -shutil.copyfile(proto, basename) - -# output filenames -without_ext = os.path.splitext(basename)[0] -pb2_py = without_ext + "_pb2.py" -pb2_pyi = without_ext + "_pb2.pyi" +prefix = "proto/" # invoke protoc subprocess.run( @@ -28,13 +21,17 @@ subprocess.run( "--plugin=protoc-gen-mypy=" + mypy_protobuf, "--python_out=.", "--mypy_out=.", - basename, + "-I" + prefix, + "-Iexternal/anki/" + prefix, + *protos, ], # mypy prints to stderr on success :-( stderr=subprocess.DEVNULL, check=True, ) -# move files into output -shutil.move(pb2_py, outdir + "/" + pb2_py) -shutil.move(pb2_pyi, outdir + "/" + pb2_pyi) +for proto in protos: + without_prefix_and_ext, _ = os.path.splitext(proto[len(prefix) :]) + for ext in "_pb2.py", "_pb2.pyi": + path = without_prefix_and_ext + ext + shutil.move(path, os.path.join(outdir, os.path.basename(path))) diff --git a/qt/dmypy.py b/qt/dmypy.py index 5e0b277a5..20f3caf0a 100755 --- a/qt/dmypy.py +++ b/qt/dmypy.py @@ -30,8 +30,8 @@ if subprocess.run( "--", "--config-file", "qt/mypy.ini", - "bazel-bin/qt/dmypy.runfiles/net_ankiweb_anki/pylib/anki", - "bazel-bin/qt/dmypy.runfiles/net_ankiweb_anki/qt/aqt", + "bazel-bin/qt/dmypy.runfiles/anki/pylib/anki", + "bazel-bin/qt/dmypy.runfiles/anki/qt/aqt", "--python-executable", os.path.abspath("pip/stubs/extendsitepkgs"), ], diff --git a/rslib/BUILD.bazel b/rslib/BUILD.bazel index ccfb727ee..27db1eaf8 100644 --- a/rslib/BUILD.bazel +++ b/rslib/BUILD.bazel @@ -13,7 +13,7 @@ cargo_build_script( name = "build_script", srcs = glob(["build/*.rs"]), build_script_env = { - "BACKEND_PROTO": "$(location //proto:backend.proto)", + "PROTO_TOP": "$(location //proto:.top_level)", "PROTOC": "$(location @com_google_protobuf//:protoc)", "RSLIB_FTL_ROOT": "$(location @rslib_ftl//:l10n.toml)", "EXTRA_FTL_ROOT": "$(location @extra_ftl//:l10n.toml)", @@ -22,9 +22,10 @@ cargo_build_script( crate_root = "build/main.rs", data = [ "//ftl", - "//proto:backend.proto", + "//proto", "@com_google_protobuf//:protoc", # bazel requires us to list these out separately + "//proto:.top_level", "@rslib_ftl//:l10n.toml", "@extra_ftl//:l10n.toml", ], diff --git a/rslib/build/protobuf.rs b/rslib/build/protobuf.rs index 23f05628d..5ea84ad7d 100644 --- a/rslib/build/protobuf.rs +++ b/rslib/build/protobuf.rs @@ -17,7 +17,7 @@ pub trait Service { write!( buf, concat!(" ", - "{idx} => {{ let input = {input_type}::decode(input)?;\n", + "{idx} => {{ let input = super::{input_type}::decode(input)?;\n", "let output = self.{rust_method}(input)?;\n", "let mut out_bytes = Vec::new(); output.encode(&mut out_bytes)?; Ok(out_bytes) }}, "), idx = idx, @@ -38,8 +38,8 @@ pub trait Service { write!( buf, concat!( - " fn {method_name}(&self, input: {input_type}) -> ", - "Result<{output_type}>;\n" + " fn {method_name}(&self, input: super::{input_type}) -> ", + "Result;\n" ), method_name = method.name, input_type = method.input_type, @@ -55,7 +55,6 @@ impl prost_build::ServiceGenerator for CustomGenerator { write!( buf, "pub mod {name}_service {{ - use super::*; use prost::Message; use crate::error::Result; ", @@ -73,16 +72,32 @@ fn service_generator() -> Box { pub fn write_backend_proto_rs() { let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - let backend_proto; - let proto_dir; - if let Ok(proto) = env::var("BACKEND_PROTO") { - backend_proto = PathBuf::from(proto); - proto_dir = backend_proto.parent().unwrap().to_owned(); + let proto_dir = if let Ok(proto) = env::var("PROTO_TOP") { + let backend_proto = PathBuf::from(proto); + backend_proto.parent().unwrap().to_owned() } else { - backend_proto = PathBuf::from("backend.proto"); - proto_dir = PathBuf::from("../proto"); + PathBuf::from("../proto") + }; + + let subfolders = &["anki"]; + let mut paths = vec![]; + + for subfolder in subfolders { + for entry in proto_dir.join(subfolder).read_dir().unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if path + .file_name() + .unwrap() + .to_str() + .unwrap() + .ends_with(".proto") + { + println!("cargo:rerun-if-changed={}", path.to_str().unwrap()); + paths.push(path); + } + } } - println!("cargo:rerun-if-changed={}", backend_proto.to_str().unwrap()); let mut config = prost_build::Config::new(); config @@ -92,6 +107,6 @@ pub fn write_backend_proto_rs() { "Deck.Filtered.SearchTerm.Order", "#[derive(strum::EnumIter)]", ) - .compile_protos(&[&backend_proto], &[&proto_dir, &out_dir]) + .compile_protos(paths.as_slice(), &[proto_dir, out_dir]) .unwrap(); } diff --git a/rslib/src/backend_proto.rs b/rslib/src/backend_proto.rs index 751d575cc..8c50836f5 100644 --- a/rslib/src/backend_proto.rs +++ b/rslib/src/backend_proto.rs @@ -1,4 +1,16 @@ // Copyright: Ankitects Pty Ltd and contributors // License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -include!(concat!(env!("OUT_DIR"), "/backend_proto.rs")); +pub mod backend { + include!(concat!(env!("OUT_DIR"), "/anki.backend.rs")); +} +pub mod i18n { + include!(concat!(env!("OUT_DIR"), "/anki.i18n.rs")); +} +pub mod generic { + include!(concat!(env!("OUT_DIR"), "/anki.generic.rs")); +} + +pub use backend::*; +pub use generic::*; +pub use i18n::*; diff --git a/ts/change-notetype/BUILD.bazel b/ts/change-notetype/BUILD.bazel index c116b3f80..017d5c46e 100644 --- a/ts/change-notetype/BUILD.bazel +++ b/ts/change-notetype/BUILD.bazel @@ -52,7 +52,6 @@ ts_library( deps = [ "//ts/components", "//ts/lib", - "//ts/lib:backend_proto", "//ts/sveltelib", "@npm//lodash-es", "@npm//svelte", @@ -76,7 +75,6 @@ esbuild( "@npm//bootstrap", "@npm//marked", "//ts/lib", - "//ts/lib:backend_proto", "//ts/sveltelib", "//ts/components", "//ts/components:svelte_components", @@ -123,7 +121,7 @@ jest_test( protobuf = True, deps = [ ":lib", - "//ts/lib:backend_proto", + "//ts/lib", "@npm//protobufjs", "@npm//svelte", ], diff --git a/ts/change-notetype/lib.test.ts b/ts/change-notetype/lib.test.ts index b9597debe..7f392a455 100644 --- a/ts/change-notetype/lib.test.ts +++ b/ts/change-notetype/lib.test.ts @@ -5,7 +5,7 @@ @typescript-eslint/no-explicit-any: "off", */ -import * as pb from "lib/backend_proto"; +import { Backend } from "lib/proto"; import { ChangeNotetypeState, negativeOneToNull, MapContext } from "./lib"; import { get } from "svelte/store"; @@ -64,15 +64,15 @@ const exampleInfoSame = { function differentState(): ChangeNotetypeState { return new ChangeNotetypeState( - pb.BackendProto.NotetypeNames.fromObject(exampleNames), - pb.BackendProto.ChangeNotetypeInfo.fromObject(exampleInfoDifferent) + Backend.NotetypeNames.fromObject(exampleNames), + Backend.ChangeNotetypeInfo.fromObject(exampleInfoDifferent) ); } function sameState(): ChangeNotetypeState { return new ChangeNotetypeState( - pb.BackendProto.NotetypeNames.fromObject(exampleNames), - pb.BackendProto.ChangeNotetypeInfo.fromObject(exampleInfoSame) + Backend.NotetypeNames.fromObject(exampleNames), + Backend.ChangeNotetypeInfo.fromObject(exampleInfoSame) ); } diff --git a/ts/change-notetype/lib.ts b/ts/change-notetype/lib.ts index 5e7fe578d..df982fa9a 100644 --- a/ts/change-notetype/lib.ts +++ b/ts/change-notetype/lib.ts @@ -5,22 +5,20 @@ @typescript-eslint/no-non-null-assertion: "off", */ -import pb from "lib/backend_proto"; +import { Backend } from "lib/proto"; import { postRequest } from "lib/postrequest"; import { readable, Readable } from "svelte/store"; import { isEqual } from "lodash-es"; -export async function getNotetypeNames(): Promise { - return pb.BackendProto.NotetypeNames.decode( - await postRequest("/_anki/notetypeNames", "") - ); +export async function getNotetypeNames(): Promise { + return Backend.NotetypeNames.decode(await postRequest("/_anki/notetypeNames", "")); } export async function getChangeNotetypeInfo( oldNotetypeId: number, newNotetypeId: number -): Promise { - return pb.BackendProto.ChangeNotetypeInfo.decode( +): Promise { + return Backend.ChangeNotetypeInfo.decode( await postRequest( "/_anki/changeNotetypeInfo", JSON.stringify({ oldNotetypeId, newNotetypeId }) @@ -29,10 +27,9 @@ export async function getChangeNotetypeInfo( } export async function changeNotetype( - input: pb.BackendProto.ChangeNotetypeRequest + input: Backend.ChangeNotetypeRequest ): Promise { - const data: Uint8Array = - pb.BackendProto.ChangeNotetypeRequest.encode(input).finish(); + const data: Uint8Array = Backend.ChangeNotetypeRequest.encode(input).finish(); await postRequest("/_anki/changeNotetype", data); return; } @@ -50,9 +47,9 @@ export function negativeOneToNull(list: number[]): (number | null)[] { export class ChangeNotetypeInfoWrapper { fields: (number | null)[]; templates?: (number | null)[]; - readonly info: pb.BackendProto.ChangeNotetypeInfo; + readonly info: Backend.ChangeNotetypeInfo; - constructor(info: pb.BackendProto.ChangeNotetypeInfo) { + constructor(info: Backend.ChangeNotetypeInfo) { this.info = info; const templates = info.input!.newTemplates!; if (templates.length > 0) { @@ -114,13 +111,13 @@ export class ChangeNotetypeInfoWrapper { ); } - input(): pb.BackendProto.ChangeNotetypeRequest { - return this.info.input as pb.BackendProto.ChangeNotetypeRequest; + input(): Backend.ChangeNotetypeRequest { + return this.info.input as Backend.ChangeNotetypeRequest; } /// Pack changes back into input message for saving. - intoInput(): pb.BackendProto.ChangeNotetypeRequest { - const input = this.info.input as pb.BackendProto.ChangeNotetypeRequest; + intoInput(): Backend.ChangeNotetypeRequest { + const input = this.info.input as Backend.ChangeNotetypeRequest; input.newFields = nullToNegativeOne(this.fields); if (this.templates) { input.newTemplates = nullToNegativeOne(this.templates); @@ -146,13 +143,10 @@ export class ChangeNotetypeState { private info_: ChangeNotetypeInfoWrapper; private infoSetter!: (val: ChangeNotetypeInfoWrapper) => void; - private notetypeNames: pb.BackendProto.NotetypeNames; + private notetypeNames: Backend.NotetypeNames; private notetypesSetter!: (val: NotetypeListEntry[]) => void; - constructor( - notetypes: pb.BackendProto.NotetypeNames, - info: pb.BackendProto.ChangeNotetypeInfo - ) { + constructor(notetypes: Backend.NotetypeNames, info: Backend.ChangeNotetypeInfo) { this.info_ = new ChangeNotetypeInfoWrapper(info); this.info = readable(this.info_, (set) => { this.infoSetter = set; @@ -203,7 +197,7 @@ export class ChangeNotetypeState { await changeNotetype(this.dataForSaving()); } - dataForSaving(): pb.BackendProto.ChangeNotetypeRequest { + dataForSaving(): Backend.ChangeNotetypeRequest { return this.info_.intoInput(); } diff --git a/ts/compile_sass.bzl b/ts/compile_sass.bzl index c8d96cfb3..256504a5c 100644 --- a/ts/compile_sass.bzl +++ b/ts/compile_sass.bzl @@ -14,7 +14,7 @@ def compile_sass(group, srcs, deps = [], visibility = ["//visibility:private"]): sourcemap = False, deps = deps, visibility = visibility, - include_paths = ["external/net_ankiweb_anki"], + include_paths = ["external/anki"], ) native.filegroup( diff --git a/ts/components/BUILD.bazel b/ts/components/BUILD.bazel index 4eaa56dbd..167e4089a 100644 --- a/ts/components/BUILD.bazel +++ b/ts/components/BUILD.bazel @@ -17,25 +17,24 @@ filegroup( compile_svelte( name = "svelte", srcs = svelte_files, + visibility = ["//visibility:public"], deps = [ "//ts/sass:button_mixins_lib", "//ts/sass/bootstrap", ], - visibility = ["//visibility:public"], ) ts_library( name = "components", - module_name = "components", srcs = glob( ["*.ts"], exclude = ["*.test.ts"], ), + module_name = "components", tsconfig = "//ts:tsconfig.json", visibility = ["//visibility:public"], deps = [ "//ts/lib", - "//ts/lib:backend_proto", "//ts/sveltelib", "@npm//@popperjs/core", "@npm//@types/bootstrap", diff --git a/ts/congrats/BUILD.bazel b/ts/congrats/BUILD.bazel index fb3ae63c4..8cba0ec3d 100644 --- a/ts/congrats/BUILD.bazel +++ b/ts/congrats/BUILD.bazel @@ -35,10 +35,7 @@ ts_library( ts_library( name = "lib", srcs = ["lib.ts"], - deps = [ - "//ts/lib", - "//ts/lib:backend_proto", - ], + deps = ["//ts/lib"], ) esbuild( @@ -56,7 +53,6 @@ esbuild( ":base_css", ":index", "//ts/lib", - "//ts/lib:backend_proto", "@npm//protobufjs", ], ) diff --git a/ts/congrats/CongratsPage.svelte b/ts/congrats/CongratsPage.svelte index df5b59ab7..d0c9d8492 100644 --- a/ts/congrats/CongratsPage.svelte +++ b/ts/congrats/CongratsPage.svelte @@ -3,11 +3,11 @@ Copyright: Ankitects Pty Ltd and contributors License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html -->