Merge pull request #1434 from ankitects/sveltecheck

Svelte build improvements

ts/card-info/BUILD.bazel

@@ -16,15 +16,18 @@ compile_sass(
     ],
 )
 
-compile_svelte()
+_ts_deps = [
+    "//ts/components",
+    "//ts/lib",
+    "@npm//@fluent",
+]
+
+compile_svelte(deps = _ts_deps)
 
 typescript(
     name = "index",
-    deps = [
+    deps = _ts_deps + [
         ":svelte",
-        "//ts/components",
-        "//ts/lib",
-        "@npm//@fluent",
     ],
 )
 

ts/change-notetype/BUILD.bazel

@@ -17,20 +17,23 @@ compile_sass(
     ],
 )
 
-compile_svelte()
+_ts_deps = [
+    "//ts/components",
+    "//ts/lib",
+    "//ts/sveltelib",
+    "@npm//@fluent",
+    "@npm//@types/jest",
+    "@npm//lodash-es",
+    "@npm//svelte",
+    "@npm//marked",
+]
+
+compile_svelte(deps = _ts_deps)
 
 typescript(
     name = "index",
-    deps = [
+    deps = _ts_deps + [
         ":svelte",
-        "//ts/components",
-        "//ts/lib",
-        "//ts/sveltelib",
-        "@npm//@fluent",
-        "@npm//@types/jest",
-        "@npm//lodash-es",
-        "@npm//svelte",
-        "@npm//svelte2tsx",
     ],
 )
 
@@ -47,12 +50,6 @@ esbuild(
         ":base_css",
         ":index",
         ":svelte",
-        "//ts/components",
-        "//ts/lib",
-        "//ts/sveltelib",
-        "@npm//bootstrap",
-        "@npm//marked",
-        "@npm//protobufjs",
     ],
 )
 

ts/components/BUILD.bazel

@@ -3,20 +3,23 @@ load("//ts:prettier.bzl", "prettier_test")
 load("//ts:eslint.bzl", "eslint_test")
 load("//ts:typescript.bzl", "typescript")
 
+_ts_deps = [
+    "//ts/lib",
+    "//ts/sveltelib",
+    "@npm//@popperjs/core",
+    "@npm//@types/bootstrap",
+    "@npm//bootstrap",
+    "@npm//svelte",
+]
+
 compile_svelte(
-    deps = ["//sass/bootstrap"],
+    deps = _ts_deps + ["//sass/bootstrap"],
 )
 
 typescript(
     name = "components",
-    deps = [
+    deps = _ts_deps + [
         ":svelte",
-        "//ts/lib",
-        "//ts/sveltelib",
-        "@npm//@popperjs/core",
-        "@npm//@types/bootstrap",
-        "@npm//bootstrap",
-        "@npm//svelte",
     ],
 )
 

ts/congrats/BUILD.bazel

@@ -1,7 +1,7 @@
 load("@npm//@bazel/typescript:index.bzl", "ts_library")
 load("//ts:prettier.bzl", "prettier_test")
 load("//ts:eslint.bzl", "eslint_test")
-load("//ts/svelte:svelte.bzl", "svelte", "svelte_check")
+load("//ts/svelte:svelte.bzl", "compile_svelte", "svelte_check")
 load("//ts:esbuild.bzl", "esbuild")
 load("//ts:compile_sass.bzl", "compile_sass")
 load("//ts:typescript.bzl", "typescript")
@@ -16,15 +16,14 @@ compile_sass(
     ],
 )
 
-svelte(
+compile_svelte(
-    name = "CongratsPage",
+    deps = ["//ts/lib"],
-    entry_point = "CongratsPage.svelte",
 )
 
 typescript(
     name = "index",
     deps = [
-        "CongratsPage",
+        ":svelte",
         "//ts/lib",
         "@npm//@fluent",
         "@npm//svelte",
@@ -41,11 +40,9 @@ esbuild(
     output_css = "congrats.css",
     visibility = ["//visibility:public"],
     deps = [
-        "CongratsPage",
         ":base_css",
         ":index",
-        "//ts/lib",
+        ":svelte",
-        "@npm//protobufjs",
     ],
 )
 

ts/deck-options/BUILD.bazel

@@ -18,25 +18,29 @@ compile_sass(
     ],
 )
 
+_ts_deps = [
+    "//ts/components",
+    "//ts/lib",
+    "//ts/sveltelib",
+    "@npm//@fluent",
+    "@npm//@popperjs",
+    "@npm//@types/jest",
+    "@npm//bootstrap-icons",
+    "@npm//lodash-es",
+    "@npm//svelte",
+    "@npm//marked",
+]
+
 compile_svelte(
-    deps = [
+    deps = _ts_deps + [
         "//sass/bootstrap",
     ],
 )
 
 typescript(
     name = "index",
-    deps = [
+    deps = _ts_deps + [
         ":svelte",
-        "//ts/components",
-        "//ts/lib",
-        "//ts/sveltelib",
-        "@npm//@fluent",
-        "@npm//@popperjs",
-        "@npm//@types/jest",
-        "@npm//bootstrap-icons",
-        "@npm//lodash-es",
-        "@npm//svelte",
     ],
 )
 
@@ -50,15 +54,9 @@ esbuild(
     output_css = "deck-options.css",
     visibility = ["//visibility:public"],
     deps = [
-        "index",
         ":base_css",
+        ":index",
         ":svelte",
-        "//ts/components",
-        "//ts/lib",
-        "//ts/sveltelib",
-        "@npm//bootstrap",
-        "@npm//marked",
-        "@npm//protobufjs",
     ],
 )
 

ts/graphs/BUILD.bazel

@@ -15,20 +15,25 @@ compile_sass(
     ],
 )
 
-compile_svelte()
+_ts_deps = [
+    "//ts/lib",
+    "//ts/sveltelib",
+    "@npm//@fluent",
+    "@npm//@types/d3",
+    "@npm//@types/lodash",
+    "@npm//d3",
+    "@npm//lodash-es",
+    "@npm//svelte",
+]
+
+compile_svelte(
+    deps = _ts_deps,
+)
 
 typescript(
     name = "index",
-    deps = [
+    deps = _ts_deps + [
         ":svelte",
-        "//ts/lib",
-        "//ts/sveltelib",
-        "@npm//@fluent",
-        "@npm//@types/d3",
-        "@npm//@types/lodash",
-        "@npm//d3",
-        "@npm//lodash-es",
-        "@npm//svelte",
     ],
 )
 

ts/reviewer/BUILD.bazel

@@ -28,8 +28,6 @@ esbuild(
     visibility = ["//visibility:public"],
     deps = [
         ":reviewer_ts",
-        "//ts/lib",
-        "@npm//protobufjs",
     ],
 )
 
@@ -39,8 +37,6 @@ esbuild(
     visibility = ["//visibility:public"],
     deps = [
         ":reviewer_ts",
-        "//ts/lib",
-        "@npm//protobufjs",
     ],
 )
 

ts/svelte/BUILD.bazel

@@ -1,28 +1,25 @@
 load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary")
-load("@npm//@bazel/typescript:index.bzl", "ts_library")
+load("//ts:typescript.bzl", "typescript")
 
 _deps = [
     "@npm//@bazel/worker",
+    "@npm//@types/node",
     "@npm//sass",
     "@npm//svelte",
     "@npm//svelte-preprocess",
     "@npm//svelte2tsx",
     "@npm//typescript",
-    "@npm//@types/node",
 ]
 
-ts_library(
+typescript(
-    name = "svelte_bin_ts",
+    name = "svelte_worker_ts",
     srcs = ["svelte.ts"],
-    tsconfig = "//ts:tsconfig_bin",
     deps = _deps,
 )
 
 nodejs_binary(
     name = "svelte_bin",
-    data = ["svelte_bin_ts"] + _deps,
+    data = [":svelte_worker_ts"] + _deps,
-    entry_point = ":svelte.ts",
+    entry_point = ":svelte.js",
-    # should fix .js files being not found on subsequent worker requests
-    templated_args = ["--bazel_patch_module_resolver"],
     visibility = ["//visibility:public"],
 )

ts/svelte/svelte.bzl

@@ -2,82 +2,97 @@ load("@npm//svelte-check:index.bzl", _svelte_check = "svelte_check_test")
 load("@build_bazel_rules_nodejs//:providers.bzl", "DeclarationInfo", "declaration_info")
 load("@io_bazel_rules_sass//:defs.bzl", "SassInfo")
 load("@build_bazel_rules_nodejs//:index.bzl", "js_library")
+load("@bazel_skylib//lib:paths.bzl", "paths")
 
-def _get_dep_sources(dep):
+def _get_declarations(dep):
     if SassInfo in dep:
         return dep[SassInfo].transitive_sources
     elif DeclarationInfo in dep:
         return dep[DeclarationInfo].transitive_declarations
     else:
-        return []
+        fail("unexpected dep", dep)
-
-def _get_sources(deps):
-    return depset([], transitive = [_get_dep_sources(dep) for dep in deps])
 
 def _svelte(ctx):
     args = ctx.actions.args()
     args.use_param_file("@%s", use_always = True)
     args.set_param_file_format("multiline")
 
-    args.add(ctx.file.entry_point.path)
+    # path to bin folder, for sass
-    args.add(ctx.outputs.mjs.path)
-    args.add(ctx.outputs.dts.path)
-    args.add(ctx.outputs.css.path)
     args.add(ctx.var["BINDIR"])
     args.add(ctx.var["GENDIR"])
-    args.add_all(ctx.files._shims)
 
-    deps = _get_sources(ctx.attr.deps).to_list()
+    # svelte and ts sources
+    outputs = []
+    dts_only = []
+    nondts_only = []
+    for src in ctx.files.srcs:
+        args.add(src.path)
+
+        if src.path.endswith(".svelte"):
+            # strip off external/ankidesktop if invoked from external workspace
+            path = src.path
+            if src.path.startswith("external/ankidesktop/"):
+                path = path[len("external/ankidesktop/"):]
+
+            # strip off package prefix, eg ts/editor/mathjax/Foo.svelte -> mathjax/Foo.svelte
+            base = path[len(ctx.label.package) + 1:]
+            for ext in ("d.ts", "css", "mjs"):
+                out = ctx.actions.declare_file(base.replace(".svelte", ".svelte." + ext))
+                args.add(out)
+                outputs.append(out)
+                if ext == "d.ts":
+                    dts_only.append(out)
+                else:
+                    nondts_only.append(out)
+
+    # dependencies
+    deps = depset([], transitive = [_get_declarations(dep) for dep in ctx.attr.deps])
+    args.add_all(deps)
 
     ctx.actions.run(
         execution_requirements = {"supports-workers": "1"},
         executable = ctx.executable._svelte_bin,
-        outputs = [ctx.outputs.mjs, ctx.outputs.dts, ctx.outputs.css],
+        outputs = outputs,
-        inputs = [ctx.file.entry_point] + deps + ctx.files._shims,
+        inputs = ctx.files.srcs + deps.to_list(),
         mnemonic = "Svelte",
+        progress_message = "Compiling Svelte {}:{}".format(ctx.label.package, ctx.attr.name),
         arguments = [args],
     )
 
     return [
-        declaration_info(depset([ctx.outputs.dts]), deps = [ctx.attr._shims]),
+        declaration_info(depset(dts_only), deps = ctx.attr.deps),
+        DefaultInfo(
+            files = depset(nondts_only),
+            runfiles = ctx.runfiles(files = outputs, transitive_files = deps),
+        ),
     ]
 
 svelte = rule(
     implementation = _svelte,
     attrs = {
-        "entry_point": attr.label(allow_single_file = True),
+        "srcs": attr.label_list(allow_files = True, doc = ".ts and .svelte files"),
         "deps": attr.label_list(),
         "_svelte_bin": attr.label(
             default = Label("//ts/svelte:svelte_bin"),
             executable = True,
             cfg = "host",
         ),
-        "_shims": attr.label(
-            default = Label("@npm//svelte2tsx:svelte2tsx__typings"),
-            allow_files = True,
-        ),
-    },
-    outputs = {
-        "mjs": "%{name}.svelte.mjs",
-        "dts": "%{name}.svelte.d.ts",
-        "css": "%{name}.svelte.css",
     },
 )
 
 def compile_svelte(name = "svelte", srcs = None, deps = [], visibility = ["//visibility:private"]):
     if not srcs:
-        srcs = native.glob(["*.svelte"])
+        srcs = native.glob([
-    for src in srcs:
+            "**/*.svelte",
-        svelte(
+            "**/*.ts",
-            name = src.replace(".svelte", ""),
+        ])
-            entry_point = src,
-            deps = deps,
-            visibility = visibility,
-        )
 
-    js_library(
+    svelte(
         name = name,
-        srcs = [s.replace(".svelte", "") for s in srcs],
+        srcs = srcs,
+        deps = deps + [
+            "@npm//svelte2tsx",
+        ],
         visibility = visibility,
     )
 

ts/svelte/svelte.ts

@@ -8,23 +8,22 @@ import { svelte2tsx } from "svelte2tsx";
 import preprocess from "svelte-preprocess";
 import { basename } from "path";
 import * as ts from "typescript";
-import * as svelte from "svelte/compiler.js";
+import * as svelte from "svelte/compiler";
 
-let parsedCommandLine: ts.ParsedCommandLine = {
+const parsedCommandLine: ts.ParsedCommandLine = {
     fileNames: [],
     errors: [],
     options: {
         jsx: ts.JsxEmit.Preserve,
         declaration: true,
         emitDeclarationOnly: true,
-        skipLibCheck: true,
+        // noEmitOnError: true,
+        paths: {
+            "*": ["*", "external/npm/node_modules/*"],
+        },
     },
 };
 
-// We avoid hitting the filesystem for ts/d.ts files after initial startup - the
-// .ts file we generate can be injected directly into our cache, and Bazel
-// should restart us if the Svelte or TS typings change.
-
 interface FileContent {
     text: string;
     version: number;
@@ -43,17 +42,17 @@ function getFileContent(path: string): FileContent {
     return content;
 }
 
-function updateFileContent(path: string, text: string): void {
+function updateFileContent(input: InputFile): void {
-    let content = fileContent.get(path);
+    let content = fileContent.get(input.path);
     if (content) {
-        content.text = text;
+        content.text = input.data;
         content.version += 1;
     } else {
         content = {
-            text,
+            text: input.data,
             version: 0,
         };
-        fileContent.set(path, content);
+        fileContent.set(input.path, content);
     }
 }
 
@@ -65,7 +64,6 @@ const languageServiceHost: ts.LanguageServiceHost = {
         return getFileContent(path).version.toString();
     },
     getScriptSnapshot: (path: string): ts.IScriptSnapshot | undefined => {
-        // if (!ts.sys.fileExists(fileName)) {
         const text = getFileContent(path).text;
         return {
             getText: (start: number, end: number) => {
@@ -78,7 +76,7 @@ const languageServiceHost: ts.LanguageServiceHost = {
             },
             getLength: () => text.length,
             getChangeRange: (
-                oldSnapshot: ts.IScriptSnapshot
+                _oldSnapshot: ts.IScriptSnapshot
             ): ts.TextChangeRange | undefined => {
                 return undefined;
             },
@@ -90,14 +88,29 @@ const languageServiceHost: ts.LanguageServiceHost = {
 
 const languageService = ts.createLanguageService(languageServiceHost);
 
-function compile(tsPath: string, tsLibs: string[]) {
+async function emitTypings(svelte: SvelteTsxFile[], deps: InputFile[]): Promise<void> {
-    parsedCommandLine.fileNames = [tsPath, ...tsLibs];
+    const allFiles = [...svelte, ...deps];
+    allFiles.forEach(updateFileContent);
+    parsedCommandLine.fileNames = allFiles.map((i) => i.path);
     const program = languageService.getProgram()!;
     const tsHost = ts.createCompilerHost(parsedCommandLine.options);
     const createdFiles = {};
-    tsHost.writeFile = (fileName, contents) => (createdFiles[fileName] = contents);
+    const cwd = ts.sys.getCurrentDirectory().replace(/\\/g, "/");
-    program.emit(undefined /* all files */, tsHost.writeFile);
+    tsHost.writeFile = (fileName, contents) => {
-    return createdFiles[parsedCommandLine.fileNames[0].replace(".tsx", ".d.ts")];
+        // tsc makes some paths absolute for some reason
+        if (fileName.startsWith(cwd)) {
+            fileName = fileName.substring(cwd.length + 1);
+        }
+        createdFiles[fileName] = contents;
+    };
+    const result = program.emit(undefined /* all files */, tsHost.writeFile);
+    // for (const diag of result.diagnostics) {
+    //     console.log(diag.messageText);
+    // }
+
+    for (const file of svelte) {
+        await writeFile(file.realDtsPath, createdFiles[file.virtualDtsPath]);
+    }
 }
 
 function writeFile(file, data): Promise<void> {
@@ -124,26 +137,8 @@ function readFile(file) {
     });
 }
 
-async function writeDts(tsPath, dtsPath, tsLibs) {
+async function compileSingleSvelte(
-    const dtsSource = compile(tsPath, tsLibs);
+    input: SvelteInput,
-    await writeFile(dtsPath, dtsSource);
-}
-
-function writeTs(svelteSource, sveltePath, tsPath): void {
-    let tsSource = svelte2tsx(svelteSource, {
-        filename: sveltePath,
-        isTsFile: true,
-        mode: "dts",
-    });
-    let codeLines = tsSource.code.split("\n");
-    updateFileContent(tsPath, codeLines.join("\n"));
-}
-
-async function writeJs(
-    source: string,
-    inputFilename: string,
-    outputJsPath: string,
-    outputCssPath: string,
     binDir: string,
     genDir: string
 ): Promise<void> {
@@ -162,14 +157,14 @@ async function writeJs(
     });
 
     try {
-        const processed = await svelte.preprocess(source, preprocessOptions, {
+        const processed = await svelte.preprocess(input.data, preprocessOptions, {
-            filename: inputFilename,
+            filename: input.path,
         });
         const result = svelte.compile(processed.toString!(), {
             format: "esm",
             css: false,
             generate: "dom",
-            filename: outputJsPath,
+            filename: input.mjsPath,
         });
         // warnings are an error
         if (result.warnings.length > 0) {
@@ -177,27 +172,123 @@ async function writeJs(
         }
         // write out the css file
         const outputCss = result.css.code ?? "";
-        await writeFile(outputCssPath, outputCss);
+        await writeFile(input.cssPath, outputCss);
         // if it was non-empty, prepend a reference to it in the js file, so that
         // it's included in the bundled .css when bundling
         const outputSource =
-            (outputCss ? `import "./${basename(outputCssPath)}";` : "") +
+            (outputCss ? `import "./${basename(input.cssPath)}";` : "") +
             result.js.code;
-        await writeFile(outputJsPath, outputSource);
+        await writeFile(input.mjsPath, outputSource);
     } catch (err) {
         console.log(`compile failed: ${err}`);
         return;
     }
 }
 
-async function compileSvelte(args) {
+interface Args {
-    const [sveltePath, mjsPath, dtsPath, cssPath, binDir, genDir, ...tsLibs] = args;
+    binDir: string;
-    const svelteSource = (await readFile(sveltePath)) as string;
+    genDir: string;
+    svelteFiles: SvelteInput[];
+    dependencies: InputFile[];
+}
 
-    const mockTsPath = sveltePath + ".tsx";
+interface InputFile {
-    writeTs(svelteSource, sveltePath, mockTsPath);
+    path: string;
-    await writeDts(mockTsPath, dtsPath, tsLibs);
+    data: string;
-    await writeJs(svelteSource, sveltePath, mjsPath, cssPath, binDir, genDir);
+}
+
+interface SvelteInput extends InputFile {
+    dtsPath: string;
+    cssPath: string;
+    mjsPath: string;
+}
+
+async function extractArgsAndData(args: string[]): Promise<Args> {
+    const [binDir, genDir, ...rest] = args;
+    const [svelteFiles, dependencies] = await extractSvelteAndDeps(rest);
+    return {
+        binDir,
+        genDir,
+        svelteFiles,
+        dependencies,
+    };
+}
+
+async function extractSvelteAndDeps(
+    files: string[]
+): Promise<[SvelteInput[], InputFile[]]> {
+    const svelte: SvelteInput[] = [];
+    const deps: InputFile[] = [];
+    files.reverse();
+    while (files.length) {
+        const file = files.pop()!;
+        const data = (await readFile(file)) as string;
+        if (file.endsWith(".svelte")) {
+            svelte.push({
+                path: file,
+                data,
+                dtsPath: files.pop()!,
+                cssPath: files.pop()!,
+                mjsPath: files.pop()!,
+            });
+        } else {
+            deps.push({ path: remapBinToSrcDir(file), data });
+        }
+    }
+    return [svelte, deps];
+}
+
+/// Our generated .tsx files sit in the bin dir,  but .ts files
+/// may be coming from the source folder, which breaks ./foo imports.
+/// Adjust the path to make it appear they're all in the same folder.
+function remapBinToSrcDir(file: string): string {
+    return file.replace(new RegExp("bazel-out/[-_a-z]+/bin/"), "");
+}
+
+/// Generate Svelte .mjs/.css files.
+async function compileSvelte(
+    svelte: SvelteInput[],
+    binDir: string,
+    genDir: string
+): Promise<void> {
+    for (const file of svelte) {
+        await compileSingleSvelte(file, binDir, genDir);
+    }
+}
+
+interface SvelteTsxFile extends InputFile {
+    // relative to src folder
+    virtualDtsPath: string;
+    // must go to bazel-out
+    realDtsPath: string;
+}
+
+function generateTsxFiles(svelteFiles: SvelteInput[]): SvelteTsxFile[] {
+    return svelteFiles.map((file) => {
+        const data = svelte2tsx(file.data, {
+            filename: file.path,
+            isTsFile: true,
+            mode: "dts",
+        }).code;
+        const path = file.path.replace(".svelte", ".svelte.tsx");
+        return {
+            path,
+            data,
+            virtualDtsPath: path.replace(".tsx", ".d.ts"),
+            realDtsPath: file.dtsPath,
+        };
+    });
+}
+
+async function compileSvelteAndGenerateTypings(argsList: string[]): Promise<boolean> {
+    const args = await extractArgsAndData(argsList);
+
+    // mjs/css
+    await compileSvelte(args.svelteFiles, args.binDir, args.genDir);
+
+    // d.ts
+    const tsxFiles = generateTsxFiles(args.svelteFiles);
+    await emitTypings(tsxFiles, args.dependencies);
 
     return true;
 }
@@ -206,12 +297,12 @@ function main() {
     if (worker.runAsWorker(process.argv)) {
         console.log = worker.log;
         worker.log("Svelte running as a Bazel worker");
-        worker.runWorkerLoop(compileSvelte);
+        worker.runWorkerLoop(compileSvelteAndGenerateTypings);
     } else {
         const paramFile = process.argv[2].replace(/^@/, "");
         const commandLineArgs = fs.readFileSync(paramFile, "utf-8").trim().split("\n");
         console.log("Svelte running as a standalone process");
-        compileSvelte(commandLineArgs);
+        compileSvelteAndGenerateTypings(commandLineArgs);
     }
 }
 

ts/svelte/tsconfig.json

@@ -0,0 +1,13 @@
+{
+    "extends": "../tsconfig.json",
+    "include": ["*"],
+    "references": [],
+    "compilerOptions": {
+        // ignore missing optional modules like postcss
+        "skipLibCheck": true,
+        // ensure node can run the resulting code
+        "noEmitHelpers": false,
+        "importHelpers": false,
+        "module": "commonjs",
+    }
+}