From 0cc06ad03bb599dc7f489d02fe4f196c0621aafd Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sat, 20 Mar 2021 20:42:29 +1000 Subject: [PATCH] switch Svelte compilation to worker model based on changes from upstream rules_svelte Their code was using run_node() instead of ctx.actions.run(), which seems to create a new worker for every CPU core, instead of respecting the standard limit of 4. --- ts/package.json | 1 + ts/svelte/BUILD.bazel | 14 +--- ts/svelte/svelte.bzl | 75 +++++++-------------- ts/svelte/svelte.js | 153 ++++++++++++++++++++++++++++++------------ ts/yarn.lock | 7 ++ 5 files changed, 144 insertions(+), 106 deletions(-) diff --git a/ts/package.json b/ts/package.json index b1f543cc9..0691eacea 100644 --- a/ts/package.json +++ b/ts/package.json @@ -8,6 +8,7 @@ "devDependencies": { "@bazel/rollup": "=3.2.2", "@bazel/typescript": "=3.2.2", + "@bazel/worker": "=3.2.2", "@pyoner/svelte-types": "^3.4.4-2", "@rollup/plugin-commonjs": "^15.1.0", "@rollup/plugin-node-resolve": "^9.0.0", diff --git a/ts/svelte/BUILD.bazel b/ts/svelte/BUILD.bazel index 730042df6..185663b94 100644 --- a/ts/svelte/BUILD.bazel +++ b/ts/svelte/BUILD.bazel @@ -1,24 +1,16 @@ load("@build_bazel_rules_nodejs//:index.bzl", "nodejs_binary") nodejs_binary( - name = "typescript", - data = [ - "@npm//typescript", - ], - entry_point = "@npm//:node_modules/typescript/lib/tsc.js", - visibility = ["//visibility:public"], -) - -nodejs_binary( - name = "svelte", + name = "svelte_bin", data = [ ":svelte.js", + "@npm//@bazel/worker", "@npm//sass", "@npm//svelte", "@npm//svelte-preprocess", "@npm//svelte2tsx", + "@npm//typescript", ], entry_point = ":svelte.js", - templated_args = ["--bazel_patch_module_resolver"], visibility = ["//visibility:public"], ) diff --git a/ts/svelte/svelte.bzl b/ts/svelte/svelte.bzl index 581305359..071871f36 100644 --- a/ts/svelte/svelte.bzl +++ b/ts/svelte/svelte.bzl @@ -1,65 +1,38 @@ load("@npm//svelte-check:index.bzl", _svelte_check = "svelte_check_test") - -"Implementation of the svelte rule" - -load("@build_bazel_rules_nodejs//:providers.bzl", "declaration_info") - -SvelteFilesInfo = provider("transitive_sources") - -def get_transitive_srcs(srcs, deps): - return depset( - srcs, - transitive = [dep[SvelteFilesInfo].transitive_sources for dep in deps], - ) +load("@build_bazel_rules_nodejs//:providers.bzl", "declaration_info", "run_node") def _svelte(ctx): - base = ctx.attr.name + ".svelte" - temp = ctx.actions.declare_directory(base + ".temp") - temptsx = temp.path + "/" + base + ".tsx" - ctx.actions.run_shell( - mnemonic = "Svelte", - command = """\ -{svelte} {input} {output_js} {temp} && \ -{tsc} {tsc_args} {temptsx} {shims} && \ -mv {temp}/{base}.d.ts {output_def} && \ -rm {temptsx}""".format( - svelte = ctx.executable._svelte.path, - input = ctx.file.entry_point.path, - output_js = ctx.outputs.build.path, - tsc = ctx.executable._typescript.path, - output_def = ctx.outputs.buildDef.path, - temp = temp.path, - temptsx = temptsx, - base = base, - tsc_args = "--jsx preserve --emitDeclarationOnly --declaration --skipLibCheck", - shims = " ".join([f.path for f in ctx.files._shims]), - ), - outputs = [ctx.outputs.build, ctx.outputs.buildDef, temp], + args = ctx.actions.args() + args.use_param_file("@%s", use_always = True) + args.set_param_file_format("multiline") + + temp_ts_path = ctx.actions.declare_file(ctx.attr.name + ".svelte.ts") + + args.add(ctx.file.entry_point.path) + args.add(ctx.outputs.mjs.path) + args.add(ctx.outputs.dts.path) + args.add(temp_ts_path) + args.add_all(ctx.files._shims) + + ctx.actions.run( + execution_requirements = {"supports-workers": "1"}, + executable = ctx.executable._svelte_bin, + outputs = [ctx.outputs.mjs, ctx.outputs.dts, temp_ts_path], inputs = [ctx.file.entry_point] + ctx.files._shims, - tools = [ctx.executable._svelte, ctx.executable._typescript], + mnemonic = "Svelte", + arguments = [args], ) - trans_srcs = get_transitive_srcs(ctx.files.srcs + [ctx.outputs.build, ctx.outputs.buildDef], ctx.attr.deps) - return [ - declaration_info(depset([ctx.outputs.buildDef]), deps = [ctx.attr._shims]), - SvelteFilesInfo(transitive_sources = trans_srcs), - DefaultInfo(files = trans_srcs), + declaration_info(depset([ctx.outputs.dts]), deps = [ctx.attr._shims]), ] svelte = rule( implementation = _svelte, attrs = { "entry_point": attr.label(allow_single_file = True), - "deps": attr.label_list(), - "srcs": attr.label_list(allow_files = True), - "_svelte": attr.label( - default = Label("//ts/svelte:svelte"), - executable = True, - cfg = "host", - ), - "_typescript": attr.label( - default = Label("//ts/svelte:typescript"), + "_svelte_bin": attr.label( + default = Label("//ts/svelte:svelte_bin"), executable = True, cfg = "host", ), @@ -69,8 +42,8 @@ svelte = rule( ), }, outputs = { - "build": "%{name}.svelte.mjs", - "buildDef": "%{name}.svelte.d.ts", + "mjs": "%{name}.svelte.mjs", + "dts": "%{name}.svelte.d.ts", }, ) diff --git a/ts/svelte/svelte.js b/ts/svelte/svelte.js index b4d9dbf02..7798b2a59 100644 --- a/ts/svelte/svelte.js +++ b/ts/svelte/svelte.js @@ -1,55 +1,120 @@ const fs = require("fs"); const process = require("process"); const path = require("path"); - -const input = process.argv[2]; -const outputJs = process.argv[3]; -const temp = process.argv[4]; - +const worker = require("@bazel/worker"); const svelte = require("svelte/compiler.js"); - -const source = fs.readFileSync(input, "utf8"); - -const preprocessOptions = require("svelte-preprocess")({}); -preprocessOptions.filename = input; - const svelte2tsx = require("svelte2tsx"); +const preprocess = require("svelte-preprocess"); +const ts = require("typescript"); -let tsoutput = svelte2tsx(source, { - filename: input, - strictMode: true, - isTsFile: true, -}); -let codeLines = tsoutput.code.split("\n"); -// replace the "///" with a line -// turning off checking, as we'll use svelte-check for that -codeLines[0] = "// @ts-nocheck"; -const outputBase = path.basename(outputJs.replace(".mjs", ".tsx")); -const outputTs = path.join(temp, outputBase); -fs.writeFileSync(outputTs, codeLines.join("\n")); +const tsOptions = { + jsx: "preserve", + declaration: true, + emitDeclarationOnly: true, + skipLibCheck: true, +}; +const tsHost = ts.createCompilerHost(tsOptions); + +function writeFile(file, data) { + return new Promise((resolve, reject) => { + fs.writeFile(file, data, (err) => { + if (err) { + reject(err); + return; + } + resolve(); + }); + }); +} + +function readFile(file) { + return new Promise((resolve, reject) => { + fs.readFile(file, "utf8", (err, data) => { + if (err) { + reject(err); + return; + } + resolve(data); + }); + }); +} + +function buildDeclarations(svelteTsFile, shims) { + const createdFiles = {}; //test2 + tsHost.writeFile = (fileName, contents) => (createdFiles[fileName] = contents); + const program = ts.createProgram([svelteTsFile, ...shims], tsOptions, tsHost); + program.emit(); + const dtsSource = createdFiles[svelteTsFile.replace(".ts", ".d.ts")]; + return dtsSource; +} + +async function writeDts(tsPath, dtsPath, shims) { + const dtsSource = buildDeclarations(tsPath, shims); + await writeFile(dtsPath, dtsSource); +} + +async function writeTs(svelteSource, sveltePath, tsPath) { + let tsSource = svelte2tsx(svelteSource, { + filename: sveltePath, + strictMode: true, + isTsFile: true, + }); + let codeLines = tsSource.code.split("\n"); + // replace the "///" with a line + // turning off checking, as we'll use svelte-check for that + codeLines[0] = "// @ts-nocheck"; + await writeFile(tsPath, codeLines.join("\n")); +} + +//test +async function writeJs(source, inputFilename, outputPath) { + const preprocessOptions = preprocess({}); + preprocessOptions.filename = inputFilename; -svelte.preprocess(source, preprocessOptions).then( - (processed) => { - let result; try { - result = svelte.compile(processed.toString(), { - format: "esm", - generate: "dom", - filename: outputJs, - }); + const processed = await svelte.preprocess(source, preprocessOptions); + const result = svelte.compile(processed.toString(), { + format: "esm", + generate: "dom", + filename: outputPath, + }); + // warnings are an error + if (result.warnings.length > 0) { + console.log(`warnings during compile: ${result.warnings}`); + return; + } + const outputSource = result.js.code; + await writeFile(outputPath, outputSource); } catch (err) { - console.log(`compile failed: ${err}`); - return; - } - if (result.warnings.length > 0) { - console.log(`warnings during compile: ${result.warnings}`); - return; + console.log(`compile failed: ${err}`); + return; } +} - let code = result.js.code; - fs.writeFileSync(outputJs, code); - }, - (error) => { - console.log(`preprocess failed: ${error}`); - } -); +async function compileSvelte(args) { + const [sveltePath, mjsPath, dtsPath, tempTsPath, ...shims] = args; + const svelteSource = await readFile(sveltePath); + + await writeTs(svelteSource, sveltePath, tempTsPath); + await writeDts(tempTsPath, dtsPath, shims); + await writeJs(svelteSource, sveltePath, mjsPath); + + return true; +} + +function main(args) { + if (worker.runAsWorker(process.argv)) { + worker.log("Svelte running as a Bazel worker"); + worker.runWorkerLoop(compileSvelte); + } else { + const paramFile = process.argv[2].replace(/^@/, ""); + const commandLineArgs = fs.readFileSync(paramFile, "utf-8").trim().split("\n"); + console.log("Svelte running as a standalone process"); + compileSvelte(commandLineArgs); + } +} + +if (require.main === module) { + main(process.argv.slice(2)); + process.exitCode = 0; +} diff --git a/ts/yarn.lock b/ts/yarn.lock index cc0e1b5ea..9c3a2df64 100644 --- a/ts/yarn.lock +++ b/ts/yarn.lock @@ -43,6 +43,13 @@ source-map-support "0.5.9" tsutils "2.27.2" +"@bazel/worker@=3.2.2": + version "3.2.2" + resolved "https://registry.yarnpkg.com/@bazel/worker/-/worker-3.2.2.tgz#16bb809aaa59bf077836da5860bfbc478c66b835" + integrity sha512-fFEdePL4r2LWOzaIVOHOP89HUHZ/cEVZCb1d7SW55uaNEIo0l9iNOx+5thpAn0Woga51LBsMADuYNPk/hmSeIg== + dependencies: + protobufjs "6.8.8" + "@fluent/bundle@^0.15.1": version "0.15.1" resolved "https://registry.yarnpkg.com/@fluent/bundle/-/bundle-0.15.1.tgz#95d3b9f836ac138b6ee8480ef8d0547dd59195b1"