Add usage output, other clean up

bench.js

@@ -53,19 +53,6 @@ function BenchContext(app, config) {
   }
 }
 
-//::node::import::native::sr25519::transfer_keep_alive::paritydb::small
-
-// const cargoRun = "cargo run --features=runtime-benchmarks --bin moonbeam -- ";
-const cargoRun = "cargo run ";
-
-var BenchConfigs = {
-  ed25519: {
-    title: "Import Benchmark (random transfers, ed25519 signed)",
-    benchCommand:
-      cargoRun + "benchmark --chain dev --execution=native --pallet \"*\" --extrinsic \"*\" --steps 32 --repeat 8 --json --record-proof"
-  },
-}
-
 const prepareBranch = async function (
   { contributor, owner, repo, bbRepo, bbRepoOwner, bbBranch, branch, baseBranch, getPushDomain, getBBPushDomain, },
   { benchContext },
@@ -144,49 +131,6 @@ const prepareBranch = async function (
     return errorResult(`Failed to "git checkout -b" our secondary branch`);
 }
 
-function benchBranch(app, config) {
-  app.log("Waiting our turn to run benchBranch...")
-
-  return mutex.runExclusive(async function () {
-    try {
-      if (config.repo != "moonbeam") {
-        return errorResult("Node benchmarks only available on Moonbeam.")
-      }
-
-      console.log(`config id: ${config.id}`);
-
-      var id = config.id
-      var benchConfig = BenchConfigs[id]
-      if (!benchConfig) {
-        return errorResult(`Bench configuration for "${id}" was not found`)
-      }
-
-      console.log(`bench command: ${benchConfig.benchCommand}`);
-
-      const collector = new libCollector.Collector()
-      var benchContext = new BenchContext(app, config)
-      var { title, benchCommand } = benchConfig
-      app.log(`Started benchmark "${title}."`)
-
-      var error = await prepareBranch(config, { benchContext })
-      if (error) return error
-
-      var { stderr, error, stdout } = benchContext.runTask(
-        benchCommand,
-        `Benching branch ${config.branch}...`,
-      )
-      if (error) return errorResult(stderr)
-
-      await collector.CollectBranchCustomRunner(stdout)
-      let output = await collector.Report()
-
-      return { title, output, extraInfo: "", benchCommand }
-    } catch (error) {
-      return errorResult("Caught exception in benchBranch", error)
-    }
-  })
-}
-
 var MoonbeamRuntimeBenchmarkConfigs = {
   pallet: {
     title: "Runtime Pallet",
@@ -213,7 +157,7 @@ var MoonbeamRuntimeBenchmarkConfigs = {
   },
 }
 
-function checkRuntimeBenchmarkCommand(command) {
+function checkPalletBenchmarkCommand(command) {
   let required = [
     "benchmark",
     "--pallet",
@@ -286,8 +230,8 @@ function matchMoonbeamPallet(palletIsh) {
   throw new Error(`Pallet argument not recognized: ${palletIsh}`);
 }
 
-function benchmarkRuntime(app, config, octokit) {
-  app.log("Waiting our turn to run benchmarkRuntime...")
+function benchmarkPallet(app, config, octokit) {
+  app.log("Waiting our turn to run benchmarkPallet...")
 
   return mutex.runExclusive(async function () {
     try {
@@ -303,7 +247,7 @@ function benchmarkRuntime(app, config, octokit) {
       // XXX: testing
       const repo = config.repo.startsWith("moonbeam") ? "moonbeam" : config.repo;
 
-      if (repo == "moonbeam" && config.id == "runtime") {
+      if (repo == "moonbeam" && config.id == "pallet") {
         benchConfig = MoonbeamRuntimeBenchmarkConfigs[command]
       } else {
         return errorResult(
@@ -345,7 +289,7 @@ function benchmarkRuntime(app, config, octokit) {
         benchCommand = benchCommand.replace("{pallet_folder}", palletInfo.dir)
       }
 
-      let missing = checkRuntimeBenchmarkCommand(benchCommand)
+      let missing = checkPalletBenchmarkCommand(benchCommand)
       if (missing.length > 0) {
         return errorResult(`Missing required flags: ${missing.toString()}`)
       }

index.js

@@ -142,6 +142,16 @@ module.exports = (app) => {
       const branch = pr.data.head.ref
       app.log.debug(`branch: ${branch}`)
 
+      // if 'help' was requested, make comment and end here
+      if (action == "help" || action == "--help") {
+        await context.octokit.issues.createComment(
+          context.issue({
+            body: usage(),
+          }),
+        );
+        return;
+      }
+
       var { stdout: toolchain, code: toolchainError } = shell.exec(
         "rustup show active-toolchain --verbose",
         { silent: false },
@@ -161,7 +171,7 @@ module.exports = (app) => {
       // generate a unique branch for our PR
       const bbBranch = `${branch}-benchbot-job-${new Date().getTime()}`;
 
-      const initialInfo = `Starting benchmark for branch: ${branch} (vs ${baseBranch})\nPR branch will be ${bbBranch}\n\nToolchain: \n${toolchain}\n\n Comment will be updated.`
+      const initialInfo = `Starting benchmark for branch: ${branch} (vs ${baseBranch})\nPR branch will be ${bbBranch}\n\nToolchain: \n${toolchain}\n\n This comment will be updated.`
       let comment_id = undefined
 
       app.log(initialInfo)
@@ -187,21 +197,24 @@ module.exports = (app) => {
       }
 
       // kick off the build/run process...
+      // TODO: match usage
       let report
-      if (action == "runtime" || action == "xcm") {
-        report = await benchmarkRuntime(app, config, context.octokit)
+      if (action == "pallet") {
+        report = await benchmarkPallet(app, config, context.octokit)
       } else if (action == "rustup") {
         report = await benchRustup(app, config)
       } else {
-        report = {
-          isError: true,
-          message: "Unsupported action",
-          error: `unsupported action: ${action}`,
-        };
+        await context.octokit.issues.updateComment({
+          owner,
+          repo,
+          comment_id,
+          body: `**Unsupported action: ${action}**\n\n${usage()}`,
+        });
+        return;
       }
       if (process.env.DEBUG) {
         console.log(report)
-        return
+        // return
       }
 
       if (report.isError) {
@@ -214,14 +227,12 @@ module.exports = (app) => {
         const output = `${report.message}${report.error ? `: ${report.error.toString()}` : ""
           }`
 
-        /*
         await context.octokit.issues.updateComment({
           owner,
           repo,
           comment_id,
           body: `Error running benchmark: **${branch}**\n\n<details><summary>stdout</summary>${output}</details>`,
         })
-        */
 
         return
       }
@@ -247,8 +258,13 @@ Toolchain: ${toolchain}
 
       const padding = 16
       const formattingLength =
-        bodyPrefix.length + bodySuffix.length + extraInfo.length + padding
-      const length = formattingLength + output.length
+        padding
+        + bodyPrefix ? bodyPrefix.length : 0
+        + bodySuffix ? bodySuffix.length : 0
+        + extraInfo ? extraInfo.length : 0;
+      const length =
+        formattingLength
+        + output ? output.length : 0;
       if (length >= githubCommentLimitLength) {
         output = `${output.slice(
           0,
@@ -289,3 +305,22 @@ ${extraInfo}
     }
   })
 }
+
+function usage() {
+  return `\`\`\`
+Benchbot usage:
+
+/bench --help
+        print this help output
+/bench <SUBCOMMAND>
+        run the given subcommand
+
+SUBCOMMAND:
+    pallet <pallet>
+            benchmark a specific pallet and update its weights.rs file
+    runtime
+            benchmark an entire runtime and update all pallet weights.rs files
+            (currently only supports moonbase runtime)
+\`\`\``;
+}
+