feat(builtin): accept any stamp vars in pkg_npm

NOTE: instead of replacing the replace_with_version with 0.0.0 in unstamped builds,
we now omit the replacement.

See #1694

Author: Alex Eagle

docs/stamping.md

@@ -7,38 +7,61 @@ toc: true
 # Stamping
 
 Bazel is generally only a build tool, and is unaware of your version control system.
-However, when publishing releases, you typically want to embed version information in the resulting distribution.
-Bazel supports this natively, using the following approach:
+However, when publishing releases, you may want to embed version information in the resulting distribution.
+Bazel supports this with the concept of a "Workspace status" which is evaluated before each build.
+See [the Bazel workspace status docs](https://docs.bazel.build/versions/master/user-manual.html#workspace_status)
 
 To stamp a build, you must pass the `--stamp` argument to Bazel.
 
-> Previous releases of rules_nodejs stamped builds always.
-> However this caused stamp-aware actions to never be remotely cached, since the volatile
-> status file is passed as an input and its checksum always changes.
+Stamping is typically performed on a later action in the graph, like on a packaging rule (`pkg_*`). This means that
+a changed status variable only causes re-packaging, not re-compilation and thus does not cause cascading re-builds.
 
-Also pass the `workspace_status_command` argument to `bazel build`.
-We prefer to do these with an entry in `.bazelrc`:
+Bazel provides a couple of statuses by default, such as `BUILD_EMBED_LABEL` which is the value of the `--embed_label`
+argument, as well as `BUILD_HOST` and `BUILD_USER`. You can supply more with the workspace status script, see below.
 
-```sh
-# This tells Bazel how to interact with the version control system
-# Enable this with --config=release
-build:release --stamp --workspace_status_command=./tools/bazel_stamp_vars.sh
+Some rules accept an attribute that uses the status variables.
+For example, in a `pkg_npm` you can use the `substitutions` attribute like:
+
+```python
+pkg_npm(
+    name = "npm_package",
+    substitutions = {"0.0.0-PLACEHOLDER": "{STABLE_GIT_COMMIT}"},
+)
 ```
 
-Then create `tools/bazel_stamp_vars.sh`.
+In a `--stamp` build, this will replace the string "0.0.0-PLACEHOLDER" in any file included in the package with the current value of the `STABLE_GIT_COMMIT` variable.
+However without stamping the placeholder will be left as-is.
 
-This is a script that prints variable/value pairs.
-Make sure you set the executable bit, eg. `chmod 755 tools/bazel_stamp_vars.sh`.
-For example, we could run `git describe` to get the current tag:
+## Stamping with a Workspace status script
+
+To define additional statuses, pass the `--workspace_status_command` argument to `bazel`.
+The value of this flag is a path to a script that prints space-separated key/value pairs, one per line, such as
 
 ```bash
 #!/usr/bin/env bash
-echo BUILD_SCM_VERSION $(git describe --abbrev=7 --tags HEAD)
+echo STABLE_GIT_COMMIT $(git rev-parse HEAD)
+```
+> For a more full-featured script, take a look at the [bazel_stamp_vars in Angular]
+
+Make sure you set the executable bit, eg. `chmod 755 tools/bazel_stamp_vars.sh`.
+
+> **NOTE** keys start start with `STABLE_` will cause a re-build when they change.
+> Other keys will NOT cause a re-build, so stale values can appear in your app.
+> Non-stable (volatile) keys should typically be things like timestamps that always vary between builds.
+
+You might like to encode your setup using an entry in `.bazelrc` such as:
+
+```sh
+# This tells Bazel how to interact with the version control system
+# Enable this with --config=release
+build:release --stamp --workspace_status_command=./tools/bazel_stamp_vars.sh
 ```
 
-For a more full-featured script, take a look at the [bazel_stamp_vars in Angular]
+## Release script
 
-Finally, we recommend a release script around Bazel. We typically have more than one npm package published from one Bazel workspace, so we do a `bazel query` to find them, and publish in a loop. Here is a template to get you started:
+If you publish more than one package from your workspace, you might want a release script around Bazel.
+A nice pattern is to do a `bazel query` to find publishable targets, build them in parallel, then publish in a loop.
+Here is a template to get you started:
 
 ```sh
 #!/usr/bin/env bash
@@ -63,8 +86,6 @@ for pkg in $PKG_NPM_LABELS ; do
 done
 ```
 
-> WARNING: Bazel can't track changes to git tags. That means it won't rebuild a target if only the result of the workspace_status_command has changed. So changes to the version information may not be reflected if you re-build the package or bundle, and nothing in the package or bundle has changed.
-
 See https://www.kchodorow.com/blog/2017/03/27/stamping-your-builds/ for more background.
 
 [bazel_stamp_vars in Angular]: https://github.com/angular/angular/blob/master/tools/bazel_stamp_vars.sh

internal/npm_version_check.js

@@ -2,7 +2,7 @@
 
 // Fetch the version of this package from its package.json
 const pkg = require('./package.json');
-const pkgVersion = pkg.version || '0.0.0';
+const pkgVersion = pkg.version ? pkg.version.split('-')[0] : '0.0.0';
 
 // BUILD_BAZEL_RULES_NODEJS_VERSION is only set when within the bazel context
 const rulesVersion = process.env['BUILD_BAZEL_RULES_NODEJS_VERSION'] || '0.0.0';
@@ -18,4 +18,4 @@ if (rulesVersion !== '0.0.0' && pkgVersion !== '0.0.0' &&
     ${pkg.name} - ${pkgVersion}  
     @build_bazel_rules_nodejs - ${rulesVersion}
   See https://github.com/bazelbuild/rules_nodejs/wiki/Avoiding-version-skew`);
-}
+}

internal/pkg_npm/packager.js

@@ -60,11 +60,29 @@ function unquoteArgs(s) {
   return s.replace(/^'(.*)'$/, '$1');
 }
 
+/**
+ * The status files are expected to look like
+ * BUILD_SCM_HASH 83c699db39cfd74526cdf9bebb75aa6f122908bb
+ * BUILD_SCM_LOCAL_CHANGES true
+ * STABLE_BUILD_SCM_VERSION 6.0.0-beta.6+12.sha-83c699d.with-local-changes
+ * BUILD_TIMESTAMP 1520021990506
+ *
+ * @param {string} p the path to the status file
+ * @returns a two-dimensional array of key/value pairs
+ */
+function parseStatusFile(p) {
+  if (!p) return [];
+  return fs.readFileSync(p, {encoding: 'utf-8'})
+      .split('\n')
+      .filter(t => !!t)
+      .map(t => t.split(' '));
+}
+
 function main(args) {
   args = fs.readFileSync(args[0], {encoding: 'utf-8'}).split('\n').map(unquoteArgs);
   const
       [outDir, baseDir, srcsArg, binDir, genDir, depsArg, packagesArg, substitutionsArg,
-       replaceWithVersion, stampFile, vendorExternalArg] = args;
+       volatileFile, infoFile, vendorExternalArg] = args;
 
   const substitutions = [
     // Strip content between BEGIN-INTERNAL / END-INTERNAL comments
@@ -74,27 +92,27 @@ function main(args) {
   for (const key of Object.keys(rawReplacements)) {
     substitutions.push([new RegExp(key, 'g'), rawReplacements[key]])
   }
-  // Replace version last so that earlier substitutions can add
-  // the version placeholder
-  if (replaceWithVersion) {
-    let version = '0.0.0';
-    if (stampFile) {
-      // The stamp file is expected to look like
-      // BUILD_SCM_HASH 83c699db39cfd74526cdf9bebb75aa6f122908bb
-      // BUILD_SCM_LOCAL_CHANGES true
-      // BUILD_SCM_VERSION 6.0.0-beta.6+12.sha-83c699d.with-local-changes
-      // BUILD_TIMESTAMP 1520021990506
-      //
-      // We want version to be the 6.0.0-beta... part
-      const versionTag = fs.readFileSync(stampFile, {encoding: 'utf-8'})
-                             .split('\n')
-                             .find(s => s.startsWith('BUILD_SCM_VERSION'));
-      // Don't assume BUILD_SCM_VERSION exists
-      if (versionTag) {
-        version = versionTag.split(' ')[1].replace(/^v/, '').trim();
+  // Replace statuses last so that earlier substitutions can add
+  // status-related placeholders
+  if (volatileFile || infoFile) {
+    const statusEntries = parseStatusFile(volatileFile)
+    statusEntries.push(...parseStatusFile(infoFile))
+    // Looks like {'BUILD_SCM_VERSION': 'v1.2.3'}
+    const statuses = new Map(statusEntries)
+    for (let idx = 0; idx < substitutions.length; idx++) {
+      const match = substitutions[idx][1].match(/\{(.*)\}/);
+      if (!match) continue;
+      const statusKey = match[1];
+      let statusValue = statuses.get(statusKey);
+      if (statusValue) {
+        // npm versions must be numeric, so if the VCS tag starts with leading 'v', strip it
+        // See https://github.com/bazelbuild/rules_nodejs/pull/1591
+        if (statusKey.endsWith('_VERSION')) {
+          statusValue = statusValue.replace(/^v/, '');
+        }
+        substitutions[idx][1] = statusValue;
       }
     }
-    substitutions.push([new RegExp(replaceWithVersion, 'g'), version]);
   }
 
   // src like baseDir/my/path is just copied to outDir/my/path

internal/pkg_npm/pkg_npm.bzl

@@ -91,16 +91,26 @@ PKG_NPM_ATTRS = {
         doc = """Optional package_name that this npm package may be imported as.""",
     ),
     "replace_with_version": attr.string(
-        doc = """If set this value is replaced with the version stamp data.
-        See the section on stamping in the README.""",
+        doc = """DEPRECATED: use substitutions instead.
+        
+        `replace_with_version = "my_version_placeholder"` is just syntax sugar for
+        `substitutions = {"my_version_placeholder": "{BUILD_SCM_VERSION}"}`.
+
+        Follow this deprecation at https://github.com/bazelbuild/rules_nodejs/issues/2158
+        """,
         default = "0.0.0-PLACEHOLDER",
     ),
     "srcs": attr.label_list(
         doc = """Files inside this directory which are simply copied into the package.""",
         allow_files = True,
     ),
     "substitutions": attr.string_dict(
-        doc = """Key-value pairs which are replaced in all the files while building the package.""",
+        doc = """Key-value pairs which are replaced in all the files while building the package.
+        
+        You can use values from the workspace status command using curly braces, for example
+        `{"0.0.0-PLACEHOLDER": "{STABLE_GIT_VERSION}"}`.
+        See the section on stamping in the README
+        """,
     ),
     "vendor_external": attr.string_list(
         doc = """External workspaces whose contents should be vendored into this workspace.
@@ -184,7 +194,15 @@ def create_package(ctx, deps_files, nested_packages):
     # current package unless explicitely specified.
     filtered_deps_sources = _filter_out_external_files(ctx, deps_files, package_path)
 
+    # Back-compat for the replace_with_version stamping
+    # see https://github.com/bazelbuild/rules_nodejs/issues/2158 for removal
+    substitutions = dict(**ctx.attr.substitutions)
+    if stamp and ctx.attr.replace_with_version:
+        substitutions[ctx.attr.replace_with_version] = "{BUILD_SCM_VERSION}"
+
     args = ctx.actions.args()
+    inputs = ctx.files.srcs + deps_files + nested_packages
+
     args.use_param_file("%s", use_always = True)
     args.add(package_dir.path)
     args.add(package_path)
@@ -193,19 +211,23 @@ def create_package(ctx, deps_files, nested_packages):
     args.add(ctx.genfiles_dir.path)
     args.add_joined(filtered_deps_sources, join_with = ",", omit_if_empty = False)
     args.add_joined([p.path for p in nested_packages], join_with = ",", omit_if_empty = False)
-    args.add(ctx.attr.substitutions)
-    args.add(ctx.attr.replace_with_version)
-    args.add(ctx.version_file.path if stamp else "")
-    args.add_joined(ctx.attr.vendor_external, join_with = ",", omit_if_empty = False)
+    args.add(substitutions)
 
-    inputs = ctx.files.srcs + deps_files + nested_packages
-
-    # The version_file is an undocumented attribute of the ctx that lets us read the volatile-status.txt file
-    # produced by the --workspace_status_command. That command will be executed whenever
-    # this action runs, so we get the latest version info on each execution.
-    # See https://github.com/bazelbuild/bazel/issues/1054
     if stamp:
+        # The version_file is an undocumented attribute of the ctx that lets us read the volatile-status.txt file
+        # produced by the --workspace_status_command.
+        # Similarly info_file reads the stable-status.txt file.
+        # That command will be executed whenever
+        # this action runs, so we get the latest version info on each execution.
+        # See https://github.com/bazelbuild/bazel/issues/1054
+        args.add(ctx.version_file.path)
         inputs.append(ctx.version_file)
+        args.add(ctx.info_file.path)
+        inputs.append(ctx.info_file)
+    else:
+        args.add_all(["", ""])
+
+    args.add_joined(ctx.attr.vendor_external, join_with = ",", omit_if_empty = False)
 
     ctx.actions.run(
         progress_message = "Assembling npm package %s" % package_dir.short_path,

packages/rollup/rollup_bundle.bzl

@@ -361,14 +361,19 @@ def _rollup_bundle(ctx):
         template = ctx.file.config_file,
         output = config,
         substitutions = {
+            "bazel_info_file": "\"%s\"" % ctx.info_file.path if stamp else "undefined",
+            # Back-compat: we used to replace a variable "bazel_stamp_file"
+            # Remove in 3.0: https://github.com/bazelbuild/rules_nodejs/issues/2158
             "bazel_stamp_file": "\"%s\"" % ctx.version_file.path if stamp else "undefined",
+            "bazel_version_file": "\"%s\"" % ctx.version_file.path if stamp else "undefined",
         },
     )
 
     args.add_all(["--config", config.path])
     inputs.append(config)
 
     if stamp:
+        inputs.append(ctx.info_file)
         inputs.append(ctx.version_file)
 
     # Prevent rollup's module resolver from hopping outside Bazel's sandbox

packages/rollup/test/integration/rollup.config.js

@@ -4,9 +4,9 @@ import nodeResolve from '@rollup/plugin-node-resolve';
 
 // Parse the stamp file produced by Bazel from the version control system
 let version = '<unknown>';
-if (bazel_stamp_file) {
+if (bazel_version_file) {
   const versionTag = require('fs')
-                         .readFileSync(bazel_stamp_file, {encoding: 'utf-8'})
+                         .readFileSync(bazel_version_file, {encoding: 'utf-8'})
                          .split('\n')
                          .find(s => s.startsWith('BUILD_SCM_VERSION'));
   // Don't assume BUILD_SCM_VERSION exists

packages/rollup/test/version_stamp/rollup.config.js

@@ -2,9 +2,9 @@ const DEBUG = process.env['COMPILATION_MODE'] === 'dbg';
 
 // Parse the stamp file produced by Bazel from the version control system
 let version = '<unknown>';
-if (bazel_stamp_file) {
+if (bazel_version_file) {
   const versionTag = require('fs')
-                         .readFileSync(bazel_stamp_file, {encoding: 'utf-8'})
+                         .readFileSync(bazel_version_file, {encoding: 'utf-8'})
                          .split('\n')
                          .find(s => s.startsWith('BUILD_SCM_VERSION'));
   // Don't assume BUILD_SCM_VERSION exists
@@ -24,4 +24,4 @@ const banner = `/**
 
 module.exports = {
   output: {banner},
-};
+};