bundle: record and read deployment state via DMS

acceptance/bin/print_requests.py

@@ -90,6 +90,15 @@
 ADD_PREFIX = "/"
 NEGATE_PREFIX = "^/"
 
+# What --nostamp deletes. A job create sends the deployment block at the top level, an
+# update nests it under new_settings.
+STAMP_FIELDS = [
+    "deployment.deployment_id",
+    "deployment.version_id",
+    "new_settings.deployment.deployment_id",
+    "new_settings.deployment.version_id",
+]
+
 
 def read_json_many(s):
     result = []
@@ -178,6 +187,17 @@ def filter_requests(requests, path_filters, include_get, should_sort, unique=Fal
     return filtered_requests
 
 
+def del_path(body, field):
+    """Delete field from body. A dotted field descends into nested objects, e.g.
+    deployment.version_id removes only that key from the deployment block."""
+    *parents, leaf = field.split(".")
+    for name in parents:
+        body = body.get(name)
+        if not isinstance(body, dict):
+            return
+    body.pop(leaf, None)
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("path_filters", nargs="*", help="Path substring filters")
@@ -210,10 +230,19 @@ def main():
         "--del-body, which edits the parsed JSON body, this drops a field of the request "
         "record itself, e.g. raw_body for a binary upload payload.",
     )
+    parser.add_argument(
+        "--nostamp",
+        action="store_true",
+        help="Drop the deployment stamp (deployment_id, version_id) from job and pipeline "
+        "bodies, so a test asserts the same requests whether or not deployment history "
+        "recording is on. Shorthand for the --del-body fields it implies.",
+    )
     parser.add_argument("--fname", default="out.requests.txt")
     args = parser.parse_args()
 
     del_body_fields = [field for group in args.del_body for field in group.split(",")]
+    if args.nostamp:
+        del_body_fields += STAMP_FIELDS
     del_fields = [field for group in args.del_field for field in group.split(",")]
 
     test_tmp_dir = os.environ.get("TEST_TMP_DIR")
@@ -238,7 +267,7 @@ def main():
         body = req.get("body")
         if isinstance(body, dict):
             for field in del_body_fields:
-                body.pop(field, None)
+                del_path(body, field)
         for field in del_fields:
             req.pop(field, None)
     if args.verbose:

acceptance/bundle/ai_runtime_task/local_code_source/output.txt

@@ -21,7 +21,7 @@ src/train.py
 
 === both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec
 
->>> print_requests.py --sort --del-field raw_body //.air_snapshots/ //jobs/create
+>>> print_requests.py --nostamp --sort --del-field raw_body //.air_snapshots/ //jobs/create
 {
   "method": "POST",
   "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",
@@ -114,7 +114,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> print_requests.py --sort --del-field raw_body //.air_snapshots/
+>>> print_requests.py --nostamp --sort --del-field raw_body //.air_snapshots/
 {
   "method": "POST",
   "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/ai-runtime-test/default/files/.air_snapshots/[SNAPSHOT].tar.gz",

acceptance/bundle/ai_runtime_task/local_code_source/script

@@ -13,16 +13,16 @@ trace list_code_snapshot.py
 title "both code_source_paths point into the bundle .air_snapshots, command_paths rewritten, deps on environments spec\n"
 # --del-field raw_body drops the binary tarball upload payload (kept readable). Filters
 # use a leading // so Git Bash on Windows does not path-convert them. --keep is not
-# passed, so print_requests.py consumes out.requests.txt.
-trace print_requests.py --sort --del-field raw_body '//.air_snapshots/' '//jobs/create'
+# passed, so print_requests.py --nostamp consumes out.requests.txt.
+trace print_requests.py --nostamp --sort --del-field raw_body '//.air_snapshots/' '//jobs/create'
 
 title "re-planning unchanged code is a no-op (no changes)\n"
 trace $CLI bundle plan
 
 title "editing a file changes the snapshot hash (content-addressed name changes)\n"
 update_file.py src/train.py 'print("training")' 'print("training v2")'
 trace $CLI bundle deploy
-trace print_requests.py --sort --del-field raw_body '//.air_snapshots/'
+trace print_requests.py --nostamp --sort --del-field raw_body '//.air_snapshots/'
 
 title "destroy removes the deployed bundle (including the synced snapshots)\n"
 trace $CLI bundle destroy --auto-approve

acceptance/bundle/artifacts/whl_dynamic/script

@@ -4,9 +4,9 @@ cp -r $TESTDIR/../whl_explicit/my_test_code .
 mkdir prebuilt
 cp -r $TESTDIR/../whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl prebuilt
 
-trace $CLI bundle validate -o json | jq .artifacts
+trace $CLI bundle validate -o json | nostamp | jq .artifacts
 
-$CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json
+$CLI bundle plan -o json | nostamp > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json
 trace $CLI bundle deploy
 
 title "There are 2 original wheels and 2 patched ones"
@@ -25,7 +25,7 @@ rm out.requests.txt
 
 title "Updating the local wheel and deploying again\n"
 touch my_test_code/src/new_module.py
-$CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json
+$CLI bundle plan -o json | nostamp > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json
 trace $CLI bundle deploy
 
 title "Verify contents, it should now have new_module.py"

acceptance/bundle/artifacts/whl_implicit_custom_path/script

@@ -3,7 +3,7 @@ trace $CLI bundle deploy
 trace find.py --expect 1 whl
 
 title "Expecting 1 wheel in libraries section in /jobs/create"
-trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt
+trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp
 
 title "Expecting 1 wheel to be uploaded"
 trace jq .path < out.requests.txt | grep import | grep whl | sort

acceptance/bundle/artifacts/whl_via_environment_key/script

@@ -3,7 +3,7 @@ trace $CLI bundle deploy
 trace find.py --expect 1 whl
 
 title "Expecting 1 wheel in environments section in /jobs/create"
-trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt
+trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body' out.requests.txt | nostamp
 
 title "Expecting 1 wheel to be uploaded"
 trace jq .path < out.requests.txt | grep import | grep whl | sort

acceptance/bundle/bundle_tag/id/output.txt

@@ -39,7 +39,7 @@ Deploying resources...
 Updating deployment state...
 Deployment complete!
 
->>> print_requests.py //jobs
+>>> print_requests.py --nostamp //jobs
 {
   "method": "POST",
   "path": "/api/2.2/jobs/create",
@@ -79,7 +79,7 @@ All files and directories at the following location will be deleted: /Workspace/
 Deleting files...
 Destroy complete!
 
->>> print_requests.py //jobs
+>>> print_requests.py --nostamp //jobs
 {
   "method": "POST",
   "path": "/api/2.2/jobs/delete",

acceptance/bundle/bundle_tag/id/script

@@ -1,8 +1,8 @@
 trace $CLI bundle validate
-trace $CLI bundle validate -o json | jq .resources
+trace $CLI bundle validate -o json | nostamp | jq .resources
 trace $CLI bundle plan
 trace $CLI bundle deploy
-trace print_requests.py //jobs
+trace print_requests.py --nostamp //jobs
 trace $CLI bundle summary
 trace $CLI bundle destroy --auto-approve
-trace print_requests.py //jobs
+trace print_requests.py --nostamp //jobs