Config remote sync command

acceptance/bin/edit_resource.py

@@ -32,6 +32,14 @@ def set(self, job_id, value):
         return run([CLI, "jobs", "reset", job_id, "--json", json.dumps(payload)])
 
 
+class pipelines:
+    def get(self, pipeline_id):
+        return run_json([CLI, "pipelines", "get", pipeline_id])["spec"]
+
+    def set(self, pipeline_id, value):
+        return run([CLI, "pipelines", "update", pipeline_id, "--json", json.dumps(value)])
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("type")

acceptance/bundle/config-remote-sync/formatting_preserved/databricks.yml

@@ -0,0 +1,25 @@
+# Top-level comment about the bundle
+bundle:
+  name: test-bundle
+
+# Resources section with extra spacing
+resources:
+  jobs:
+    my_job:
+      # Comment about max concurrent runs
+      max_concurrent_runs: 1
+
+      # Task configuration
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 1 # inline comment about workers
+
+      # Tags for categorization
+      tags:
+        env: dev # environment tag
+        team: data-eng

acceptance/bundle/config-remote-sync/formatting_preserved/output.txt

@@ -0,0 +1,35 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Modify max_concurrent_runs from 1 to 5
+=== Detect and save changesDetected changes in 1 resource(s):
+
+Resource: resources.jobs.my_job
+  max_concurrent_runs: update
+
+
+=== Updated configuration
+# Top-level comment about the bundle
+bundle:
+  name: test-bundle
+# Resources section with extra spacing
+resources:
+  jobs:
+    my_job:
+      # Comment about max concurrent runs
+      max_concurrent_runs: 5
+      # Task configuration
+      tasks:
+        - task_key: main
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/notebook
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: [NODE_TYPE_ID]
+            num_workers: 1 # inline comment about workers
+      # Tags for categorization
+      tags:
+        env: dev # environment tag
+        team: data-eng

acceptance/bundle/config-remote-sync/formatting_preserved/script

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+touch dummy.whl
+$CLI bundle deploy
+job_id="$(read_id.py my_job)"
+
+title "Modify max_concurrent_runs from 1 to 5"
+edit_resource.py jobs $job_id <<EOF
+r["max_concurrent_runs"] = 5
+EOF
+
+title "Detect and save changes"
+$CLI bundle config-remote-sync --save
+
+title "Updated configuration"
+echo
+cat databricks.yml

acceptance/bundle/config-remote-sync/formatting_preserved/test.toml

@@ -0,0 +1,8 @@
+RecordRequests = false
+Ignore = [".databricks", "dummy.whl"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/config-remote-sync/job_email_notifications/databricks.yml

@@ -0,0 +1,9 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      email_notifications:
+        on_success:
+          - success@example.com

acceptance/bundle/config-remote-sync/job_email_notifications/output.txt

@@ -0,0 +1,25 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Add on_failure notifications
+=== Detect and save changesDetected changes in 1 resource(s):
+
+Resource: resources.jobs.my_job
+  email_notifications.no_alert_for_skipped_runs: skip
+  email_notifications.on_failure: skip
+
+
+=== Updated configuration
+bundle:
+  name: test-bundle
+resources:
+  jobs:
+    my_job:
+      email_notifications:
+        on_success:
+          - success@example.com
+        no_alert_for_skipped_runs: true
+        on_failure:
+          - failure@example.com

acceptance/bundle/config-remote-sync/job_email_notifications/script

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+touch dummy.whl
+$CLI bundle deploy
+job_id="$(read_id.py my_job)"
+
+
+title "Add on_failure notifications"
+edit_resource.py jobs $job_id <<EOF
+r["email_notifications"]["on_failure"] = ["failure@example.com"]
+r["email_notifications"]["no_alert_for_skipped_runs"] = True
+EOF
+
+title "Detect and save changes"
+$CLI bundle config-remote-sync --save
+
+title "Updated configuration"
+echo
+cat databricks.yml

acceptance/bundle/config-remote-sync/job_email_notifications/test.toml

@@ -0,0 +1,8 @@
+RecordRequests = false
+Ignore = [".databricks", "dummy.whl"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/config-remote-sync/job_environments/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "3"

acceptance/bundle/config-remote-sync/job_environments/output.txt

@@ -0,0 +1,22 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Modify job environment version from 3 to 4
+=== Detect and save changesDetected changes in 1 resource(s):
+
+Resource: resources.jobs.my_job
+  environments[0].spec.environment_version: update
+
+
+=== Updated configuration
+bundle:
+  name: test-bundle
+resources:
+  jobs:
+    my_job:
+      environments:
+        - environment_key: default
+          spec:
+            environment_version: "4"

acceptance/bundle/config-remote-sync/job_environments/script

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+touch dummy.whl
+$CLI bundle deploy
+job_id="$(read_id.py my_job)"
+
+title "Modify job environment version from 3 to 4"
+edit_resource.py jobs $job_id <<EOF
+r["environments"][0]["spec"]["environment_version"] = "4"
+EOF
+
+title "Detect and save changes"
+$CLI bundle config-remote-sync --save
+
+title "Updated configuration"
+echo
+cat databricks.yml

acceptance/bundle/config-remote-sync/job_environments/test.toml

@@ -0,0 +1,8 @@
+RecordRequests = false
+Ignore = [".databricks", "dummy.whl"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/config-remote-sync/job_multiple_tasks/databricks.yml

@@ -0,0 +1,32 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      tasks:
+        - task_key: setup
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/setup
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 1
+        - task_key: process
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/process
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 2
+          depends_on:
+            - task_key: setup
+        - task_key: cleanup
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/cleanup
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: i3.xlarge
+            num_workers: 1
+          depends_on:
+            - task_key: process

acceptance/bundle/config-remote-sync/job_multiple_tasks/output.txt

@@ -0,0 +1,46 @@
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Modify only 'process' task num_workers and add timeout
+=== Detect and save changesDetected changes in 1 resource(s):
+
+Resource: resources.jobs.my_job
+  tasks[task_key='process'].new_cluster.num_workers: update
+  tasks[task_key='process'].timeout_seconds: skip
+
+
+=== Updated configuration
+bundle:
+  name: test-bundle
+resources:
+  jobs:
+    my_job:
+      tasks:
+        - task_key: setup
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/setup
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: [NODE_TYPE_ID]
+            num_workers: 1
+        - task_key: process
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/process
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: [NODE_TYPE_ID]
+            num_workers: 5
+          depends_on:
+            - task_key: setup
+          timeout_seconds: 3600
+        - task_key: cleanup
+          notebook_task:
+            notebook_path: /Users/{{workspace_user_name}}/cleanup
+          new_cluster:
+            spark_version: 13.3.x-scala2.12
+            node_type_id: [NODE_TYPE_ID]
+            num_workers: 1
+          depends_on:
+            - task_key: process

acceptance/bundle/config-remote-sync/job_multiple_tasks/script

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+touch dummy.whl
+$CLI bundle deploy
+job_id="$(read_id.py my_job)"
+
+
+title "Modify only 'process' task num_workers and add timeout"
+edit_resource.py jobs $job_id <<EOF
+for task in r["tasks"]:
+    if task["task_key"] == "process":
+        task["new_cluster"]["num_workers"] = 5
+        task["timeout_seconds"] = 3600
+EOF
+
+title "Detect and save changes"
+$CLI bundle config-remote-sync --save
+
+title "Updated configuration"
+echo
+cat databricks.yml

acceptance/bundle/config-remote-sync/job_multiple_tasks/test.toml

@@ -0,0 +1,8 @@
+RecordRequests = false
+Ignore = [".databricks", "dummy.whl"]
+
+[Env]
+DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC = "true"
+
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/config-remote-sync/job_parameters/databricks.yml

@@ -0,0 +1,11 @@
+bundle:
+  name: test-bundle
+
+resources:
+  jobs:
+    my_job:
+      parameters:
+        - name: catalog
+          default: main
+        - name: env
+          default: dev