Added support for UC catalogs (only in direct mode)

acceptance/bundle/refschema/out.fields.txt

@@ -208,6 +208,52 @@ resources.apps.*.permissions.permissions[*].group_name	string	ALL
 resources.apps.*.permissions.permissions[*].permission_level	iam.PermissionLevel	ALL
 resources.apps.*.permissions.permissions[*].service_principal_name	string	ALL
 resources.apps.*.permissions.permissions[*].user_name	string	ALL
+resources.catalogs.*.browse_only	bool	REMOTE
+resources.catalogs.*.catalog_type	catalog.CatalogType	REMOTE
+resources.catalogs.*.comment	string	ALL
+resources.catalogs.*.connection_name	string	ALL
+resources.catalogs.*.created_at	int64	REMOTE
+resources.catalogs.*.created_by	string	REMOTE
+resources.catalogs.*.effective_predictive_optimization_flag	*catalog.EffectivePredictiveOptimizationFlag	REMOTE
+resources.catalogs.*.effective_predictive_optimization_flag.inherited_from_name	string	REMOTE
+resources.catalogs.*.effective_predictive_optimization_flag.inherited_from_type	catalog.EffectivePredictiveOptimizationFlagInheritedFromType	REMOTE
+resources.catalogs.*.effective_predictive_optimization_flag.value	catalog.EnablePredictiveOptimization	REMOTE
+resources.catalogs.*.enable_predictive_optimization	catalog.EnablePredictiveOptimization	REMOTE
+resources.catalogs.*.full_name	string	REMOTE
+resources.catalogs.*.grants	[]resources.CatalogGrant	INPUT
+resources.catalogs.*.grants[*]	resources.CatalogGrant	INPUT
+resources.catalogs.*.grants[*].principal	string	INPUT
+resources.catalogs.*.grants[*].privileges	[]resources.CatalogGrantPrivilege	INPUT
+resources.catalogs.*.grants[*].privileges[*]	resources.CatalogGrantPrivilege	INPUT
+resources.catalogs.*.id	string	INPUT
+resources.catalogs.*.isolation_mode	catalog.CatalogIsolationMode	REMOTE
+resources.catalogs.*.lifecycle	resources.Lifecycle	INPUT
+resources.catalogs.*.lifecycle.prevent_destroy	bool	INPUT
+resources.catalogs.*.metastore_id	string	REMOTE
+resources.catalogs.*.modified_status	string	INPUT
+resources.catalogs.*.name	string	ALL
+resources.catalogs.*.options	map[string]string	ALL
+resources.catalogs.*.options.*	string	ALL
+resources.catalogs.*.owner	string	REMOTE
+resources.catalogs.*.properties	map[string]string	ALL
+resources.catalogs.*.properties.*	string	ALL
+resources.catalogs.*.provider_name	string	ALL
+resources.catalogs.*.provisioning_info	*catalog.ProvisioningInfo	REMOTE
+resources.catalogs.*.provisioning_info.state	catalog.ProvisioningInfoState	REMOTE
+resources.catalogs.*.securable_type	catalog.SecurableType	REMOTE
+resources.catalogs.*.share_name	string	ALL
+resources.catalogs.*.storage_location	string	REMOTE
+resources.catalogs.*.storage_root	string	ALL
+resources.catalogs.*.updated_at	int64	REMOTE
+resources.catalogs.*.updated_by	string	REMOTE
+resources.catalogs.*.url	string	INPUT
+resources.catalogs.*.grants.full_name	string	ALL
+resources.catalogs.*.grants.grants	[]dresources.GrantAssignment	ALL
+resources.catalogs.*.grants.grants[*]	dresources.GrantAssignment	ALL
+resources.catalogs.*.grants.grants[*].principal	string	ALL
+resources.catalogs.*.grants.grants[*].privileges	[]catalog.Privilege	ALL
+resources.catalogs.*.grants.grants[*].privileges[*]	catalog.Privilege	ALL
+resources.catalogs.*.grants.securable_type	string	ALL
 resources.clusters.*.apply_policy_default_values	bool	INPUT	STATE
 resources.clusters.*.autoscale	*compute.AutoScale	ALL
 resources.clusters.*.autoscale.max_workers	int	ALL

acceptance/bundle/resources/catalogs/basic/databricks.yml.tmpl

@@ -0,0 +1,17 @@
+bundle:
+  name: uc-catalog-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  catalogs:
+    test_catalog:
+      name: test_catalog_$UNIQUE_NAME
+      comment: "This catalog was created from DABs"
+      properties:
+        owner: "dabs"
+
+targets:
+  development:
+    default: true

acceptance/bundle/resources/catalogs/basic/output.txt

@@ -0,0 +1,44 @@
+
+=== Deploy bundle with catalog
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Assert the catalog is created
+>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME]
+{
+  "name": "test_catalog_[UNIQUE_NAME]",
+  "comment": "This catalog was created from DABs",
+  "properties": {
+    "owner": "dabs"
+  }
+}
+
+=== Update catalog comment
+=== Redeploy with updated comment
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Assert the catalog comment is updated
+>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME]
+{
+  "name": "test_catalog_[UNIQUE_NAME]",
+  "comment": "Updated comment from DABs"
+}
+
+=== Test cleanup
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.catalogs.test_catalog
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Deleting files...
+Destroy complete!
+
+=== Assert the catalog is deleted

acceptance/bundle/resources/catalogs/basic/script

@@ -0,0 +1,27 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+CATALOG_NAME="test_catalog_${UNIQUE_NAME}"
+
+cleanup() {
+  title "Test cleanup"
+  trace $CLI bundle destroy --auto-approve
+
+  title "Assert the catalog is deleted"
+  trace errcode $CLI catalogs get "${CATALOG_NAME}" 2>/dev/null
+}
+trap cleanup EXIT
+
+title "Deploy bundle with catalog"
+trace $CLI bundle deploy
+
+title "Assert the catalog is created"
+trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment, properties}"
+
+title "Update catalog comment"
+update_file.py databricks.yml "This catalog was created from DABs" "Updated comment from DABs"
+
+title "Redeploy with updated comment"
+trace $CLI bundle deploy
+
+title "Assert the catalog comment is updated"
+trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment}"

acceptance/bundle/resources/catalogs/basic/test.toml

@@ -0,0 +1,12 @@
+Local = true
+Cloud = true
+RecordRequests = false
+RequiresUnityCatalog = true
+
+Ignore = [
+    ".databricks",
+    "databricks.yml",
+]
+
+[EnvMatrix]
+  DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/resources/catalogs/with-schemas/databricks.yml.tmpl

@@ -0,0 +1,23 @@
+bundle:
+  name: uc-catalog-schemas-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+resources:
+  catalogs:
+    test_catalog:
+      name: test_catalog_$UNIQUE_NAME
+      comment: "Catalog created from DABs"
+      properties:
+        purpose: "testing"
+
+  schemas:
+    test_schema:
+      name: schema1
+      catalog_name: ${resources.catalogs.test_catalog.name}
+      comment: "Schema in custom catalog"
+
+targets:
+  development:
+    default: true

acceptance/bundle/resources/catalogs/with-schemas/output.txt

@@ -0,0 +1,64 @@
+
+=== Deploy bundle with catalog and schema
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Assert the catalog is created
+>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME]
+{
+  "name": "test_catalog_[UNIQUE_NAME]",
+  "comment": "Catalog created from DABs",
+  "properties": {
+    "purpose": "testing"
+  }
+}
+
+=== Assert schema is created in the custom catalog
+>>> [CLI] schemas get test_catalog_[UNIQUE_NAME].schema1
+{
+  "full_name": "test_catalog_[UNIQUE_NAME].schema1",
+  "catalog_name": "test_catalog_[UNIQUE_NAME]",
+  "comment": "Schema in custom catalog"
+}
+
+=== Verify schema belongs to the test catalog
+=== Update catalog comment
+=== Redeploy with updated catalog
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Assert catalog comment is updated
+>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME]
+{
+  "name": "test_catalog_[UNIQUE_NAME]",
+  "comment": "Updated catalog comment"
+}
+
+=== Assert schema still exists after catalog update
+>>> [CLI] schemas get test_catalog_[UNIQUE_NAME].schema1
+{
+  "full_name": "test_catalog_[UNIQUE_NAME].schema1"
+}
+
+=== Test cleanup
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.catalogs.test_catalog
+  delete resources.schemas.test_schema
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete resources.schemas.test_schema
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Deleting files...
+Destroy complete!
+
+=== Assert schema is deleted
+=== Assert the catalog is deleted

acceptance/bundle/resources/catalogs/with-schemas/script

@@ -0,0 +1,45 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+CATALOG_NAME="test_catalog_${UNIQUE_NAME}"
+SCHEMA_FULL_NAME="${CATALOG_NAME}.schema1"
+
+cleanup() {
+  title "Test cleanup"
+  trace $CLI bundle destroy --auto-approve
+
+  title "Assert schema is deleted"
+  trace errcode $CLI schemas get "${SCHEMA_FULL_NAME}" 2>/dev/null
+
+  title "Assert the catalog is deleted"
+  trace errcode $CLI catalogs get "${CATALOG_NAME}" 2>/dev/null
+}
+trap cleanup EXIT
+
+title "Deploy bundle with catalog and schema"
+trace $CLI bundle deploy
+
+title "Assert the catalog is created"
+trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment, properties}"
+
+title "Assert schema is created in the custom catalog"
+trace $CLI schemas get "${SCHEMA_FULL_NAME}" | jq "{full_name, catalog_name, comment}"
+
+title "Verify schema belongs to the test catalog"
+SCHEMA_CATALOG=$($CLI schemas get "${SCHEMA_FULL_NAME}" | jq -r ".catalog_name")
+
+if [ "$SCHEMA_CATALOG" != "$CATALOG_NAME" ]; then
+  echo "Error: Schema catalog_name is $SCHEMA_CATALOG, expected $CATALOG_NAME"
+  exit 1
+fi
+
+title "Update catalog comment"
+update_file.py databricks.yml "Catalog created from DABs" "Updated catalog comment"
+
+title "Redeploy with updated catalog"
+trace $CLI bundle deploy
+
+title "Assert catalog comment is updated"
+trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment}"
+
+title "Assert schema still exists after catalog update"
+trace $CLI schemas get "${SCHEMA_FULL_NAME}" | jq "{full_name}"

acceptance/bundle/resources/catalogs/with-schemas/test.toml

@@ -0,0 +1,12 @@
+Local = true
+Cloud = true
+RecordRequests = false
+RequiresUnityCatalog = true
+
+Ignore = [
+    ".databricks",
+    "databricks.yml",
+]
+
+[EnvMatrix]
+  DATABRICKS_BUNDLE_ENGINE = ["direct"]

acceptance/bundle/resources/grants/catalogs/databricks.yml

@@ -0,0 +1,13 @@
+bundle:
+  name: catalog-grants-test_cli_12345
+
+resources:
+  catalogs:
+    grants_catalog:
+      name: catalog_grants_test_cli_12345
+      comment: "Test catalog for grants"
+      grants:
+        - principal: deco-test-user@databricks.com
+          privileges:
+            - USE_CATALOG
+            - CREATE_SCHEMA

acceptance/bundle/resources/grants/catalogs/databricks.yml.tmpl

@@ -0,0 +1,13 @@
+bundle:
+  name: catalog-grants-$UNIQUE_NAME
+
+resources:
+  catalogs:
+    grants_catalog:
+      name: catalog_grants_$UNIQUE_NAME
+      comment: "Test catalog for grants"
+      grants:
+        - principal: deco-test-user@databricks.com
+          privileges:
+            - USE_CATALOG
+            - CREATE_SCHEMA

acceptance/bundle/resources/grants/catalogs/out.deploy1.requests.direct.json

@@ -0,0 +1,18 @@
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/catalog/catalog_grants_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "add": [
+          "CREATE_SCHEMA",
+          "USE_CATALOG"
+        ],
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "ALL_PRIVILEGES"
+        ]
+      }
+    ]
+  }
+}

acceptance/bundle/resources/grants/catalogs/out.deploy2.requests.direct.json

@@ -0,0 +1,18 @@
+{
+  "method": "PATCH",
+  "path": "/api/2.1/unity-catalog/permissions/catalog/catalog_grants_[UNIQUE_NAME]",
+  "body": {
+    "changes": [
+      {
+        "add": [
+          "USE_CATALOG",
+          "USE_SCHEMA"
+        ],
+        "principal": "deco-test-user@databricks.com",
+        "remove": [
+          "ALL_PRIVILEGES"
+        ]
+      }
+    ]
+  }
+}