diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 2c48fdb1f20..ceea2deeca7 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,6 +7,7 @@ ### CLI ### Bundles +* Add the `genie_spaces` bundle resource for managing Databricks Genie spaces as code, plus `bundle generate genie-space` to import an existing space. Direct deployment engine only ([#5282](https://github.com/databricks/cli/pull/5282)). ### Dependency updates diff --git a/acceptance/.gitattributes b/acceptance/.gitattributes index 8d48122750e..8380bb6ae0b 100644 --- a/acceptance/.gitattributes +++ b/acceptance/.gitattributes @@ -4,6 +4,7 @@ # uploading the file's content to a workspace. *.txt text eol=lf *.lvdash.json text eol=lf +*.geniespace.json text eol=lf # The out.test.toml file is autogenerated based on the merged test.toml view. out.test.toml linguist-generated=true diff --git a/acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl b/acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl new file mode 100644 index 00000000000..a31d53c1785 --- /dev/null +++ b/acceptance/bundle/deployment/bind/genie_space/databricks.yml.tmpl @@ -0,0 +1,13 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + genie_spaces: + genie_space1: + title: $GENIE_SPACE_TITLE + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + # Inline body matches the out-of-band-created space byte-for-byte, so the + # only possible post-bind drift signal is the etag. + serialized_space: + version: 1 diff --git a/acceptance/bundle/deployment/bind/genie_space/out.test.toml b/acceptance/bundle/deployment/bind/genie_space/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/deployment/bind/genie_space/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/deployment/bind/genie_space/output.txt b/acceptance/bundle/deployment/bind/genie_space/output.txt new file mode 100644 index 00000000000..34c6295bb51 --- /dev/null +++ b/acceptance/bundle/deployment/bind/genie_space/output.txt @@ -0,0 +1,37 @@ + +>>> [CLI] bundle deployment bind genie_space1 [GENIE_SPACE_ID] --auto-approve +Updating deployment state... +Successfully bound genie_space with an id '[GENIE_SPACE_ID]' +Run 'bundle deploy' to deploy changes to your workspace + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] genie get-space [GENIE_SPACE_ID] +{ + "parent_path": "/Users/[USERNAME]", + "title": "test genie space [UNIQUE_NAME]", + "warehouse_id": "test-warehouse-id" +} + +>>> [CLI] bundle deployment unbind genie_space1 +Updating deployment state... + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! + +>>> [CLI] genie get-space [GENIE_SPACE_ID] +{ + "parent_path": "/Users/[USERNAME]", + "title": "test genie space [UNIQUE_NAME]", + "warehouse_id": "test-warehouse-id" +} diff --git a/acceptance/bundle/deployment/bind/genie_space/script b/acceptance/bundle/deployment/bind/genie_space/script new file mode 100644 index 00000000000..896685a2311 --- /dev/null +++ b/acceptance/bundle/deployment/bind/genie_space/script @@ -0,0 +1,30 @@ +GENIE_SPACE_TITLE="test genie space $UNIQUE_NAME" + +export GENIE_SPACE_TITLE +envsubst < databricks.yml.tmpl > databricks.yml + +# Create a Genie space out of band, then bind the bundle resource to it. +GENIE_SPACE_ID=$($CLI genie create-space "test-warehouse-id" '{"version":1}' --title "${GENIE_SPACE_TITLE}" | jq -r '.space_id') + +cleanupRemoveGenieSpace() { + $CLI genie trash-space "${GENIE_SPACE_ID}" +} +trap cleanupRemoveGenieSpace EXIT + +trace $CLI bundle deployment bind genie_space1 "${GENIE_SPACE_ID}" --auto-approve + +# Bind must copy the remote etag into state, so the first plan after bind is +# clean. Without that, the etag drift signal (empty stored vs remote) would +# produce a bogus update here. +trace $CLI bundle plan + +trace $CLI bundle deploy + +trace $CLI genie get-space "${GENIE_SPACE_ID}" | jq --sort-keys '{title, parent_path, warehouse_id}' + +trace $CLI bundle deployment unbind genie_space1 + +trace $CLI bundle destroy --auto-approve + +# Read the Genie space again (expecting it still exists and is not deleted): +trace $CLI genie get-space "${GENIE_SPACE_ID}" | jq --sort-keys '{title, parent_path, warehouse_id}' diff --git a/acceptance/bundle/deployment/bind/genie_space/test.toml b/acceptance/bundle/deployment/bind/genie_space/test.toml new file mode 100644 index 00000000000..f46c8906df6 --- /dev/null +++ b/acceptance/bundle/deployment/bind/genie_space/test.toml @@ -0,0 +1,13 @@ +Local = true + +# Genie spaces are only deployed via the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +# Uses a literal warehouse id and a version-sensitive serialized_space, so this +# lifecycle test runs against the local mock server only (overrides the Cloud=true +# inherited from acceptance/bundle/deployment/test.toml). +Cloud = false + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[GENIE_SPACE_ID]" diff --git a/acceptance/bundle/generate/genie_space/databricks.yml b/acceptance/bundle/generate/genie_space/databricks.yml new file mode 100644 index 00000000000..c533b2b6f98 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: genie-space-generate diff --git a/acceptance/bundle/generate/genie_space/genie_space.json.tmpl b/acceptance/bundle/generate/genie_space/genie_space.json.tmpl new file mode 100644 index 00000000000..2056f3061ef --- /dev/null +++ b/acceptance/bundle/generate/genie_space/genie_space.json.tmpl @@ -0,0 +1,7 @@ +{ + "title": "test genie space", + "description": "test description", + "parent_path": "/Workspace/test-$UNIQUE_NAME", + "warehouse_id": "test-warehouse-id", + "serialized_space": "{\"tables\":[],\"questions\":[]}" +} diff --git a/acceptance/bundle/generate/genie_space/out.test.toml b/acceptance/bundle/generate/genie_space/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json new file mode 100644 index 00000000000..2c12b3c032c --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out/genie_space/test_genie_space.geniespace.json @@ -0,0 +1,4 @@ +{ + "questions": [], + "tables": [] +} diff --git a/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml new file mode 100644 index 00000000000..1471a901344 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/out/resource/test_genie_space.genie_space.yml @@ -0,0 +1,8 @@ +resources: + genie_spaces: + test_genie_space: + title: "test genie space" + warehouse_id: test-warehouse-id + file_path: ../genie_space/test_genie_space.geniespace.json + description: test description + parent_path: /Workspace/test-[UNIQUE_NAME] diff --git a/acceptance/bundle/generate/genie_space/output.txt b/acceptance/bundle/generate/genie_space/output.txt new file mode 100644 index 00000000000..a313a51adc3 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/output.txt @@ -0,0 +1,6 @@ + +>>> [CLI] workspace mkdirs /Workspace/test-[UNIQUE_NAME] + +>>> [CLI] bundle generate genie-space --existing-id [GENIE_SPACE_ID] --genie-space-dir out/genie_space --resource-dir out/resource +Writing genie space to out/genie_space/test_genie_space.geniespace.json +Writing configuration to out/resource/test_genie_space.genie_space.yml diff --git a/acceptance/bundle/generate/genie_space/script b/acceptance/bundle/generate/genie_space/script new file mode 100644 index 00000000000..8e2fa32170a --- /dev/null +++ b/acceptance/bundle/generate/genie_space/script @@ -0,0 +1,8 @@ +trace $CLI workspace mkdirs /Workspace/test-$UNIQUE_NAME + +# create a genie space to import +envsubst < genie_space.json.tmpl > genie_space.json +genie_space_id=$($CLI genie create-space --json @genie_space.json | jq -r '.space_id') +rm genie_space.json + +trace $CLI bundle generate genie-space --existing-id $genie_space_id --genie-space-dir out/genie_space --resource-dir out/resource diff --git a/acceptance/bundle/generate/genie_space/test.toml b/acceptance/bundle/generate/genie_space/test.toml new file mode 100644 index 00000000000..e389c33c277 --- /dev/null +++ b/acceptance/bundle/generate/genie_space/test.toml @@ -0,0 +1,10 @@ +[[Repls]] +Old = '\\\\' +New = '/' + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[GENIE_SPACE_ID]" + +[Env] +MSYS_NO_PATHCONV = "1" diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/databricks.yml b/acceptance/bundle/generate/genie_space_existing_id_not_found/databricks.yml new file mode 100644 index 00000000000..576d7a9ef25 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/databricks.yml @@ -0,0 +1,2 @@ +bundle: + name: test-bundle diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/output.txt b/acceptance/bundle/generate/genie_space_existing_id_not_found/output.txt new file mode 100644 index 00000000000..7ccd778a6b1 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/output.txt @@ -0,0 +1,4 @@ +Error: genie space with ID f00dcafe not found + + +Exit code: 1 diff --git a/acceptance/bundle/generate/genie_space_existing_id_not_found/script b/acceptance/bundle/generate/genie_space_existing_id_not_found/script new file mode 100644 index 00000000000..94d8b8594bc --- /dev/null +++ b/acceptance/bundle/generate/genie_space_existing_id_not_found/script @@ -0,0 +1,2 @@ +# Test that bundle generate genie-space fails when the existing ID is not found +exec $CLI bundle generate genie-space --existing-id f00dcafe diff --git a/acceptance/bundle/generate/genie_space_inplace/databricks.yml b/acceptance/bundle/generate/genie_space_inplace/databricks.yml new file mode 100644 index 00000000000..e5a76861842 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/databricks.yml @@ -0,0 +1,9 @@ +bundle: + name: genie space update inplace + +resources: + genie_spaces: + test_genie_space: + title: "test genie space" + warehouse_id: "my-warehouse-1234" + file_path: ./space.geniespace.json diff --git a/acceptance/bundle/generate/genie_space_inplace/out.test.toml b/acceptance/bundle/generate/genie_space_inplace/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/generate/genie_space_inplace/output.txt b/acceptance/bundle/generate/genie_space_inplace/output.txt new file mode 100644 index 00000000000..9beebf12605 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/output.txt @@ -0,0 +1,30 @@ + +>>> cat space.geniespace.json +{} + +=== deploy initial genie space +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/genie space update inplace/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== update the genie space +>>> [CLI] genie update-space [GENIE_SPACE_ID] --serialized-space {"a":"b"} +{ + "etag": "2", + "parent_path": "/Users/[USERNAME]/.bundle/genie space update inplace/default/resources", + "serialized_space": "{\"a\":\"b\"}", + "space_id": "[GENIE_SPACE_ID]", + "title": "test genie space", + "warehouse_id": "my-warehouse-1234" +} + +=== update the genie space file using bundle generate +>>> [CLI] bundle generate genie-space --resource test_genie_space --force +Writing genie space to space.geniespace.json + +>>> cat space.geniespace.json +{ + "a": "b" +} diff --git a/acceptance/bundle/generate/genie_space_inplace/script b/acceptance/bundle/generate/genie_space_inplace/script new file mode 100644 index 00000000000..18ca58d2986 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/script @@ -0,0 +1,13 @@ +trace cat space.geniespace.json + +title "deploy initial genie space" +trace $CLI bundle deploy +genie_space_id=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.test_genie_space.id') + +title "update the genie space" +trace $CLI genie update-space $genie_space_id --serialized-space '{"a":"b"}' + +title "update the genie space file using bundle generate" +trace $CLI bundle generate genie-space --resource test_genie_space --force + +trace cat space.geniespace.json diff --git a/acceptance/bundle/generate/genie_space_inplace/space.geniespace.json b/acceptance/bundle/generate/genie_space_inplace/space.geniespace.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/space.geniespace.json @@ -0,0 +1 @@ +{} diff --git a/acceptance/bundle/generate/genie_space_inplace/test.toml b/acceptance/bundle/generate/genie_space_inplace/test.toml new file mode 100644 index 00000000000..723c9907976 --- /dev/null +++ b/acceptance/bundle/generate/genie_space_inplace/test.toml @@ -0,0 +1,6 @@ +# Genie spaces are only deployed via the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] + +[[Repls]] +Old = "[0-9a-f]{32}" +New = "[GENIE_SPACE_ID]" diff --git a/acceptance/bundle/help/bundle-generate/output.txt b/acceptance/bundle/help/bundle-generate/output.txt index 97e8667ac78..39ce9293539 100644 --- a/acceptance/bundle/help/bundle-generate/output.txt +++ b/acceptance/bundle/help/bundle-generate/output.txt @@ -30,6 +30,7 @@ Available Commands: alert Generate configuration for an alert app Generate bundle configuration for a Databricks app dashboard Generate configuration for a dashboard + genie-space Generate configuration for a Genie space job Generate bundle configuration for a job pipeline Generate bundle configuration for a pipeline diff --git a/acceptance/bundle/invariant/configs/genie_space.yml.tmpl b/acceptance/bundle/invariant/configs/genie_space.yml.tmpl new file mode 100644 index 00000000000..fc4b6490263 --- /dev/null +++ b/acceptance/bundle/invariant/configs/genie_space.yml.tmpl @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + genie_spaces: + foo: + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID + title: test-genie-space-$UNIQUE_NAME + # Inline (structured) serialized_space is marshalled to a JSON string by + # ConfigureGenieSpaceSerializedSpace; this config doubles as a regression + # guard that the normalization produces a drift-free deploy. + serialized_space: + version: 1 + display_name: test-genie-space-$UNIQUE_NAME diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index 023feb47cdc..62de900a9ce 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -12,6 +12,7 @@ EnvMatrix.INPUT_CONFIG = [ "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", + "genie_space.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", diff --git a/acceptance/bundle/invariant/continue_293/test.toml b/acceptance/bundle/invariant/continue_293/test.toml index 2afbcbf5e31..6887ada9a71 100644 --- a/acceptance/bundle/invariant/continue_293/test.toml +++ b/acceptance/bundle/invariant/continue_293/test.toml @@ -9,6 +9,9 @@ EnvMatrixExclude.no_model_with_permissions = ["INPUT_CONFIG=model_with_permissio # vector_search_endpoints resource is not supported on v0.293.0 EnvMatrixExclude.no_vector_search_endpoint = ["INPUT_CONFIG=vector_search_endpoint.yml.tmpl"] +# genie_spaces resource is not supported on v0.293.0 +EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] + # Dotted pipeline configuration keys are not supported on v0.293.0 EnvMatrixExclude.no_pipeline_config_dots = ["INPUT_CONFIG=pipeline_config_dots.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 023feb47cdc..62de900a9ce 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -12,6 +12,7 @@ EnvMatrix.INPUT_CONFIG = [ "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", + "genie_space.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index 5fa381832e7..9492aff72cd 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -5,6 +5,8 @@ EnvMatrixExclude.no_vector_search_index = ["INPUT_CONFIG=vector_search_index.yml # Error: Catalog resources are only supported with direct deployment mode EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"] EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"] +# Genie spaces are direct-only too; the terraform deploy that seeds the migration fails for them. +EnvMatrixExclude.no_genie_space = ["INPUT_CONFIG=genie_space.yml.tmpl"] # Cross-resource permission references (e.g. ${resources.jobs.job_b.permissions[0].level}) # don't work in terraform mode: the terraform interpolator converts the path to diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 023feb47cdc..62de900a9ce 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -12,6 +12,7 @@ EnvMatrix.INPUT_CONFIG = [ "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", + "genie_space.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 6b0ee25fc61..345d18883a9 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -30,6 +30,7 @@ EnvMatrix.INPUT_CONFIG = [ "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", + "genie_space.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", @@ -80,6 +81,10 @@ no_external_location_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=external_loc # External volumes reference external locations; excluded from cloud for the same reason no_external_volume_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=volume_external.yml.tmpl"] +# Genie space serialized_space schema is workspace-version-sensitive; we only +# exercise it against the local mock server, not a real cloud workspace. +no_genie_space_on_cloud = ["CONFIG_Cloud=true", "INPUT_CONFIG=genie_space.yml.tmpl"] + # Fake SQL endpoint for local tests [[Server]] Pattern = "POST /api/2.0/sql/statements/" diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 07cbaac3a63..b6f0f177b3b 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -734,6 +734,25 @@ resources.external_locations.*.grants[*] catalog.PrivilegeAssignment ALL resources.external_locations.*.grants[*].principal string ALL resources.external_locations.*.grants[*].privileges []catalog.Privilege ALL resources.external_locations.*.grants[*].privileges[*] catalog.Privilege ALL +resources.genie_spaces.*.description string ALL +resources.genie_spaces.*.etag string ALL +resources.genie_spaces.*.file_path string INPUT +resources.genie_spaces.*.id string INPUT +resources.genie_spaces.*.lifecycle resources.Lifecycle INPUT +resources.genie_spaces.*.lifecycle.prevent_destroy bool INPUT +resources.genie_spaces.*.modified_status string INPUT +resources.genie_spaces.*.parent_path string ALL +resources.genie_spaces.*.serialized_space any ALL +resources.genie_spaces.*.space_id string ALL +resources.genie_spaces.*.title string ALL +resources.genie_spaces.*.url string INPUT +resources.genie_spaces.*.warehouse_id string ALL +resources.genie_spaces.*.permissions.object_id string ALL +resources.genie_spaces.*.permissions[*] dresources.StatePermission ALL +resources.genie_spaces.*.permissions[*].group_name string ALL +resources.genie_spaces.*.permissions[*].level iam.PermissionLevel ALL +resources.genie_spaces.*.permissions[*].service_principal_name string ALL +resources.genie_spaces.*.permissions[*].user_name string ALL resources.jobs.*.budget_policy_id string ALL resources.jobs.*.continuous *jobs.Continuous ALL resources.jobs.*.continuous.pause_status jobs.PauseStatus ALL diff --git a/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl new file mode 100644 index 00000000000..ea045097468 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: deploy-genie-space-inline-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Sales Analytics Inline Genie" + description: "Inline serialized_space test" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + serialized_space: + version: 1 + config: + sample_questions: + - id: "sq-001" + question: ["What is the total revenue?"] + data_sources: + tables: + - identifier: "main.sales.orders" + column_configs: + - column_name: "amount" + get_example_values: true diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.plan.json b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json new file mode 100644 index 00000000000..8cfcc539a44 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/out.plan.json @@ -0,0 +1,28 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.genie_spaces.sales_analytics": { + "action": "skip", + "remote_state": { + "description": "Inline serialized_space test", + "etag": "1", + "parent_path": "/Workspace/Users/[USERNAME]", + "serialized_space": "{\"config\":{\"sample_questions\":[{\"id\":\"sq-001\",\"question\":[\"What is the total revenue?\"]}]},\"data_sources\":{\"tables\":[{\"column_configs\":[{\"column_name\":\"amount\",\"get_example_values\":true}],\"identifier\":\"main.sales.orders\"}]},\"version\":1}", + "space_id": "[GENIE_SPACE_ID]", + "title": "Sales Analytics Inline Genie", + "warehouse_id": "test-warehouse-id" + }, + "changes": { + "etag": { + "action": "skip", + "reason": "custom", + "old": "1", + "remote": "1" + } + } + } + } +} diff --git a/acceptance/bundle/resources/genie_spaces/inline/out.test.toml b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/inline/output.txt b/acceptance/bundle/resources/genie_spaces/inline/output.txt new file mode 100644 index 00000000000..99fef197dd0 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/output.txt @@ -0,0 +1,17 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-inline-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.sales_analytics + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-inline-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/inline/script b/acceptance/bundle/resources/genie_spaces/inline/script new file mode 100644 index 00000000000..8f2f625b7b0 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') + +# Capture the genie space ID as a replacement. +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +# Plan after deploy must be drift-free aside from input_only fields. +# Without normalization the inline serialized_space leaves a map in the +# config struct while state holds a string, and structdiff reports false +# drift on every plan. +trace $CLI bundle plan -o json > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/inline/test.toml b/acceptance/bundle/resources/genie_spaces/inline/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/inline/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/parent_path_update/databricks.yml.tmpl new file mode 100644 index 00000000000..d479c3eef35 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/databricks.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: update-genie-space-$UNIQUE_NAME + +resources: + genie_spaces: + update_target: + title: "Update Target" + warehouse_id: "test-warehouse-id" + parent_path: PARENT_PATH_PLACEHOLDER + serialized_space: "{}" diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt b/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt new file mode 100644 index 00000000000..76f2dd99b32 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/output.txt @@ -0,0 +1,21 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/update-genie-space-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Plan after changing parent_path should show update +>>> [CLI] bundle plan +update genie_spaces.update_target + +Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.update_target + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/update-genie-space-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/script b/acceptance/bundle/resources/genie_spaces/parent_path_update/script new file mode 100644 index 00000000000..cc7359b67d9 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/script @@ -0,0 +1,14 @@ +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +# Deploy with the original parent_path. +envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-old|" > databricks.yml +trace $CLI bundle deploy + +# Change parent_path. The Genie update API moves the space to the new parent, +# so the plan should show an update rather than a recreate (delete + create). +envsubst < databricks.yml.tmpl | sed "s|PARENT_PATH_PLACEHOLDER|/Users/$CURRENT_USER_NAME/genie-new|" > databricks.yml +title "Plan after changing parent_path should show update" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/genie_spaces/parent_path_update/test.toml b/acceptance/bundle/resources/genie_spaces/parent_path_update/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/parent_path_update/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/databricks.yml.tmpl new file mode 100644 index 00000000000..fe135e8b627 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: recreate-gone-genie-space-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Recreate When Gone" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + serialized_space: + version: 1 diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt new file mode 100644 index 00000000000..aeaa5d7964d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/output.txt @@ -0,0 +1,20 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/recreate-gone-genie-space-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] genie trash-space [GENIE_SPACE_ID] + +=== Plan after remote deletion should recreate the space +>>> [CLI] bundle plan +create genie_spaces.sales_analytics + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle destroy --auto-approve +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/recreate-gone-genie-space-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script new file mode 100644 index 00000000000..c965df9fcc1 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +# Delete the space out-of-band, simulating a UI deletion. The GET API then +# returns 403; the resource maps that to "gone" so the plan recreates the +# space instead of failing. +trace $CLI genie trash-space $GENIE_SPACE_ID + +title "Plan after remote deletion should recreate the space" +trace $CLI bundle plan diff --git a/acceptance/bundle/resources/genie_spaces/recreate_when_gone/test.toml b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/recreate_when_gone/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/serialized_space/databricks.yml.tmpl new file mode 100644 index 00000000000..6b426d5c117 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: genie-serialized-space-$UNIQUE_NAME + +resources: + genie_spaces: + from_file: + title: "From File" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + file_path: "space.geniespace.json" + from_inline: + title: "From Inline" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + serialized_space: + version: 1 + data_sources: + tables: + - identifier: "main.sales.orders" diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/out.create_requests.json b/acceptance/bundle/resources/genie_spaces/serialized_space/out.create_requests.json new file mode 100644 index 00000000000..1ed6f9baf50 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/out.create_requests.json @@ -0,0 +1,30 @@ +[ + { + "title": "From File", + "serialized_space": "{\"version\": 1, \"data_sources\": {\"tables\": [{\"identifier\": \"main.sales.customers\"}]}}\n", + "parsed": { + "version": 1, + "data_sources": { + "tables": [ + { + "identifier": "main.sales.customers" + } + ] + } + } + }, + { + "title": "From Inline", + "serialized_space": "{\"data_sources\":{\"tables\":[{\"identifier\":\"main.sales.orders\"}]},\"version\":1}", + "parsed": { + "data_sources": { + "tables": [ + { + "identifier": "main.sales.orders" + } + ] + }, + "version": 1 + } + } +] diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/output.txt b/acceptance/bundle/resources/genie_spaces/serialized_space/output.txt new file mode 100644 index 00000000000..ff660039377 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/output.txt @@ -0,0 +1,16 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/genie-serialized-space-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.from_file + delete resources.genie_spaces.from_inline + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/genie-serialized-space-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/script b/acceptance/bundle/resources/genie_spaces/serialized_space/script new file mode 100644 index 00000000000..bf5aea4c21d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/script @@ -0,0 +1,21 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +rm -f out.requests.txt +trace $CLI bundle deploy + +# Capture the serialized_space sent to the Genie create API: +# - from_file must be sent verbatim from space.geniespace.json. +# - from_inline must be the inline YAML rendered as valid JSON (the fromjson +# below fails the test if the payload is not valid JSON). +jq -s '[.[] | select(.method == "POST" and .path == "/api/2.0/genie/spaces") + | {title: .body.title, + serialized_space: .body.serialized_space, + parsed: (.body.serialized_space | fromjson)}] + | sort_by(.title)' out.requests.txt > out.create_requests.json +rm -f out.requests.txt diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/space.geniespace.json b/acceptance/bundle/resources/genie_spaces/serialized_space/space.geniespace.json new file mode 100644 index 00000000000..ef4a1508015 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/space.geniespace.json @@ -0,0 +1 @@ +{"version": 1, "data_sources": {"tables": [{"identifier": "main.sales.customers"}]}} diff --git a/acceptance/bundle/resources/genie_spaces/serialized_space/test.toml b/acceptance/bundle/resources/genie_spaces/serialized_space/test.toml new file mode 100644 index 00000000000..a4821187360 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/serialized_space/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = true + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl new file mode 100644 index 00000000000..f30e4991de9 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: deploy-genie-space-test-$UNIQUE_NAME + +resources: + genie_spaces: + sales_analytics: + title: "Sales Analytics Genie" + description: "AI assistant for sales data analysis" + warehouse_id: "test-warehouse-id" + parent_path: /Users/$CURRENT_USER_NAME + file_path: "sales_analytics.geniespace.json" diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.plan.json b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json new file mode 100644 index 00000000000..77d3669d8ab --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/out.plan.json @@ -0,0 +1,28 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.genie_spaces.sales_analytics": { + "action": "skip", + "remote_state": { + "description": "AI assistant for sales data analysis", + "etag": "1", + "parent_path": "/Workspace/Users/[USERNAME]", + "serialized_space": "{\n \"benchmarks\": {\n \"questions\": [\n {\n \"answer\": [\n {\n \"content\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ],\n \"format\": \"SQL\"\n }\n ],\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Show all names and countries\"\n ]\n }\n ]\n },\n \"config\": {\n \"sample_questions\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ]\n },\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"Which names are in Canada?\"\n ]\n }\n ]\n },\n \"data_sources\": {\n \"tables\": [\n {\n \"column_configs\": [\n {\n \"column_name\": \"country\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n },\n {\n \"column_name\": \"name\",\n \"enable_entity_matching\": true,\n \"enable_format_assistance\": true\n }\n ],\n \"identifier\": \"main.default.countries\"\n }\n ]\n },\n \"instructions\": {\n \"example_question_sqls\": [\n {\n \"id\": \"[NUMID]\",\n \"question\": [\n \"List the names and countries\"\n ],\n \"sql\": [\n \"SELECT\\n\",\n \" name,\\n\",\n \" country\\n\",\n \"FROM main.default.countries\\n\",\n \"ORDER BY name\"\n ]\n }\n ],\n \"text_instructions\": [\n {\n \"content\": [\n \"This genie space answers simple questions about people and their countries.\\n\",\n \"Use only the main.default.countries table.\\n\",\n \"Prefer returning the name and country columns directly.\"\n ],\n \"id\": \"[NUMID]\"\n }\n ]\n },\n \"version\": 2\n}\n", + "space_id": "[GENIE_SPACE_ID]", + "title": "Sales Analytics Genie", + "warehouse_id": "test-warehouse-id" + }, + "changes": { + "etag": { + "action": "skip", + "reason": "custom", + "old": "1", + "remote": "1" + } + } + } + } +} diff --git a/acceptance/bundle/resources/genie_spaces/simple/out.test.toml b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/genie_spaces/simple/output.txt b/acceptance/bundle/resources/genie_spaces/simple/output.txt new file mode 100644 index 00000000000..f65d9583e18 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-test-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] genie get-space [GENIE_SPACE_ID] +{ + "title": "Sales Analytics Genie", + "description": "AI assistant for sales data analysis", + "warehouse_id": "test-warehouse-id" +} + +>>> [CLI] bundle plan -o json + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.sales_analytics + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-genie-space-test-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json new file mode 100644 index 00000000000..fb62b7c4859 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/sales_analytics.geniespace.json @@ -0,0 +1,87 @@ +{ + "benchmarks": { + "questions": [ + { + "answer": [ + { + "content": [ + "SELECT\n", + " name,\n", + " country\n", + "FROM main.default.countries\n", + "ORDER BY name" + ], + "format": "SQL" + } + ], + "id": "88888888888888888888888888888888", + "question": [ + "Show all names and countries" + ] + } + ] + }, + "config": { + "sample_questions": [ + { + "id": "11111111111111111111111111111111", + "question": [ + "List the names and countries" + ] + }, + { + "id": "22222222222222222222222222222222", + "question": [ + "Which names are in Canada?" + ] + } + ] + }, + "data_sources": { + "tables": [ + { + "column_configs": [ + { + "column_name": "country", + "enable_entity_matching": true, + "enable_format_assistance": true + }, + { + "column_name": "name", + "enable_entity_matching": true, + "enable_format_assistance": true + } + ], + "identifier": "main.default.countries" + } + ] + }, + "instructions": { + "example_question_sqls": [ + { + "id": "44444444444444444444444444444444", + "question": [ + "List the names and countries" + ], + "sql": [ + "SELECT\n", + " name,\n", + " country\n", + "FROM main.default.countries\n", + "ORDER BY name" + ] + } + ], + "text_instructions": [ + { + "content": [ + "This genie space answers simple questions about people and their countries.\n", + "Use only the main.default.countries table.\n", + "Prefer returning the name and country columns directly." + ], + "id": "33333333333333333333333333333333" + } + ] + }, + "version": 2 +} diff --git a/acceptance/bundle/resources/genie_spaces/simple/script b/acceptance/bundle/resources/genie_spaces/simple/script new file mode 100644 index 00000000000..4db9ef03078 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/script @@ -0,0 +1,17 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +trace $CLI bundle deploy +GENIE_SPACE_ID=$($CLI bundle summary --output json | jq -r '.resources.genie_spaces.sales_analytics.id') + +# Capture the genie space ID as a replacement. +echo "$GENIE_SPACE_ID:GENIE_SPACE_ID" >> ACC_REPLS + +trace $CLI genie get-space $GENIE_SPACE_ID | jq '{title, description, warehouse_id}' + +# Verify that there is no drift right after deploy. +trace $CLI bundle plan -o json > out.plan.json diff --git a/acceptance/bundle/resources/genie_spaces/simple/test.toml b/acceptance/bundle/resources/genie_spaces/simple/test.toml new file mode 100644 index 00000000000..bbdf2380b2d --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/simple/test.toml @@ -0,0 +1,6 @@ +Local = true +RecordRequests = false + +Ignore = [ + "databricks.yml", +] diff --git a/acceptance/bundle/resources/genie_spaces/test.toml b/acceptance/bundle/resources/genie_spaces/test.toml new file mode 100644 index 00000000000..7f397c47833 --- /dev/null +++ b/acceptance/bundle/resources/genie_spaces/test.toml @@ -0,0 +1,2 @@ +# Genie spaces are only deployed via the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml new file mode 100644 index 00000000000..d9e1c56c35c --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bundle + +resources: + genie_spaces: + foo: + title: "Permissions Test Space" + warehouse_id: test-warehouse-id + parent_path: /Workspace/Users/tester@databricks.com + serialized_space: "{}" + permissions: + - level: CAN_READ + user_name: viewer@example.com + - level: CAN_MANAGE + group_name: data-team + - level: CAN_MANAGE + service_principal_name: f37d18cd-98a8-4db5-8112-12dd0a6bfe38 + - level: CAN_MANAGE + user_name: tester@databricks.com diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json new file mode 100644 index 00000000000..5e1e8dd17b3 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.plan.direct.json @@ -0,0 +1,52 @@ +{ + "plan_version": 2, + "cli_version": "[DEV_VERSION]", + "plan": { + "resources.genie_spaces.foo": { + "action": "create", + "new_state": { + "value": { + "parent_path": "/Workspace/Users/[USERNAME]", + "serialized_space": "{}", + "title": "Permissions Test Space", + "warehouse_id": "test-warehouse-id" + } + } + }, + "resources.genie_spaces.foo.permissions": { + "depends_on": [ + { + "node": "resources.genie_spaces.foo", + "label": "${resources.genie_spaces.foo.id}" + } + ], + "action": "create", + "new_state": { + "value": { + "object_id": "", + "__embed__": [ + { + "level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "level": "CAN_MANAGE", + "group_name": "data-team" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + }, + "vars": { + "object_id": "/genie/${resources.genie_spaces.foo.id}" + } + } + } + } +} diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json new file mode 100644 index 00000000000..abcba718ab7 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.deploy.direct.json @@ -0,0 +1,24 @@ +{ + "method": "PUT", + "path": "/api/2.0/permissions/genie/[FOO_ID]", + "body": { + "access_control_list": [ + { + "permission_level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "permission_level": "CAN_MANAGE" + }, + { + "permission_level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "permission_level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } + ] + } +} diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.requests.destroy.direct.json new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt new file mode 100644 index 00000000000..e4e8cf0189d --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/output.txt @@ -0,0 +1,35 @@ + +>>> [CLI] bundle validate -o json +[ + { + "level": "CAN_READ", + "user_name": "viewer@example.com" + }, + { + "group_name": "data-team", + "level": "CAN_MANAGE" + }, + { + "level": "CAN_MANAGE", + "service_principal_name": "[UUID]" + }, + { + "level": "CAN_MANAGE", + "user_name": "[USERNAME]" + } +] + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.genie_spaces.foo + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script new file mode 100644 index 00000000000..1b20af07543 --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/current_can_manage/script @@ -0,0 +1,18 @@ +trace $CLI bundle validate -o json | jq .resources.genie_spaces.foo.permissions +rm out.requests.txt + +$CLI bundle plan -o json > out.plan.$DATABRICKS_BUNDLE_ENGINE.json + +print_requests() { + jq -c < out.requests.txt | jq 'select(.method != "GET" and (.path | contains("permissions")))' + rm out.requests.txt +} + +rm out.requests.txt +trace $CLI bundle deploy +# Genie space IDs are random; normalize them in the recorded requests. +replace_ids.py +print_requests > out.requests.deploy.$DATABRICKS_BUNDLE_ENGINE.json + +trace $CLI bundle destroy --auto-approve +print_requests > out.requests.destroy.$DATABRICKS_BUNDLE_ENGINE.json diff --git a/acceptance/bundle/resources/permissions/genie_spaces/test.toml b/acceptance/bundle/resources/permissions/genie_spaces/test.toml new file mode 100644 index 00000000000..390388f04aa --- /dev/null +++ b/acceptance/bundle/resources/permissions/genie_spaces/test.toml @@ -0,0 +1,4 @@ +Env.RESOURCE = "genie_spaces" # for ../_script + +# Genie spaces only support the direct deployment engine. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/permissions/output.txt b/acceptance/bundle/resources/permissions/output.txt index 85eea2e6e9c..12a35fcfd86 100644 --- a/acceptance/bundle/resources/permissions/output.txt +++ b/acceptance/bundle/resources/permissions/output.txt @@ -99,6 +99,8 @@ DIFF experiments/current_can_manage/out.requests.destroy.direct.json + "path": "/api/2.0/permissions/experiments/[NUMID]" + } +] +DIRECT_ONLY genie_spaces/current_can_manage/out.requests.deploy.direct.json +DIRECT_ONLY genie_spaces/current_can_manage/out.requests.destroy.direct.json MATCH jobs/current_can_manage/out.requests.deploy.direct.json DIFF jobs/current_can_manage/out.requests.destroy.direct.json --- jobs/current_can_manage/out.requests.destroy.direct.json diff --git a/acceptance/bundle/validate/genie_space_complex/databricks.yml b/acceptance/bundle/validate/genie_space_complex/databricks.yml new file mode 100644 index 00000000000..94c5fc7188c --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/databricks.yml @@ -0,0 +1,51 @@ +bundle: + name: genie-space-complex + +workspace: + resource_path: /foo/bar + +resources: + genie_spaces: + # Test with all features enabled + full_featured: + warehouse_id: "my-warehouse-1234" + title: "Full Featured Genie Space" + description: "A comprehensive test of all genie space features" + file_path: ./full_featured.geniespace.json + + # Test with inline serialized_space (YAML syntax) + inline_yaml: + warehouse_id: "my-warehouse-1234" + title: "Inline YAML Genie Space" + serialized_space: + version: 1 + data_sources: + tables: + - identifier: main.schema.table1 + column_configs: + - column_name: id + get_example_values: true + build_value_dictionary: true + - column_name: name + get_example_values: true + instructions: + text_instructions: + - id: inst-001 + content: + - "This is a text instruction.\n" + - "It spans multiple lines." + example_question_sqls: + - id: eq-001 + question: + - "How many records are there?" + sql: + - "SELECT COUNT(*) FROM main.schema.table1" + + # Test with empty but valid structure + minimal_valid: + warehouse_id: "my-warehouse-1234" + title: "Minimal Valid" + serialized_space: + version: 1 + data_sources: + tables: [] diff --git a/acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json b/acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json new file mode 100644 index 00000000000..9c9221d8328 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/full_featured.geniespace.json @@ -0,0 +1,160 @@ +{ + "version": 1, + "config": { + "sample_questions": [ + { + "id": "sq-001", + "question": ["What is the total revenue?"] + }, + { + "id": "sq-002", + "question": ["Show me the top customers"] + } + ] + }, + "data_sources": { + "tables": [ + { + "identifier": "main.sales.orders", + "column_configs": [ + { + "column_name": "order_id", + "get_example_values": true, + "build_value_dictionary": true + }, + { + "column_name": "customer_id", + "get_example_values": true + }, + { + "column_name": "amount", + "get_example_values": false + } + ] + }, + { + "identifier": "main.sales.customers" + } + ] + }, + "instructions": { + "text_instructions": [ + { + "id": "ti-001", + "content": [ + "This genie space analyzes sales data.\n", + "Always filter by date when querying orders.\n", + "Use customer_name instead of customer_id in results." + ] + } + ], + "example_question_sqls": [ + { + "id": "eq-001", + "question": ["What are the top customers by revenue?"], + "sql": [ + "SELECT\n", + " c.customer_name,\n", + " SUM(o.amount) AS total_revenue\n", + "FROM main.sales.orders o\n", + "JOIN main.sales.customers c ON o.customer_id = c.id\n", + "WHERE o.order_date >= :start_date\n", + "GROUP BY c.customer_name\n", + "ORDER BY total_revenue DESC\n", + "LIMIT :limit" + ], + "parameters": [ + { + "name": "start_date", + "type_hint": "STRING", + "description": ["Start date for the analysis period"], + "default_value": { + "values": ["2024-01-01"] + } + }, + { + "name": "limit", + "type_hint": "INTEGER", + "description": ["Number of customers to return"], + "default_value": { + "values": ["10"] + } + } + ] + }, + { + "id": "eq-002", + "question": ["Calculate daily revenue"], + "sql": [ + "SELECT\n", + " order_date,\n", + " SUM(amount) AS daily_revenue\n", + "FROM main.sales.orders\n", + "GROUP BY order_date\n", + "ORDER BY order_date" + ] + } + ], + "sql_snippets": { + "measures": [ + { + "id": "m-001", + "alias": "total_revenue", + "sql": ["SUM(orders.amount)"], + "display_name": "Total Revenue" + }, + { + "id": "m-002", + "alias": "order_count", + "sql": ["COUNT(orders.order_id)"], + "display_name": "Order Count" + } + ] + }, + "sql_functions": [ + { + "id": "sf-001", + "identifier": "main.analytics.calculate_churn" + } + ] + }, + "benchmarks": { + "questions": [ + { + "id": "bq-001", + "question": ["What is the monthly revenue trend?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " DATE_TRUNC('month', order_date) AS month,\n", + " SUM(amount) AS revenue\n", + "FROM main.sales.orders\n", + "GROUP BY 1\n", + "ORDER BY 1" + ] + } + ] + }, + { + "id": "bq-002", + "question": ["Which customers have the highest average order value?"], + "answer": [ + { + "format": "SQL", + "content": [ + "SELECT\n", + " customer_id,\n", + " AVG(amount) AS avg_order_value\n", + "FROM main.sales.orders\n", + "GROUP BY customer_id\n", + "ORDER BY avg_order_value DESC\n", + "LIMIT 10" + ] + } + ] + } + ] + } +} diff --git a/acceptance/bundle/validate/genie_space_complex/out.test.toml b/acceptance/bundle/validate/genie_space_complex/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_complex/output.txt b/acceptance/bundle/validate/genie_space_complex/output.txt new file mode 100644 index 00000000000..4a0747210cc --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/output.txt @@ -0,0 +1,22 @@ +{ + "full_featured": { + "title": "Full Featured Genie Space", + "warehouse_id": "my-warehouse-1234", + "serialized_space_is_string": true + }, + "inline_yaml": { + "title": "Inline YAML Genie Space", + "serialized_space_type": "string", + "parsed": { + "tables_count": 1, + "has_column_configs": true, + "has_text_instructions": true + } + }, + "minimal_valid": { + "title": "Minimal Valid", + "parsed": { + "tables_count": 0 + } + } +} diff --git a/acceptance/bundle/validate/genie_space_complex/script b/acceptance/bundle/validate/genie_space_complex/script new file mode 100644 index 00000000000..4feab1e373b --- /dev/null +++ b/acceptance/bundle/validate/genie_space_complex/script @@ -0,0 +1,26 @@ +# Validate complex genie spaces. ConfigureGenieSpaceSerializedSpace +# normalizes inline serialized_space YAML to a JSON string so the field has +# the same shape as the file_path code path; the script parses the JSON +# back to verify that the original structure is preserved. +$CLI bundle validate -o json | jq '{ + full_featured: .resources.genie_spaces.full_featured | { + title, + warehouse_id, + serialized_space_is_string: (.serialized_space | type == "string") + }, + inline_yaml: .resources.genie_spaces.inline_yaml | { + title, + serialized_space_type: (.serialized_space | type), + parsed: (.serialized_space | fromjson) | { + tables_count: (.data_sources.tables | length), + has_column_configs: ((.data_sources.tables[0].column_configs | length) > 0), + has_text_instructions: ((.instructions.text_instructions | length) > 0) + } + }, + minimal_valid: .resources.genie_spaces.minimal_valid | { + title, + parsed: (.serialized_space | fromjson) | { + tables_count: (.data_sources.tables | length) + } + } +}' diff --git a/acceptance/bundle/validate/genie_space_defaults/databricks.yml b/acceptance/bundle/validate/genie_space_defaults/databricks.yml new file mode 100644 index 00000000000..7e4b87d35e6 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/databricks.yml @@ -0,0 +1,30 @@ +bundle: + name: test-bundle + +workspace: + resource_path: /foo/bar + +resources: + genie_spaces: + empty_string: + warehouse_id: "my-warehouse-1234" + title: "empty-string" + serialized_space: "{}" + + # unchanged + parent_path: "" + + non_empty_string: + warehouse_id: "my-warehouse-1234" + title: "non-empty-string" + serialized_space: "{}" + + # unchanged + parent_path: "already-set" + + default_parent_path: + warehouse_id: "my-warehouse-1234" + title: "default-parent-path" + serialized_space: "{}" + + # parent_path set to default diff --git a/acceptance/bundle/validate/genie_space_defaults/out.test.toml b/acceptance/bundle/validate/genie_space_defaults/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_defaults/output.txt b/acceptance/bundle/validate/genie_space_defaults/output.txt new file mode 100644 index 00000000000..13105c91b76 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/output.txt @@ -0,0 +1,14 @@ +{ + "default_parent_path": { + "title": "default-parent-path", + "parent_path": "/Workspace/foo/bar" + }, + "empty_string": { + "title": "empty-string", + "parent_path": "/Workspace" + }, + "non_empty_string": { + "title": "non-empty-string", + "parent_path": "/Workspace/already-set" + } +} diff --git a/acceptance/bundle/validate/genie_space_defaults/script b/acceptance/bundle/validate/genie_space_defaults/script new file mode 100644 index 00000000000..eebc582f136 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_defaults/script @@ -0,0 +1 @@ +$CLI bundle validate -o json | jq '.resources.genie_spaces | map_values({title: .title, parent_path: .parent_path})' diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json new file mode 100644 index 00000000000..cb608f6e9c4 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/contents.geniespace.json @@ -0,0 +1 @@ +{"version": 1} diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml new file mode 100644 index 00000000000..ca57e978d83 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: genie-space-file-path-and-inline + +resources: + genie_spaces: + both_set: + title: "Both set" + warehouse_id: "test-warehouse-id" + file_path: "./contents.geniespace.json" + serialized_space: + version: 1 diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt new file mode 100644 index 00000000000..9402a3ff711 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/output.txt @@ -0,0 +1,12 @@ +Error: both file_path and serialized_space are set; specify only one + in databricks.yml:11:9 + +Name: genie-space-file-path-and-inline +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/genie-space-file-path-and-inline/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/script b/acceptance/bundle/validate/genie_space_file_path_and_inline/script new file mode 100644 index 00000000000..72555b332a4 --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/script @@ -0,0 +1 @@ +$CLI bundle validate diff --git a/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml new file mode 100644 index 00000000000..97900adac7a --- /dev/null +++ b/acceptance/bundle/validate/genie_space_file_path_and_inline/test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false + +# Genie spaces only support direct deployment engine. +[Env] +DATABRICKS_BUNDLE_ENGINE = "direct" diff --git a/acceptance/bundle/validate/no_genie_space_etag/databricks.yml b/acceptance/bundle/validate/no_genie_space_etag/databricks.yml new file mode 100644 index 00000000000..846608ec580 --- /dev/null +++ b/acceptance/bundle/validate/no_genie_space_etag/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: test-bundle + +resources: + genie_spaces: + foobar: + title: foobar + etag: "1234567890" + warehouse_id: "my-warehouse-1234" + serialized_space: "{}" diff --git a/acceptance/bundle/validate/no_genie_space_etag/out.test.toml b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml new file mode 100644 index 00000000000..e90b6d5d1ba --- /dev/null +++ b/acceptance/bundle/validate/no_genie_space_etag/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/validate/no_genie_space_etag/output.txt b/acceptance/bundle/validate/no_genie_space_etag/output.txt new file mode 100644 index 00000000000..58a7648f9aa --- /dev/null +++ b/acceptance/bundle/validate/no_genie_space_etag/output.txt @@ -0,0 +1,15 @@ + +>>> [CLI] bundle validate +Error: genie space "foobar" has an etag set. Etags must not be set in bundle configuration + at resources.genie_spaces.foobar + in databricks.yml:7:7 + +Name: test-bundle +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Found 1 error + +Exit code: 1 diff --git a/acceptance/bundle/validate/no_genie_space_etag/script b/acceptance/bundle/validate/no_genie_space_etag/script new file mode 100644 index 00000000000..5350876150f --- /dev/null +++ b/acceptance/bundle/validate/no_genie_space_etag/script @@ -0,0 +1 @@ +trace $CLI bundle validate diff --git a/acceptance/bundle/validate/no_genie_space_etag/test.toml b/acceptance/bundle/validate/no_genie_space_etag/test.toml new file mode 100644 index 00000000000..d573410a054 --- /dev/null +++ b/acceptance/bundle/validate/no_genie_space_etag/test.toml @@ -0,0 +1,3 @@ +# Genie spaces are only supported by the direct deployment engine; under +# terraform the direct-only validation error would diverge the output. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/experimental/open/output.txt b/acceptance/experimental/open/output.txt index af12759c8a4..c395bb4967e 100644 --- a/acceptance/experimental/open/output.txt +++ b/acceptance/experimental/open/output.txt @@ -9,7 +9,7 @@ === unknown resource type >>> [CLI] experimental open --url unknown 123 -Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses +Error: unknown resource type "unknown", must be one of: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses Exit code: 1 @@ -23,6 +23,7 @@ dashboards database_catalogs database_instances experiments +genie_spaces jobs model_serving_endpoints models diff --git a/bundle/config/mutator/paths/genie_space_paths_visitor.go b/bundle/config/mutator/paths/genie_space_paths_visitor.go new file mode 100644 index 00000000000..edd6ff2d8df --- /dev/null +++ b/bundle/config/mutator/paths/genie_space_paths_visitor.go @@ -0,0 +1,18 @@ +package paths + +import ( + "github.com/databricks/cli/libs/dyn" +) + +func VisitGenieSpacePaths(value dyn.Value, fn VisitFunc) (dyn.Value, error) { + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("genie_spaces"), + dyn.AnyKey(), + dyn.Key("file_path"), + ) + + return dyn.MapByPattern(value, pattern, func(path dyn.Path, value dyn.Value) (dyn.Value, error) { + return fn(path, TranslateModeLocalRelative, value) + }) +} diff --git a/bundle/config/mutator/paths/visitor.go b/bundle/config/mutator/paths/visitor.go index 0e3d59d43f5..bdf42188fde 100644 --- a/bundle/config/mutator/paths/visitor.go +++ b/bundle/config/mutator/paths/visitor.go @@ -15,6 +15,7 @@ func VisitPaths(root dyn.Value, fn VisitFunc) (dyn.Value, error) { VisitArtifactPaths, VisitAlertPaths, VisitDashboardPaths, + VisitGenieSpacePaths, VisitPipelinePaths, VisitPipelineLibrariesPaths, } diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index fd019479d77..cbb05d7d622 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -47,6 +47,10 @@ var ( permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_READ", }, + "genie_spaces": { + permissions.CAN_MANAGE: "CAN_MANAGE", + permissions.CAN_VIEW: "CAN_READ", + }, "apps": { permissions.CAN_MANAGE: "CAN_MANAGE", permissions.CAN_VIEW: "CAN_USE", diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 663adf4a281..79c0ebbec3c 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -237,6 +237,14 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos dashboard.DisplayName = prefix + dashboard.DisplayName } + // Genie Spaces: Prefix + for _, genieSpace := range r.GenieSpaces { + if genieSpace == nil { + continue + } + genieSpace.Title = prefix + genieSpace.Title + } + // Apps: No presets // Alerts: Prefix, TriggerPauseStatus diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index f8063ddf346..440221001a7 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -154,6 +154,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "geniespace1": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "geniespace1", + }, + }, + }, Apps: map[string]*resources.App{ "app1": { App: apps.App{ @@ -350,6 +357,9 @@ func TestProcessTargetModeDevelopment(t *testing.T) { // Dashboards assert.Equal(t, "[dev lennart] dashboard1", b.Config.Resources.Dashboards["dashboard1"].DisplayName) + // Genie Spaces + assert.Equal(t, "[dev lennart] geniespace1", b.Config.Resources.GenieSpaces["geniespace1"].Title) + // Alert 1: has schedule without pause status set - should be paused assert.Equal(t, "[dev lennart] alert1", b.Config.Resources.Alerts["alert1"].DisplayName) assert.Equal(t, sql.SchedulePauseStatusPaused, b.Config.Resources.Alerts["alert1"].Schedule.PauseStatus) diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go new file mode 100644 index 00000000000..c51c97a51e4 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -0,0 +1,87 @@ +package resourcemutator + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +const serializedSpaceFieldName = "serialized_space" + +type configureGenieSpaceSerializedSpace struct{} + +func ConfigureGenieSpaceSerializedSpace() bundle.Mutator { + return &configureGenieSpaceSerializedSpace{} +} + +func (c configureGenieSpaceSerializedSpace) Name() string { + return "ConfigureGenieSpaceSerializedSpace" +} + +func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + + pattern := dyn.NewPattern( + dyn.Key("resources"), + dyn.Key("genie_spaces"), + dyn.AnyKey(), + ) + + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + return dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + filePath, hasFilePath := v.Get(filePathFieldName).AsString() + ss := v.Get(serializedSpaceFieldName) + + if hasFilePath { + // file_path and serialized_space are two ways to provide the same + // content. Accepting both is ambiguous, so reject it instead of + // silently picking one. + if ss.IsValid() && ss.Kind() != dyn.KindNil { + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: "both file_path and serialized_space are set; specify only one", + Locations: ss.Locations(), + }) + return v, nil + } + contents, err := b.SyncRoot.ReadFile(filePath) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to read serialized genie space from file_path %s: %w", filePath, err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(contents))) + } + + // Marshal an inline structured serialized_space to a JSON string so + // both config-side and state-side carry the same plain string. + // Otherwise YAML decodes small ints as Go `int` while state JSON + // round-trip decodes them as `float64`, and structdiff reports + // false drift on every plan. + switch ss.Kind() { + case dyn.KindInvalid, dyn.KindNil, dyn.KindString: + // KindInvalid means serialized_space is absent (neither it nor + // file_path is set); leave it for backend validation to reject. + return v, nil + case dyn.KindMap, dyn.KindSequence: + jsonBytes, err := json.Marshal(ss.AsAny()) + if err != nil { + return dyn.InvalidValue, fmt.Errorf("failed to marshal inline serialized_space: %w", err) + } + return dyn.Set(v, serializedSpaceFieldName, dyn.V(string(jsonBytes))) + default: + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("serialized_space must be a string, map, or sequence, got %s", ss.Kind()), + Locations: ss.Locations(), + }) + return v, nil + } + }) + }) + + diags = diags.Extend(diag.FromErr(err)) + return diags +} diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space_test.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space_test.go new file mode 100644 index 00000000000..19c1b685f10 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space_test.go @@ -0,0 +1,120 @@ +package resourcemutator_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestConfigureGenieSpaceSerializedSpace(t *testing.T) { + const fileName = "space.geniespace.json" + + tests := []struct { + name string + // filePath is set on the resource as-is (already sync-root-relative). + filePath string + // writeFile creates filePath with fileContents before the mutator runs. + writeFile bool + fileContents string + setSerialized bool + serializedSpace any + // wantSerialized is the expected serialized_space after a successful run. + wantSerialized any + // wantErr, when non-empty, is a substring expected in the diagnostics. + wantErr string + }{ + { + // The file is read verbatim, so formatting and the trailing newline + // are preserved (unlike the inline path, which re-marshals). + name: "file_path reads file contents verbatim", + filePath: fileName, + writeFile: true, + fileContents: `{"version": 1}` + "\n", + wantSerialized: `{"version": 1}` + "\n", + }, + { + // Inline maps are marshaled to a compact JSON string with sorted keys + // so config and state hold an identical string and don't drift. + name: "inline map is marshaled to a JSON string", + setSerialized: true, + serializedSpace: map[string]any{"version": 1}, + wantSerialized: `{"version":1}`, + }, + { + name: "inline string is left unchanged", + setSerialized: true, + serializedSpace: `{"version":1}`, + wantSerialized: `{"version":1}`, + }, + { + // Neither field set: the absent field must pass through, not error. + name: "neither file_path nor serialized_space passes through", + wantSerialized: nil, + }, + { + name: "both file_path and serialized_space is rejected", + filePath: fileName, + setSerialized: true, + serializedSpace: map[string]any{"version": 1}, + wantErr: "both file_path and serialized_space are set; specify only one", + }, + { + name: "non-structured serialized_space is rejected", + setSerialized: true, + serializedSpace: true, + wantErr: "serialized_space must be a string, map, or sequence, got bool", + }, + { + name: "unreadable file_path is an error", + filePath: "does_not_exist.json", + wantErr: "failed to read serialized genie space", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := t.TempDir() + if tt.writeFile { + require.NoError(t, os.WriteFile(filepath.Join(dir, tt.filePath), []byte(tt.fileContents), 0o600)) + } + + gs := &resources.GenieSpace{ + GenieSpaceConfig: resources.GenieSpaceConfig{Title: "My Genie Space"}, + FilePath: tt.filePath, + } + if tt.setSerialized { + gs.SerializedSpace = tt.serializedSpace + } + + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{"my_space": gs}, + }, + }, + } + + diags := bundle.ApplySeq(t.Context(), b, resourcemutator.ConfigureGenieSpaceSerializedSpace()) + + if tt.wantErr != "" { + require.Error(t, diags.Error()) + assert.ErrorContains(t, diags.Error(), tt.wantErr) + return + } + + require.NoError(t, diags.Error()) + assert.Equal(t, tt.wantSerialized, b.Config.Resources.GenieSpaces["my_space"].SerializedSpace) + }) + } +} diff --git a/bundle/config/mutator/resourcemutator/genie_space_fixups.go b/bundle/config/mutator/resourcemutator/genie_space_fixups.go new file mode 100644 index 00000000000..85e1bb7e745 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/genie_space_fixups.go @@ -0,0 +1,30 @@ +package resourcemutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" +) + +type genieSpaceFixups struct{} + +func GenieSpaceFixups() bundle.Mutator { + return &genieSpaceFixups{} +} + +func (m *genieSpaceFixups) Name() string { + return "GenieSpaceFixups" +} + +func (m *genieSpaceFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, genieSpace := range b.Config.Resources.GenieSpaces { + if genieSpace == nil { + continue + } + + genieSpace.ParentPath = ensureWorkspacePrefix(genieSpace.ParentPath) + } + + return nil +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 209bbcb06a0..45740f53599 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -52,6 +52,7 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { }{ {"resources.dashboards.*.parent_path", b.Config.Workspace.ResourcePath}, {"resources.dashboards.*.embed_credentials", false}, + {"resources.genie_spaces.*.parent_path", b.Config.Workspace.ResourcePath}, {"resources.volumes.*.volume_type", "MANAGED"}, {"resources.alerts.*.parent_path", b.Config.Workspace.ResourcePath}, @@ -115,6 +116,11 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { // Ensures dashboard parent paths have the required /Workspace prefix DashboardFixups(), + // Reads (typed): b.Config.Resources.GenieSpaces (checks genie space configurations) + // Updates (typed): b.Config.Resources.GenieSpaces[].ParentPath (ensures /Workspace prefix is present) + // Ensures genie space parent paths have the required /Workspace prefix + GenieSpaceFixups(), + // Reads (typed): b.Config.Permissions (validates permission levels) // Reads (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (reads existing permissions) // Updates (dynamic): resources.{jobs,pipelines,experiments,models,model_serving_endpoints,dashboards,apps,vector_search_endpoints,...}.*.permissions (adds permissions from bundle-level configuration) @@ -182,6 +188,10 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Drops (dynamic): resources.dashboards.*.file_path ConfigureDashboardSerializedDashboard(), + // Reads (dynamic): resources.genie_spaces.*.file_path + // Updates (dynamic): resources.genie_spaces.*.serialized_space + ConfigureGenieSpaceSerializedSpace(), + // Reads (typed): resources.alerts.*.file_path // Updates (typed): resources.alerts.* (loads alert configuration from .dbalert.json file) mutator.LoadDBAlertFiles(), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 9ef2db077ac..af1470848d7 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -41,6 +41,7 @@ func allResourceTypes(t *testing.T) []string { "database_instances", "experiments", "external_locations", + "genie_spaces", "jobs", "model_serving_endpoints", "models", @@ -183,6 +184,7 @@ var allowList = []string{ "postgres_synced_tables", "registered_models", "experiments", + "genie_spaces", "schemas", "secret_scopes", "sql_warehouses", diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index a0bb76e2e6e..b36ec094447 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -368,6 +368,7 @@ func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ t.applyDashboardTranslations, + t.applyGenieSpaceTranslations, }) } diff --git a/bundle/config/mutator/translate_paths_genie_spaces.go b/bundle/config/mutator/translate_paths_genie_spaces.go new file mode 100644 index 00000000000..4e6d41f1cec --- /dev/null +++ b/bundle/config/mutator/translate_paths_genie_spaces.go @@ -0,0 +1,21 @@ +package mutator + +import ( + "context" + + "github.com/databricks/cli/bundle/config/mutator/paths" + "github.com/databricks/cli/libs/dyn" +) + +func (t *translateContext) applyGenieSpaceTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { + // Rewrite the `file_path` field to a path relative to the bundle sync root. + // We load the file at this path and use its contents for the genie space contents. + + return paths.VisitGenieSpacePaths(v, func(p dyn.Path, mode paths.TranslateMode, v dyn.Value) (dyn.Value, error) { + opts := translateOptions{ + Mode: mode, + } + + return t.rewriteValue(ctx, p, v, t.b.BundleRootPath, opts) + }) +} diff --git a/bundle/config/mutator/translate_paths_genie_spaces_test.go b/bundle/config/mutator/translate_paths_genie_spaces_test.go new file mode 100644 index 00000000000..a1ac0b160b1 --- /dev/null +++ b/bundle/config/mutator/translate_paths_genie_spaces_test.go @@ -0,0 +1,55 @@ +package mutator_test + +import ( + "path/filepath" + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/internal/bundletest" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/vfs" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTranslatePathsGenieSpaces_FilePathRelativeSubDirectory(t *testing.T) { + dir := t.TempDir() + touchEmptyFile(t, filepath.Join(dir, "src", "my_space.geniespace.json")) + + b := &bundle.Bundle{ + SyncRootPath: dir, + BundleRootPath: dir, + SyncRoot: vfs.MustNew(dir), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Genie Space", + }, + FilePath: "../src/my_space.geniespace.json", + }, + }, + }, + }, + } + + bundletest.SetLocation(b, "resources.genie_spaces", []dyn.Location{{ + File: filepath.Join(dir, "resources", "genie_space.yml"), + }}) + + // Genie space paths reuse the dashboard translator; there is no separate + // genie_space mutator. The dashboard translator walks all resource types + // that need path translation, so calling it covers genie_spaces too. + diags := bundle.ApplySeq(t.Context(), b, mutator.NormalizePaths(), mutator.TranslatePathsDashboards()) + require.NoError(t, diags.Error()) + + assert.Equal( + t, + filepath.ToSlash(filepath.Join("src", "my_space.geniespace.json")), + b.Config.Resources.GenieSpaces["genie_space"].FilePath, + ) +} diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 2e840a5ef80..3dc7dc295d3 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -25,6 +25,7 @@ type Resources struct { ExternalLocations map[string]*resources.ExternalLocation `json:"external_locations,omitempty"` Clusters map[string]*resources.Cluster `json:"clusters,omitempty"` Dashboards map[string]*resources.Dashboard `json:"dashboards,omitempty"` + GenieSpaces map[string]*resources.GenieSpace `json:"genie_spaces,omitempty"` Apps map[string]*resources.App `json:"apps,omitempty"` SecretScopes map[string]*resources.SecretScope `json:"secret_scopes,omitempty"` Alerts map[string]*resources.Alert `json:"alerts,omitempty"` @@ -104,6 +105,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["external_locations"], r.ExternalLocations), collectResourceMap(descriptions["clusters"], r.Clusters), collectResourceMap(descriptions["dashboards"], r.Dashboards), + collectResourceMap(descriptions["genie_spaces"], r.GenieSpaces), collectResourceMap(descriptions["volumes"], r.Volumes), collectResourceMap(descriptions["apps"], r.Apps), collectResourceMap(descriptions["alerts"], r.Alerts), @@ -162,6 +164,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "external_locations": (&resources.ExternalLocation{}).ResourceDescription(), "clusters": (&resources.Cluster{}).ResourceDescription(), "dashboards": (&resources.Dashboard{}).ResourceDescription(), + "genie_spaces": (&resources.GenieSpace{}).ResourceDescription(), "volumes": (&resources.Volume{}).ResourceDescription(), "apps": (&resources.App{}).ResourceDescription(), "secret_scopes": (&resources.SecretScope{}).ResourceDescription(), diff --git a/bundle/config/resources/genie_space.go b/bundle/config/resources/genie_space.go new file mode 100644 index 00000000000..b0a5efbf304 --- /dev/null +++ b/bundle/config/resources/genie_space.go @@ -0,0 +1,100 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/workspaceurls" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +type GenieSpaceConfig struct { + // Description of the Genie Space + Description string `json:"description,omitempty"` + // Etag for change detection. The bundle persists the value the backend + // returned on the last Create/Update and uses it both as an If-Match for + // the next Update and as the signal for `bundle plan` to detect remote + // drift (see OverrideChangeDesc in bundle/direct/dresources/genie_space.go). + // Mirrors dashboards.DashboardConfig.Etag. + Etag string `json:"etag,omitempty"` + // Genie space ID + SpaceId string `json:"space_id,omitempty"` + // Title of the Genie Space + Title string `json:"title,omitempty"` + // Warehouse associated with the Genie Space + WarehouseId string `json:"warehouse_id,omitempty"` + // Parent folder path where the space will be registered + ParentPath string `json:"parent_path,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` + + // ============================================== + // === overrides over [dashboards.GenieSpace] === + // ============================================== + + // SerializedSpace holds the contents of the Genie Space in serialized JSON form. + // Even though the SDK represents this as a string, we override it as any to allow for inlining as YAML. + // If the value is a string, it is used as is. + // If it is not a string, its contents is marshalled as JSON. + SerializedSpace any `json:"serialized_space,omitempty"` +} + +func (c *GenieSpaceConfig) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c GenieSpaceConfig) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +type GenieSpace struct { + BaseResource + GenieSpaceConfig + + Permissions []Permission `json:"permissions,omitempty"` + + // FilePath points to the local `.geniespace.json` file containing the Genie Space definition. + // This is inlined into serialized_space during deployment. The file_path is kept around + // as metadata which is needed for `databricks bundle generate genie-space --resource ` to work. + // This is not part of GenieSpaceConfig because we don't need to store this in the resource state. + FilePath string `json:"file_path,omitempty"` +} + +func (*GenieSpace) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: id, + }) + if err != nil { + log.Debugf(ctx, "genie space %s does not exist", id) + return false, err + } + return true, nil +} + +func (*GenieSpace) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "genie_space", + PluralName: "genie_spaces", + SingularTitle: "Genie Space", + PluralTitle: "Genie Spaces", + } +} + +func (r *GenieSpace) InitializeURL(baseURL url.URL) { + if r.ID == "" { + return + } + + r.URL = workspaceurls.ResourceURL(baseURL, "genie_spaces", r.ID) +} + +func (r *GenieSpace) GetName() string { + return r.Title +} + +func (r *GenieSpace) GetURL() string { + return r.URL +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 4f51476536f..8e610ba9418 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -201,6 +201,9 @@ func TestResourcesBindSupport(t *testing.T) { Dashboards: map[string]*resources.Dashboard{ "my_dashboard": {}, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "my_genie_space": {}, + }, Volumes: map[string]*resources.Volume{ "my_volume": { CreateVolumeRequestContent: catalog.CreateVolumeRequestContent{}, @@ -328,6 +331,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockSchemasAPI().EXPECT().GetByFullName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockClustersAPI().EXPECT().GetByClusterId(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockLakeviewAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockGenieAPI().EXPECT().GetSpace(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockVolumesAPI().EXPECT().Read(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAppsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockAlertsV2API().EXPECT().GetAlertById(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go new file mode 100644 index 00000000000..84e9626c0f4 --- /dev/null +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -0,0 +1,39 @@ +package validate + +import ( + "context" + "fmt" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/dyn" +) + +func ValidateGenieSpaceEtags() bundle.ReadOnlyMutator { + return &validateGenieSpaceEtags{} +} + +type validateGenieSpaceEtags struct{ bundle.RO } + +func (v *validateGenieSpaceEtags) Name() string { + return "validate:validate_genie_space_etags" +} + +func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + // No genie spaces should have etags set. They are purely internal state + // (persisted by the direct engine for drift detection), never authored by + // the user. Mirrors ValidateDashboardEtags. + for k, genieSpace := range b.Config.Resources.GenieSpaces { + if genieSpace.Etag != "" { + return diag.Diagnostics{ + { + Severity: diag.Error, + Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), + Paths: []dyn.Path{dyn.MustPathFromString("resources.genie_spaces." + k)}, + Locations: b.Config.GetLocations("resources.genie_spaces." + k), + }, + } + } + } + return nil +} diff --git a/bundle/deploy/terraform/lifecycle_test.go b/bundle/deploy/terraform/lifecycle_test.go index 2986a9cc8de..b60bff612c7 100644 --- a/bundle/deploy/terraform/lifecycle_test.go +++ b/bundle/deploy/terraform/lifecycle_test.go @@ -17,6 +17,7 @@ func TestConvertLifecycleForAllResources(t *testing.T) { ignoredResources := []string{ "catalogs", "external_locations", + "genie_spaces", "vector_search_endpoints", "vector_search_indexes", } diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index f1c534bea9d..9760ce95666 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -132,10 +132,11 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac dependsOn = entry.DependsOn } - // Copy etag from remote state for dashboards. - // Dashboards store "etag" in state which is not provided by user but comes from remote. - // If we don't store "etag" in state, we won't detect remote drift correctly. - if strings.Contains(resourceKey, ".dashboards.") && entry != nil && entry.RemoteState != nil { + // Copy etag from remote state for resources that use etag-based drift + // detection (dashboards and genie spaces). The etag is not provided by the + // user; it comes from remote. If we don't store it in state, we won't + // detect remote drift correctly and the next plan shows a bogus update. + if (strings.Contains(resourceKey, ".dashboards.") || strings.Contains(resourceKey, ".genie_spaces.")) && entry != nil && entry.RemoteState != nil { etag, err := structaccess.Get(entry.RemoteState, structpath.NewStringKey(nil, "etag")) if err == nil && etag != nil { if etagStr, ok := etag.(string); ok && etagStr != "" { diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index a263a25f127..6cc1eb55437 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -29,6 +29,7 @@ var SupportedResources = map[string]any{ "clusters": (*ResourceCluster)(nil), "registered_models": (*ResourceRegisteredModel)(nil), "dashboards": (*ResourceDashboard)(nil), + "genie_spaces": (*ResourceGenieSpace)(nil), "secret_scopes": (*ResourceSecretScope)(nil), "model_serving_endpoints": (*ResourceModelServingEndpoint)(nil), "quality_monitors": (*ResourceQualityMonitor)(nil), @@ -49,6 +50,7 @@ var SupportedResources = map[string]any{ "secret_scopes.permissions": (*ResourceSecretScopeAcls)(nil), "model_serving_endpoints.permissions": (*ResourcePermissions)(nil), "dashboards.permissions": (*ResourcePermissions)(nil), + "genie_spaces.permissions": (*ResourcePermissions)(nil), "vector_search_endpoints.permissions": (*ResourcePermissions)(nil), // Grants diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index f18a84d0efc..30adb4640cc 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -254,6 +254,15 @@ var testConfig map[string]any = map[string]any{ }, }, + "genie_spaces": &resources.GenieSpace{ + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "my-genie-space", + WarehouseId: "test-warehouse-id", + ParentPath: "/Workspace/Users/user@example.com", + SerializedSpace: "{}", + }, + }, + "vector_search_endpoints": &resources.VectorSearchEndpoint{ CreateEndpoint: vectorsearch.CreateEndpoint{ Name: "my-endpoint", @@ -495,6 +504,25 @@ var testDeps = map[string]prepareWorkspace{ }, nil }, + "genie_spaces.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { + resp, err := client.Genie.CreateSpace(ctx, dashboards.GenieCreateSpaceRequest{ + Title: "genie-space-permissions", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }) + if err != nil { + return nil, err + } + + return &PermissionsState{ + ObjectID: "/genie/" + resp.SpaceId, + EmbeddedSlice: []StatePermission{{ + Level: "CAN_MANAGE", + UserName: "user@example.com", + }}, + }, nil + }, + "model_serving_endpoints.permissions": func(ctx context.Context, client *databricks.WorkspaceClient) (any, error) { waiter, err := client.ServingEndpoints.Create(ctx, serving.CreateServingEndpoint{ Name: "endpoint-permissions", diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index b65d9ac41d4..069f8dbbe5f 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -18,6 +18,8 @@ experiments: ml.CreateExperiment external_locations: catalog.CreateExternalLocation +genie_spaces: dashboards.GenieSpace + jobs: jobs.JobSettings model_serving_endpoints: serving.CreateServingEndpoint diff --git a/bundle/direct/dresources/genie_space.go b/bundle/direct/dresources/genie_space.go new file mode 100644 index 00000000000..dcc03a8af33 --- /dev/null +++ b/bundle/direct/dresources/genie_space.go @@ -0,0 +1,313 @@ +package dresources + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/cli/libs/utils" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +var pathSerializedSpace = structpath.MustParsePath("serialized_space") + +// ResourceGenieSpace mirrors the dashboard resource pattern (see dashboard.go), +// with these intentional divergences: +// - No Published wrapper: Genie spaces have no publish lifecycle, so +// PrepareState returns the config directly. +// - RemapState filters fewer fields: Genie has no LifecycleState / CreateTime / +// Path / UpdateTime output-only fields to scrub. +// - DoUpdate omits serialized_space when unchanged: serialized_space is in +// ignore_remote_changes (see resources.yml), so a UI edit produces no plan +// entry. Sending the local body anyway would clobber the UI edit on every +// unrelated update. +// - DoUpdate omits the etag (dashboard sends it as an If-Match guard): the +// backend bumps the etag when it migrates serialized_space to a newer +// schema version, so sending a stale etag would 409 the update after a +// migration. Drift is still detected on read via OverrideChangeDesc. +// - DoCreate has expanded missing-parent-path detection: see +// isMissingGenieParentPathError below. +// +// Permissions follow the standard /permissions/genie/{id} endpoint and are wired +// up via the generic permissions adapter (permissions.go). +type ResourceGenieSpace struct { + client *databricks.WorkspaceClient +} + +func (*ResourceGenieSpace) New(client *databricks.WorkspaceClient) *ResourceGenieSpace { + return &ResourceGenieSpace{client: client} +} + +func (*ResourceGenieSpace) PrepareState(input *resources.GenieSpace) *resources.GenieSpaceConfig { + return &input.GenieSpaceConfig +} + +func (r *ResourceGenieSpace) RemapState(state *resources.GenieSpaceConfig) *resources.GenieSpaceConfig { + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](state.ForceSendFields, []string{ + "SpaceId", + "SerializedSpace", + }...) + + return &resources.GenieSpaceConfig{ + Description: state.Description, + Etag: state.Etag, + Title: state.Title, + WarehouseId: state.WarehouseId, + ParentPath: state.ParentPath, + SerializedSpace: state.SerializedSpace, + + ForceSendFields: forceSendFields, + + // Clear output only fields. They should not show up on remote diff computation. + SpaceId: "", + } +} + +func (r *ResourceGenieSpace) DoRead(ctx context.Context, id string) (*resources.GenieSpaceConfig, error) { + space, err := r.client.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: id, + IncludeSerializedSpace: true, // otherwise etag isn't returned + ForceSendFields: nil, + }) + if err != nil { + return nil, genieSpaceGoneError(err) + } + return responseToGenieSpaceConfig(space, space.SerializedSpace), nil +} + +func prepareGenieSpaceRequest(config *resources.GenieSpaceConfig) (string, error) { + v := config.SerializedSpace + if serializedSpace, ok := v.(string); ok { + return serializedSpace, nil + } else if v != nil { + b, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("failed to marshal serialized_space: %w", err) + } + return string(b), nil + } + return "", nil +} + +func responseToGenieSpaceConfig(space *dashboards.GenieSpace, serializedSpace string) *resources.GenieSpaceConfig { + forceSendFields := utils.FilterFields[resources.GenieSpaceConfig](space.ForceSendFields) + + return &resources.GenieSpaceConfig{ + Description: space.Description, + Etag: space.Etag, + Title: space.Title, + WarehouseId: space.WarehouseId, + ParentPath: ensureWorkspacePrefix(space.ParentPath), + SerializedSpace: serializedSpace, + + // Output only fields + SpaceId: space.SpaceId, + ForceSendFields: forceSendFields, + } +} + +// isMissingGenieParentPathError reports whether the given Create error means +// "the parent workspace folder does not exist", so DoCreate can mkdir and retry. +// +// Dashboard handles the equivalent condition with a plain apierr.IsMissing +// check (see ResourceDashboard.DoCreate). Genie cannot, because it surfaces +// the same condition in two different shapes depending on the workspace's +// backend version: +// +// 1. Standard missing-resource error: HTTP 404, ErrorCode RESOURCE_DOES_NOT_EXIST. +// Caught by apierr.IsMissing. Observed on workspaces running the newer +// Genie service implementation. +// 2. HTTP 400 with ErrorCode INVALID_PARAMETER_VALUE and a message of the +// form "Tree node with path '' does not exist". Observed on +// workspaces still backed by the legacy implementation during integration +// testing in early 2026 (aws-prod-ucws and azure-prod-ucws clusters at +// the time). The string match is intentional: there is no distinct error +// code to key on. +// +// Both forms unambiguously mean "create the parent and retry once". +func isMissingGenieParentPathError(err error) bool { + if apierr.IsMissing(err) { + return true + } + + apiErr, ok := errors.AsType[*apierr.APIError](err) + if !ok { + return false + } + + return apiErr.StatusCode == http.StatusBadRequest && + apiErr.ErrorCode == "INVALID_PARAMETER_VALUE" && + strings.Contains(apiErr.Message, "Tree node with path") && + strings.Contains(apiErr.Message, "does not exist") +} + +// genieSpaceGoneError maps the Genie API's "space does not exist" response to +// the framework's gone sentinel (apierr.ErrResourceDoesNotExist). The Genie API +// returns 403 (not 404) for a missing space, so without this translation +// isResourceGone would treat a remotely-deleted space as a hard permission +// error instead of recreating it (on read) or tolerating it (on delete). +func genieSpaceGoneError(err error) error { + if errors.Is(err, apierr.ErrPermissionDenied) { + return errors.Join(err, apierr.ErrResourceDoesNotExist) + } + return err +} + +func (r *ResourceGenieSpace) DoCreate(ctx context.Context, config *resources.GenieSpaceConfig) (string, *resources.GenieSpaceConfig, error) { + serializedSpace, err := prepareGenieSpaceRequest(config) + if err != nil { + return "", nil, err + } + + req := dashboards.GenieCreateSpaceRequest{ + Description: config.Description, + Title: config.Title, + WarehouseId: config.WarehouseId, + ParentPath: config.ParentPath, + SerializedSpace: serializedSpace, + + ForceSendFields: utils.FilterFields[dashboards.GenieCreateSpaceRequest](config.ForceSendFields), + } + + createResp, err := r.client.Genie.CreateSpace(ctx, req) + + // Retry once after creating the parent directory when the workspace folder + // is missing. Genie can surface this either as a standard missing-resource + // error or as INVALID_PARAMETER_VALUE with a "Tree node ... does not exist" + // message depending on the backend. + if err != nil && isMissingGenieParentPathError(err) { + err = r.client.Workspace.MkdirsByPath(ctx, config.ParentPath) //nolint:staticcheck // Deprecated in SDK v0.127.0. Migration to WorkspaceHierarchyService tracked separately. + if err != nil { + return "", nil, fmt.Errorf("failed to create parent directory: %w", err) + } + createResp, err = r.client.Genie.CreateSpace(ctx, req) + } + if err != nil { + return "", nil, err + } + + // Persist the etag in state. The deploy framework saves `config` (the input + // to DoCreate) as the state record, so mutating it here is what gets the + // backend-returned etag onto disk for the next plan's drift check. + // Matches the dashboard pattern (dashboard.go DoCreate). + config.Etag = createResp.Etag + + return createResp.SpaceId, responseToGenieSpaceConfig(createResp, serializedSpace), nil +} + +func (r *ResourceGenieSpace) DoUpdate(ctx context.Context, id string, config *resources.GenieSpaceConfig, entry *PlanEntry) (*resources.GenieSpaceConfig, error) { + serializedSpace, err := prepareGenieSpaceRequest(config) + if err != nil { + return nil, err + } + + // serialized_space is in ignore_remote_changes (we cannot diff structured + // local YAML against remote JSON), so a UI edit produces no plan entry. + // If we still sent the unchanged local body on every update, the next + // update triggered by another field would clobber the UI edit. Only + // send it when the user actually changed it locally. + var excludeForceSend []string + sentSerialized := true + if !hasUpdate(entry, pathSerializedSpace) { + serializedSpace = "" + sentSerialized = false + excludeForceSend = append(excludeForceSend, "SerializedSpace") + } + + updateResp, err := r.client.Genie.UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: id, + Description: config.Description, + Title: config.Title, + WarehouseId: config.WarehouseId, + ParentPath: config.ParentPath, + SerializedSpace: serializedSpace, + // Intentionally empty: we do not send an If-Match guard. The backend + // bumps the etag when it migrates serialized_space to a newer schema + // version, so sending the last-observed etag would fail the update with + // 409 after such a migration. Drift is still detected on read via + // OverrideChangeDesc, which compares the stored and remote etags. + Etag: "", + + ForceSendFields: utils.FilterFields[dashboards.GenieUpdateSpaceRequest](config.ForceSendFields, excludeForceSend...), + }) + if err != nil { + return nil, err + } + + // Persist the new etag in state (see DoCreate for the rationale). + config.Etag = updateResp.Etag + + // Decide what to record as the new state's serialized_space. + // - If we sent a new body, use it. + // - If we omitted it (UI-edit protection above) but the API echoed back + // a value, record that — it's the most up-to-date view we have. + // - If neither side carries a value, keep whatever was already in state. + // Otherwise RemapState would blank the field on every unrelated update. + respSerialized := serializedSpace + if !sentSerialized { + respSerialized = updateResp.SerializedSpace + if respSerialized == "" { + if prior, ok := config.SerializedSpace.(string); ok { + respSerialized = prior + } + } + } + + return responseToGenieSpaceConfig(updateResp, respSerialized), nil +} + +// OverrideChangeDesc handles the etag field. The user never sets it directly; +// we compare the stored etag against the remote one and Skip if they match. +// This mirrors ResourceDashboard.OverrideChangeDesc. +func (r *ResourceGenieSpace) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *resources.GenieSpaceConfig) error { + switch path.String() { + case "etag": + // change.New is always nil for etag because it's not present in the + // user-authored config. Compare stored etag with remote one to decide + // whether anything changed out-of-band since the last deploy. + if change.Old == change.Remote { + change.Action = deployplan.Skip + } else { + change.Action = deployplan.Update + } + } + return nil +} + +// hasUpdate reports whether entry has an Update-action change at the given path. +// HasChange alone matches Skip-action changes too, which we cannot use to drive +// request shaping for fields covered by ignore_remote_changes. +func hasUpdate(entry *PlanEntry, path *structpath.PathNode) bool { + if entry == nil { + return false + } + for s, change := range entry.Changes { + if change.Action != deployplan.Update { + continue + } + node, err := structpath.ParsePath(s) + if err != nil { + continue + } + if node.HasPrefix(path) { + return true + } + } + return false +} + +func (r *ResourceGenieSpace) DoDelete(ctx context.Context, id string, _ *resources.GenieSpaceConfig) error { + // TrashSpace returns 403 when the space is already gone; map that to the + // gone sentinel so deletion is idempotent. + return genieSpaceGoneError(r.client.Genie.TrashSpace(ctx, dashboards.GenieTrashSpaceRequest{ + SpaceId: id, + })) +} diff --git a/bundle/direct/dresources/genie_space_test.go b/bundle/direct/dresources/genie_space_test.go new file mode 100644 index 00000000000..deb476b0204 --- /dev/null +++ b/bundle/direct/dresources/genie_space_test.go @@ -0,0 +1,311 @@ +package dresources + +import ( + "errors" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestIsMissingGenieParentPathError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "standard missing error", + err: &apierr.APIError{ + StatusCode: 404, + ErrorCode: "NOT_FOUND", + Message: "not found", + }, + want: true, + }, + { + name: "invalid parameter tree node missing error", + err: &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "NOT_FOUND: Tree node with path /Workspace/foo does not exist", + }, + want: true, + }, + { + name: "other invalid parameter error", + err: &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "some other validation failure", + }, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, isMissingGenieParentPathError(tt.err)) + }) + } +} + +func TestGenieSpaceDoCreateRetriesWhenParentPathLooksMissing(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + req := dashboards.GenieCreateSpaceRequest{ + Title: "test genie space", + Description: "test description", + ParentPath: "/Workspace/test-parent", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + } + + m.GetMockGenieAPI().EXPECT(). + CreateSpace(ctx, req). + Return(nil, &apierr.APIError{ + StatusCode: 400, + ErrorCode: "INVALID_PARAMETER_VALUE", + Message: "NOT_FOUND: Tree node with path /Workspace/test-parent does not exist", + }). + Once() + + m.GetMockWorkspaceAPI().EXPECT(). + MkdirsByPath(ctx, "/Workspace/test-parent"). + Return(nil). + Once() + + m.GetMockGenieAPI().EXPECT(). + CreateSpace(ctx, req). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "test genie space", + Description: "test description", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }, nil). + Once() + + id, state, err := r.DoCreate(ctx, &resources.GenieSpaceConfig{ + Title: "test genie space", + Description: "test description", + ParentPath: "/Workspace/test-parent", + WarehouseId: "test-warehouse-id", + SerializedSpace: "{}", + }) + require.NoError(t, err) + assert.Equal(t, "space-id", id) + require.NotNil(t, state) + assert.Equal(t, "test genie space", state.Title) +} + +func TestGenieSpaceDoUpdateOmitsSerializedSpaceWhenUnchanged(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + // Plan entry indicates only title changed; serialized_space is absent. + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + SerializedSpace: "{\"remote\":\"edit\"}", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + SerializedSpace: "{\"local\":\"stale\"}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"remote\":\"edit\"}", state.SerializedSpace) +} + +func TestGenieSpaceDoUpdateSendsSerializedSpaceWhenChanged(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "serialized_space": {Action: deployplan.Update, Old: "{}", New: "{\"v\":1}"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + SerializedSpace: "{\"v\":1}", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + SerializedSpace: "{\"v\":1}", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + SerializedSpace: "{\"v\":1}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"v\":1}", state.SerializedSpace) +} + +func TestGenieSpaceDoUpdateRoundTripsEtag(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + // The stored etag (etag-7) must NOT be sent as an If-Match guard — it would + // 409 after a backend serialized_space schema migration. Only the etag from + // the response is persisted, for drift detection on the next plan. + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + Etag: "etag-8", + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + Etag: "etag-7", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "etag-8", state.Etag) +} + +func TestGenieSpaceDoUpdateKeepsPriorSerializedSpaceWhenBothEmpty(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + // Only title changed; serialized_space should be omitted from the request. + entry := &deployplan.PlanEntry{ + Changes: deployplan.Changes{ + "title": {Action: deployplan.Update, Old: "old", New: "new"}, + }, + } + + m.GetMockGenieAPI().EXPECT(). + UpdateSpace(ctx, dashboards.GenieUpdateSpaceRequest{ + SpaceId: "space-id", + Title: "new", + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id", + Title: "new", + // API also omits serialized_space; we should keep the prior local value. + }, nil). + Once() + + state, err := r.DoUpdate(ctx, "space-id", &resources.GenieSpaceConfig{ + Title: "new", + SerializedSpace: "{\"keep\":\"me\"}", + }, entry) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, "{\"keep\":\"me\"}", state.SerializedSpace) +} + +func TestGenieSpaceOverrideChangeDescEtag(t *testing.T) { + r := &ResourceGenieSpace{} + etagPath := structpath.MustParsePath("etag") + + t.Run("Skip when stored matches remote", func(t *testing.T) { + change := &ChangeDesc{Old: "etag-7", Remote: "etag-7"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), etagPath, change, nil)) + assert.Equal(t, deployplan.Skip, change.Action) + }) + + t.Run("Update when stored differs from remote", func(t *testing.T) { + change := &ChangeDesc{Old: "etag-7", Remote: "etag-8"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), etagPath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) + + t.Run("Other paths are untouched", func(t *testing.T) { + titlePath := structpath.MustParsePath("title") + change := &ChangeDesc{Action: deployplan.Update, Old: "a", Remote: "b"} + require.NoError(t, r.OverrideChangeDesc(t.Context(), titlePath, change, nil)) + assert.Equal(t, deployplan.Update, change.Action) + }) +} + +func TestGenieSpaceGoneErrorMapsForbidden(t *testing.T) { + // The Genie API returns 403 (not 404) for a missing space; it must surface + // as the framework's gone sentinel so a deleted space is recreated (read) + // or tolerated (delete). + forbidden := &apierr.APIError{StatusCode: 403, Message: "dataRoom is not user-facing"} + assert.ErrorIs(t, genieSpaceGoneError(forbidden), apierr.ErrResourceDoesNotExist) + + // Unrelated errors pass through unchanged. + other := errors.New("boom") + assert.Equal(t, other, genieSpaceGoneError(other)) + + // A nil error stays nil. + assert.NoError(t, genieSpaceGoneError(nil)) +} + +func TestGenieSpaceDoReadTreatsForbiddenAsGone(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + m.GetMockGenieAPI().EXPECT(). + GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id", + IncludeSerializedSpace: true, + }). + Return(nil, &apierr.APIError{StatusCode: 403, Message: "dataRoom is not user-facing"}). + Once() + + _, err := r.DoRead(ctx, "space-id") + require.Error(t, err) + assert.ErrorIs(t, err, apierr.ErrResourceDoesNotExist) +} + +func TestGenieSpaceDoDeleteToleratesForbidden(t *testing.T) { + ctx := t.Context() + m := mocks.NewMockWorkspaceClient(t) + r := (&ResourceGenieSpace{}).New(m.WorkspaceClient) + + m.GetMockGenieAPI().EXPECT(). + TrashSpace(ctx, dashboards.GenieTrashSpaceRequest{SpaceId: "space-id"}). + Return(&apierr.APIError{StatusCode: 403, Message: "dataRoom is not user-facing"}). + Once() + + err := r.DoDelete(ctx, "space-id", nil) + require.Error(t, err) + assert.ErrorIs(t, err, apierr.ErrResourceDoesNotExist) +} diff --git a/bundle/direct/dresources/permissions.go b/bundle/direct/dresources/permissions.go index 91ca9000aaf..6e1fa79d811 100644 --- a/bundle/direct/dresources/permissions.go +++ b/bundle/direct/dresources/permissions.go @@ -18,6 +18,7 @@ var permissionResourceToObjectType = map[string]string{ "apps": "/apps/", "clusters": "/clusters/", "dashboards": "/dashboards/", + "genie_spaces": "/genie/", "database_instances": "/database-instances/", "postgres_projects": "/database-projects/", "jobs": "/jobs/", diff --git a/bundle/direct/dresources/resources.generated.yml b/bundle/direct/dresources/resources.generated.yml index b73eb8931b2..61994d371a9 100644 --- a/bundle/direct/dresources/resources.generated.yml +++ b/bundle/direct/dresources/resources.generated.yml @@ -175,6 +175,12 @@ resources: - field: effective_file_event_queue reason: spec:output_only + genie_spaces: + + ignore_remote_changes: + - field: etag + reason: spec:output_only + # jobs: no api field behaviors # model_serving_endpoints: no api field behaviors diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index c283e6f0b68..fde20f5af59 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -359,6 +359,13 @@ resources: - field: dataset_schema reason: input_only + genie_spaces: + ignore_remote_changes: + # serialized_space locally (structured YAML) and remotely (JSON string) will differ + # textually, so we cannot meaningfully compare them for drift. + - field: serialized_space + reason: etag_based + apps: recreate_on_changes: - field: name diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index c9fefb7c1f2..fee09ed119f 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -85,6 +85,9 @@ var knownMissingInStateType = map[string][]string{ "dashboards": { "file_path", }, + "genie_spaces": { + "file_path", + }, "secret_scopes": { "backend_type", "keyvault_metadata", diff --git a/bundle/direct/dstate/state.go b/bundle/direct/dstate/state.go index 5b2a70adbb3..7f719674ebb 100644 --- a/bundle/direct/dstate/state.go +++ b/bundle/direct/dstate/state.go @@ -439,9 +439,14 @@ func ExportStateFromData(data Database) resourcestate.ExportedResourcesMap { result := make(resourcestate.ExportedResourcesMap) for key, entry := range data.State { var etag string - // Extract etag for dashboards. - // covered by test case: bundle/deploy/dashboard/detect-change - if strings.Contains(key, ".dashboards.") && len(entry.State) > 0 { + // Extract etag for resources that use it for drift detection + // (dashboards and genie_spaces). Both follow the same pattern of + // persisting the backend-returned etag in state and comparing it + // against the remote on the next plan via OverrideChangeDesc. + // covered by test cases: + // - bundle/deploy/dashboard/detect-change + // - bundle/resources/genie_spaces/simple + if (strings.Contains(key, ".dashboards.") || strings.Contains(key, ".genie_spaces.")) && len(entry.State) > 0 { var holder struct { Etag string `json:"etag"` } diff --git a/bundle/docsgen/output/reference.md b/bundle/docsgen/output/reference.md index 16b7020cecd..92228810a38 100644 --- a/bundle/docsgen/output/reference.md +++ b/bundle/docsgen/output/reference.md @@ -1,7 +1,7 @@ --- description: 'Configuration reference for databricks.yml' last_update: - date: 2026-06-03 + date: 2026-06-04 --- @@ -479,6 +479,10 @@ resources: - Map - See [\_](#resourcesexternal_locations). +- - `genie_spaces` + - Map + - See [\_](#resourcesgenie_spaces). + - - `jobs` - Map - The job definitions for the bundle, where each key is the name of the job. See [\_](/dev-tools/bundles/resources.md#jobs). @@ -1502,6 +1506,122 @@ external_locations: ::: +### resources.genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - Description of the Genie space shown alongside the title in the Databricks UI. + +- - `etag` + - String + - + +- - `file_path` + - String + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + +- - `lifecycle` + - Map + - See [\_](#resourcesgenie_spacesnamelifecycle). + +- - `parent_path` + - String + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + +- - `permissions` + - Sequence + - See [\_](#resourcesgenie_spacesnamepermissions). + +- - `serialized_space` + - Any + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + +- - `space_id` + - String + - + +- - `title` + - String + - Title of the Genie space shown in the Databricks UI. + +- - `warehouse_id` + - String + - ID of the SQL warehouse used to run queries for this Genie space. + +::: + + +### resources.genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### resources.genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ### resources.pipelines **`Type: Map`** @@ -2018,6 +2138,10 @@ postgres_projects: - String - +- - `purge_on_delete` + - Boolean + - + ::: @@ -3026,6 +3150,10 @@ The resource definitions for the target. - Map - See [\_](#targetsnameresourcesexternal_locations). +- - `genie_spaces` + - Map + - See [\_](#targetsnameresourcesgenie_spaces). + - - `jobs` - Map - The job definitions for the bundle, where each key is the name of the job. See [\_](/dev-tools/bundles/resources.md#jobs). @@ -4049,6 +4177,122 @@ external_locations: ::: +### targets._name_.resources.genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - Description of the Genie space shown alongside the title in the Databricks UI. + +- - `etag` + - String + - + +- - `file_path` + - String + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + +- - `lifecycle` + - Map + - See [\_](#targetsnameresourcesgenie_spacesnamelifecycle). + +- - `parent_path` + - String + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + +- - `permissions` + - Sequence + - See [\_](#targetsnameresourcesgenie_spacesnamepermissions). + +- - `serialized_space` + - Any + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + +- - `space_id` + - String + - + +- - `title` + - String + - Title of the Genie space shown in the Databricks UI. + +- - `warehouse_id` + - String + - ID of the SQL warehouse used to run queries for this Genie space. + +::: + + +### targets._name_.resources.genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### targets._name_.resources.genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ### targets._name_.resources.pipelines **`Type: Map`** @@ -4565,6 +4809,10 @@ postgres_projects: - String - +- - `purge_on_delete` + - Boolean + - + ::: diff --git a/bundle/docsgen/output/resources.md b/bundle/docsgen/output/resources.md index e4f902e6146..bce0e3f9fb7 100644 --- a/bundle/docsgen/output/resources.md +++ b/bundle/docsgen/output/resources.md @@ -1,7 +1,7 @@ --- description: 'Learn about resources supported by Declarative Automation Bundles and how to configure them.' last_update: - date: 2026-06-03 + date: 2026-06-04 --- @@ -3086,6 +3086,122 @@ The privileges assigned to the principal. ::: +## genie_spaces + +**`Type: Map`** + + + +```yaml +genie_spaces: + : + : +``` + + +:::list-table + +- - Key + - Type + - Description + +- - `description` + - String + - Description of the Genie space shown alongside the title in the Databricks UI. + +- - `etag` + - String + - + +- - `file_path` + - String + - Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + +- - `lifecycle` + - Map + - See [\_](#genie_spacesnamelifecycle). + +- - `parent_path` + - String + - Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + +- - `permissions` + - Sequence + - See [\_](#genie_spacesnamepermissions). + +- - `serialized_space` + - Any + - Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + +- - `space_id` + - String + - + +- - `title` + - String + - Title of the Genie space shown in the Databricks UI. + +- - `warehouse_id` + - String + - ID of the SQL warehouse used to run queries for this Genie space. + +::: + + +### genie_spaces._name_.lifecycle + +**`Type: Map`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `prevent_destroy` + - Boolean + - Lifecycle setting to prevent the resource from being destroyed. + +::: + + +### genie_spaces._name_.permissions + +**`Type: Sequence`** + + + + + +:::list-table + +- - Key + - Type + - Description + +- - `group_name` + - String + - The name of the group that has the permission set in level. + +- - `level` + - String + - The allowed permission for user, group, service principal defined for this permission. + +- - `service_principal_name` + - String + - The name of the service principal that has the permission set in level. + +- - `user_name` + - String + - The name of the user that has the permission set in level. + +::: + + ## jobs **`Type: Map`** @@ -10771,6 +10887,10 @@ postgres_projects: - String - +- - `purge_on_delete` + - Boolean + - + ::: @@ -11909,6 +12029,10 @@ Lifecycle is a struct that contains the lifecycle settings for a resource. It co - Boolean - Lifecycle setting to prevent the resource from being destroyed. +- - `started` + - Boolean + - Lifecycle setting to deploy the resource in started mode. Only supported for apps, clusters, and sql_warehouses in direct deployment mode. + ::: diff --git a/bundle/generate/genie_space.go b/bundle/generate/genie_space.go new file mode 100644 index 00000000000..bf00abbc66e --- /dev/null +++ b/bundle/generate/genie_space.go @@ -0,0 +1,42 @@ +package generate + +import ( + "path" + "strings" + + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +func ConvertGenieSpaceToValue(genieSpace *dashboards.GenieSpace, filePath string) (dyn.Value, error) { + // Emit only the fields a user authors in a bundle. serialized_space is + // written to a separate file and referenced via file_path, and output-only + // fields (e.g. space_id, etag) must not appear in the generated config, so + // we build the value field by field rather than marshaling the struct. + dv := map[string]dyn.Value{ + "title": dyn.NewValue(genieSpace.Title, []dyn.Location{{Line: 1}}), + "warehouse_id": dyn.NewValue(genieSpace.WarehouseId, []dyn.Location{{Line: 2}}), + "file_path": dyn.NewValue(filePath, []dyn.Location{{Line: 3}}), + } + + if genieSpace.Description != "" { + dv["description"] = dyn.NewValue(genieSpace.Description, []dyn.Location{{Line: 4}}) + } + + if genieSpace.ParentPath != "" { + dv["parent_path"] = dyn.NewValue(ensureWorkspacePrefix(genieSpace.ParentPath), []dyn.Location{{Line: 5}}) + } + + return dyn.V(dv), nil +} + +// ensureWorkspacePrefix re-adds the /Workspace prefix that the Genie GET API +// strips from parent_path, so the generated config matches the convention used +// in hand-written bundles and in deployment state (mirrors the equivalent +// helper in bundle/direct/dresources/dashboard.go). +func ensureWorkspacePrefix(parentPath string) string { + if parentPath == "/Workspace" || strings.HasPrefix(parentPath, "/Workspace/") { + return parentPath + } + return path.Join("/Workspace", parentPath) +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index fd5f4275d12..69d7c9d025d 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -195,6 +195,9 @@ github.com/databricks/cli/bundle/config.Resources: "external_locations": "description": |- PLACEHOLDER + "genie_spaces": + "description": |- + PLACEHOLDER "jobs": "description": |- The job definitions for the bundle, where each key is the name of the job. @@ -668,6 +671,37 @@ github.com/databricks/cli/bundle/config/resources.ExternalLocation: "url": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.GenieSpace: + "description": + "description": |- + Description of the Genie space shown alongside the title in the Databricks UI. + "etag": + "description": |- + PLACEHOLDER + "file_path": + "description": |- + Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`. + "lifecycle": + "description": |- + PLACEHOLDER + "parent_path": + "description": |- + Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource. + "permissions": + "description": |- + PLACEHOLDER + "serialized_space": + "description": |- + Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`. + "space_id": + "description": |- + PLACEHOLDER + "title": + "description": |- + Title of the Genie space shown in the Databricks UI. + "warehouse_id": + "description": |- + ID of the SQL warehouse used to run queries for this Genie space. github.com/databricks/cli/bundle/config/resources.JobPermission: "group_name": "description": |- diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index 9e8f45fbd9a..8beb3e25a4a 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -62,6 +62,8 @@ var EnumFields = map[string][]string{ "resources.external_locations.*.encryption_details.sse_encryption_details.algorithm": {"AWS_SSE_KMS", "AWS_SSE_S3"}, "resources.external_locations.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, + "resources.genie_spaces.*.permissions[*].level": {"CAN_ATTACH_TO", "CAN_BIND", "CAN_CREATE", "CAN_CREATE_APP", "CAN_EDIT", "CAN_EDIT_METADATA", "CAN_MANAGE", "CAN_MANAGE_PRODUCTION_VERSIONS", "CAN_MANAGE_RUN", "CAN_MANAGE_STAGING_VERSIONS", "CAN_MONITOR", "CAN_MONITOR_ONLY", "CAN_QUERY", "CAN_READ", "CAN_RESTART", "CAN_RUN", "CAN_USE", "CAN_VIEW", "CAN_VIEW_METADATA", "IS_OWNER"}, + "resources.jobs.*.continuous.pause_status": {"PAUSED", "UNPAUSED"}, "resources.jobs.*.continuous.task_retry_mode": {"NEVER", "ON_FAILURE"}, "resources.jobs.*.deployment.kind": {"BUNDLE", "SYSTEM_MANAGED"}, diff --git a/bundle/internal/validation/generated/required_fields.go b/bundle/internal/validation/generated/required_fields.go index ad6437f1887..3d47858587e 100644 --- a/bundle/internal/validation/generated/required_fields.go +++ b/bundle/internal/validation/generated/required_fields.go @@ -64,6 +64,8 @@ var RequiredFields = map[string][]string{ "resources.external_locations.*": {"credential_name", "name", "url"}, + "resources.genie_spaces.*.permissions[*]": {"level"}, + "resources.jobs.*.deployment": {"kind"}, "resources.jobs.*.environments[*]": {"environment_key"}, "resources.jobs.*.git_source": {"git_provider", "git_url"}, diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index a40506ebb18..1b86e37dcee 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -158,6 +158,9 @@ func Initialize(ctx context.Context, b *bundle.Bundle) { // Validate that no dashboard etags are set. They are purely internal state and should not be set by the user. validate.ValidateDashboardEtags(), + // Validate that no genie space etags are set. They are purely internal state and should not be set by the user. + validate.ValidateGenieSpaceEtags(), + // Reads (dynamic): * (strings) (searches for ${resources.*} references) // Warns (TF engine) or errors (direct engine) when a cross-resource reference // points to a Terraform-only field with no DABs equivalent. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 476eaaf3637..1a8b4369e83 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -760,6 +760,56 @@ } ] }, + "resources.GenieSpace": { + "oneOf": [ + { + "type": "object", + "properties": { + "description": { + "description": "Description of the Genie space shown alongside the title in the Databricks UI.", + "$ref": "#/$defs/string" + }, + "etag": { + "$ref": "#/$defs/string" + }, + "file_path": { + "description": "Local path to a `.geniespace.json` file holding the serialized Genie space definition. The contents are inlined into `serialized_space` at deploy time. Mutually exclusive with an inline `serialized_space`.", + "$ref": "#/$defs/string" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "parent_path": { + "description": "Workspace folder under which to create the Genie space. Immutable: changing this field recreates the resource.", + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "serialized_space": { + "description": "Serialized Genie space body. May be provided inline as a JSON string (or YAML that will be marshalled to JSON) or referenced via `file_path`. To round-trip an existing space into a bundle, use `databricks bundle generate genie-space`.", + "$ref": "#/$defs/interface" + }, + "space_id": { + "$ref": "#/$defs/string" + }, + "title": { + "description": "Title of the Genie space shown in the Databricks UI.", + "$ref": "#/$defs/string" + }, + "warehouse_id": { + "description": "ID of the SQL warehouse used to run queries for this Genie space.", + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { @@ -2655,6 +2705,9 @@ "external_locations": { "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation" }, + "genie_spaces": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -12206,6 +12259,20 @@ } ] }, + "resources.GenieSpace": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.GenieSpace" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Job": { "oneOf": [ { diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index 198d9191196..04e3930f36c 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -741,6 +741,39 @@ "url" ] }, + "resources.GenieSpace": { + "type": "object", + "properties": { + "description": { + "$ref": "#/$defs/string" + }, + "file_path": { + "$ref": "#/$defs/string" + }, + "lifecycle": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" + }, + "parent_path": { + "$ref": "#/$defs/string" + }, + "permissions": { + "$ref": "#/$defs/slice/github.com/databricks/cli/bundle/config/resources.Permission" + }, + "serialized_space": { + "$ref": "#/$defs/interface" + }, + "space_id": { + "$ref": "#/$defs/string" + }, + "title": { + "$ref": "#/$defs/string" + }, + "warehouse_id": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, "resources.Job": { "type": "object", "properties": { @@ -2615,6 +2648,9 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.ExternalLocation", "x-since-version": "v0.289.0" }, + "genie_spaces": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.GenieSpace" + }, "jobs": { "description": "The job definitions for the bundle, where each key is the name of the job.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Job", @@ -10211,6 +10247,12 @@ "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.ExternalLocation" } }, + "resources.GenieSpace": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.GenieSpace" + } + }, "resources.Job": { "type": "object", "additionalProperties": { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 57f31450d93..672cd9855b2 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -39,6 +39,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.volumes.test_volume": {ID: "1"}, "resources.clusters.test_cluster": {ID: "1"}, "resources.dashboards.test_dashboard": {ID: "1"}, + "resources.genie_spaces.test_genie_space": {ID: "1"}, "resources.apps.test_app": {ID: "app1"}, "resources.secret_scopes.test_secret_scope": {ID: "secret_scope1"}, "resources.sql_warehouses.test_sql_warehouse": {ID: "1"}, @@ -96,6 +97,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { assert.Equal(t, "1", config.Resources.Dashboards["test_dashboard"].ID) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "app1", config.Resources.Apps["test_app"].ID) assert.Empty(t, config.Resources.Apps["test_app"].Name) assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.Apps["test_app"].ModifiedStatus) @@ -228,6 +232,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "test_genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space", + }, + }, + }, Apps: map[string]*resources.App{ "test_app": { App: apps.App{ @@ -374,6 +385,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Empty(t, config.Resources.Dashboards["test_dashboard"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard"].ModifiedStatus) + assert.Empty(t, config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Empty(t, config.Resources.Apps["test_app"].Name) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Apps["test_app"].ModifiedStatus) @@ -571,6 +585,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + GenieSpaces: map[string]*resources.GenieSpace{ + "test_genie_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space", + }, + }, + "test_genie_space_new": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "test_genie_space_new", + }, + }, + }, Apps: map[string]*resources.App{ "test_app": { App: apps.App{ @@ -767,6 +793,8 @@ func TestStateToBundleModifiedResources(t *testing.T) { "resources.clusters.test_cluster_old": {ID: "2"}, "resources.dashboards.test_dashboard": {ID: "1"}, "resources.dashboards.test_dashboard_old": {ID: "2"}, + "resources.genie_spaces.test_genie_space": {ID: "1"}, + "resources.genie_spaces.test_genie_space_old": {ID: "2"}, "resources.apps.test_app": {ID: "test_app"}, "resources.apps.test_app_old": {ID: "test_app_old"}, "resources.secret_scopes.test_secret_scope": {ID: "test_secret_scope"}, @@ -877,6 +905,13 @@ func TestStateToBundleModifiedResources(t *testing.T) { assert.Empty(t, config.Resources.Dashboards["test_dashboard_new"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.Dashboards["test_dashboard_new"].ModifiedStatus) + assert.Equal(t, "1", config.Resources.GenieSpaces["test_genie_space"].ID) + assert.Empty(t, config.Resources.GenieSpaces["test_genie_space"].ModifiedStatus) + assert.Equal(t, "2", config.Resources.GenieSpaces["test_genie_space_old"].ID) + assert.Equal(t, resources.ModifiedStatusDeleted, config.Resources.GenieSpaces["test_genie_space_old"].ModifiedStatus) + assert.Empty(t, config.Resources.GenieSpaces["test_genie_space_new"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.GenieSpaces["test_genie_space_new"].ModifiedStatus) + assert.Equal(t, "test_app", config.Resources.Apps["test_app"].Name) assert.Empty(t, config.Resources.Apps["test_app"].ModifiedStatus) assert.Equal(t, "test_app_old", config.Resources.Apps["test_app_old"].ID) diff --git a/cmd/bundle/generate.go b/cmd/bundle/generate.go index 9caf7fa1e37..12452293bc6 100644 --- a/cmd/bundle/generate.go +++ b/cmd/bundle/generate.go @@ -40,6 +40,7 @@ Use --bind to automatically bind the generated resource to the existing workspac cmd.AddCommand(generate.NewGenerateDashboardCommand()) cmd.AddCommand(generate.NewGenerateAlertCommand()) cmd.AddCommand(generate.NewGenerateAppCommand()) + cmd.AddCommand(generate.NewGenerateGenieSpaceCommand()) cmd.PersistentFlags().StringVar(&key, "key", "", `resource key to use for the generated configuration`) return cmd } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 7af4e01e92f..250d05e3693 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -275,7 +275,11 @@ func waitForChanges(ctx context.Context, w *databricks.WorkspaceClient, dashboar break } - time.Sleep(1 * time.Second) + select { + case <-ctx.Done(): + return + case <-time.After(1 * time.Second): + } } } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go new file mode 100644 index 00000000000..36ebdf54800 --- /dev/null +++ b/cmd/bundle/generate/genie_space.go @@ -0,0 +1,486 @@ +package generate + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "maps" + "os" + "path" + "path/filepath" + "slices" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/deploy/terraform" + "github.com/databricks/cli/bundle/direct/dstate" + "github.com/databricks/cli/bundle/generate" + "github.com/databricks/cli/bundle/phases" + "github.com/databricks/cli/bundle/resources" + "github.com/databricks/cli/bundle/statemgmt" + "github.com/databricks/cli/cmd/bundle/deployment" + "github.com/databricks/cli/cmd/bundle/utils" + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/textutil" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" +) + +const genieSpaceWatchInterval = 1 * time.Second + +type genieSpace struct { + // Lookup flag for one-time generate. + existingID string + + // Lookup flag for existing bundle resource. + resource string + + // Where to write the configuration and genie space representation. + resourceDir string + genieSpaceDir string + + // Force overwrite of existing files. + force bool + + // Watch for changes to the genie space. + watch bool + + // Relative path from the resource directory to the genie space directory. + relativeGenieSpaceDir string + + // Command. + cmd *cobra.Command + + // Automatically bind the generated resource to the existing resource. + bind bool + + // Output and error streams. + out io.Writer + err io.Writer +} + +func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string { + w := b.WorkspaceClient(ctx) + obj, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: g.existingID, + }) + if err != nil { + // The Genie API returns 403 (not 404) when a space does not exist. + if apierr.IsMissing(err) || errors.Is(err, apierr.ErrPermissionDenied) { + logdiag.LogError(ctx, fmt.Errorf("genie space with ID %s not found", g.existingID)) + return "" + } + logdiag.LogError(ctx, err) + return "" + } + + return obj.SpaceId +} + +func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, filename string) error { + if genieSpace.SerializedSpace == "" { + return fmt.Errorf("genie space response did not include serialized_space; refusing to write %s", filepath.ToSlash(filename)) + } + + // Unmarshal and remarshal the serialized genie space to ensure it is formatted correctly. + // The result will have alphabetically sorted keys and be indented. + data, err := remarshalJSON([]byte(genieSpace.SerializedSpace)) + if err != nil { + return err + } + + // Make sure the output directory exists. + if err := os.MkdirAll(filepath.Dir(filename), 0o755); err != nil { + return err + } + + // Clean the filename to ensure it is a valid path (and can be used on this OS). + filename = filepath.Clean(filename) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, filename) + if err != nil { + rel = filename + } + + // Verify that the file does not already exist. + info, err := os.Stat(filename) + if err == nil { + if info.IsDir() { + return fmt.Errorf("%s is a directory", filepath.ToSlash(rel)) + } + if !g.force { + return fmt.Errorf("%s already exists. Use --force to overwrite", filepath.ToSlash(rel)) + } + } + + cmdio.LogString(ctx, "Writing genie space to "+filepath.ToSlash(rel)) + return os.WriteFile(filename, data, 0o644) +} + +func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, key string) error { + // Save serialized genie space definition to the genie space directory. + genieSpaceBasename := key + ".geniespace.json" + genieSpacePath := filepath.Join(g.genieSpaceDir, genieSpaceBasename) + err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath) + if err != nil { + return err + } + + // Synthesize resource configuration. + v, err := generate.ConvertGenieSpaceToValue(genieSpace, path.Join(g.relativeGenieSpaceDir, genieSpaceBasename)) + if err != nil { + return err + } + + result := map[string]dyn.Value{ + "resources": dyn.V(map[string]dyn.Value{ + "genie_spaces": dyn.V(map[string]dyn.Value{ + key: v, + }), + }), + } + + // Make sure the output directory exists. + if err := os.MkdirAll(g.resourceDir, 0o755); err != nil { + return err + } + + // Save the configuration to the resource directory. + resourcePath := filepath.Join(g.resourceDir, key+".genie_space.yml") + saver := yamlsaver.NewSaverWithStyle(map[string]yaml.Style{ + "title": yaml.DoubleQuotedStyle, + }) + + // Attempt to make the path relative to the bundle root. + rel, err := filepath.Rel(b.BundleRootPath, resourcePath) + if err != nil { + rel = resourcePath + } + + cmdio.LogString(ctx, "Writing configuration to "+filepath.ToSlash(rel)) + err = saver.SaveAsYAML(result, resourcePath, g.force) + if err != nil { + return err + } + + return nil +} + +func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle.Bundle) { + resource, ok := b.Config.Resources.GenieSpaces[g.resource] + if !ok { + logdiag.LogError(ctx, fmt.Errorf("genie space resource %q is not defined", g.resource)) + return + } + + if resource.FilePath == "" { + logdiag.LogError(ctx, fmt.Errorf("genie space resource %q has no file path defined", g.resource)) + return + } + + genieSpaceID := resource.ID + genieSpacePath := resource.FilePath + + w := b.WorkspaceClient(ctx) + + first := true + for { + genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: genieSpaceID, + IncludeSerializedSpace: true, + }) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + // Genie has no remote modification timestamp we can poll. Compare + // the canonicalized remote body against the on-disk body and only + // re-save when they differ. The first iteration always saves, to + // match the prior behavior of an unconditional initial sync. + shouldSave := first + if !first { + differs, err := genieSpaceBodyDiffersFromDisk(genieSpace.SerializedSpace, genieSpacePath) + if err != nil { + logdiag.LogError(ctx, err) + return + } + shouldSave = differs + } + + if shouldSave { + if err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath); err != nil { + logdiag.LogError(ctx, err) + return + } + } + + if !g.watch { + return + } + + first = false + select { + case <-ctx.Done(): + return + case <-time.After(genieSpaceWatchInterval): + } + } +} + +// genieSpaceBodyDiffersFromDisk reports whether the canonicalized remote +// serialized_space differs from the contents of filename. +func genieSpaceBodyDiffersFromDisk(remoteSerialized, filename string) (bool, error) { + if remoteSerialized == "" { + return false, nil + } + canonical, err := remarshalJSON([]byte(remoteSerialized)) + if err != nil { + return false, err + } + onDisk, err := os.ReadFile(filename) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return true, nil + } + return false, err + } + return !bytes.Equal(canonical, onDisk), nil +} + +func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, genieSpaceID string) { + w := b.WorkspaceClient(ctx) + genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ + SpaceId: genieSpaceID, + IncludeSerializedSpace: true, + }) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + key := g.cmd.Flag("key").Value.String() + if key == "" { + key = textutil.NormalizeString(genieSpace.Title) + } + err = g.saveConfiguration(ctx, b, genieSpace, key) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + if g.bind { + err = deployment.BindResource(g.cmd, key, genieSpaceID, true, false, true) + if err != nil { + logdiag.LogError(ctx, err) + return + } + cmdio.LogString(ctx, fmt.Sprintf("Successfully bound genie space with an id '%s'", genieSpaceID)) + } +} + +func (g *genieSpace) initialize(ctx context.Context, b *bundle.Bundle) { + // Make the paths absolute if they aren't already. + if !filepath.IsAbs(g.resourceDir) { + g.resourceDir = filepath.Join(b.BundleRootPath, g.resourceDir) + } + if !filepath.IsAbs(g.genieSpaceDir) { + g.genieSpaceDir = filepath.Join(b.BundleRootPath, g.genieSpaceDir) + } + + // Make sure we know how the genie space path is relative to the resource path. + rel, err := filepath.Rel(g.resourceDir, g.genieSpaceDir) + if err != nil { + logdiag.LogError(ctx, err) + return + } + + g.relativeGenieSpaceDir = filepath.ToSlash(rel) +} + +func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { + phases.Initialize(ctx, b) + if logdiag.HasError(ctx) { + return + } + + requiredEngine, err := utils.ResolveEngineSetting(ctx, b) + if err != nil { + logdiag.LogError(ctx, err) + return + } + ctx, stateDesc := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true), requiredEngine) + if logdiag.HasError(ctx) { + return + } + + var state statemgmt.ExportedResourcesMap + if stateDesc.Engine.IsDirect() { + _, localPath := b.StateFilenameDirect(ctx) + if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { + logdiag.LogError(ctx, err) + return + } + state = b.DeploymentBundle.ExportState(ctx) + } else { + var err error + state, err = terraform.ParseResourcesState(ctx, b) + if err != nil { + logdiag.LogError(ctx, err) + return + } + } + + bundle.ApplySeqContext(ctx, b, + statemgmt.Load(state), + ) + if logdiag.HasError(ctx) { + return + } + + g.updateGenieSpaceForResource(ctx, b) +} + +func (g *genieSpace) runForExisting(ctx context.Context, b *bundle.Bundle) { + // Resolve the ID of the genie space to generate configuration for. + genieSpaceID := g.resolveFromID(ctx, b) + if logdiag.HasError(ctx) { + return + } + + g.generateForExisting(ctx, b, genieSpaceID) +} + +func (g *genieSpace) RunE(cmd *cobra.Command, args []string) error { + ctx := logdiag.InitContext(cmd.Context()) + cmd.SetContext(ctx) + + b := root.MustConfigureBundle(cmd) + if b == nil || logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + g.initialize(ctx, b) + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + if g.resource != "" { + g.runForResource(ctx, b) + } else { + g.runForExisting(ctx, b) + } + + if logdiag.HasError(ctx) { + return root.ErrAlreadyPrinted + } + + return nil +} + +// filterGenieSpaces returns a filter that only includes genie spaces. +func filterGenieSpaces(ref resources.Reference) bool { + return ref.Description.SingularName == "genie_space" +} + +// genieSpaceResourceCompletion executes to autocomplete the argument to the resource flag. +func genieSpaceResourceCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + b := root.MustConfigureBundle(cmd) + if logdiag.HasError(cmd.Context()) { + return nil, cobra.ShellCompDirectiveError + } + + if b == nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + return slices.Collect(maps.Keys(resources.Completions(b, filterGenieSpaces))), cobra.ShellCompDirectiveNoFileComp +} + +func NewGenerateGenieSpaceCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "genie-space", + Short: "Generate configuration for a Genie space", + Long: `Generate bundle configuration for an existing Databricks Genie space. + +This command downloads an existing Genie space and creates bundle files +that you can use to deploy the Genie space to other environments or manage it as code. + +Examples: + # Import Genie space by ID + databricks bundle generate genie-space --existing-id abc123 --key my_genie_space + + # Watch for changes to keep bundle in sync with UI modifications + databricks bundle generate genie-space --resource my_genie_space --watch --force + +Use --key to set the resource name in the generated configuration. + +What gets generated: +- Genie space configuration YAML file with settings and a reference to the Genie space definition +- Genie space definition (.geniespace.json) file with the serialized space content + +Sync workflow for Genie space development: +When developing Genie spaces, you can modify them in the Databricks UI and sync +changes back to your bundle: + +1. Make changes to Genie space in the Databricks UI +2. Run: databricks bundle generate genie-space --resource my_genie_space --force +3. Commit changes to version control +4. Deploy to other environments with: databricks bundle deploy --target prod + +The --watch flag continuously polls for remote changes and updates your local +bundle files automatically, useful during active Genie space development.`, + } + + g := &genieSpace{ + out: cmd.OutOrStdout(), + err: cmd.ErrOrStderr(), + } + + // Lookup flags. + cmd.Flags().StringVar(&g.existingID, "existing-id", "", `ID of the Genie space to generate configuration for`) + cmd.Flags().StringVar(&g.resource, "resource", "", `resource key of Genie space to watch for changes`) + + // Alias lookup flag that includes the resource type name. + cmd.Flags().StringVar(&g.existingID, "existing-genie-space-id", "", `ID of the Genie space to generate configuration for`) + cmd.Flags().MarkHidden("existing-genie-space-id") + + // Output flags. + cmd.Flags().StringVarP(&g.resourceDir, "resource-dir", "d", "resources", `directory to write the configuration to`) + cmd.Flags().StringVarP(&g.genieSpaceDir, "genie-space-dir", "s", "src", `directory to write the Genie space representation to`) + cmd.Flags().BoolVarP(&g.force, "force", "f", false, `force overwrite existing files in the output directory`) + + cmd.Flags().BoolVarP(&g.bind, "bind", "b", false, `automatically bind the generated Genie space config to the existing Genie space`) + cmd.Flags().MarkHidden("bind") + + // Exactly one of the lookup flags must be provided. + cmd.MarkFlagsOneRequired( + "existing-id", + "resource", + ) + + // Watch flag. This is relevant only in combination with the resource flag. + cmd.Flags().BoolVar(&g.watch, "watch", false, `watch for changes to the Genie space and update the configuration`) + + // Make sure the watch flag is only used with the existing-resource flag. + cmd.MarkFlagsMutuallyExclusive("watch", "existing-id") + + // Make sure the bind flag is only used with the existing-resource flag. + cmd.MarkFlagsMutuallyExclusive("bind", "resource") + + // Completion for the resource flag. + cmd.RegisterFlagCompletionFunc("resource", genieSpaceResourceCompletion) + + cmd.RunE = g.RunE + g.cmd = cmd + return cmd +} diff --git a/cmd/bundle/generate/genie_space_test.go b/cmd/bundle/generate/genie_space_test.go new file mode 100644 index 00000000000..746b2b1c3af --- /dev/null +++ b/cmd/bundle/generate/genie_space_test.go @@ -0,0 +1,126 @@ +package generate + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/databricks-sdk-go/experimental/mocks" + "github.com/databricks/databricks-sdk-go/service/dashboards" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +func newGenieSpaceTestBundle(t *testing.T, m *mocks.MockWorkspaceClient, filePath string) *bundle.Bundle { + t.Helper() + b := &bundle.Bundle{ + BundleRootPath: t.TempDir(), + Config: config.Root{ + Resources: config.Resources{ + GenieSpaces: map[string]*resources.GenieSpace{ + "my_space": { + GenieSpaceConfig: resources.GenieSpaceConfig{ + Title: "My Space", + }, + FilePath: filePath, + }, + }, + }, + }, + } + b.Config.Resources.GenieSpaces["my_space"].ID = "space-id-1" + b.SetWorkpaceClient(m.WorkspaceClient) + return b +} + +func TestGenieSpace_UpdateForResource_WritesFileWhenNotWatching(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "my_space.geniespace.json") + + m := mocks.NewMockWorkspaceClient(t) + m.GetMockGenieAPI().EXPECT(). + GetSpace(mock.Anything, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id-1", + IncludeSerializedSpace: true, + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id-1", + Title: "My Space", + SerializedSpace: `{"version":1}`, + }, nil). + Once() + + g := &genieSpace{ + resource: "my_space", + force: true, + } + b := newGenieSpaceTestBundle(t, m, filePath) + + ctx, _ := cmdio.NewTestContextWithStdout(t.Context()) + ctx = logdiag.InitContext(ctx) + logdiag.SetCollect(ctx, true) + g.updateGenieSpaceForResource(ctx, b) + + require.Empty(t, logdiag.FlushCollected(ctx)) + + contents, err := os.ReadFile(filePath) + require.NoError(t, err) + assert.Contains(t, string(contents), `"version"`) +} + +func TestGenieSpace_UpdateForResource_WatchExitsOnCancel(t *testing.T) { + dir := t.TempDir() + filePath := filepath.Join(dir, "my_space.geniespace.json") + + m := mocks.NewMockWorkspaceClient(t) + // Allow any number of GetSpace calls; we don't know how many fire before cancel. + m.GetMockGenieAPI().EXPECT(). + GetSpace(mock.Anything, dashboards.GenieGetSpaceRequest{ + SpaceId: "space-id-1", + IncludeSerializedSpace: true, + }). + Return(&dashboards.GenieSpace{ + SpaceId: "space-id-1", + Title: "My Space", + SerializedSpace: `{"version":1}`, + }, nil). + Maybe() + + g := &genieSpace{ + resource: "my_space", + force: true, + watch: true, + } + b := newGenieSpaceTestBundle(t, m, filePath) + + base, _ := cmdio.NewTestContextWithStdout(t.Context()) + ctx, cancel := context.WithCancel(logdiag.InitContext(base)) + logdiag.SetCollect(ctx, true) + + done := make(chan struct{}) + go func() { + g.updateGenieSpaceForResource(ctx, b) + close(done) + }() + + // First iteration always saves. Wait until the file lands, then cancel. + require.Eventually(t, func() bool { + _, err := os.Stat(filePath) + return err == nil + }, 2*time.Second, 10*time.Millisecond, "expected initial save to write file") + + cancel() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watch loop did not exit promptly after ctx cancel") + } +} diff --git a/cmd/experimental/workspace_open_test.go b/cmd/experimental/workspace_open_test.go index f81e79be52b..9589b48a126 100644 --- a/cmd/experimental/workspace_open_test.go +++ b/cmd/experimental/workspace_open_test.go @@ -67,7 +67,7 @@ func TestBuildWorkspaceURLFragmentBasedResources(t *testing.T) { func TestBuildWorkspaceURLUnknownResourceType(t *testing.T) { _, err := workspaceurls.BuildResourceURL("https://myworkspace.databricks.com", "unknown", "123", "") assert.ErrorContains(t, err, "unknown resource type \"unknown\"") - assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") + assert.ErrorContains(t, err, "alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses") } func TestBuildWorkspaceURLHostWithTrailingSlash(t *testing.T) { @@ -115,6 +115,7 @@ func TestWorkspaceOpenCommandCompletion(t *testing.T) { "database_catalogs", "database_instances", "experiments", + "genie_spaces", "jobs", "model_serving_endpoints", "models", @@ -145,7 +146,7 @@ func TestWorkspaceOpenCommandCompletionSecondArg(t *testing.T) { func TestWorkspaceOpenCommandHelpText(t *testing.T) { cmd := newWorkspaceOpenCommand() - assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") + assert.Contains(t, cmd.Long, "Supported resource types: alerts, apps, catalogs, clusters, dashboards, database_catalogs, database_instances, experiments, genie_spaces, jobs, model_serving_endpoints, models, notebooks, pipelines, postgres_catalogs, postgres_synced_tables, quality_monitors, queries, registered_models, schemas, synced_database_tables, vector_search_endpoints, vector_search_indexes, volumes, warehouses.") assert.Contains(t, cmd.Long, "databricks experimental open jobs 123456789") assert.Contains(t, cmd.Long, "databricks experimental open notebooks /Users/user@example.com/my-notebook") assert.Contains(t, cmd.Long, "databricks experimental open registered_models catalog.schema.my_model") diff --git a/libs/structs/structwalk/walktype_test.go b/libs/structs/structwalk/walktype_test.go index 5db64efc0d7..51069fc1f42 100644 --- a/libs/structs/structwalk/walktype_test.go +++ b/libs/structs/structwalk/walktype_test.go @@ -136,7 +136,7 @@ func TestTypeJobSettings(t *testing.T) { func TestTypeRoot(t *testing.T) { testStruct(t, reflect.TypeFor[config.Root](), - 5000, 5800, // 5651 after SDK v0.136.0 bump + 5000, 6000, // 5814 after genie_space resource added map[string]any{ "bundle.target": "", `variables.*.lookup.dashboard`: "", diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index ff70f6b0505..6c3766ccaeb 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -141,6 +141,7 @@ type FakeWorkspace struct { Volumes map[string]catalog.VolumeInfo Dashboards map[string]fakeDashboard PublishedDashboards map[string]dashboards.PublishedDashboard + GenieSpaces map[string]dashboards.GenieSpace SqlWarehouses map[string]sql.GetWarehouseResponse Alerts map[string]sql.AlertV2 Experiments map[string]ml.GetExperimentResponse @@ -288,6 +289,7 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { Volumes: map[string]catalog.VolumeInfo{}, Dashboards: map[string]fakeDashboard{}, PublishedDashboards: map[string]dashboards.PublishedDashboard{}, + GenieSpaces: map[string]dashboards.GenieSpace{}, SqlWarehouses: map[string]sql.GetWarehouseResponse{ TestDefaultWarehouseId: { Id: TestDefaultWarehouseId, diff --git a/libs/testserver/genie_spaces.go b/libs/testserver/genie_spaces.go new file mode 100644 index 00000000000..cded1497611 --- /dev/null +++ b/libs/testserver/genie_spaces.go @@ -0,0 +1,208 @@ +package testserver + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "strconv" + "strings" + + "github.com/databricks/databricks-sdk-go/service/dashboards" +) + +// generateGenieSpaceId returns a random 32-character hex string. +func generateGenieSpaceId() (string, error) { + randomBytes := make([]byte, 16) + _, err := rand.Read(randomBytes) + if err != nil { + return "", err + } + return hex.EncodeToString(randomBytes), nil +} + +func (s *FakeWorkspace) GenieSpaceCreate(req Request) Response { + defer s.LockUnlock()() + + var createReq dashboards.GenieCreateSpaceRequest + if err := json.Unmarshal(req.Body, &createReq); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "message": "Invalid request body: " + err.Error(), + }, + } + } + + // Default to user's home directory if parent_path is not provided (matches cloud behavior) + if createReq.ParentPath == "" { + createReq.ParentPath = "/Users/" + s.CurrentUser().UserName + } + + spaceId, err := generateGenieSpaceId() + if err != nil { + return Response{ + StatusCode: 500, + Body: map[string]string{ + "message": "Failed to generate genie space ID", + }, + } + } + + // Strip the /Workspace prefix from parent_path before storing. This matches + // the remote behavior: the GET API returns parent_path without the prefix, + // mirroring DashboardCreate. + if strings.HasPrefix(createReq.ParentPath, "/Workspace/") { + createReq.ParentPath = strings.TrimPrefix(createReq.ParentPath, "/Workspace") + } + + genieSpace := dashboards.GenieSpace{ + SpaceId: spaceId, + Title: createReq.Title, + Description: createReq.Description, + ParentPath: createReq.ParentPath, + WarehouseId: createReq.WarehouseId, + SerializedSpace: createReq.SerializedSpace, + // Mirror libs/testserver/dashboards.go: initialize etag to a numeric + // string so each subsequent update can bump it monotonically. + Etag: "1", + } + + s.GenieSpaces[spaceId] = genieSpace + + // Genie spaces are not exposed as workspace files ("dataRoom is not + // user-facing"), so unlike dashboards we do not register a workspace path + // entry — there is nothing to resolve via the Workspace API. + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceGet(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + genieSpace, ok := s.GenieSpaces[spaceId] + if !ok { + // The real API returns 403 (not 404) when a Genie space does not exist. + return Response{ + StatusCode: 403, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + // The GET API only returns the etag when serialized_space is requested + // (include_serialized_space=true). genieSpace is a copy, so clearing the + // field here only affects the response. + if req.URL.Query().Get("include_serialized_space") != "true" { + genieSpace.Etag = "" + } + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceUpdate(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + genieSpace, ok := s.GenieSpaces[spaceId] + if !ok { + // The real API returns 403 (not 404) when a Genie space does not exist. + return Response{ + StatusCode: 403, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + var updateReq dashboards.GenieUpdateSpaceRequest + if err := json.Unmarshal(req.Body, &updateReq); err != nil { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "message": "Invalid request body: " + err.Error(), + }, + } + } + + // Optimistic concurrency: if the caller sent an etag, it must match the + // current one. Empty etag means apply unconditionally. + if updateReq.Etag != "" && updateReq.Etag != genieSpace.Etag { + return Response{ + StatusCode: 409, + Body: map[string]string{ + "message": "Etag mismatch: expected " + genieSpace.Etag + ", got " + updateReq.Etag, + }, + } + } + + prev := genieSpace + if updateReq.Title != "" { + genieSpace.Title = updateReq.Title + } + if updateReq.Description != "" { + genieSpace.Description = updateReq.Description + } + if updateReq.WarehouseId != "" { + genieSpace.WarehouseId = updateReq.WarehouseId + } + if updateReq.ParentPath != "" { + parentPath := updateReq.ParentPath + if strings.HasPrefix(parentPath, "/Workspace/") { + parentPath = strings.TrimPrefix(parentPath, "/Workspace") + } + genieSpace.ParentPath = parentPath + } + if updateReq.SerializedSpace != "" { + genieSpace.SerializedSpace = updateReq.SerializedSpace + } + + // The backend bumps the etag only when serialized_space changes; updates to + // other fields (title, description, warehouse_id, parent_path) leave it + // unchanged. This mirrors the GET API, which only returns the etag + // alongside serialized_space. + if prev.SerializedSpace != genieSpace.SerializedSpace { + prevEtag, err := strconv.Atoi(genieSpace.Etag) + if err != nil { + return Response{ + StatusCode: 500, + Body: map[string]string{ + "message": "Invalid stored etag: " + genieSpace.Etag, + }, + } + } + genieSpace.Etag = strconv.Itoa(prevEtag + 1) + } + + s.GenieSpaces[spaceId] = genieSpace + + return Response{ + Body: genieSpace, + } +} + +func (s *FakeWorkspace) GenieSpaceTrash(req Request) Response { + defer s.LockUnlock()() + + spaceId := req.Vars["space_id"] + if _, ok := s.GenieSpaces[spaceId]; !ok { + // The real API returns 403 (not 404) when a Genie space does not exist. + return Response{ + StatusCode: 403, + Body: map[string]string{ + "message": "Genie space not found", + }, + } + } + + delete(s.GenieSpaces, spaceId) + + return Response{ + StatusCode: 200, + } +} diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index b1ec9b2e3d8..3b0a154a6ee 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -309,6 +309,20 @@ func AddDefaultHandlers(server *Server) { return req.Workspace.DashboardUnpublish(req) }) + // Genie Spaces: + server.Handle("GET", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceGet(req) + }) + server.Handle("POST", "/api/2.0/genie/spaces", func(req Request) any { + return req.Workspace.GenieSpaceCreate(req) + }) + server.Handle("PATCH", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceUpdate(req) + }) + server.Handle("DELETE", "/api/2.0/genie/spaces/{space_id}", func(req Request) any { + return req.Workspace.GenieSpaceTrash(req) + }) + // Pipelines: server.Handle("GET", "/api/2.0/pipelines/{pipeline_id}", func(req Request) any { diff --git a/libs/testserver/permissions.go b/libs/testserver/permissions.go index e7983b1afa7..312c88e9029 100644 --- a/libs/testserver/permissions.go +++ b/libs/testserver/permissions.go @@ -25,6 +25,7 @@ var requestObjectTypeToObjectType = map[string]string{ "sql/alerts": "alert", "sql/queries": "query", "dashboards": "dashboard", + "genie": "genie-space", "experiments": "mlflowExperiment", "registered-models": "registered-model", "serving-endpoints": "serving-endpoint", diff --git a/libs/workspaceurls/urls.go b/libs/workspaceurls/urls.go index be9a41fc959..46930c92239 100644 --- a/libs/workspaceurls/urls.go +++ b/libs/workspaceurls/urls.go @@ -16,6 +16,7 @@ var resourceURLPatterns = map[string]string{ "database_catalogs": "explore/data/%s", "database_instances": "compute/database-instances/%s", "experiments": "ml/experiments/%s", + "genie_spaces": "genie/rooms/%s", "jobs": "jobs/%s", "models": "ml/models/%s", "model_serving_endpoints": "ml/endpoints/%s", diff --git a/libs/workspaceurls/urls_test.go b/libs/workspaceurls/urls_test.go index af7b5b08ea4..d0a86a0f247 100644 --- a/libs/workspaceurls/urls_test.go +++ b/libs/workspaceurls/urls_test.go @@ -111,6 +111,7 @@ func TestResourceURL(t *testing.T) { {"jobs", "jobs", "123", "https://host.com/jobs/123"}, {"experiments", "experiments", "exp-1", "https://host.com/ml/experiments/exp-1"}, {"dashboards", "dashboards", "d-1", "https://host.com/dashboardsv3/d-1/published"}, + {"genie_spaces", "genie_spaces", "space-1", "https://host.com/genie/rooms/space-1"}, {"notebooks", "notebooks", "12345", "https://host.com/#notebook/12345"}, {"notebooks with path", "notebooks", "/Users/u/nb", "https://host.com/#notebook//Users/u/nb"}, {"registered_models normalizes dots", "registered_models", "cat.sch.model", "https://host.com/explore/data/models/cat/sch/model"},