fix(databricks): restore the dynamic connection form

superset/db_engine_specs/__init__.py

@@ -239,10 +239,12 @@ def get_available_engine_specs() -> dict[type[BaseEngineSpec], set[str]]:  # noq
 
     for engine_spec in load_engine_specs():
         driver = drivers[engine_spec.engine]
-        if (
-            engine_spec.engine in dbs_denylist_engines
-            and hasattr(engine_spec, "default_driver")
-            and engine_spec.default_driver in dbs_denylist[engine_spec.engine]
+        if engine_spec.engine in dbs_denylist_engines and any(
+            driver in dbs_denylist[engine_spec.engine]
+            for driver in (
+                engine_spec.default_driver,
+                *engine_spec.default_driver_aliases,
+            )
         ):
             # do not add denied db engine specs to available list
             continue

superset/db_engine_specs/base.py

@@ -420,6 +420,8 @@ class BaseEngineSpec:  # pylint: disable=too-many-public-methods
     engine_aliases: set[str] = set()
     drivers: dict[str, str] = {}
     default_driver: str | None = None
+    # Historical default names that must continue to match DBS_AVAILABLE_DENYLIST.
+    default_driver_aliases: tuple[str, ...] = ()
 
     # placeholder with the SQLAlchemy URI template
     sqlalchemy_uri_placeholder = (

superset/db_engine_specs/databricks.py

@@ -743,6 +743,7 @@ class DatabricksPythonConnectorEngineSpec(DatabricksDynamicBaseEngineSpec):
     engine = "databricks"
     engine_name = "Databricks"
     default_driver = "databricks"
+    default_driver_aliases: tuple[str, ...] = ("databricks-sql-python",)
     drivers = {
         "databricks": "Databricks SQL Python",
         "databricks-sql-python": "Databricks SQL Python",

tests/unit_tests/databases/api_test.py

@@ -33,6 +33,7 @@
 from freezegun import freeze_time
 from pytest_mock import MockerFixture
 from sqlalchemy.orm.session import Session
+from werkzeug.test import TestResponse
 
 from superset import db
 from superset.commands.database.uploaders.base import UploadCommand
@@ -3046,6 +3047,33 @@ def test_related_objects_limits_dataset_details(
     assert can_access.call_count == MAX_RELATED_DATASETS
 
 
+def test_available_databricks_includes_dynamic_form(
+    client: FlaskClient,
+    full_api_access: None,
+    mocker: MockerFixture,
+) -> None:
+    """An installed official Databricks driver exposes the connection form."""
+    from superset.db_engine_specs.databricks import DatabricksPythonConnectorEngineSpec
+
+    mocker.patch(
+        "superset.databases.api.get_available_engine_specs",
+        return_value={DatabricksPythonConnectorEngineSpec: {"databricks"}},
+    )
+
+    response: TestResponse = client.get("/api/v1/database/available/")
+
+    assert response.status_code == 200
+    assert response.json is not None
+    entry: dict[str, Any] = response.json["databases"][0]
+    assert entry["engine"] == "databricks"
+    assert entry["available_drivers"] == ["databricks"]
+    assert entry["default_driver"] == "databricks"
+    assert entry["parameters"]
+    assert {"host", "http_path_field", "access_token"} <= set(
+        entry["parameters"]["properties"]
+    )
+
+
 def test_available_hides_legacy_databricks(
     mocker: MockerFixture,
     client: FlaskClient,

tests/unit_tests/db_engine_specs/test_databricks.py

@@ -1404,11 +1404,16 @@ def test_get_engine_spec_unrecognized_driver_prefers_python_connector() -> None:
         is DatabricksPythonConnectorEngineSpec
     )
 
+    assert (
+        get_engine_spec("databricks", "databricks-sql-python")
+        is DatabricksPythonConnectorEngineSpec
+    )
+
     # Exact driver matches must still resolve to their specific spec; the
     # relocation must not disturb exact-match resolution.
     assert get_engine_spec("databricks", "pyhive") is DatabricksHiveEngineSpec
     assert (
-        get_engine_spec("databricks", "databricks-sql-python")
+        get_engine_spec("databricks", "databricks")
         is DatabricksPythonConnectorEngineSpec
     )
 

tests/unit_tests/db_engine_specs/test_init.py

@@ -392,3 +392,25 @@ def test_get_available_engine_specs_with_denylist(mocker: MockerFixture) -> None
     )
     available = get_available_engine_specs()
     assert list(available.keys()) == [DatabricksNativeEngineSpec]
+
+
+@pytest.mark.parametrize(
+    "app",
+    [
+        {"DBS_AVAILABLE_DENYLIST": {"databricks": {"databricks-sql-python"}}},
+        {"DBS_AVAILABLE_DENYLIST": {"databricks": {"databricks"}}},
+    ],
+    indirect=True,
+)
+def test_databricks_default_driver_denylist_keeps_legacy_name(
+    mocker: MockerFixture,
+) -> None:
+    """Either name for the native Databricks driver hides its connection option."""
+    from superset.db_engine_specs.databricks import DatabricksPythonConnectorEngineSpec
+
+    mocker.patch(
+        "superset.db_engine_specs.load_engine_specs",
+        return_value=iter([DatabricksPythonConnectorEngineSpec]),
+    )
+
+    assert DatabricksPythonConnectorEngineSpec not in get_available_engine_specs()