feat(schema-engine): require an external shadow database to be empty or consented before reset

libs/user-facing-errors/src/schema_engine.rs

@@ -307,6 +307,18 @@ pub struct UnexpectedNamespaceInExternalTables;
 )]
 pub struct ShadowDbSameAsMainDb;
 
+#[derive(Debug, UserFacingError, Serialize)]
+#[user_facing(
+    code = "P3026",
+    message = "The shadow database at `{shadow_database_location}` is not empty. Prisma Migrate resets the shadow database before it replays your migration history into it, which would destroy the data it currently holds.\n\nPoint `datasource.shadowDatabaseUrl` in your `prisma.config.ts` at an empty database, or, if you are certain that the data in this one can be destroyed, run the command again with `--reset-shadow-database`."
+)]
+pub struct ShadowDbNotEmpty {
+    /// The shadow database, with credentials removed. Reads
+    /// `(driver adapter shadow database)` when the shadow database is reached through a driver
+    /// adapter and has no connection string of its own.
+    pub shadow_database_location: String,
+}
+
 #[derive(Debug, SimpleUserFacingError)]
 #[user_facing(code = "P4001", message = "The introspected database was empty.")]
 pub struct IntrospectionResultEmpty;

schema-engine/cli/tests/cli_tests.rs

@@ -944,6 +944,7 @@ async fn test_missing_datasource_url_gives_proper_error() {
     let datasource_urls = DatasourceUrls {
         url: None,
         shadow_database_url: None,
+        reset_shadow_database: false,
     };
 
     let output = Command::new(schema_engine_bin_path())

schema-engine/commands/src/commands/dev_diagnostic.rs

@@ -17,6 +17,7 @@ pub async fn dev_diagnostic(
     namespaces: Option<Namespaces>,
     connector: &mut dyn SchemaConnector,
     adapter_factory: Arc<dyn ExternalConnectorFactory>,
+    reset_shadow_database: bool,
     migration_schema_cache: &mut MigrationSchemaCache,
 ) -> ConnectorResult<DevDiagnosticOutput> {
     migrations_directory::error_on_changed_provider(&input.migrations_list.lockfile, connector.connector_type())?;
@@ -32,6 +33,7 @@ pub async fn dev_diagnostic(
         namespaces,
         connector,
         adapter_factory,
+        reset_shadow_database,
         migration_schema_cache,
     )
     .await?;

schema-engine/commands/src/commands/diagnose_migration_history.rs

@@ -73,6 +73,7 @@ pub async fn diagnose_migration_history(
     namespaces: Option<Namespaces>,
     connector: &mut dyn SchemaConnector,
     adapter_factory: Arc<dyn ExternalConnectorFactory>,
+    reset_shadow_database: bool,
     migration_schema_cache: &mut MigrationSchemaCache,
 ) -> CoreResult<DiagnoseMigrationHistoryOutput> {
     tracing::debug!("Diagnosing migration history");
@@ -149,6 +150,7 @@ pub async fn diagnose_migration_history(
             let target = ExternalShadowDatabase::DriverAdapter {
                 factory: adapter_factory,
                 preview_features: connector.preview_features(),
+                reset_allowed: reset_shadow_database,
             };
             let from = migration_schema_cache
                 .get_or_insert(&applied_migrations.migration_directories, || async {

schema-engine/commands/src/commands/diff.rs

@@ -18,6 +18,7 @@ pub async fn diff(
     params: DiffParams,
     connector: &mut dyn SchemaConnector,
     adapter_factory: Arc<dyn ExternalConnectorFactory>,
+    reset_shadow_database: bool,
     extension_types: &dyn ExtensionTypes,
 ) -> CoreResult<DiffResult> {
     // In order to properly handle MultiSchema, we need to make sure the preview feature is
@@ -30,10 +31,16 @@ pub async fn diff(
     let filter: SchemaFilter = params.filters.into();
     filter.validate(&*connector.schema_dialect())?;
 
+    let shadow_database = ExternalShadowDatabase::DriverAdapter {
+        factory: adapter_factory,
+        preview_features,
+        reset_allowed: reset_shadow_database,
+    };
+
     let (conn_from, schema_from) = diff_target_to_dialect(
         &params.from,
         connector,
-        adapter_factory.clone(),
+        shadow_database.clone(),
         namespaces.clone(),
         &filter,
         preview_features,
@@ -45,7 +52,7 @@ pub async fn diff(
     let (conn_to, schema_to) = diff_target_to_dialect(
         &params.to,
         connector,
-        adapter_factory,
+        shadow_database,
         namespaces,
         &filter,
         preview_features,
@@ -118,7 +125,7 @@ fn namespaces_and_preview_features_from_diff_targets(
 async fn diff_target_to_dialect(
     target: &DiffTarget,
     connector: &mut dyn SchemaConnector,
-    adapter_factory: Arc<dyn ExternalConnectorFactory>,
+    shadow_database: ExternalShadowDatabase,
     namespaces: Option<Namespaces>,
     filter: &SchemaFilter,
     preview_features: BitFlags<psl::PreviewFeature>,
@@ -160,15 +167,7 @@ async fn diff_target_to_dialect(
 
                     // TODO: enable Driver Adapter for shadow database, using the AdapterFactory.
                     let schema = dialect
-                        .schema_from_migrations_with_target(
-                            &migrations,
-                            namespaces,
-                            filter,
-                            ExternalShadowDatabase::DriverAdapter {
-                                factory: adapter_factory,
-                                preview_features,
-                            },
-                        )
+                        .schema_from_migrations_with_target(&migrations, namespaces, filter, shadow_database)
                         .await?;
                     Ok(Some((dialect, schema)))
                 }

schema-engine/connectors/mongodb-schema-connector/tests/introspection/test_api/mod.rs

@@ -114,6 +114,7 @@ where
             connection_string: connection_string.clone(),
             preview_features,
             shadow_database_connection_string: None,
+            reset_shadow_database: false,
         };
 
         let connector = MongoDbSchemaConnector::new(params);

schema-engine/connectors/mongodb-schema-connector/tests/migrations/test_api.rs

@@ -82,6 +82,7 @@ fn new_connector(preview_features: BitFlags<PreviewFeature>) -> (String, MongoDb
         connection_string: url.to_string(),
         preview_features,
         shadow_database_connection_string: None,
+        reset_shadow_database: false,
     };
     (db_name, MongoDbSchemaConnector::new(params))
 }

schema-engine/connectors/schema-connector/src/connector_params.rs

@@ -10,10 +10,12 @@ pub struct ConnectorParams {
     pub preview_features: BitFlags<PreviewFeature>,
     /// The shadow database connection string.
     pub shadow_database_connection_string: Option<String>,
+    /// Whether the user consented to the shadow database being reset even when it is not empty.
+    pub reset_shadow_database: bool,
 }
 
 impl ConnectorParams {
-    /// Creates new [`ConnectorParams`].
+    /// Creates new [`ConnectorParams`], without consent to reset a non-empty shadow database.
     pub fn new(
         connection_string: String,
         preview_features: BitFlags<PreviewFeature>,
@@ -23,6 +25,7 @@ impl ConnectorParams {
             connection_string,
             preview_features,
             shadow_database_connection_string,
+            reset_shadow_database: false,
         }
     }
 }

schema-engine/connectors/schema-connector/src/schema_connector.rs

@@ -231,12 +231,16 @@ pub enum ExternalShadowDatabase {
         factory: Arc<dyn ExternalConnectorFactory>,
         /// The preview features to use while building the shadow schema.
         preview_features: PreviewFeatures,
+        /// Whether the user consented to the shadow database being reset even when it is not empty.
+        reset_allowed: bool,
     },
     /// A shadow database connection string and preview features.
     ConnectionString {
         /// The shadow database connection string.
         connection_string: String,
         /// The preview features.
         preview_features: PreviewFeatures,
+        /// Whether the user consented to the shadow database being reset even when it is not empty.
+        reset_allowed: bool,
     },
 }

schema-engine/connectors/sql-schema-connector/src/external_shadow_db.rs

@@ -0,0 +1,146 @@
+//! Replaying a migration history into a shadow database the user provided.
+//!
+//! Such a database is reset before the history is replayed into it, which destroys whatever it
+//! holds, so it is only ever used when it holds nothing or when the user said that its contents
+//! can go. It is also left empty afterwards, which is what makes the promise hold for the next
+//! replay: a command that replays the history several times finds an empty database every time
+//! after the first, and so does the command after it.
+
+use crate::flavour::{SqlConnector, UsingExternalShadowDb};
+use schema_connector::{ConnectorError, ConnectorResult, Namespaces, SchemaFilter, migrations_directory::Migrations};
+use sql_schema_describer::SqlSchema;
+use user_facing_errors::schema_engine::ShadowDbNotEmpty;
+
+/// Replays `migrations` into the shadow database `shadow_db` is connected to, and leaves it empty.
+///
+/// `reset_allowed` carries the user's consent to destroy the contents of a shadow database that is
+/// not empty; `location` names the database in the error raised when that consent is missing, and
+/// must be safe to show (see [`crate::sanitize_connection_string`]).
+///
+/// Disposing of `shadow_db` is the caller's business, on this function's success and failure alike.
+pub(crate) async fn replay_migration_history(
+    shadow_db: &mut (dyn SqlConnector + Send + Sync),
+    migrations: &Migrations,
+    namespaces: Option<Namespaces>,
+    filter: &SchemaFilter,
+    reset_allowed: bool,
+    location: &str,
+) -> ConnectorResult<SqlSchema> {
+    if reset_allowed {
+        // Consent is exercised here rather than left to the flavour: SQLite and the Wasm PostgreSQL
+        // connector replay a migration history into an external shadow database without resetting
+        // it first, and the contents the user agreed to part with would collide with the replay.
+        reset(shadow_db, namespaces.clone(), filter).await?;
+    } else {
+        ensure_shadow_db_is_empty(shadow_db, namespaces.clone(), location).await?;
+    }
+
+    let schema = shadow_db
+        .sql_schema_from_migration_history(migrations, namespaces.clone(), filter, UsingExternalShadowDb::Yes)
+        .await?;
+
+    reset(shadow_db, namespaces, filter).await?;
+
+    Ok(schema)
+}
+
+/// Empties the shadow database. A user may well have granted the engine the right to create and to
+/// drop tables in it without making it the owner of anything, which is not enough to drop and
+/// recreate the schema itself, so the objects are dropped one by one when that fails.
+async fn reset(
+    shadow_db: &mut (dyn SqlConnector + Send + Sync),
+    namespaces: Option<Namespaces>,
+    filter: &SchemaFilter,
+) -> ConnectorResult<()> {
+    if shadow_db.reset(namespaces.clone()).await.is_err() {
+        crate::best_effort_reset(shadow_db, namespaces, filter).await?;
+    }
+
+    Ok(())
+}
+
+/// A failed replay leaves the shadow database as it was when it failed: the next run finding it
+/// dirty, and asking about it, is the signal that something went wrong here.
+async fn ensure_shadow_db_is_empty(
+    shadow_db: &mut (dyn SqlConnector + Send + Sync),
+    namespaces: Option<Namespaces>,
+    location: &str,
+) -> ConnectorResult<()> {
+    let schema = shadow_db.describe_schema(namespaces).await?;
+
+    if holds_no_objects(&schema) {
+        return Ok(());
+    }
+
+    Err(ConnectorError::user_facing(ShadowDbNotEmpty {
+        shadow_database_location: location.to_owned(),
+    }))
+}
+
+/// Whether the described schema holds nothing at all — no table (`_prisma_migrations` included, a
+/// database that only holds one is the remains of an earlier replay), no view, no enum, no
+/// user-defined type, no stored procedure.
+fn holds_no_objects(schema: &SqlSchema) -> bool {
+    schema.tables_count() == 0
+        && schema.views_count() == 0
+        && schema.enum_walkers().len() == 0
+        && schema.udt_walkers().next().is_none()
+        && schema.procedures_count() == 0
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn schema_with_a_namespace() -> SqlSchema {
+        let mut schema = SqlSchema::default();
+        schema.push_namespace("public".to_owned());
+        schema
+    }
+
+    #[test]
+    fn a_schema_without_objects_holds_no_objects() {
+        assert!(holds_no_objects(&SqlSchema::default()));
+        assert!(holds_no_objects(&schema_with_a_namespace()));
+    }
+
+    #[test]
+    fn a_table_counts() {
+        let mut schema = schema_with_a_namespace();
+        schema.push_table("Cat".to_owned(), Default::default(), None);
+
+        assert!(!holds_no_objects(&schema));
+    }
+
+    #[test]
+    fn the_migrations_table_alone_counts() {
+        let mut schema = schema_with_a_namespace();
+        schema.push_table(crate::MIGRATIONS_TABLE_NAME.to_owned(), Default::default(), None);
+
+        assert!(!holds_no_objects(&schema));
+    }
+
+    #[test]
+    fn a_view_counts() {
+        let mut schema = schema_with_a_namespace();
+        schema.push_view("Cats".to_owned(), Default::default(), None, None);
+
+        assert!(!holds_no_objects(&schema));
+    }
+
+    #[test]
+    fn an_enum_counts() {
+        let mut schema = schema_with_a_namespace();
+        schema.push_enum(Default::default(), "Mood".to_owned(), None);
+
+        assert!(!holds_no_objects(&schema));
+    }
+
+    #[test]
+    fn a_user_defined_type_counts() {
+        let mut schema = schema_with_a_namespace();
+        schema.push_udt(Default::default(), "Whiskers".to_owned(), None);
+
+        assert!(!holds_no_objects(&schema));
+    }
+}

schema-engine/connectors/sql-schema-connector/src/flavour.rs

@@ -165,6 +165,9 @@ pub(crate) trait SqlConnector: Send + Sync + Debug {
 
     fn shadow_db_url(&self) -> Option<&str>;
 
+    /// Whether the user consented to the shadow database being reset even when it is not empty.
+    fn reset_shadow_database(&self) -> bool;
+
     fn acquire_lock(&mut self) -> BoxFuture<'_, ConnectorResult<()>>;
 
     fn apply_migration_script<'a>(

schema-engine/connectors/sql-schema-connector/src/flavour/mssql.rs

@@ -180,6 +180,12 @@ impl SqlConnector for MssqlConnector {
             .as_deref()
     }
 
+    fn reset_shadow_database(&self) -> bool {
+        self.state
+            .params()
+            .is_some_and(|params| params.connector_params.reset_shadow_database)
+    }
+
     fn acquire_lock(&mut self) -> BoxFuture<'_, ConnectorResult<()>> {
         // see
         // https://docs.microsoft.com/en-us/sql/relational-databases/system-stored-procedures/sp-getapplock-transact-sql?view=sql-server-ver15

schema-engine/connectors/sql-schema-connector/src/flavour/mysql.rs

@@ -160,6 +160,12 @@ impl SqlConnector for MysqlConnector {
             .as_deref()
     }
 
+    fn reset_shadow_database(&self) -> bool {
+        self.state
+            .params()
+            .is_some_and(|params| params.connector_params.reset_shadow_database)
+    }
+
     fn acquire_lock(&mut self) -> BoxFuture<'_, ConnectorResult<()>> {
         with_connection(&mut self.state, |params, _, connection| async move {
             // We do not acquire advisory locks on PlanetScale instances.

schema-engine/connectors/sql-schema-connector/src/flavour/postgres.rs

@@ -360,6 +360,10 @@ impl SqlConnector for PostgresConnector {
         imp::get_shadow_db_url(&self.state)
     }
 
+    fn reset_shadow_database(&self) -> bool {
+        imp::get_reset_shadow_database(&self.state)
+    }
+
     fn acquire_lock(&mut self) -> BoxFuture<'_, ConnectorResult<()>> {
         // They do not support advisory locking:
         // https://github.com/cockroachdb/cockroach/issues/13546

schema-engine/connectors/sql-schema-connector/src/flavour/postgres/connector/native/mod.rs

@@ -421,6 +421,12 @@ pub fn get_shadow_db_url(state: &State) -> Option<&str> {
         .as_deref()
 }
 
+pub fn get_reset_shadow_database(state: &State) -> bool {
+    state
+        .params()
+        .is_some_and(|params| params.connector_params.reset_shadow_database)
+}
+
 pub async fn dispose(state: &mut State) -> ConnectorResult<()> {
     if let State::Connected(_, (_, conn)) = std::mem::replace(state, State::Initial) {
         conn.close().await;