Skip to content

Commit 828e13c

Browse files
ralyodioclaude
andauthored
dbbackup: discover every database on every server over ssh (#64)
* feat(dbbackup): discover every database on every server over ssh `diskpush dbbackup discover [SERVER...] [--save]` looks on each saved connection and ~/.ssh/config host with one read-only probe per server and finds Postgres/Supabase/MySQL in Docker or native, SQLite files, and the app env files that use them. --save keeps them; running it again adds only what is new and names saved ones a reachable server no longer has. Databases can now live on a server: the dump runs there (docker exec pg_dump / mysqldump, sudo -u postgres pg_dump, sqlite3 .backup + quick_check) and only the bytes come back over `ssh -o BatchMode=yes`, through the same gzip + completion-line check. No database port opened, no password copied, no local pg_dump needed. - plugin-api: optional, additive BaseContext.servers (ServerRef list, known_hosts files, ssh_config path); wired in the CLI and desktop hosts - ssh-core: toServerRef() - git forges and duplicate logins are skipped; aliases that reach the same machine (/etc/machine-id) are listed once - desktop task "Find databases on every server"; setting sqliteRoots - docs/plugins.md, README Live-tested read-only against the fleet (370 -> ~180 real databases after noise filters on 7 of 8 servers) and end to end on dev2: a Supabase stack, a shared-cluster app database, a plain Postgres container and a SQLite volume backed up over ssh and verified. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(dbbackup): discovery skips hidden, backups and tmp folders Vienna tenant homes held .retired/ app copies and backups/ full of dated copies; the live fleet count drops from 175 to 142 real databases. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent d1d8e88 commit 828e13c

16 files changed

Lines changed: 1602 additions & 29 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ diskpush mediaanalyzer login
112112
diskpush mediaanalyzer analyze ~/Pictures/2024 --sort
113113

114114
# Back up Postgres, Supabase, MySQL and SQLite: tagged, verified, pruned, pushed
115+
diskpush dbbackup discover --save # every database on every server you have, over ssh
115116
diskpush dbbackup add shop --kind postgres --push backup@nas:/srv/db
116117
diskpush dbbackup run --tags nightly
117118
diskpush dbbackup restore shop --to-env STAGING_DATABASE_URL

‎apps/cli/src/bin.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import { ArgvError, hasFlag, isKnownCommand, looksLikeEndpoint, parseArgv } from
1818
import { autoUpdate, reexec } from './self-update.js'
1919
import { findPluginCall, runPluginCommand, runPlugins } from './commands/plugins.js'
2020
import { pluginHelp } from './help.js'
21-
import { loadPlugins } from './plugins.js'
21+
import { loadPlugins, serverDirectory } from './plugins.js'
2222
import { PluginError } from '@diskpush/plugin-api'
2323
import { existsSync } from 'node:fs'
2424

@@ -41,7 +41,7 @@ async function runPlugin(argv: readonly string[]): Promise<number | null> {
4141
})
4242
for (const { name, error } of failures) output.warn(`plugin ${name} did not load: ${error}`)
4343
if (await autoUpdate(call.pluginId, output) === 'updated') reexec()
44-
return await runPluginCommand(registry, call.pluginId, call.args, output)
44+
return await runPluginCommand(registry, call.pluginId, call.args, output, serverDirectory(store))
4545
} catch (error) {
4646
if (error instanceof PluginError) {
4747
process.stderr.write(`${error.message}\n`)

‎apps/cli/src/commands/plugins.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import {
99
type DiskpushPlugin,
1010
type PluginRegistry,
1111
type ProgressSink,
12+
type ServerDirectory,
1213
} from '@diskpush/plugin-api'
1314
import { EXIT } from '../exit-codes.js'
1415
import { failure, type Output } from '../output.js'
@@ -174,6 +175,7 @@ export async function runPluginCommand(
174175
pluginId: string,
175176
argv: readonly string[],
176177
output: Output,
178+
servers?: ServerDirectory,
177179
): Promise<number> {
178180
const plugin = await registry.require(pluginId)
179181
const args = stripGlobalFlags(argv)
@@ -208,6 +210,7 @@ export async function runPluginCommand(
208210
signal: controller.signal,
209211
surface: 'cli',
210212
env: process.env,
213+
...(servers ? { servers } : {}),
211214
cwd: process.cwd(),
212215
json: output.isJson,
213216
print: (text) => output.line(text),

‎apps/cli/src/plugins.ts‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,9 @@
77
* the user's privileges, exactly like the CLI itself.
88
*/
99
import { spawn } from 'node:child_process'
10-
import { diskpushHome } from '@diskpush/database'
10+
import { homedir } from 'node:os'
11+
import { join } from 'node:path'
12+
import { diskpushHome, knownHostsPath, type DiskPushStore } from '@diskpush/database'
1113
import {
1214
PluginRegistry,
1315
loadExternalPlugins,
@@ -18,10 +20,13 @@ import {
1820
type EntryRef,
1921
type LoadFailure,
2022
type ProgressSink,
23+
type ServerDirectory,
2124
type SettingsBackend,
2225
} from '@diskpush/plugin-api'
26+
import { toServerRef } from '@diskpush/ssh-core'
2327
import { dbBackupPlugin } from '@diskpush/plugin-dbbackup'
2428
import { mediaAnalyzerPlugin } from '@diskpush/plugin-mediaanalyzer'
29+
import { sshConfigHosts } from './resolve.js'
2530

2631
/** Plugins that ship with DiskPush. */
2732
export const BUILTIN_PLUGINS = [mediaAnalyzerPlugin, dbBackupPlugin]
@@ -43,6 +48,23 @@ export async function loadPlugins(
4348
return { registry, failures }
4449
}
4550

51+
/**
52+
* The servers a plugin may work on: the same list `diskpush fleet servers`
53+
* shows (saved SSH connections, then ~/.ssh/config hosts, saved names
54+
* winning), and the known_hosts files DiskPush and OpenSSH keep.
55+
*/
56+
export function serverDirectory(store: DiskPushStore, env: NodeJS.ProcessEnv = process.env): ServerDirectory {
57+
return {
58+
async list() {
59+
const saved = (await store.listConnections()).filter((connection) => connection.kind !== 's3')
60+
const savedNames = new Set(saved.map((connection) => connection.name))
61+
return [...saved, ...sshConfigHosts(env).filter((host) => !savedNames.has(host.name))].map(toServerRef)
62+
},
63+
knownHostsFiles: [join(homedir(), '.ssh', 'known_hosts'), knownHostsPath(env)],
64+
sshConfigFile: env.DISKPUSH_SSH_CONFIG ?? null,
65+
}
66+
}
67+
4668
/**
4769
* Opens a URL in the browser. http(s) only: a plugin must not be able to
4870
* hand the desktop opener a file path or a custom scheme.

‎apps/desktop/electron/main/services/plugins.ts‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
import { homedir } from 'node:os'
1111
import { isAbsolute, join, resolve } from 'node:path'
1212
import { shell, type WebContents } from 'electron'
13-
import { diskpushHome } from '@diskpush/database'
13+
import { diskpushHome, knownHostsPath } from '@diskpush/database'
1414
import {
1515
PluginError,
1616
PluginRegistry,
@@ -24,11 +24,15 @@ import {
2424
type LoadFailure,
2525
type LogLevel,
2626
type ProgressSink,
27+
type ServerDirectory,
2728
type SettingDef,
2829
} from '@diskpush/plugin-api'
30+
import { toServerRef } from '@diskpush/ssh-core'
2931
import { dbBackupPlugin } from '@diskpush/plugin-dbbackup'
3032
import { mediaAnalyzerPlugin } from '@diskpush/plugin-mediaanalyzer'
3133
import { ExternalUrlSchema, IPC, type PluginActionRequest } from '../../shared/contract.js'
34+
import { fleetServers } from './fleet.js'
35+
import { isLocalConnectionId } from './local-session.js'
3236
import { store } from './store.js'
3337

3438
/** The same built-in list as the CLI's (apps/cli/src/plugins.ts). */
@@ -125,13 +129,22 @@ async function openUrl(url: string): Promise<void> {
125129
await shell.openExternal(ExternalUrlSchema.parse(url))
126130
}
127131

132+
/** The Fleet tab's servers, without its synthetic localhost: a plugin runs here already. */
133+
const servers: ServerDirectory = {
134+
async list() {
135+
return (await fleetServers()).filter((connection) => !isLocalConnectionId(connection.id)).map(toServerRef)
136+
},
137+
knownHostsFiles: [join(homedir(), '.ssh', 'known_hosts'), knownHostsPath()],
138+
sshConfigFile: process.env.DISKPUSH_SSH_CONFIG ?? null,
139+
}
140+
128141
/** Starts `work` under a fresh AbortController, reports its outcome, and returns at once. */
129142
function launch(jobId: string, sender: WebContents, work: (host: HostContext) => Promise<ActionResult>): { jobId: string } {
130143
if (running.has(jobId)) throw new PluginError('That job id is already running.')
131144
const controller = new AbortController()
132145
running.set(jobId, controller)
133146
const progress = sink(jobId, sender)
134-
const host: HostContext = { progress, openUrl, signal: controller.signal, surface: 'desktop', env: process.env }
147+
const host: HostContext = { progress, openUrl, signal: controller.signal, surface: 'desktop', env: process.env, servers }
135148
void (async () => {
136149
let result: ActionResult
137150
try {

‎docs/plugins.md‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,55 @@ diskpush dbbackup restore shop --to-env STAGING_URL # asks before it writ
118118
diskpush dbbackup cron --at "15 3 * * *" # a crontab line to paste
119119
```
120120

121+
### Every database on every server: `discover`
122+
123+
You do not have to list your databases. `discover` looks on every server
124+
DiskPush already knows (saved connections and `~/.ssh/config` hosts, the same
125+
list as `diskpush fleet servers`) over the ssh keys and agent you already use,
126+
and lists what it finds; `--save` keeps them for backup.
127+
128+
```bash
129+
diskpush dbbackup discover # look everywhere, change nothing
130+
diskpush dbbackup discover dev2 vienna # just these
131+
diskpush dbbackup discover --save --tags nightly --keep-last 14 --push backup@nas:/srv/db
132+
diskpush dbbackup run --tags nightly # back them all up
133+
```
134+
135+
On each server one read-only shell script, sent in a single ssh call, finds:
136+
137+
| What | How it is found | How it is backed up |
138+
| --- | --- | --- |
139+
| Postgres / Supabase in Docker | `docker ps` (images `postgres`, `postgis`, `supabase/postgres`, `timescale`, `pgvector`…; a compose project ending `-supabase` makes it Supabase), then each container's databases with `psql` inside it, signing in the way the image allows (`POSTGRES_USER`, `postgres`, `supabase_admin`, over the socket or `127.0.0.1`) | `docker exec <container> pg_dump …` on the server |
140+
| MySQL / MariaDB in Docker | `docker ps`, then `mysql -uroot` inside the container with the container's own `MYSQL_ROOT_PASSWORD` | `docker exec <container> mysqldump …`, with that same password from the container's environment |
141+
| Native Postgres | every online cluster (`pg_lsclusters`), as the `postgres` OS user | `sudo -n -u postgres pg_dump -p <port> …` |
142+
| Native MySQL / MariaDB | root over the local socket | `mysqldump` as root |
143+
| SQLite | files under `/srv`, `/opt`, `/var/www`, `/root`, `/home` (setting `sqliteRoots`) whose first bytes are SQLite's header, skipping dependency, cache, source-checkout, worktree, `backups` and `tmp` folders, hidden folders and files, `*.bak*`/replica copies and `opencode.db`; files over 4 GB are listed as skipped (`--max-sqlite-mb`) | SQLite's online backup on the server, `PRAGMA quick_check`, then the file itself comes back |
144+
145+
**The dump runs on the server; only its bytes come back over ssh.** No
146+
database port is opened to this machine, no `pg_dump` of the right version is
147+
needed here, and **no database password is copied anywhere**: a container is
148+
entered with `docker exec`, a native install through its socket. A database
149+
that discovery lists is one that it reached exactly the way the backup will.
150+
The same completion-line rule applies, so a dump cut short by a dropped
151+
connection is still thrown away.
152+
153+
Each database is named after its server and what a person would call it
154+
(`dev2-crawlproof`, `dev2-nichedb-supabase`, `bbs-agentbbs-bbs`), and app
155+
env files (`~/www/*/app.env`, `*/.env`) that point at it are shown beside it,
156+
passwords masked on the server before they are read. Running `discover` again
157+
adds only what is new, and names anything saved that a reachable server no
158+
longer has.
159+
160+
Git forges (`github.com`, `gitlab.com`, …) and second aliases of the same
161+
`user@host:port` are not looked on, and two aliases that turn out to reach the
162+
same machine (same `/etc/machine-id`) are listed once, under the one with the
163+
most access. A server that cannot be reached is reported and the rest carry on.
164+
Probing uses `ssh -o BatchMode=yes`: an unknown host key fails rather than
165+
asks, unless you pass `--accept-new`. The login needs to be root, in the
166+
`docker` group, or have passwordless `sudo` for what it should see; the report
167+
says when it cannot get in somewhere. In the desktop, *Find databases on every
168+
server* under Plugins… does the same and saves what it finds.
169+
121170
**File names are the catalogue.** `shop--20261017T031500Z--nightly+prod.sql.gz`
122171
is database `shop`, taken at that UTC time, tagged `nightly` and `prod`. Each
123172
database gets its own folder under the backup folder (`~/Backups/diskpush`
@@ -174,7 +223,7 @@ the file really is SQLite), and *Verify backup* on a backup. *Back up every
174223
database now* is under Plugins… in the desktop.
175224

176225
**Settings:** `backupDir`, `keepLast`, `keepDays`, `push`, `rsh` (the ssh
177-
command for pushes), and `pg_dumpPath` / `mysqldumpPath` / `psqlPath` /
226+
command for pushes), `sqliteRoots` (where discovery looks for SQLite files), and `pg_dumpPath` / `mysqldumpPath` / `psqlPath` /
178227
`mysqlPath` / `sqlite3Path` to pin a tool.
179228

180229
## Writing a plugin
@@ -237,6 +286,7 @@ it (`ctx.surface` says which: `cli`, `tui` or `desktop`).
237286
| `openUrl(url)` | the user's browser; http and https only |
238287
| `signal` | aborted on Ctrl+C, esc, or Cancel. Stop, save what you have, return |
239288
| `env` | environment variables |
289+
| `servers` | optional: the user's servers (`list()` → name, host, port, username, keyPath, jumpHost, sshConfigAlias, passwordAuth; never a password), plus `knownHostsFiles` and `sshConfigFile` to hand to `ssh`. Absent where a host has none to offer |
240290
| `print`, `warn`, `printJson`, `json`, `prompt`, `cwd` | commands only |
241291

242292
An action returns `{ ok, message, changed? }`. `changed: true` makes the file

‎packages/plugin-api/src/types.ts‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,36 @@ export interface ProgressSink {
7171
log(level: LogLevel, message: string): void
7272
}
7373

74+
/**
75+
* A server DiskPush can reach over SSH, as the host knows it: a saved
76+
* connection or a `~/.ssh/config` host. Enough to build an `ssh` command
77+
* line; never a password.
78+
*/
79+
export type ServerRef = {
80+
/** What the user calls it: the connection name or the ssh_config alias. */
81+
name: string
82+
host: string
83+
port: number
84+
username: string
85+
/** An identity file. Null: the agent and ssh's own defaults. */
86+
keyPath: string | null
87+
/** ProxyJump, e.g. `bastion` or `user@bastion:2222`. */
88+
jumpHost: string | null
89+
/** Set for a ~/.ssh/config host: connect by the alias so the rest of that entry applies. */
90+
sshConfigAlias: string | null
91+
/** The connection signs in with a password, which nothing non-interactive can use. */
92+
passwordAuth: boolean
93+
}
94+
95+
/** The servers the user has given DiskPush, for plugins that work on them over ssh. */
96+
export interface ServerDirectory {
97+
list(): Promise<ServerRef[]>
98+
/** known_hosts files to trust, in order: pass them to ssh as UserKnownHostsFile. */
99+
knownHostsFiles: string[]
100+
/** An ssh_config other than ~/.ssh/config (DISKPUSH_SSH_CONFIG), for `ssh -F`. */
101+
sshConfigFile: string | null
102+
}
103+
74104
/** Everything any plugin code is handed, whichever surface is running it. */
75105
export interface BaseContext {
76106
settings: PluginSettings
@@ -86,6 +116,8 @@ export interface BaseContext {
86116
surface: Surface
87117
/** Environment variables, so a plugin can take an API key from one. */
88118
env: Readonly<Record<string, string | undefined>>
119+
/** The user's servers. Absent on a host that has none to offer; optional, so older plugins and hosts are unaffected. */
120+
servers?: ServerDirectory
89121
}
90122

91123
export interface ActionContext extends BaseContext {

0 commit comments

Comments
 (0)