diff --git a/.github/workflows/deploy-raindex-api.yaml b/.github/workflows/deploy-raindex-api.yaml new file mode 100644 index 0000000000..ca5002b514 --- /dev/null +++ b/.github/workflows/deploy-raindex-api.yaml @@ -0,0 +1,89 @@ +name: Deploy Raindex API +on: + workflow_dispatch: + inputs: + scope: + description: Deployment scope + required: true + default: service + type: choice + options: + - service + - all + - system +concurrency: + group: deploy-raindex-api-production + cancel-in-progress: false +permissions: + contents: read +jobs: + deploy: + runs-on: ubuntu-latest + environment: Production + env: + RAINDEX_API_HOSTNAME: ${{ vars.RAINDEX_API_HOSTNAME }} + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + - uses: nixbuild/nix-quick-install-action@v30 + with: + nix_conf: | + accept-flake-config = true + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + - uses: cachix/cachix-action@v15 + continue-on-error: true + with: + name: rainlanguage + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + useDaemon: false + - uses: nix-community/cache-nix-action@v7 + with: + primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/flake.lock') }} + restore-prefixes-first-match: nix-${{ runner.os }}- + gc-max-store-size-linux: 8G + - name: Validate deployment definitions + run: | + nix flake check --no-build --impure + nix build .#raindex-api --no-link + nix shell .#opentofu -c tofu -chdir=infra init -backend=false + nix shell .#opentofu -c tofu -chdir=infra validate + - name: Configure authenticated SSH + env: + SSH_KEY: ${{ secrets.RAINDEX_API_SSH_KEY }} + run: | + test -n "$SSH_KEY" + install -d -m 700 ~/.ssh + printf '%s\n' "$SSH_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + + host_ip=$(nix run .#resolveIp -- -i ~/.ssh/id_ed25519) + host_key=$(nix eval --raw --file keys.nix keys.host) + test "$host_key" != "ssh-ed25519 REPLACE_AFTER_BOOTSTRAP" + printf '%s %s\n' "$host_ip" "$host_key" > ~/.ssh/known_hosts + - name: Deploy + env: + SCOPE: ${{ inputs.scope }} + run: | + case "$SCOPE" in + service) nix run .#deployService -- -i ~/.ssh/id_ed25519 rest-api ;; + system) nix run .#deployNixos -- -i ~/.ssh/id_ed25519 ;; + all) nix run .#deployAll -- -i ~/.ssh/id_ed25519 ;; + *) echo "Unsupported deployment scope: $SCOPE" >&2; exit 1 ;; + esac + - name: Verify deployment + run: | + api_hostname="${RAINDEX_API_HOSTNAME:-api.raindex.finance}" + health_url="https://$api_hostname/health/detailed" + for _ in $(seq 1 60); do + if curl --fail --silent --show-error --max-time 5 "$health_url" \ + | jq --exit-status '.status == "ok"' >/dev/null; then + exit 0 + fi + sleep 5 + done + + host_ip=$(nix run .#resolveIp -- -i ~/.ssh/id_ed25519) + ssh -i ~/.ssh/id_ed25519 "root@$host_ip" \ + 'systemctl status rest-api.service --no-pager || true; journalctl -u rest-api.service -n 200 --no-pager || true' + exit 1 diff --git a/.github/workflows/raindex-api.yaml b/.github/workflows/raindex-api.yaml new file mode 100644 index 0000000000..2599936bc3 --- /dev/null +++ b/.github/workflows/raindex-api.yaml @@ -0,0 +1,63 @@ +name: Raindex API +on: + push: + branches: + - main + paths: + - "Cargo.lock" + - "Cargo.toml" + - "crates/**" + - "*.nix" + - "flake.lock" + - "infra/**" + - ".github/workflows/raindex-api.yaml" + - ".github/workflows/deploy-raindex-api.yaml" + pull_request: + paths: + - "Cargo.lock" + - "Cargo.toml" + - "crates/**" + - "*.nix" + - "flake.lock" + - "infra/**" + - ".github/workflows/raindex-api.yaml" + - ".github/workflows/deploy-raindex-api.yaml" +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} +permissions: + contents: read +jobs: + production-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + submodules: recursive + - uses: nixbuild/nix-quick-install-action@v30 + with: + nix_conf: | + accept-flake-config = true + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + - uses: cachix/cachix-action@v15 + continue-on-error: true + with: + name: rainlanguage + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + useDaemon: false + - uses: nix-community/cache-nix-action@v7 + with: + primary-key: nix-${{ runner.os }}-${{ hashFiles('**/*.nix', '**/flake.lock', 'Cargo.lock') }} + restore-prefixes-first-match: nix-${{ runner.os }}- + gc-max-store-size-linux: 8G + - name: Validate reproducible service and host definitions + run: | + nix flake check --no-build --impure + nix build .#raindex-api --no-link + - name: Validate infrastructure and workflows + run: | + nix shell .#opentofu -c tofu -chdir=infra init -backend=false + nix shell .#opentofu -c tofu -chdir=infra validate + nix shell .#actionlint -c actionlint \ + .github/workflows/raindex-api.yaml \ + .github/workflows/deploy-raindex-api.yaml diff --git a/.gitignore b/.gitignore index 1772bbdc6a..dba2ab86d9 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ packages/raindex/cjs.d.ts packages/raindex/esm.js packages/raindex/esm.d.ts .direnv/ +.raindex/ # Audit proposed fixes .fixes/ diff --git a/Cargo.lock b/Cargo.lock index e9b5509c72..8b1704bc35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2295,6 +2295,24 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -4541,6 +4559,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener 5.4.0", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + [[package]] name = "multer" version = "3.1.0" @@ -5104,6 +5142,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + [[package]] name = "potential_utf" version = "0.1.2" @@ -5742,15 +5786,22 @@ name = "raindex_rest_api" version = "0.0.0-alpha.0" dependencies = [ "alloy", + "async-trait", + "moka", "raindex_common", "rocket", "rocket_cors", "serde", "serde_json", + "tempfile", "thiserror 1.0.69", "tokio", + "tracing", + "tracing-appender", + "tracing-subscriber 0.3.19", "utoipa", "utoipa-swagger-ui", + "uuid", ] [[package]] @@ -7449,6 +7500,12 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "symlink" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7973cce6668464ea31f176d85b13c7ab3bba2cb3b77a2ed26abd7801688010a" + [[package]] name = "syn" version = "1.0.109" @@ -7563,6 +7620,12 @@ dependencies = [ "libc", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" @@ -7903,6 +7966,19 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-appender" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050686193eb999b4bb3bc2acfa891a13da00f79734704c4b8b4ef1a10b368a3c" +dependencies = [ + "crossbeam-channel", + "symlink", + "thiserror 2.0.12", + "time", + "tracing-subscriber 0.3.19", +] + [[package]] name = "tracing-attributes" version = "0.1.30" @@ -7935,6 +8011,16 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.2.25" @@ -7954,12 +8040,15 @@ dependencies = [ "nu-ansi-term", "once_cell", "regex", + "serde", + "serde_json", "sharded-slab", "smallvec", "thread_local", "tracing", "tracing-core", "tracing-log", + "tracing-serde", ] [[package]] @@ -8205,15 +8294,23 @@ dependencies = [ "serde_json", "url", "utoipa", + "utoipa-swagger-ui-vendored", "zip", ] +[[package]] +name = "utoipa-swagger-ui-vendored" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2eebbbfe4093922c2b6734d7c679ebfebd704a0d7e56dfcb0d05818ce28977d" + [[package]] name = "uuid" version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" dependencies = [ + "getrandom 0.3.3", "js-sys", "wasm-bindgen", ] diff --git a/REUSE.toml b/REUSE.toml index 07b8bfab31..a1e16c44e2 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -25,9 +25,16 @@ path = [ "package-lock.json", "Cargo.lock", "Cargo.toml", + "deploy.nix", + "disko.nix", ".devcontainer.json", ".env.example", + "infra/**/", + "keys.nix", + "os.nix", "packages/**/", + "rust.nix", + "services.nix", "ai_commands/**/", "subgraph/**/", "crates/**/", diff --git a/crates/rest_api/Cargo.toml b/crates/rest_api/Cargo.toml index a2d72a553b..3b57614c3e 100644 --- a/crates/rest_api/Cargo.toml +++ b/crates/rest_api/Cargo.toml @@ -13,13 +13,19 @@ path = "src/main.rs" [dependencies] raindex_common = { workspace = true } +async-trait = "0.1" +moka = { version = "0.12", features = ["future"] } rocket = { version = "0.5.1", features = ["json"] } rocket_cors = "0.6" serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } thiserror = { workspace = true } +tracing = { workspace = true } +tracing-appender = "0.2" +tracing-subscriber = { workspace = true, features = ["env-filter", "json"] } utoipa = { version = "5", features = ["rocket_extras"] } -utoipa-swagger-ui = { version = "9", features = ["rocket"] } +utoipa-swagger-ui = { version = "9", features = ["rocket", "vendored"] } +uuid = { version = "1", features = ["v4"] } [target.'cfg(not(target_family = "wasm"))'.dependencies] tokio = { workspace = true, features = ["full"] } @@ -27,3 +33,4 @@ tokio = { workspace = true, features = ["full"] } [dev-dependencies] tokio = { workspace = true, features = ["full", "macros"] } alloy = { workspace = true } +tempfile = "3" diff --git a/crates/rest_api/README.md b/crates/rest_api/README.md new file mode 100644 index 0000000000..f50caeb1b5 --- /dev/null +++ b/crates/rest_api/README.md @@ -0,0 +1,62 @@ +# Raindex Market Data API + +Public market data service backed by the Raindex Rust SDK and its persistent +local indexer. It does not require API keys. + +## Run locally + +```sh +cargo run -p raindex_rest_api +``` + +The first startup creates `.raindex/market-data.sqlite`, synchronizes the +configured orderbook, and warms the market cache before accepting requests. +Swagger UI is served at `http://127.0.0.1:8000/swagger/`. + +## Routes + +- `GET /tickers` — 24-hour ticker data for every direct quote-token market found + in active indexed Raindex orders, including its stable unique market ID as + `pool_id` for the orderbook DEX schema. +- `GET /orderbook?ticker_id=...&depth=100` — executable bids and asks for one + market. +- `GET /v1/markets` — complete active-market overview for the Raindex UI. +- `GET /v1/markets?ticker_id=...` — one market with statistics, trades, and its + executable book. +- `GET /health` and `GET /health/detailed` — service and local-indexer health. + +## Configuration + +All settings are optional environment variables: + +| Variable | Default | +| ---------------------------------------- | ---------------------------------------- | +| `RAINDEX_REGISTRY_URL` | Pinned `rain.strategies` registry commit | +| `RAINDEX_LOCAL_DB_PATH` | `.raindex/market-data.sqlite` | +| `RAINDEX_LOG_DIR` | `.raindex/logs` | +| `RAINDEX_LOCAL_DB_READY_TIMEOUT_SECONDS` | `600` | +| `RAINDEX_CACHE_TTL_SECONDS` | `60` | +| `RAINDEX_RATE_LIMIT_GLOBAL_RPM` | `6000` | +| `RAINDEX_RATE_LIMIT_PER_IP_RPM` | `120` | +| `RAINDEX_SNAPSHOT_RECENT_TRADES_LIMIT` | `20` | +| `RAINDEX_TRUSTED_PROXY_IP_HEADER` | unset; direct socket IP is used | + +The all-market overview is refreshed in the background. Market discovery and +statistics use the persistent local index first, with the SDK's configured +subgraph fallback until the local index is ready. Executable orderbooks are +cached per ticker so an orderbook request quotes only the requested market. +Overview responses omit per-market trade arrays to keep minutely UI reads small; +requesting one `ticker_id` includes its configured recent-trade window. +Orderbooks use a fixed 1000-level SDK snapshot. A `depth` of `0` returns that +entire snapshot; positive values return at most that many levels split evenly +between bids and asks. The optional historical-trades compatibility endpoint is +intentionally deferred until the SDK can serve arbitrary indexed time ranges +without truncating to a cached 24-hour window. + +Console and file logs are emitted as JSON. File logs rotate daily and retain the +most recent 14 files. Production should place `RAINDEX_LOG_DIR` on the +persistent data volume. + +Set `RAINDEX_TRUSTED_PROXY_IP_HEADER` only when a trusted ingress overwrites +that header before forwarding requests. Client-provided forwarding headers are +ignored by default so public rate limits cannot be bypassed by spoofing them. diff --git a/crates/rest_api/src/cache.rs b/crates/rest_api/src/cache.rs new file mode 100644 index 0000000000..df32508e78 --- /dev/null +++ b/crates/rest_api/src/cache.rs @@ -0,0 +1,265 @@ +use crate::provider::RaindexProvider; +use moka::future::Cache; +use raindex_common::raindex_client::markets::{ + MarketSnapshotOptions, RaindexMarketDataErrorSeverity, RaindexMarketSnapshot, +}; +use std::{ + collections::HashSet, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, +}; +use tokio::sync::RwLock; + +pub const MAX_ORDERBOOK_DEPTH: u16 = 1_000; +const OVERVIEW_ORDERBOOK_DEPTH: u16 = 2; +const DETAIL_FAILURE_TTL: Duration = Duration::from_secs(5); + +#[derive(Debug, Default)] +struct OverviewState { + snapshots: Option>>, + last_success_at: Option, + refresh_healthy: bool, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct OverviewHealth { + pub ready: bool, + pub last_success_at: Option, + pub refresh_healthy: bool, +} + +#[derive(Debug, thiserror::Error)] +pub enum MarketCacheError { + #[error(transparent)] + Sdk(#[from] raindex_common::raindex_client::RaindexError), + #[error("invalid market snapshot: {0}")] + InvalidSnapshot(String), +} + +#[derive(Clone)] +pub struct MarketSnapshotCache { + overview: Arc>, + details: Cache>>, + detail_failures: Cache>, + orderbook_depth: u16, + recent_trades_limit: u16, +} + +impl MarketSnapshotCache { + pub fn new(ttl: Duration, recent_trades_limit: u16) -> Self { + Self { + overview: Arc::new(RwLock::new(OverviewState::default())), + details: Cache::builder() + .max_capacity(1_000) + .time_to_live(ttl) + .build(), + detail_failures: Cache::builder() + .max_capacity(1_000) + .time_to_live(DETAIL_FAILURE_TTL) + .build(), + orderbook_depth: MAX_ORDERBOOK_DEPTH, + recent_trades_limit, + } + } + + pub async fn get_overview( + &self, + provider: &RaindexProvider, + ) -> Result>, MarketCacheError> { + match self.overview.read().await.snapshots.clone() { + Some(snapshots) => Ok(snapshots), + None => self.refresh_overview(provider).await, + } + } + + pub async fn get_market( + &self, + provider: &RaindexProvider, + ticker_id: &str, + ) -> Result>, Arc> { + let key = ticker_id.to_ascii_lowercase(); + if let Some(error) = self.detail_failures.get(&key).await { + return Err(error); + } + let options = self.market_options(&key); + let result = self + .details + .try_get_with(key.clone(), async move { + provider + .client() + .get_market_snapshots(Some(options)) + .await + .map_err(MarketCacheError::Sdk) + .and_then(validate_detail) + .map(Arc::new) + }) + .await; + match result { + Ok(snapshots) => { + self.detail_failures.invalidate(&key).await; + Ok(snapshots) + } + Err(error) => { + self.detail_failures.insert(key, Arc::clone(&error)).await; + Err(error) + } + } + } + + pub async fn refresh_overview( + &self, + provider: &RaindexProvider, + ) -> Result>, MarketCacheError> { + let result = provider + .client() + .get_market_snapshots(Some(self.overview_options())) + .await + .map_err(MarketCacheError::Sdk) + .and_then(validate_overview) + .map(Arc::new); + let mut state = self.overview.write().await; + match result { + Ok(snapshots) => { + state.snapshots = Some(Arc::clone(&snapshots)); + state.last_success_at = Some(unix_now()); + state.refresh_healthy = true; + Ok(snapshots) + } + Err(error) => { + state.refresh_healthy = false; + Err(error) + } + } + } + + pub async fn overview_health(&self) -> OverviewHealth { + let state = self.overview.read().await; + OverviewHealth { + ready: state.snapshots.is_some(), + last_success_at: state.last_success_at, + refresh_healthy: state.refresh_healthy, + } + } + + fn overview_options(&self) -> MarketSnapshotOptions { + MarketSnapshotOptions { + orderbook_depth: Some(OVERVIEW_ORDERBOOK_DEPTH), + recent_trades_limit: Some(self.recent_trades_limit), + include_orderbook: Some(true), + ..Default::default() + } + } + + fn market_options(&self, ticker_id: &str) -> MarketSnapshotOptions { + let ticker_id = ticker_id.to_ascii_lowercase(); + MarketSnapshotOptions { + markets: raindex_common::raindex_client::markets::MarketListOptions { + ticker_ids: Some(vec![ticker_id]), + ..Default::default() + }, + orderbook_depth: Some(self.orderbook_depth), + recent_trades_limit: Some(0), + include_trades: Some(false), + ..Default::default() + } + } +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +pub(crate) fn validate_overview( + mut snapshots: Vec, +) -> Result, MarketCacheError> { + let snapshot_count = snapshots.len(); + let rejected_ticker_ids = snapshots + .iter() + .filter(|snapshot| has_critical_snapshot_error(snapshot)) + .map(|snapshot| snapshot.market.ticker_id.clone()) + .collect::>(); + snapshots.retain(|snapshot| !has_critical_snapshot_error(snapshot)); + if snapshot_count > 0 && snapshots.is_empty() { + return Err(MarketCacheError::InvalidSnapshot( + "critical market data reads failed for every market".into(), + )); + } + if !rejected_ticker_ids.is_empty() { + tracing::warn!( + rejected_market_count = rejected_ticker_ids.len(), + rejected_ticker_ids = ?rejected_ticker_ids, + "omitting markets with critical data read failures from the overview" + ); + } + validate_unique_ticker_ids(&snapshots)?; + Ok(snapshots) +} + +pub(crate) fn validate_detail( + snapshots: Vec, +) -> Result, MarketCacheError> { + if snapshots.iter().any(has_critical_snapshot_error) { + return Err(MarketCacheError::InvalidSnapshot( + "a critical market data read failed".into(), + )); + } + validate_unique_ticker_ids(&snapshots)?; + Ok(snapshots) +} + +fn has_critical_snapshot_error(snapshot: &RaindexMarketSnapshot) -> bool { + snapshot.errors.iter().any(|error| { + error.severity == RaindexMarketDataErrorSeverity::Error + && matches!( + error.source.as_str(), + "registry" | "trades" | "orderbook" | "ratios" + ) + }) +} + +fn validate_unique_ticker_ids(snapshots: &[RaindexMarketSnapshot]) -> Result<(), MarketCacheError> { + let mut ticker_ids = HashSet::new(); + if snapshots + .iter() + .map(|snapshot| snapshot.market.ticker_id.to_ascii_lowercase()) + .any(|ticker_id| !ticker_ids.insert(ticker_id)) + { + return Err(MarketCacheError::InvalidSnapshot( + "ticker IDs must be unique across configured chains".into(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn detail_cache_uses_canonical_ticker_for_sdk_filter() { + let cache = MarketSnapshotCache::new(Duration::from_secs(60), 100); + assert_eq!( + cache.market_options("0xAB_0xCD").markets.ticker_ids, + Some(vec!["0xab_0xcd".to_string()]) + ); + } + + #[test] + fn overview_reads_only_the_best_level_on_each_side() { + let cache = MarketSnapshotCache::new(Duration::from_secs(60), 100); + let options = cache.overview_options(); + assert_eq!(options.orderbook_depth, Some(2)); + assert_eq!(options.include_orderbook, Some(true)); + } + + #[test] + fn overview_validation_accepts_empty_market_configuration() { + assert_eq!( + validate_overview(Vec::new()).expect("valid overview"), + vec![] + ); + } +} diff --git a/crates/rest_api/src/catchers.rs b/crates/rest_api/src/catchers.rs new file mode 100644 index 0000000000..54b8b8a632 --- /dev/null +++ b/crates/rest_api/src/catchers.rs @@ -0,0 +1,36 @@ +use crate::error::ApiError; + +#[catch(400)] +fn bad_request() -> ApiError { + ApiError::BadRequest("the request could not be understood".into()) +} + +#[catch(404)] +fn not_found() -> ApiError { + ApiError::NotFound("route not found".into()) +} + +#[catch(422)] +fn unprocessable() -> ApiError { + ApiError::BadRequest("invalid path or query parameters".into()) +} + +#[catch(429)] +fn rate_limited() -> ApiError { + ApiError::RateLimited("too many requests; try again later".into()) +} + +#[catch(500)] +fn internal_error() -> ApiError { + ApiError::Internal("internal server error".into()) +} + +pub fn catchers() -> Vec { + catchers![ + bad_request, + not_found, + unprocessable, + rate_limited, + internal_error + ] +} diff --git a/crates/rest_api/src/config.rs b/crates/rest_api/src/config.rs new file mode 100644 index 0000000000..1bc6c2f8d2 --- /dev/null +++ b/crates/rest_api/src/config.rs @@ -0,0 +1,122 @@ +use std::path::PathBuf; + +pub const DEFAULT_REGISTRY_URL: &str = "https://raw.githubusercontent.com/rainlanguage/rain.strategies/de1d6990052d5003cadf4550fed4a2bfdf9560ff/registry"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Config { + pub registry_url: String, + pub local_db_path: PathBuf, + pub log_dir: PathBuf, + pub local_db_ready_timeout_seconds: u64, + pub cache_ttl_seconds: u64, + pub rate_limit_global_rpm: u64, + pub rate_limit_per_ip_rpm: u64, + pub snapshot_recent_trades_limit: u16, + pub trusted_proxy_ip_header: Option, +} + +impl Default for Config { + fn default() -> Self { + Self { + registry_url: DEFAULT_REGISTRY_URL.to_string(), + local_db_path: PathBuf::from(".raindex/market-data.sqlite"), + log_dir: PathBuf::from(".raindex/logs"), + local_db_ready_timeout_seconds: 600, + cache_ttl_seconds: 60, + rate_limit_global_rpm: 6_000, + rate_limit_per_ip_rpm: 120, + snapshot_recent_trades_limit: 20, + trusted_proxy_ip_header: None, + } + } +} + +impl Config { + pub fn from_env() -> Result { + let defaults = Self::default(); + let config = Self { + registry_url: env_string("RAINDEX_REGISTRY_URL", defaults.registry_url), + local_db_path: PathBuf::from(env_string( + "RAINDEX_LOCAL_DB_PATH", + defaults.local_db_path.to_string_lossy().into_owned(), + )), + log_dir: PathBuf::from(env_string( + "RAINDEX_LOG_DIR", + defaults.log_dir.to_string_lossy().into_owned(), + )), + local_db_ready_timeout_seconds: env_parse( + "RAINDEX_LOCAL_DB_READY_TIMEOUT_SECONDS", + defaults.local_db_ready_timeout_seconds, + )?, + cache_ttl_seconds: env_parse("RAINDEX_CACHE_TTL_SECONDS", defaults.cache_ttl_seconds)?, + rate_limit_global_rpm: env_parse( + "RAINDEX_RATE_LIMIT_GLOBAL_RPM", + defaults.rate_limit_global_rpm, + )?, + rate_limit_per_ip_rpm: env_parse( + "RAINDEX_RATE_LIMIT_PER_IP_RPM", + defaults.rate_limit_per_ip_rpm, + )?, + snapshot_recent_trades_limit: env_parse( + "RAINDEX_SNAPSHOT_RECENT_TRADES_LIMIT", + defaults.snapshot_recent_trades_limit, + )?, + trusted_proxy_ip_header: std::env::var("RAINDEX_TRUSTED_PROXY_IP_HEADER") + .ok() + .filter(|value| !value.trim().is_empty()), + }; + if config.cache_ttl_seconds == 0 { + return Err(ConfigError::InvalidValue( + "RAINDEX_CACHE_TTL_SECONDS", + "0".into(), + )); + } + if config.local_db_ready_timeout_seconds == 0 { + return Err(ConfigError::InvalidValue( + "RAINDEX_LOCAL_DB_READY_TIMEOUT_SECONDS", + "0".into(), + )); + } + Ok(config) + } +} + +fn env_string(name: &'static str, default: String) -> String { + std::env::var(name).unwrap_or(default) +} + +fn env_parse(name: &'static str, default: T) -> Result +where + T: std::str::FromStr, +{ + std::env::var(name) + .ok() + .map(|value| { + value + .parse() + .map_err(|_| ConfigError::InvalidValue(name, value)) + }) + .transpose() + .map(|value| value.unwrap_or(default)) +} + +#[derive(Debug, thiserror::Error)] +pub enum ConfigError { + #[error("invalid value for {0}: {1}")] + InvalidValue(&'static str, String), +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_are_suitable_for_minutely_public_polling() { + let config = Config::default(); + assert_eq!(config.cache_ttl_seconds, 60); + assert_eq!(config.local_db_ready_timeout_seconds, 600); + assert!(config.rate_limit_per_ip_rpm >= 60); + assert_eq!(config.snapshot_recent_trades_limit, 20); + assert_eq!(crate::cache::MAX_ORDERBOOK_DEPTH, 1_000); + } +} diff --git a/crates/rest_api/src/error.rs b/crates/rest_api/src/error.rs index 106e79804d..9077f6bc68 100644 --- a/crates/rest_api/src/error.rs +++ b/crates/rest_api/src/error.rs @@ -1,225 +1,106 @@ -use raindex_common::raindex_client::RaindexError; -use rocket::http::Status; -use rocket::response::{self, Responder}; -use rocket::serde::json::Json; -use rocket::Request; +use crate::fairings::{request_id_for, request_span_for}; +use rocket::{ + http::{Header, Status}, + response::Responder, + serde::json::Json, + Request, Response, +}; use serde::{Deserialize, Serialize}; -use thiserror::Error; use utoipa::ToSchema; -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "camelCase")] -#[schema(example = json!({ - "error": "No liquidity available for the given token pair", - "readableMessage": "No liquidity available for the given token pair on the specified chain" -}))] -pub struct ApiErrorResponse { - #[schema(example = "No liquidity available for the given token pair")] - pub error: String, - #[schema(example = "No liquidity available for the given token pair on the specified chain")] - pub readable_message: String, +#[derive(Debug, Clone, Copy, Serialize, Deserialize, ToSchema, PartialEq, Eq)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub enum ApiErrorCode { + BadRequest, + NotFound, + RateLimited, + UpstreamUnavailable, + InternalError, } -#[derive(Error, Debug)] -pub enum ApiError { - #[error(transparent)] - Raindex(#[from] RaindexError), - - #[error("Internal server error: {0}")] - Internal(String), -} - -impl ApiError { - fn status_code(&self) -> Status { +impl ApiErrorCode { + fn status(self) -> Status { match self { - ApiError::Raindex(e) => match e { - RaindexError::InvalidYamlConfig - | RaindexError::YamlError(_) - | RaindexError::FromHexError(_) - | RaindexError::U256ParseError(_) - | RaindexError::I256ParseError(_) - | RaindexError::ZeroAmount - | RaindexError::NegativeAmount - | RaindexError::NonPositiveAmount - | RaindexError::NegativePriceCap - | RaindexError::SameTokenPair - | RaindexError::Float(_) - | RaindexError::ParseInt(_) => Status::BadRequest, - - RaindexError::NoLiquidity | RaindexError::InsufficientLiquidity { .. } => { - Status::NotFound - } - - RaindexError::ChainIdNotFound(_) - | RaindexError::RaindexNotFound(_, _) - | RaindexError::OrderNotFound(_, _, _) - | RaindexError::VaultNotFound(_, _, _) - | RaindexError::SubgraphNotFound(_, _) - | RaindexError::SubgraphNotConfigured(_) - | RaindexError::NoNetworksConfigured => Status::NotFound, - - _ => Status::InternalServerError, - }, - ApiError::Internal(_) => Status::InternalServerError, - } - } - - fn to_response(&self) -> ApiErrorResponse { - let readable_message = match self { - ApiError::Raindex(e) => e.to_readable_msg(), - ApiError::Internal(msg) => msg.clone(), - }; - - ApiErrorResponse { - error: self.to_string(), - readable_message, + Self::BadRequest => Status::BadRequest, + Self::NotFound => Status::NotFound, + Self::RateLimited => Status::TooManyRequests, + Self::UpstreamUnavailable => Status::ServiceUnavailable, + Self::InternalError => Status::InternalServerError, } } } -impl<'r> Responder<'r, 'static> for ApiError { - fn respond_to(self, request: &'r Request<'_>) -> response::Result<'static> { - let status = self.status_code(); - let body = self.to_response(); - - response::Response::build_from(Json(body).respond_to(request)?) - .status(status) - .ok() - } +#[derive(Debug, Serialize, Deserialize, ToSchema)] +pub struct ApiErrorDetail { + pub code: ApiErrorCode, + pub message: String, } -#[cfg(test)] -mod tests { - use super::*; - use alloy::primitives::B256; - - #[test] - fn test_status_code_bad_request_errors() { - let bad_request_errors = vec![ - ApiError::Raindex(RaindexError::InvalidYamlConfig), - ApiError::Raindex(RaindexError::ZeroAmount), - ApiError::Raindex(RaindexError::NegativeAmount), - ApiError::Raindex(RaindexError::NonPositiveAmount), - ApiError::Raindex(RaindexError::NegativePriceCap), - ApiError::Raindex(RaindexError::SameTokenPair), - ]; - - for error in bad_request_errors { - assert_eq!( - error.status_code(), - Status::BadRequest, - "Expected BadRequest for {:?}", - error - ); - } - } +#[derive(Debug, Serialize, Deserialize, ToSchema)] +pub struct ApiErrorResponse { + pub request_id: String, + pub error: ApiErrorDetail, +} - #[test] - fn test_status_code_not_found_liquidity_errors() { - let not_found_errors = vec![ - ApiError::Raindex(RaindexError::NoLiquidity), - ApiError::Raindex(RaindexError::InsufficientLiquidity { - requested: "100".to_string(), - available: "50".to_string(), - }), - ]; +#[derive(Debug, thiserror::Error)] +pub enum ApiError { + #[error("bad request: {0}")] + BadRequest(String), + #[error("not found: {0}")] + NotFound(String), + #[error("rate limited: {0}")] + RateLimited(String), + #[error("upstream unavailable: {0}")] + UpstreamUnavailable(String), + #[error("internal error: {0}")] + Internal(String), +} - for error in not_found_errors { - assert_eq!( - error.status_code(), - Status::NotFound, - "Expected NotFound for {:?}", - error - ); +impl ApiError { + fn code(&self) -> ApiErrorCode { + match self { + Self::BadRequest(_) => ApiErrorCode::BadRequest, + Self::NotFound(_) => ApiErrorCode::NotFound, + Self::RateLimited(_) => ApiErrorCode::RateLimited, + Self::UpstreamUnavailable(_) => ApiErrorCode::UpstreamUnavailable, + Self::Internal(_) => ApiErrorCode::InternalError, } } - #[test] - fn test_status_code_not_found_config_errors() { - let not_found_errors = vec![ - ApiError::Raindex(RaindexError::ChainIdNotFound(1)), - ApiError::Raindex(RaindexError::RaindexNotFound("0x123".to_string(), 1)), - ApiError::Raindex(RaindexError::OrderNotFound( - "0x123".to_string(), - 1, - B256::ZERO, - )), - ApiError::Raindex(RaindexError::VaultNotFound( - "0x123".to_string(), - 1, - "1".to_string(), - )), - ApiError::Raindex(RaindexError::SubgraphNotFound( - "test".to_string(), - "order".to_string(), - )), - ApiError::Raindex(RaindexError::SubgraphNotConfigured("1".to_string())), - ApiError::Raindex(RaindexError::NoNetworksConfigured), - ]; - - for error in not_found_errors { - assert_eq!( - error.status_code(), - Status::NotFound, - "Expected NotFound for {:?}", - error - ); + fn public_message(&self) -> String { + match self { + Self::BadRequest(message) + | Self::NotFound(message) + | Self::RateLimited(message) + | Self::UpstreamUnavailable(message) + | Self::Internal(message) => message.clone(), } } +} - #[test] - fn test_status_code_internal_server_error() { - let internal_error = ApiError::Internal("Something went wrong".to_string()); - assert_eq!(internal_error.status_code(), Status::InternalServerError); - } - - #[test] - fn test_status_code_preflight_error_is_internal() { - let preflight_error = ApiError::Raindex(RaindexError::PreflightError( - "Simulation failed".to_string(), - )); - assert_eq!(preflight_error.status_code(), Status::InternalServerError); - } - - #[test] - fn test_to_response_raindex_error() { - let error = ApiError::Raindex(RaindexError::NoLiquidity); - let response = error.to_response(); - - assert!(response.error.contains("No liquidity")); - assert!(response.readable_message.contains("No liquidity available")); - } - - #[test] - fn test_to_response_internal_error() { - let error = ApiError::Internal("Custom error message".to_string()); - let response = error.to_response(); - - assert!(response.error.contains("Internal server error")); - assert_eq!(response.readable_message, "Custom error message"); - } - - #[test] - fn test_api_error_response_serialization() { - let response = ApiErrorResponse { - error: "Test error".to_string(), - readable_message: "A readable message".to_string(), +impl<'r> Responder<'r, 'static> for ApiError { + fn respond_to(self, request: &'r Request<'_>) -> rocket::response::Result<'static> { + let code = self.code(); + let status = code.status(); + request_span_for(request).in_scope(|| { + if status.code >= 500 { + tracing::error!(error = %self, "request failed"); + } else { + tracing::warn!(error = %self, "request failed"); + } + }); + let body = ApiErrorResponse { + request_id: request_id_for(request), + error: ApiErrorDetail { + code, + message: self.public_message(), + }, }; - - let json = serde_json::to_string(&response).unwrap(); - - assert!(json.contains("\"error\":\"Test error\"")); - assert!(json.contains("\"readableMessage\":\"A readable message\"")); - } - - #[test] - fn test_api_error_from_raindex_error() { - let raindex_error = RaindexError::NoLiquidity; - let api_error: ApiError = raindex_error.into(); - - assert!(matches!( - api_error, - ApiError::Raindex(RaindexError::NoLiquidity) - )); + let response = Json(body).respond_to(request)?; + let mut response = Response::build_from(response).status(status).finalize(); + if matches!(code, ApiErrorCode::RateLimited) { + response.set_header(Header::new("Retry-After", "60")); + } + Ok(response) } } diff --git a/crates/rest_api/src/fairings.rs b/crates/rest_api/src/fairings.rs new file mode 100644 index 0000000000..4b51f65232 --- /dev/null +++ b/crates/rest_api/src/fairings.rs @@ -0,0 +1,313 @@ +use crate::error::ApiError; +use rocket::{ + fairing::{Fairing, Info, Kind}, + http::{Header, Status}, + request::{FromRequest, Outcome}, + Data, Request, Response, +}; +use std::{ + collections::{HashMap, VecDeque}, + net::IpAddr, + sync::{ + atomic::{AtomicU64, Ordering}, + Mutex, + }, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; +use uuid::Uuid; + +const WINDOW: Duration = Duration::from_secs(60); +const REQUEST_ID_HEADER: &str = "X-Request-Id"; +const PER_IP_CLEANUP_EVERY: u64 = 1_024; + +struct RequestMeta { + started: Instant, + request_id: String, + span: tracing::Span, +} + +fn fallback_meta() -> RequestMeta { + RequestMeta { + started: Instant::now(), + request_id: "unknown".into(), + span: tracing::Span::none(), + } +} + +pub fn request_span_for(request: &Request<'_>) -> tracing::Span { + request.local_cache(fallback_meta).span.clone() +} + +pub fn request_id_for(request: &Request<'_>) -> String { + request.local_cache(fallback_meta).request_id.clone() +} + +pub struct RequestLogger; + +#[rocket::async_trait] +impl Fairing for RequestLogger { + fn info(&self) -> Info { + Info { + name: "Request logger", + kind: Kind::Request | Kind::Response, + } + } + + async fn on_request(&self, request: &mut Request<'_>, _data: &mut Data<'_>) { + let request_id = request + .headers() + .get_one(REQUEST_ID_HEADER) + .map(str::trim) + .filter(|value| { + !value.is_empty() + && value.len() <= 128 + && value.is_ascii() + && !value.chars().any(char::is_control) + }) + .map(str::to_owned) + .unwrap_or_else(|| Uuid::new_v4().to_string()); + let span = tracing::info_span!( + "request", + method = %request.method(), + uri = %request.uri(), + request_id = %request_id + ); + span.in_scope(|| tracing::info!("request started")); + request.local_cache(|| RequestMeta { + started: Instant::now(), + request_id, + span, + }); + } + + async fn on_response<'r>(&self, request: &'r Request<'_>, response: &mut Response<'r>) { + let meta = request.local_cache(fallback_meta); + let duration_ms = meta.started.elapsed().as_secs_f64() * 1_000.0; + meta.span.in_scope(|| { + tracing::info!( + status = response.status().code, + duration_ms, + "request completed" + ) + }); + response.set_header(Header::new(REQUEST_ID_HEADER, meta.request_id.clone())); + } +} + +#[derive(Clone)] +pub struct RateLimitInfo { + limit: u64, + remaining: u64, + reset: u64, + allowed: bool, +} + +struct CachedRateLimitInfo(Mutex>); + +pub struct RateLimiter { + global_rpm: u64, + per_ip_rpm: u64, + state: Mutex, + per_ip_check_count: AtomicU64, +} + +#[derive(Default)] +struct RateLimitState { + global: VecDeque, + per_ip: HashMap>, +} + +impl RateLimiter { + pub fn new(global_rpm: u64, per_ip_rpm: u64) -> Self { + Self { + global_rpm, + per_ip_rpm, + state: Mutex::new(RateLimitState::default()), + per_ip_check_count: AtomicU64::new(0), + } + } + + fn check(&self, ip: IpAddr) -> Result { + let mut state = self + .state + .lock() + .map_err(|_| ApiError::Internal("rate limiter unavailable".into()))?; + let now = Instant::now(); + let cutoff = now - WINDOW; + prune(&mut state.global, cutoff); + let check_count = self.per_ip_check_count.fetch_add(1, Ordering::Relaxed) + 1; + if check_count.is_multiple_of(PER_IP_CLEANUP_EVERY) { + state.per_ip.retain(|_, window| { + prune(window, cutoff); + !window.is_empty() + }); + } + + let global_allowed = self.global_rpm == 0 || state.global.len() < self.global_rpm as usize; + let per_ip = state.per_ip.entry(ip).or_default(); + prune(per_ip, cutoff); + let per_ip_allowed = self.per_ip_rpm == 0 || per_ip.len() < self.per_ip_rpm as usize; + if !global_allowed || !per_ip_allowed { + let (limit, reset) = if !per_ip_allowed { + (self.per_ip_rpm, reset_at(per_ip, now)) + } else { + (self.global_rpm, reset_at(&state.global, now)) + }; + return Ok(RateLimitInfo { + limit, + remaining: 0, + reset, + allowed: false, + }); + } + + if self.global_rpm > 0 { + state.global.push_back(now); + } + if self.per_ip_rpm > 0 { + state.per_ip.entry(ip).or_default().push_back(now); + } + let (limit, remaining, reset) = if self.per_ip_rpm > 0 { + let window = state.per_ip.get(&ip); + ( + self.per_ip_rpm, + self.per_ip_rpm + .saturating_sub(window.map_or(0, |window| window.len() as u64)), + window.map_or_else(|| unix_now() + 60, |window| reset_at(window, now)), + ) + } else { + ( + self.global_rpm, + self.global_rpm.saturating_sub(state.global.len() as u64), + reset_at(&state.global, now), + ) + }; + Ok(RateLimitInfo { + limit, + remaining, + reset, + allowed: true, + }) + } +} + +fn prune(window: &mut VecDeque, cutoff: Instant) { + while window.front().is_some_and(|instant| *instant < cutoff) { + window.pop_front(); + } +} + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +fn reset_at(window: &VecDeque, now: Instant) -> u64 { + window.front().map_or_else( + || unix_now() + WINDOW.as_secs(), + |oldest| unix_now() + (*oldest + WINDOW).saturating_duration_since(now).as_secs(), + ) +} + +pub struct PublicRateLimit; + +#[rocket::async_trait] +impl<'r> FromRequest<'r> for PublicRateLimit { + type Error = ApiError; + + async fn from_request(request: &'r Request<'_>) -> Outcome { + let Some(limiter) = request.rocket().state::() else { + return Outcome::Error(( + Status::InternalServerError, + ApiError::Internal("rate limiter unavailable".into()), + )); + }; + let ip = request + .client_ip() + .unwrap_or(IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED)); + match limiter.check(ip) { + Ok(info) if info.allowed => { + let cache = request.local_cache(|| CachedRateLimitInfo(Mutex::new(None))); + if let Ok(mut value) = cache.0.lock() { + *value = Some(info); + } + Outcome::Success(Self) + } + Ok(info) => { + let cache = request.local_cache(|| CachedRateLimitInfo(Mutex::new(None))); + if let Ok(mut value) = cache.0.lock() { + *value = Some(info); + } + Outcome::Error(( + Status::TooManyRequests, + ApiError::RateLimited("too many requests; try again later".into()), + )) + } + Err(error) => Outcome::Error((Status::InternalServerError, error)), + } + } +} + +pub struct RateLimitHeaders; + +#[rocket::async_trait] +impl Fairing for RateLimitHeaders { + fn info(&self) -> Info { + Info { + name: "Rate limit headers", + kind: Kind::Response, + } + } + + async fn on_response<'r>(&self, request: &'r Request<'_>, response: &mut Response<'r>) { + let cache = request.local_cache(|| CachedRateLimitInfo(Mutex::new(None))); + if let Ok(value) = cache.0.lock() { + if let Some(info) = value.as_ref() { + response.set_header(Header::new("X-RateLimit-Limit", info.limit.to_string())); + response.set_header(Header::new( + "X-RateLimit-Remaining", + info.remaining.to_string(), + )); + response.set_header(Header::new("X-RateLimit-Reset", info.reset.to_string())); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn limiter_enforces_per_ip_limit() { + let limiter = RateLimiter::new(100, 2); + let ip = IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); + assert!(limiter.check(ip).expect("first").allowed); + assert!(limiter.check(ip).expect("second").allowed); + assert!(!limiter.check(ip).expect("blocked").allowed); + } + + #[test] + fn limiter_amortizes_stale_ip_cleanup() { + let limiter = RateLimiter::new(0, 1); + let stale = Instant::now() - Duration::from_secs(61); + { + let mut state = limiter.state.lock().expect("rate-limit state"); + (1..=5).for_each(|last_octet| { + state.per_ip.insert( + IpAddr::V4(std::net::Ipv4Addr::new(192, 0, 2, last_octet)), + VecDeque::from([stale]), + ); + }); + } + let active_ip = IpAddr::V4(std::net::Ipv4Addr::LOCALHOST); + (0..PER_IP_CLEANUP_EVERY).for_each(|_| { + let _ = limiter.check(active_ip).expect("rate-limit check"); + }); + let state = limiter.state.lock().expect("rate-limit state"); + assert_eq!(state.per_ip.len(), 1); + assert!(state.per_ip.contains_key(&active_ip)); + } +} diff --git a/crates/rest_api/src/main.rs b/crates/rest_api/src/main.rs index f292a5042d..dc2c373611 100644 --- a/crates/rest_api/src/main.rs +++ b/crates/rest_api/src/main.rs @@ -1,638 +1,166 @@ +#[macro_use] +extern crate rocket; + +mod cache; +mod catchers; +mod config; mod error; +mod fairings; +mod provider; mod routes; - -use error::ApiErrorResponse; -use rocket::http::Method; -use rocket::{launch, Build, Rocket}; -use rocket_cors::{AllowedHeaders, AllowedOrigins, CorsOptions}; -use routes::take_orders::{ - ApprovalApiResponse, BuyRequest, SellRequest, TakeOrdersApiResponse, TakeOrdersReadyResponse, -}; +mod service; +mod telemetry; + +use cache::MarketSnapshotCache; +use config::Config; +use provider::RaindexProvider; +use rocket_cors::{AllowedHeaders, AllowedMethods, AllowedOrigins, CorsOptions}; +use std::{collections::HashSet, str::FromStr, sync::Arc, time::Duration}; use utoipa::OpenApi; use utoipa_swagger_ui::SwaggerUi; +pub struct AppState { + source: Arc, +} + #[derive(OpenApi)] #[openapi( - info( - title = "Rain Raindex API", - description = "REST API for interacting with Rain Raindex." + paths( + routes::health::health, + routes::health::detailed_health, + routes::markets::tickers, + routes::markets::orderbook, + routes::markets::markets, ), - paths(routes::take_orders::buy, routes::take_orders::sell), components(schemas( - BuyRequest, - SellRequest, - TakeOrdersApiResponse, - ApprovalApiResponse, - TakeOrdersReadyResponse, - ApiErrorResponse + error::ApiErrorCode, + error::ApiErrorDetail, + error::ApiErrorResponse, + routes::health::HealthResponse, + routes::health::DetailedHealthResponse, + routes::markets::TickerResponse, + routes::markets::OrderbookResponse, )), tags( - (name = "Take Orders", description = "Endpoints for generating take orders calldata") + (name = "Markets", description = "Public ticker and orderbook compatibility endpoints"), + (name = "Raindex", description = "Raindex market data"), + (name = "Health", description = "Service and indexer health") + ), + info( + title = "Raindex Market Data API", + description = "Public cached market data backed by the Raindex local indexer", ) )] struct ApiDoc; -fn configure_cors() -> CorsOptions { - CorsOptions { +#[derive(Debug, thiserror::Error)] +enum StartupError { + #[error(transparent)] + Config(#[from] config::ConfigError), + #[error(transparent)] + Provider(#[from] provider::ProviderError), + #[error("failed to warm market cache: {0}")] + CacheWarm(String), + #[error("invalid CORS method: {0}")] + CorsMethod(String), + #[error(transparent)] + Cors(#[from] rocket_cors::Error), + #[error("Rocket failed: {0}")] + Rocket(String), + #[error("failed to initialize logging: {0}")] + Logging(String), +} + +fn cors() -> Result { + let methods = ["Get", "Options"] + .into_iter() + .map(|method| { + rocket_cors::Method::from_str(method) + .map_err(|_| StartupError::CorsMethod(method.to_string())) + }) + .collect::>()?; + Ok(CorsOptions { allowed_origins: AllowedOrigins::all(), - allowed_methods: vec![Method::Get, Method::Post, Method::Options] - .into_iter() - .map(From::from) - .collect(), + allowed_methods: methods, allowed_headers: AllowedHeaders::all(), + expose_headers: HashSet::from([ + "X-Request-Id".to_string(), + "Retry-After".to_string(), + "X-RateLimit-Limit".to_string(), + "X-RateLimit-Remaining".to_string(), + "X-RateLimit-Reset".to_string(), + ]), ..Default::default() } + .to_cors()?) } -fn rocket() -> Rocket { - let cors = configure_cors() - .to_cors() - .expect("CORS configuration failed"); - - rocket::build() - .attach(cors.clone()) - .mount("/", routes::take_orders::routes()) - .mount("/", rocket_cors::catch_all_options_routes()) - .mount( - "/", - SwaggerUi::new("/swagger/").url("/swagger/openapi.json", ApiDoc::openapi()), - ) - .manage(cors) -} - -#[launch] -fn launch() -> Rocket { - rocket() -} - -#[cfg(test)] -mod tests { - use super::*; - use rocket::http::{ContentType, Status}; - use rocket::local::blocking::Client; - - fn client() -> Client { - Client::tracked(rocket()).expect("valid rocket instance") - } - - #[test] - fn test_cors_preflight_buy() { - let client = client(); - let response = client - .options("/take-orders/buy") - .header(rocket::http::Header::new( - "Access-Control-Request-Method", - "POST", - )) - .header(rocket::http::Header::new( - "Access-Control-Request-Headers", - "content-type", - )) - .dispatch(); - - assert_eq!(response.status(), Status::Ok); +fn rocket_figment(trusted_proxy_ip_header: Option<&str>) -> rocket::figment::Figment { + let figment = rocket::Config::figment(); + match trusted_proxy_ip_header { + Some(header) => figment.merge((rocket::Config::IP_HEADER, header)), + None => figment.merge((rocket::Config::IP_HEADER, false)), } +} - #[test] - fn test_cors_preflight_sell() { - let client = client(); - let response = client - .options("/take-orders/sell") - .header(rocket::http::Header::new( - "Access-Control-Request-Method", - "POST", +async fn build(config: Config) -> Result, StartupError> { + let provider = + Arc::new(RaindexProvider::load(&config.registry_url, config.local_db_path.clone()).await?); + let cache = MarketSnapshotCache::new( + Duration::from_secs(config.cache_ttl_seconds), + config.snapshot_recent_trades_limit, + ); + let market_data = Arc::new(service::CachedMarketData::new(provider, cache)); + tracing::info!( + timeout_seconds = config.local_db_ready_timeout_seconds, + "waiting for local market index readiness" + ); + market_data + .wait_for_local_index(Duration::from_secs(config.local_db_ready_timeout_seconds)) + .await + .map_err(|error| StartupError::CacheWarm(error.to_string()))?; + tracing::info!("warming market snapshot cache"); + market_data + .warm() + .await + .map_err(|error| StartupError::CacheWarm(error.to_string()))?; + tracing::info!("market snapshot cache is ready"); + market_data + .clone() + .start_background_refresh(Duration::from_secs(config.cache_ttl_seconds)); + let state = AppState { + source: market_data, + }; + Ok( + rocket::custom(rocket_figment(config.trusted_proxy_ip_header.as_deref())) + .manage(state) + .manage(fairings::RateLimiter::new( + config.rate_limit_global_rpm, + config.rate_limit_per_ip_rpm, )) - .header(rocket::http::Header::new( - "Access-Control-Request-Headers", - "content-type", - )) - .dispatch(); - - assert_eq!(response.status(), Status::Ok); - } - - #[test] - fn test_buy_missing_yaml() { - let client = client(); - let response = client - .post("/take-orders/buy") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "100", - "maxRatio": "2.5" - }"#, - ) - .dispatch(); - - assert_eq!(response.status(), Status::BadRequest); - } - - #[test] - fn test_buy_invalid_address() { - let client = client(); - let response = client - .post("/take-orders/buy") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "version: 1", - "taker": "invalid-address", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "100", - "maxRatio": "2.5" - }"#, - ) - .dispatch(); - - assert_eq!(response.status(), Status::BadRequest); - } - - #[test] - fn test_buy_same_token_pair() { - let client = client(); - let response = client - .post("/take-orders/buy") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "amount": "100", - "maxRatio": "2.5" - }"#, - ) - .dispatch(); - - assert_eq!(response.status(), Status::BadRequest); - } - - #[test] - fn test_buy_zero_amount() { - let client = client(); - let response = client - .post("/take-orders/buy") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "0", - "maxRatio": "2.5" - }"#, - ) - .dispatch(); - - assert_eq!(response.status(), Status::BadRequest); - } - - #[test] - fn test_buy_negative_max_ratio() { - let client = client(); - let response = client - .post("/take-orders/buy") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "100", - "maxRatio": "-1" - }"#, - ) - .dispatch(); - - assert_eq!(response.status(), Status::BadRequest); - } - - #[test] - fn test_buy_missing_field() { - let client = client(); - let response = client - .post("/take-orders/buy") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "amount": "100", - "maxRatio": "2.5" - }"#, - ) - .dispatch(); - - assert_eq!(response.status(), Status::UnprocessableEntity); - } - - #[test] - fn test_sell_missing_yaml() { - let client = client(); - let response = client - .post("/take-orders/sell") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "100", - "maxRatio": "2.5" - }"#, - ) - .dispatch(); - - assert_eq!(response.status(), Status::BadRequest); - } - - #[test] - fn test_sell_same_token_pair() { - let client = client(); - let response = client - .post("/take-orders/sell") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "amount": "100", - "maxRatio": "2.5" - }"#, - ) - .dispatch(); - - assert_eq!(response.status(), Status::BadRequest); - } - - #[test] - fn test_sell_invalid_address() { - let client = client(); - let response = client - .post("/take-orders/sell") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "version: 1", - "taker": "invalid-address", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "100", - "maxRatio": "2.5" - }"#, - ) - .dispatch(); - - assert_eq!(response.status(), Status::BadRequest); - } - - #[test] - fn test_sell_zero_amount() { - let client = client(); - let response = client - .post("/take-orders/sell") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "0", - "maxRatio": "2.5" - }"#, - ) - .dispatch(); - - assert_eq!(response.status(), Status::BadRequest); - } - - #[test] - fn test_sell_negative_max_ratio() { - let client = client(); - let response = client - .post("/take-orders/sell") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "100", - "maxRatio": "-1" - }"#, - ) - .dispatch(); - - assert_eq!(response.status(), Status::BadRequest); - } - - #[test] - fn test_sell_missing_field() { - let client = client(); - let response = client - .post("/take-orders/sell") - .header(ContentType::JSON) - .body( - r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "amount": "100", - "maxRatio": "2.5" - }"#, + .mount("/", routes::health::routes()) + .mount("/", routes::markets::compatibility_routes()) + .mount("/v1", routes::markets::raindex_routes()) + .mount( + "/", + SwaggerUi::new("/swagger/<_..>").url("/api-doc/openapi.json", ApiDoc::openapi()), ) - .dispatch(); - - assert_eq!(response.status(), Status::UnprocessableEntity); - } - - #[test] - fn test_swagger_ui_returns_html() { - let client = client(); - let response = client.get("/swagger/").dispatch(); - - assert_eq!(response.status(), Status::Ok); - let body = response.into_string().unwrap(); - assert!(body.contains("")); - assert!(body.contains("swagger-ui")); - } - - #[test] - fn test_openapi_json_returns_valid_spec() { - let client = client(); - let response = client.get("/swagger/openapi.json").dispatch(); - - assert_eq!(response.status(), Status::Ok); - let body = response.into_string().unwrap(); - let spec: serde_json::Value = serde_json::from_str(&body).unwrap(); - - assert_eq!(spec["openapi"], "3.1.0"); - assert_eq!(spec["info"]["title"], "Rain Raindex API"); - } - - #[test] - fn test_openapi_json_contains_buy_and_sell_paths() { - let client = client(); - let response = client.get("/swagger/openapi.json").dispatch(); - let body = response.into_string().unwrap(); - let spec: serde_json::Value = serde_json::from_str(&body).unwrap(); - - assert!(spec["paths"]["/take-orders/buy"]["post"].is_object()); - assert_eq!( - spec["paths"]["/take-orders/buy"]["post"]["tags"][0], - "Take Orders" - ); - - assert!(spec["paths"]["/take-orders/sell"]["post"].is_object()); - assert_eq!( - spec["paths"]["/take-orders/sell"]["post"]["tags"][0], - "Take Orders" - ); - } - - #[test] - fn test_openapi_json_contains_schemas() { - let client = client(); - let response = client.get("/swagger/openapi.json").dispatch(); - let body = response.into_string().unwrap(); - let spec: serde_json::Value = serde_json::from_str(&body).unwrap(); - - let schemas = &spec["components"]["schemas"]; - assert!(schemas["BuyRequest"].is_object()); - assert!(schemas["SellRequest"].is_object()); - assert!(schemas["TakeOrdersApiResponse"].is_object()); - assert!(schemas["ApprovalApiResponse"].is_object()); - assert!(schemas["TakeOrdersReadyResponse"].is_object()); - assert!(schemas["ApiErrorResponse"].is_object()); - } - - #[test] - fn test_openapi_json_contains_response_codes() { - let client = client(); - let response = client.get("/swagger/openapi.json").dispatch(); - let body = response.into_string().unwrap(); - let spec: serde_json::Value = serde_json::from_str(&body).unwrap(); - - let buy_responses = &spec["paths"]["/take-orders/buy"]["post"]["responses"]; - assert!(buy_responses["200"].is_object()); - assert!(buy_responses["400"].is_object()); - assert!(buy_responses["404"].is_object()); - assert!(buy_responses["500"].is_object()); - - let sell_responses = &spec["paths"]["/take-orders/sell"]["post"]["responses"]; - assert!(sell_responses["200"].is_object()); - assert!(sell_responses["400"].is_object()); - assert!(sell_responses["404"].is_object()); - assert!(sell_responses["500"].is_object()); - } - - #[test] - fn test_openapi_buy_request_field_descriptions() { - let client = client(); - let response = client.get("/swagger/openapi.json").dispatch(); - let body = response.into_string().unwrap(); - let spec: serde_json::Value = serde_json::from_str(&body).unwrap(); - - let buy_schema = &spec["components"]["schemas"]["BuyRequest"]["properties"]; - - assert_eq!( - buy_schema["yamlContent"]["description"], - "YAML configuration containing network RPC endpoints, subgraph URLs, and raindex addresses" - ); - assert_eq!( - buy_schema["taker"]["description"], - "Address that will execute the transaction" - ); - assert_eq!( - buy_schema["chainId"]["description"], - "Chain ID where the trade will be executed" - ); - assert_eq!( - buy_schema["tokenIn"]["description"], - "Token address you are giving (spending)" - ); - assert_eq!( - buy_schema["tokenOut"]["description"], - "Token address you are receiving (buying)" - ); - assert_eq!( - buy_schema["amount"]["description"], - "Amount of tokenOut to receive (human-readable decimal string)" - ); - assert_eq!( - buy_schema["maxRatio"]["description"], - "Maximum price ratio (tokenIn per 1 tokenOut). Trade fails if actual ratio exceeds this." - ); - assert_eq!( - buy_schema["exact"]["description"], - "If true, transaction reverts unless exactly the specified amount is received. If false (default), receives up to the specified amount." - ); - } - - #[test] - fn test_openapi_sell_request_field_descriptions() { - let client = client(); - let response = client.get("/swagger/openapi.json").dispatch(); - let body = response.into_string().unwrap(); - let spec: serde_json::Value = serde_json::from_str(&body).unwrap(); - - let sell_schema = &spec["components"]["schemas"]["SellRequest"]["properties"]; - - assert_eq!( - sell_schema["yamlContent"]["description"], - "YAML configuration containing network RPC endpoints, subgraph URLs, and raindex addresses" - ); - assert_eq!( - sell_schema["taker"]["description"], - "Address that will execute the transaction" - ); - assert_eq!( - sell_schema["chainId"]["description"], - "Chain ID where the trade will be executed" - ); - assert_eq!( - sell_schema["tokenIn"]["description"], - "Token address you are giving (selling)" - ); - assert_eq!( - sell_schema["tokenOut"]["description"], - "Token address you are receiving" - ); - assert_eq!( - sell_schema["amount"]["description"], - "Amount of tokenIn to spend (human-readable decimal string)" - ); - assert_eq!( - sell_schema["maxRatio"]["description"], - "Maximum price ratio (tokenIn per 1 tokenOut). Trade fails if actual ratio exceeds this." - ); - assert_eq!( - sell_schema["exact"]["description"], - "If true, transaction reverts unless exactly the specified amount is spent. If false (default), spends up to the specified amount." - ); - } - - #[test] - fn test_openapi_response_field_descriptions() { - let client = client(); - let response = client.get("/swagger/openapi.json").dispatch(); - let body = response.into_string().unwrap(); - let spec: serde_json::Value = serde_json::from_str(&body).unwrap(); - - let ready_schema = &spec["components"]["schemas"]["TakeOrdersReadyResponse"]["properties"]; - - assert_eq!( - ready_schema["raindex"]["description"], - "Address of the raindex contract to call" - ); - assert_eq!( - ready_schema["calldata"]["description"], - "ABI-encoded calldata for the takeOrders4 function" - ); - assert_eq!( - ready_schema["effectivePrice"]["description"], - "Blended effective price across all selected orders (tokenIn per 1 tokenOut)" - ); - assert_eq!( - ready_schema["prices"]["description"], - "Individual prices for each order leg, sorted from best to worst" - ); - assert_eq!( - ready_schema["expectedSell"]["description"], - "Expected amount of tokenIn to spend based on current quotes" - ); - assert_eq!( - ready_schema["maxSellCap"]["description"], - "Maximum tokenIn that could be spent (worst-case based on maxRatio)" - ); - - let approval_schema = &spec["components"]["schemas"]["ApprovalApiResponse"]["properties"]; - - assert_eq!( - approval_schema["token"]["description"], - "Token address that needs approval" - ); - assert_eq!( - approval_schema["spender"]["description"], - "Spender address (the raindex contract)" - ); - assert_eq!( - approval_schema["amount"]["description"], - "Amount to approve (raw value)" - ); - assert_eq!( - approval_schema["formattedAmount"]["description"], - "Human-readable formatted amount" - ); - assert_eq!( - approval_schema["calldata"]["description"], - "ABI-encoded approval calldata" - ); - } - - #[test] - fn test_openapi_json_contains_response_examples() { - let client = client(); - let response = client.get("/swagger/openapi.json").dispatch(); - let body = response.into_string().unwrap(); - let spec: serde_json::Value = serde_json::from_str(&body).unwrap(); - - for endpoint in ["/take-orders/buy", "/take-orders/sell"] { - let examples = &spec["paths"][endpoint]["post"]["responses"]["200"]["content"] - ["application/json"]["examples"]; - - assert!( - examples["Ready"].is_object(), - "Ready example should exist for {endpoint}" - ); - assert!( - examples["NeedsApproval"].is_object(), - "NeedsApproval example should exist for {endpoint}" - ); - - let ready_value = &examples["Ready"]["value"]; - assert_eq!(ready_value["status"], "ready"); - assert!(ready_value["data"]["raindex"].is_string()); - assert!(ready_value["data"]["calldata"].is_string()); - assert!(ready_value["data"]["effectivePrice"].is_string()); - assert!(ready_value["data"]["prices"].is_array()); - assert!(ready_value["data"]["expectedSell"].is_string()); - assert!(ready_value["data"]["maxSellCap"].is_string()); + .register("/", catchers::catchers()) + .attach(fairings::RequestLogger) + .attach(fairings::RateLimitHeaders) + .attach(cors()?), + ) +} - let needs_approval_value = &examples["NeedsApproval"]["value"]; - assert_eq!(needs_approval_value["status"], "needsApproval"); - assert!(needs_approval_value["data"]["token"].is_string()); - assert!(needs_approval_value["data"]["spender"].is_string()); - assert!(needs_approval_value["data"]["amount"].is_string()); - assert!(needs_approval_value["data"]["formattedAmount"].is_string()); - assert!(needs_approval_value["data"]["calldata"].is_string()); - } - } +#[rocket::main] +async fn main() -> Result<(), StartupError> { + let config = Config::from_env()?; + let _logging_guard = telemetry::init(&config.log_dir) + .map_err(|error| StartupError::Logging(error.to_string()))?; + build(config) + .await? + .launch() + .await + .map_err(|error| StartupError::Rocket(error.to_string()))?; + Ok(()) } diff --git a/crates/rest_api/src/provider.rs b/crates/rest_api/src/provider.rs new file mode 100644 index 0000000000..478f108742 --- /dev/null +++ b/crates/rest_api/src/provider.rs @@ -0,0 +1,44 @@ +use raindex_common::{raindex_client::RaindexClient, registry::DotrainRegistry}; +use std::path::{Path, PathBuf}; + +#[derive(Debug)] +pub struct RaindexProvider { + client: RaindexClient, +} + +impl RaindexProvider { + pub async fn load(registry_url: &str, db_path: PathBuf) -> Result { + ensure_parent_directory(&db_path)?; + let registry = DotrainRegistry::new(registry_url.to_string()) + .await + .map_err(|error| ProviderError::Registry(error.to_string()))?; + let client = registry + .get_raindex_client(Some(db_path.clone())) + .await + .map_err(|error| ProviderError::Client(error.to_string()))?; + Ok(Self { client }) + } + + pub fn client(&self) -> &RaindexClient { + &self.client + } +} + +fn ensure_parent_directory(path: &Path) -> Result<(), ProviderError> { + path.parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .map(std::fs::create_dir_all) + .transpose() + .map(|_| ()) + .map_err(ProviderError::DatabaseDirectory) +} + +#[derive(Debug, thiserror::Error)] +pub enum ProviderError { + #[error("failed to prepare local database directory: {0}")] + DatabaseDirectory(std::io::Error), + #[error("failed to load registry: {0}")] + Registry(String), + #[error("failed to initialize Raindex client: {0}")] + Client(String), +} diff --git a/crates/rest_api/src/routes/health.rs b/crates/rest_api/src/routes/health.rs new file mode 100644 index 0000000000..1a2bbb384f --- /dev/null +++ b/crates/rest_api/src/routes/health.rs @@ -0,0 +1,54 @@ +use crate::{error::ApiError, fairings::PublicRateLimit, AppState}; +use rocket::{serde::json::Json, State}; +use serde::Serialize; +use utoipa::ToSchema; + +#[derive(Debug, Serialize, ToSchema)] +pub struct HealthResponse { + pub status: &'static str, +} + +#[derive(Debug, Serialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct DetailedHealthResponse { + pub status: &'static str, + pub sync_configured: bool, + pub sync_healthy: bool, + pub network_count: usize, + pub orderbook_count: usize, + pub snapshot_ready: bool, + pub snapshot_last_success_at: Option, + pub snapshot_refresh_healthy: bool, +} + +#[utoipa::path(get, path = "/health", responses((status = 200, body = HealthResponse)))] +#[get("/health")] +pub fn health(_limit: PublicRateLimit) -> Json { + Json(HealthResponse { status: "ok" }) +} + +#[utoipa::path(get, path = "/health/detailed", responses((status = 200, body = DetailedHealthResponse)))] +#[get("/health/detailed")] +pub async fn detailed_health( + _limit: PublicRateLimit, + state: &State, +) -> Result, ApiError> { + let snapshot = state.source.health().await.map_err(|error| { + tracing::error!(error = %error, "detailed health check failed"); + ApiError::Internal("health data is temporarily unavailable".into()) + })?; + Ok(Json(DetailedHealthResponse { + status: if snapshot.healthy { "ok" } else { "degraded" }, + sync_configured: snapshot.configured, + sync_healthy: snapshot.sync_healthy, + network_count: snapshot.network_count, + orderbook_count: snapshot.orderbook_count, + snapshot_ready: snapshot.snapshot_ready, + snapshot_last_success_at: snapshot.snapshot_last_success_at, + snapshot_refresh_healthy: snapshot.snapshot_refresh_healthy, + })) +} + +pub fn routes() -> Vec { + routes![health, detailed_health] +} diff --git a/crates/rest_api/src/routes/markets.rs b/crates/rest_api/src/routes/markets.rs new file mode 100644 index 0000000000..59cadf4e3e --- /dev/null +++ b/crates/rest_api/src/routes/markets.rs @@ -0,0 +1,624 @@ +use crate::{ + cache::MAX_ORDERBOOK_DEPTH, error::ApiError, fairings::PublicRateLimit, service::MarketDetail, + AppState, +}; +use raindex_common::raindex_client::markets::RaindexMarketSnapshot; +use rocket::{serde::json::Json, State}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use utoipa::ToSchema; + +const DEFAULT_DEPTH: usize = 100; + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema, PartialEq, Eq)] +pub struct TickerResponse { + pub ticker_id: String, + pub base_currency: String, + pub target_currency: String, + pub pool_id: String, + pub last_price: String, + pub base_volume: String, + pub target_volume: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub bid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ask: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub high: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub low: Option, +} + +impl TickerResponse { + fn from_snapshot(snapshot: &RaindexMarketSnapshot) -> Option { + Some(Self { + ticker_id: snapshot.market.ticker_id.clone(), + base_currency: format!("{:#x}", snapshot.market.base.address), + target_currency: format!("{:#x}", snapshot.market.quote.address), + pool_id: snapshot.market.id.clone(), + last_price: snapshot.stats.last_price.clone()?, + base_volume: snapshot.stats.base_volume_24h.clone(), + target_volume: snapshot.stats.target_volume_24h.clone(), + bid: snapshot.orderbook.best_bid.clone(), + ask: snapshot.orderbook.best_ask.clone(), + high: snapshot.stats.high_24h.clone(), + low: snapshot.stats.low_24h.clone(), + }) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, ToSchema, PartialEq, Eq)] +pub struct OrderbookResponse { + pub ticker_id: String, + pub timestamp: u64, + pub bids: Vec<[String; 2]>, + pub asks: Vec<[String; 2]>, +} + +async fn snapshots(state: &AppState) -> Result>, ApiError> { + state.source.snapshots().await.map_err(|error| { + tracing::error!(error = %error, "market snapshot refresh failed"); + ApiError::UpstreamUnavailable("market data is temporarily unavailable".into()) + }) +} + +async fn market_snapshot(state: &AppState, ticker_id: &str) -> Result { + state + .source + .market(ticker_id) + .await + .map_err(|error| { + tracing::error!(ticker_id, error = %error, "market detail refresh failed"); + ApiError::UpstreamUnavailable("market data is temporarily unavailable".into()) + })? + .ok_or_else(|| ApiError::NotFound(format!("unknown ticker_id: {ticker_id}"))) +} + +fn canonical_ticker_id(ticker_id: &str) -> String { + ticker_id.to_ascii_lowercase() +} + +fn validate_depth(depth: Option) -> Result { + let depth = depth.unwrap_or(DEFAULT_DEPTH); + let max_depth = usize::from(MAX_ORDERBOOK_DEPTH); + (depth <= max_depth) + .then_some(depth) + .ok_or_else(|| ApiError::BadRequest(format!("depth must be between 0 and {max_depth}"))) +} + +fn parse_depth(depth: Option<&str>) -> Result { + depth + .map(|value| { + value + .parse::() + .map_err(|_| ApiError::BadRequest("depth must be an unsigned integer".into())) + }) + .transpose() + .and_then(validate_depth) +} + +#[utoipa::path(get, path = "/tickers", tag = "Markets", responses((status = 200, body = [TickerResponse])))] +#[get("/tickers")] +pub async fn tickers( + _limit: PublicRateLimit, + state: &State, +) -> Result>, ApiError> { + Ok(Json( + snapshots(state) + .await? + .iter() + .filter_map(TickerResponse::from_snapshot) + .collect(), + )) +} + +#[utoipa::path( + get, + path = "/orderbook", + tag = "Markets", + params(("ticker_id" = String, Query), ("depth" = Option, Query)), + responses((status = 200, body = OrderbookResponse)) +)] +#[get("/orderbook?&")] +pub async fn orderbook( + ticker_id: &str, + depth: Option<&str>, + _limit: PublicRateLimit, + state: &State, +) -> Result, ApiError> { + let depth = parse_depth(depth)?; + let ticker_id = canonical_ticker_id(ticker_id); + let detail = market_snapshot(state, &ticker_id).await?; + let snapshot = &detail.snapshot; + let per_side = if depth == 0 { + usize::MAX + } else { + depth.div_ceil(2) + }; + Ok(Json(OrderbookResponse { + ticker_id: snapshot.market.ticker_id.clone(), + timestamp: detail.orderbook_observed_at.saturating_mul(1_000), + bids: snapshot + .orderbook + .bids + .iter() + .take(per_side) + .map(|level| [level.price.clone(), level.base_quantity.clone()]) + .collect(), + asks: snapshot + .orderbook + .asks + .iter() + .take(per_side) + .map(|level| [level.price.clone(), level.base_quantity.clone()]) + .collect(), + })) +} + +#[utoipa::path(get, path = "/v1/markets", tag = "Raindex", responses((status = 200, description = "Cached market overviews or one complete market snapshot")))] +#[get("/markets?&")] +pub async fn markets( + chain_id: Option, + ticker_id: Option<&str>, + _limit: PublicRateLimit, + state: &State, +) -> Result>, ApiError> { + let ticker_id = ticker_id.map(canonical_ticker_id); + let include_recent_trades = ticker_id.is_some(); + let snapshots = match ticker_id.as_deref() { + Some(ticker_id) => Arc::new(vec![market_snapshot(state, ticker_id) + .await? + .snapshot + .as_ref() + .clone()]), + None => snapshots(state).await?, + }; + Ok(Json( + snapshots + .iter() + .filter(|snapshot| chain_id.is_none_or(|id| snapshot.market.chain_id == id)) + .filter(|snapshot| { + ticker_id + .as_deref() + .is_none_or(|id| snapshot.market.ticker_id == id) + }) + .map(|snapshot| { + let mut snapshot = snapshot.clone(); + if !include_recent_trades { + snapshot.recent_trades.clear(); + } + snapshot + }) + .collect(), + )) +} + +pub fn compatibility_routes() -> Vec { + routes![tickers, orderbook] +} + +pub fn raindex_routes() -> Vec { + routes![markets] +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{fairings::RateLimiter, service::MarketDataSource}; + use alloy::primitives::{Address, B256}; + use async_trait::async_trait; + use raindex_common::raindex_client::markets::{ + RaindexMarket, RaindexMarketOrderbook, RaindexMarketOrderbookLevel, RaindexMarketStats, + RaindexMarketToken, RaindexMarketTrade, RaindexMarketTradeSide, + }; + use rocket::{ + http::{Header, Status}, + local::asynchronous::Client, + }; + use std::collections::HashMap; + + struct MockMarketData { + snapshots: Arc>, + orderbook_observed_at: u64, + } + + #[async_trait] + impl MarketDataSource for MockMarketData { + async fn snapshots( + &self, + ) -> Result>, crate::service::MarketDataError> { + Ok(Arc::clone(&self.snapshots)) + } + + async fn market( + &self, + _ticker_id: &str, + ) -> Result, crate::service::MarketDataError> { + Ok(self + .snapshots + .first() + .cloned() + .map(|snapshot| MarketDetail { + snapshot: Arc::new(snapshot), + orderbook_observed_at: self.orderbook_observed_at, + })) + } + + async fn health( + &self, + ) -> Result { + Err(crate::service::MarketDataError::Unavailable( + "unused".into(), + )) + } + } + + fn token(address_byte: u8, symbol: &str) -> RaindexMarketToken { + RaindexMarketToken { + chain_id: 8453, + address: Address::from([address_byte; 20]), + name: symbol.to_string(), + symbol: symbol.to_string(), + decimals: Some(18), + logo_uri: None, + extensions: Some(HashMap::new()), + unwrapped_address: None, + legacy_address: None, + receipt_address: None, + variants: Vec::new(), + } + } + + fn level(price: &str, quantity: &str) -> RaindexMarketOrderbookLevel { + RaindexMarketOrderbookLevel { + price: price.into(), + base_quantity: quantity.into(), + target_quantity: "10".into(), + chain_id: 8453, + raindex: Address::ZERO, + order_hash: B256::ZERO, + source_token: Address::ZERO, + block_number: 10, + } + } + + fn trade(side: RaindexMarketTradeSide, timestamp: u64) -> RaindexMarketTrade { + RaindexMarketTrade { + trade_id: format!("trade-{timestamp}"), + price: "2".into(), + base_volume: "3".into(), + target_volume: "6".into(), + timestamp, + block_number: 10, + trade_event_id: format!("event-{timestamp}"), + trade_event_kind: "take-order".into(), + side, + chain_id: 8453, + raindex: Address::ZERO, + order_hash: B256::ZERO, + source_token: Address::ZERO, + } + } + + fn snapshot() -> RaindexMarketSnapshot { + let base = token(1, "BASE"); + let quote = token(2, "QUOTE"); + RaindexMarketSnapshot { + market: RaindexMarket { + id: "8453:base_quote".into(), + ticker_id: format!("{:#x}_{:#x}", base.address, quote.address), + chain_id: 8453, + base, + quote, + raindex_addresses: vec![Address::ZERO], + }, + orderbook: RaindexMarketOrderbook { + best_bid: Some("1.9".into()), + best_ask: Some("2.1".into()), + midpoint: Some("2".into()), + bids: vec![level("1.9", "5"), level("1.8", "4")], + asks: vec![level("2.1", "6"), level("2.2", "7")], + }, + stats: RaindexMarketStats { + last_price: Some("2".into()), + high_24h: Some("3".into()), + low_24h: Some("1".into()), + base_volume_24h: "30".into(), + target_volume_24h: "60".into(), + trade_count_24h: 2, + }, + recent_trades: vec![ + trade(RaindexMarketTradeSide::Buy, 200), + trade(RaindexMarketTradeSide::Sell, 100), + ], + observed_at: 300, + block_number: Some(10), + ratio_block_number: None, + assets_per_share: None, + errors: Vec::new(), + } + } + + async fn client_with_snapshots_and_rate_limit( + snapshots: Vec, + per_ip_rpm: u64, + ) -> Client { + let orderbook_observed_at = snapshots.first().map_or(0, |snapshot| snapshot.observed_at); + client_with_snapshots_rate_limit_and_orderbook_time( + snapshots, + per_ip_rpm, + orderbook_observed_at, + ) + .await + } + + async fn client_with_snapshots_rate_limit_and_orderbook_time( + snapshots: Vec, + per_ip_rpm: u64, + orderbook_observed_at: u64, + ) -> Client { + let state = AppState { + source: Arc::new(MockMarketData { + snapshots: Arc::new(snapshots), + orderbook_observed_at, + }), + }; + Client::tracked( + rocket::custom(crate::rocket_figment(None)) + .manage(state) + .manage(RateLimiter::new(10_000, per_ip_rpm)) + .mount("/", compatibility_routes()) + .mount("/v1", raindex_routes()) + .register("/", crate::catchers::catchers()) + .attach(crate::fairings::RequestLogger) + .attach(crate::fairings::RateLimitHeaders), + ) + .await + .expect("test Rocket client") + } + + async fn client_with_rate_limit(per_ip_rpm: u64) -> Client { + client_with_snapshots_and_rate_limit(vec![snapshot()], per_ip_rpm).await + } + + async fn client() -> Client { + client_with_rate_limit(10_000).await + } + + #[test] + fn validates_depth_bounds() { + assert_eq!(validate_depth(None).expect("default depth"), 100); + assert_eq!(validate_depth(Some(0)).expect("full depth"), 0); + assert!(validate_depth(Some(usize::from(MAX_ORDERBOOK_DEPTH) + 1)).is_err()); + } + + #[rocket::async_test] + async fn ticker_route_maps_cached_snapshot() { + let client = client().await; + let response = client.get("/tickers").dispatch().await; + assert_eq!(response.status(), Status::Ok); + let body = response + .into_json::>() + .await + .expect("json"); + assert_eq!(body.len(), 1); + assert_eq!(body[0].pool_id, "8453:base_quote"); + assert_eq!(body[0].last_price, "2"); + assert_eq!(body[0].base_volume, "30"); + assert_eq!(body[0].bid.as_deref(), Some("1.9")); + } + + #[rocket::async_test] + async fn ticker_route_excludes_markets_without_a_genuine_last_trade() { + let mut never_traded = snapshot(); + never_traded.market.id = "8453:never_traded".into(); + never_traded.market.ticker_id = "never_traded_quote".into(); + never_traded.stats.last_price = None; + let client = + client_with_snapshots_and_rate_limit(vec![snapshot(), never_traded.clone()], 10_000) + .await; + + let tickers = client + .get("/tickers") + .dispatch() + .await + .into_json::>() + .await + .expect("tickers json"); + assert_eq!(tickers.len(), 1); + assert_eq!(tickers[0].last_price, "2"); + + let markets = client + .get("/v1/markets") + .dispatch() + .await + .into_json::>() + .await + .expect("markets json"); + assert_eq!(markets.len(), 2); + assert!(markets.iter().any(|snapshot| { + snapshot.market.ticker_id == never_traded.market.ticker_id + && snapshot.stats.last_price.is_none() + })); + } + + #[rocket::async_test] + async fn empty_market_configuration_returns_successful_empty_collections() { + let client = client_with_snapshots_and_rate_limit(vec![], 10_000).await; + + let tickers = client.get("/tickers").dispatch().await; + assert_eq!(tickers.status(), Status::Ok); + assert_eq!( + tickers + .into_json::>() + .await + .expect("tickers json"), + vec![] + ); + + let markets = client.get("/v1/markets").dispatch().await; + assert_eq!(markets.status(), Status::Ok); + assert_eq!( + markets + .into_json::>() + .await + .expect("markets json"), + vec![] + ); + } + + #[rocket::async_test] + async fn orderbook_route_applies_total_depth_across_both_sides() { + let ticker_id = snapshot().market.ticker_id; + let client = client().await; + let response = client + .get(format!("/orderbook?ticker_id={ticker_id}&depth=2")) + .dispatch() + .await; + assert_eq!(response.status(), Status::Ok); + let body = response + .into_json::() + .await + .expect("json"); + assert_eq!(body.timestamp, 300_000); + assert_eq!(body.bids, vec![[String::from("1.9"), String::from("5")]]); + assert_eq!(body.asks, vec![[String::from("2.1"), String::from("6")]]); + } + + #[rocket::async_test] + async fn orderbook_route_uses_detail_observation_timestamp() { + let snapshot = snapshot(); + let ticker_id = snapshot.market.ticker_id.clone(); + let client = + client_with_snapshots_rate_limit_and_orderbook_time(vec![snapshot], 10_000, 450).await; + + let body = client + .get(format!("/orderbook?ticker_id={ticker_id}")) + .dispatch() + .await + .into_json::() + .await + .expect("json"); + + assert_eq!(body.timestamp, 450_000); + } + + #[rocket::async_test] + async fn orderbook_route_rounds_odd_depth_up_per_side() { + let ticker_id = snapshot().market.ticker_id; + let client = client().await; + let response = client + .get(format!("/orderbook?ticker_id={ticker_id}&depth=3")) + .dispatch() + .await; + assert_eq!(response.status(), Status::Ok); + let body = response + .into_json::() + .await + .expect("json"); + assert_eq!(body.bids.len(), 2); + assert_eq!(body.asks.len(), 2); + } + + #[rocket::async_test] + async fn raindex_overview_omits_per_market_trade_payloads() { + let client = client().await; + let response = client.get("/v1/markets?chain_id=8453").dispatch().await; + assert_eq!(response.status(), Status::Ok); + let body = response + .into_json::>() + .await + .expect("json"); + assert!(body[0].recent_trades.is_empty()); + assert_eq!(body[0].stats, snapshot().stats); + } + + #[rocket::async_test] + async fn raindex_market_detail_preserves_recent_trades() { + let expected = snapshot(); + let client = client().await; + let response = client + .get(format!( + "/v1/markets?ticker_id={}", + expected.market.ticker_id + )) + .dispatch() + .await; + assert_eq!(response.status(), Status::Ok); + let body = response + .into_json::>() + .await + .expect("json"); + assert_eq!(body, vec![expected]); + } + + #[rocket::async_test] + async fn public_routes_return_structured_rate_limit_errors() { + let client = client_with_rate_limit(1).await; + assert_eq!(client.get("/tickers").dispatch().await.status(), Status::Ok); + let response = client.get("/tickers").dispatch().await; + assert_eq!(response.status(), Status::TooManyRequests); + assert_eq!(response.headers().get_one("Retry-After"), Some("60")); + assert_eq!( + response.headers().get_one("X-RateLimit-Remaining"), + Some("0") + ); + let body = response + .into_json::() + .await + .expect("structured error"); + assert_eq!(body.error.code, crate::error::ApiErrorCode::RateLimited); + } + + #[rocket::async_test] + async fn spoofed_ip_headers_do_not_bypass_public_rate_limit() { + let client = client_with_rate_limit(1).await; + let first = client + .get("/tickers") + .header(Header::new("X-Real-IP", "192.0.2.1")) + .dispatch() + .await; + assert_eq!(first.status(), Status::Ok); + let second = client + .get("/tickers") + .header(Header::new("X-Real-IP", "192.0.2.2")) + .dispatch() + .await; + assert_eq!(second.status(), Status::TooManyRequests); + } + + #[rocket::async_test] + async fn unknown_routes_return_structured_errors_with_request_ids() { + let client = client().await; + let response = client.get("/missing").dispatch().await; + assert_eq!(response.status(), Status::NotFound); + let request_id = response + .headers() + .get_one("X-Request-Id") + .expect("request ID") + .to_string(); + assert!(uuid::Uuid::parse_str(&request_id).is_ok()); + let body = response + .into_json::() + .await + .expect("structured error"); + assert_eq!(body.error.code, crate::error::ApiErrorCode::NotFound); + assert_eq!(body.request_id, request_id); + } + + #[rocket::async_test] + async fn malformed_query_parameters_return_structured_bad_requests() { + let ticker_id = snapshot().market.ticker_id; + let client = client().await; + let response = client + .get(format!("/orderbook?ticker_id={ticker_id}&depth=invalid")) + .dispatch() + .await; + assert_eq!(response.status(), Status::BadRequest); + let body = response + .into_json::() + .await + .expect("structured error"); + assert_eq!(body.error.code, crate::error::ApiErrorCode::BadRequest); + } +} diff --git a/crates/rest_api/src/routes/mod.rs b/crates/rest_api/src/routes/mod.rs index 47d76071b0..38a7831546 100644 --- a/crates/rest_api/src/routes/mod.rs +++ b/crates/rest_api/src/routes/mod.rs @@ -1 +1,2 @@ -pub mod take_orders; +pub mod health; +pub mod markets; diff --git a/crates/rest_api/src/routes/take_orders.rs b/crates/rest_api/src/routes/take_orders.rs deleted file mode 100644 index 1528e2abc2..0000000000 --- a/crates/rest_api/src/routes/take_orders.rs +++ /dev/null @@ -1,504 +0,0 @@ -use crate::error::{ApiError, ApiErrorResponse}; -use raindex_common::raindex_client::take_orders::TakeOrdersRequest; -use raindex_common::raindex_client::RaindexClient; -use raindex_common::take_orders::TakeOrdersMode; -use rocket::serde::json::Json; -use rocket::{post, Route}; -use serde::{Deserialize, Serialize}; -use utoipa::ToSchema; - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "camelCase")] -pub struct BuyRequest { - /// YAML configuration containing network RPC endpoints, subgraph URLs, and raindex addresses - #[schema( - example = "networks:\n base:\n rpc: https://mainnet.base.org\n chain-id: 8453\nsubgraphs:\n base: https://api.goldsky.com/api/public/project_clv14x04y9kzi01saerx7bxpg/subgraphs/raindex-base/0.9/gn\nraindexes:\n base:\n address: 0xd2938e7c9fe3597f78832ce780feb61945c377d7\n network: base\n subgraph: base" - )] - pub yaml_content: String, - /// Address that will execute the transaction - #[schema(example = "0x1111111111111111111111111111111111111111")] - pub taker: String, - /// Chain ID where the trade will be executed - #[schema(example = 8453)] - pub chain_id: u32, - /// Token address you are giving (spending) - #[schema(example = "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913")] - pub token_in: String, - /// Token address you are receiving (buying) - #[schema(example = "0x4200000000000000000000000000000000000006")] - pub token_out: String, - /// Amount of tokenOut to receive (human-readable decimal string) - #[schema(example = "1000")] - pub amount: String, - /// Maximum price ratio (tokenIn per 1 tokenOut). Trade fails if actual ratio exceeds this. - #[schema(example = "0.0005")] - pub max_ratio: String, - /// If true, transaction reverts unless exactly the specified amount is received. If false (default), receives up to the specified amount. - #[serde(default)] - #[schema(example = false)] - pub exact: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "camelCase")] -pub struct SellRequest { - /// YAML configuration containing network RPC endpoints, subgraph URLs, and raindex addresses - #[schema( - example = "networks:\n base:\n rpc: https://mainnet.base.org\n chain-id: 8453\nsubgraphs:\n base: https://api.goldsky.com/api/public/project_clv14x04y9kzi01saerx7bxpg/subgraphs/raindex-base/0.9/gn\nraindexes:\n base:\n address: 0xd2938e7c9fe3597f78832ce780feb61945c377d7\n network: base\n subgraph: base" - )] - pub yaml_content: String, - /// Address that will execute the transaction - #[schema(example = "0x1111111111111111111111111111111111111111")] - pub taker: String, - /// Chain ID where the trade will be executed - #[schema(example = 8453)] - pub chain_id: u32, - /// Token address you are giving (selling) - #[schema(example = "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913")] - pub token_in: String, - /// Token address you are receiving - #[schema(example = "0x4200000000000000000000000000000000000006")] - pub token_out: String, - /// Amount of tokenIn to spend (human-readable decimal string) - #[schema(example = "500")] - pub amount: String, - /// Maximum price ratio (tokenIn per 1 tokenOut). Trade fails if actual ratio exceeds this. - #[schema(example = "0.0005")] - pub max_ratio: String, - /// If true, transaction reverts unless exactly the specified amount is spent. If false (default), spends up to the specified amount. - #[serde(default)] - #[schema(example = false)] - pub exact: bool, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "camelCase")] -#[schema(example = json!({ - "token": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", - "spender": "0xd2938e7c9fe3597f78832ce780feb61945c377d7", - "amount": "1000", - "formattedAmount": "1000", - "calldata": "0x095ea7b3..." -}))] -pub struct ApprovalApiResponse { - /// Token address that needs approval - #[schema(example = "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913")] - pub token: String, - /// Spender address (the raindex contract) - #[schema(example = "0xd2938e7c9fe3597f78832ce780feb61945c377d7")] - pub spender: String, - /// Amount to approve (raw value) - #[schema(example = "1000")] - pub amount: String, - /// Human-readable formatted amount - #[schema(example = "1000")] - pub formatted_amount: String, - /// ABI-encoded approval calldata - #[schema(example = "0x095ea7b3...")] - pub calldata: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "camelCase")] -#[schema(example = json!({ - "raindex": "0xd2938e7c9fe3597f78832ce780feb61945c377d7", - "calldata": "0x...", - "effectivePrice": "0.00045", - "prices": ["0.00044", "0.00046"], - "expectedSell": "450", - "maxSellCap": "500" -}))] -pub struct TakeOrdersReadyResponse { - /// Address of the raindex contract to call - #[schema(example = "0xd2938e7c9fe3597f78832ce780feb61945c377d7")] - pub raindex: String, - /// ABI-encoded calldata for the takeOrders4 function - #[schema(example = "0x...")] - pub calldata: String, - /// Blended effective price across all selected orders (tokenIn per 1 tokenOut) - #[schema(example = "0.00045")] - pub effective_price: String, - /// Individual prices for each order leg, sorted from best to worst - #[schema(example = json!(["0.00044", "0.00046"]))] - pub prices: Vec, - /// Expected amount of tokenIn to spend based on current quotes - #[schema(example = "450")] - pub expected_sell: String, - /// Maximum tokenIn that could be spent (worst-case based on maxRatio) - #[schema(example = "500")] - pub max_sell_cap: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] -#[serde(rename_all = "camelCase", tag = "status", content = "data")] -pub enum TakeOrdersApiResponse { - #[schema(title = "NeedsApproval")] - NeedsApproval(ApprovalApiResponse), - #[schema(title = "Ready")] - Ready(TakeOrdersReadyResponse), -} - -async fn execute_take_orders( - yaml_content: String, - request: TakeOrdersRequest, -) -> Result { - let client = RaindexClient::new(vec![yaml_content], None, None).await?; - - let result = client.get_take_orders_calldata(request).await?; - - if let Some(approval_info) = result.approval_info() { - let amount = approval_info.amount().format().map_err(|e| { - ApiError::Raindex(raindex_common::raindex_client::RaindexError::Float(e)) - })?; - - Ok(TakeOrdersApiResponse::NeedsApproval(ApprovalApiResponse { - token: approval_info.token().to_string(), - spender: approval_info.spender().to_string(), - amount, - formatted_amount: approval_info.formatted_amount().to_string(), - calldata: approval_info.calldata().to_string(), - })) - } else if let Some(take_orders_info) = result.take_orders_info() { - let effective_price = take_orders_info.effective_price().format().map_err(|e| { - ApiError::Raindex(raindex_common::raindex_client::RaindexError::Float(e)) - })?; - - let prices: Result, _> = take_orders_info - .prices() - .iter() - .map(|p| { - p.format().map_err(|e| { - ApiError::Raindex(raindex_common::raindex_client::RaindexError::Float(e)) - }) - }) - .collect(); - - let expected_sell = take_orders_info.expected_sell().format().map_err(|e| { - ApiError::Raindex(raindex_common::raindex_client::RaindexError::Float(e)) - })?; - - let max_sell_cap = take_orders_info.max_sell_cap().format().map_err(|e| { - ApiError::Raindex(raindex_common::raindex_client::RaindexError::Float(e)) - })?; - - Ok(TakeOrdersApiResponse::Ready(TakeOrdersReadyResponse { - raindex: take_orders_info.raindex().to_string(), - calldata: take_orders_info.calldata().to_string(), - effective_price, - prices: prices?, - expected_sell, - max_sell_cap, - })) - } else { - unreachable!("TakeOrdersCalldataResult must be either NeedsApproval or Ready") - } -} - -#[utoipa::path( - post, - path = "/take-orders/buy", - tag = "Take Orders", - request_body = BuyRequest, - responses( - (status = 200, description = "Successfully generated buy calldata. Returns either approval info if token approval is needed, or ready calldata if approval is sufficient.", body = TakeOrdersApiResponse, - examples( - ("Ready" = ( - summary = "Calldata ready to execute", - description = "Returned when the taker has sufficient token approval. The calldata can be submitted directly to the raindex contract.", - value = json!({ - "status": "ready", - "data": { - "raindex": "0xd2938e7c9fe3597f78832ce780feb61945c377d7", - "calldata": "0x...", - "effectivePrice": "0.00045", - "prices": ["0.00044", "0.00046"], - "expectedSell": "450", - "maxSellCap": "500" - } - }) - )), - ("NeedsApproval" = ( - summary = "Token approval required", - description = "Returned when the taker needs to approve token spending before executing. Submit the approval calldata first, then retry the request.", - value = json!({ - "status": "needsApproval", - "data": { - "token": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", - "spender": "0xd2938e7c9fe3597f78832ce780feb61945c377d7", - "amount": "1000", - "formattedAmount": "1000", - "calldata": "0x095ea7b3..." - } - }) - )) - ) - ), - (status = 400, description = "Invalid request parameters", body = ApiErrorResponse), - (status = 404, description = "No liquidity found or configuration not found", body = ApiErrorResponse), - (status = 500, description = "Internal server error", body = ApiErrorResponse) - ) -)] -#[post("/take-orders/buy", data = "")] -pub async fn buy(request: Json) -> Result, ApiError> { - let mode = if request.exact { - TakeOrdersMode::BuyExact - } else { - TakeOrdersMode::BuyUpTo - }; - - let yaml_content = request.yaml_content.clone(); - let take_request = TakeOrdersRequest { - taker: request.taker.clone(), - chain_id: request.chain_id, - sell_token: request.token_in.clone(), - buy_token: request.token_out.clone(), - mode, - amount: request.amount.clone(), - price_cap: request.max_ratio.clone(), - }; - - // RaindexClient contains Rc> which is not Send, but Rocket requires - // Send futures. We use spawn_blocking with a dedicated runtime to run everything - // on a single thread where Rc is safe. - let response = tokio::task::spawn_blocking(move || { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|e| ApiError::Internal(format!("Failed to create runtime: {}", e)))?; - - rt.block_on(execute_take_orders(yaml_content, take_request)) - }) - .await - .map_err(|e| ApiError::Internal(format!("Task execution failed: {}", e)))??; - - Ok(Json(response)) -} - -#[utoipa::path( - post, - path = "/take-orders/sell", - tag = "Take Orders", - request_body = SellRequest, - responses( - (status = 200, description = "Successfully generated sell calldata. Returns either approval info if token approval is needed, or ready calldata if approval is sufficient.", body = TakeOrdersApiResponse, - examples( - ("Ready" = ( - summary = "Calldata ready to execute", - description = "Returned when the taker has sufficient token approval. The calldata can be submitted directly to the raindex contract.", - value = json!({ - "status": "ready", - "data": { - "raindex": "0xd2938e7c9fe3597f78832ce780feb61945c377d7", - "calldata": "0x...", - "effectivePrice": "0.00045", - "prices": ["0.00044", "0.00046"], - "expectedSell": "450", - "maxSellCap": "500" - } - }) - )), - ("NeedsApproval" = ( - summary = "Token approval required", - description = "Returned when the taker needs to approve token spending before executing. Submit the approval calldata first, then retry the request.", - value = json!({ - "status": "needsApproval", - "data": { - "token": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", - "spender": "0xd2938e7c9fe3597f78832ce780feb61945c377d7", - "amount": "1000", - "formattedAmount": "1000", - "calldata": "0x095ea7b3..." - } - }) - )) - ) - ), - (status = 400, description = "Invalid request parameters", body = ApiErrorResponse), - (status = 404, description = "No liquidity found or configuration not found", body = ApiErrorResponse), - (status = 500, description = "Internal server error", body = ApiErrorResponse) - ) -)] -#[post("/take-orders/sell", data = "")] -pub async fn sell(request: Json) -> Result, ApiError> { - let mode = if request.exact { - TakeOrdersMode::SpendExact - } else { - TakeOrdersMode::SpendUpTo - }; - - let yaml_content = request.yaml_content.clone(); - let take_request = TakeOrdersRequest { - taker: request.taker.clone(), - chain_id: request.chain_id, - sell_token: request.token_in.clone(), - buy_token: request.token_out.clone(), - mode, - amount: request.amount.clone(), - price_cap: request.max_ratio.clone(), - }; - - // RaindexClient contains Rc> which is not Send, but Rocket requires - // Send futures. We use spawn_blocking with a dedicated runtime to run everything - // on a single thread where Rc is safe. - let response = tokio::task::spawn_blocking(move || { - let rt = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|e| ApiError::Internal(format!("Failed to create runtime: {}", e)))?; - - rt.block_on(execute_take_orders(yaml_content, take_request)) - }) - .await - .map_err(|e| ApiError::Internal(format!("Task execution failed: {}", e)))??; - - Ok(Json(response)) -} - -pub fn routes() -> Vec { - rocket::routes![buy, sell] -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_buy_request_deserialization() { - let json = r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "100", - "maxRatio": "2.5" - }"#; - - let request: BuyRequest = serde_json::from_str(json).unwrap(); - - assert_eq!(request.yaml_content, "version: 1"); - assert_eq!(request.taker, "0x1111111111111111111111111111111111111111"); - assert_eq!(request.chain_id, 1); - assert_eq!( - request.token_in, - "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - ); - assert_eq!( - request.token_out, - "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - ); - assert_eq!(request.amount, "100"); - assert_eq!(request.max_ratio, "2.5"); - assert!(!request.exact); - } - - #[test] - fn test_buy_request_deserialization_exact() { - let json = r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 137, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "50.5", - "maxRatio": "1.0", - "exact": true - }"#; - - let request: BuyRequest = serde_json::from_str(json).unwrap(); - - assert!(request.exact); - } - - #[test] - fn test_sell_request_deserialization() { - let json = r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "100", - "maxRatio": "2.5" - }"#; - - let request: SellRequest = serde_json::from_str(json).unwrap(); - - assert_eq!(request.yaml_content, "version: 1"); - assert_eq!(request.taker, "0x1111111111111111111111111111111111111111"); - assert_eq!(request.chain_id, 1); - assert_eq!( - request.token_in, - "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - ); - assert_eq!( - request.token_out, - "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - ); - assert_eq!(request.amount, "100"); - assert_eq!(request.max_ratio, "2.5"); - assert!(!request.exact); - } - - #[test] - fn test_sell_request_deserialization_exact() { - let json = r#"{ - "yamlContent": "version: 1", - "taker": "0x1111111111111111111111111111111111111111", - "chainId": 1, - "tokenIn": "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "tokenOut": "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "amount": "75", - "maxRatio": "3.0", - "exact": true - }"#; - - let request: SellRequest = serde_json::from_str(json).unwrap(); - - assert!(request.exact); - } - - #[test] - fn test_ready_response_serialization() { - let response = TakeOrdersApiResponse::Ready(TakeOrdersReadyResponse { - raindex: "0x1234567890123456789012345678901234567890".to_string(), - calldata: "0xabcdef".to_string(), - effective_price: "1.5".to_string(), - prices: vec!["1.4".to_string(), "1.6".to_string()], - expected_sell: "150".to_string(), - max_sell_cap: "200".to_string(), - }); - - let json = serde_json::to_string(&response).unwrap(); - - assert!(json.contains("\"status\":\"ready\"")); - assert!(json.contains("\"data\":")); - assert!(json.contains("\"raindex\":")); - assert!(json.contains("\"calldata\":")); - assert!(json.contains("\"effectivePrice\":")); - assert!(json.contains("\"prices\":")); - assert!(json.contains("\"expectedSell\":")); - assert!(json.contains("\"maxSellCap\":")); - } - - #[test] - fn test_needs_approval_response_serialization() { - let response = TakeOrdersApiResponse::NeedsApproval(ApprovalApiResponse { - token: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(), - spender: "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(), - amount: "1000".to_string(), - formatted_amount: "1000".to_string(), - calldata: "0xabcdef".to_string(), - }); - - let json = serde_json::to_string(&response).unwrap(); - - assert!(json.contains("\"status\":\"needsApproval\"")); - assert!(json.contains("\"data\":")); - assert!(json.contains("\"token\":")); - assert!(json.contains("\"spender\":")); - assert!(json.contains("\"amount\":")); - assert!(json.contains("\"formattedAmount\":")); - assert!(json.contains("\"calldata\":")); - } -} diff --git a/crates/rest_api/src/service.rs b/crates/rest_api/src/service.rs new file mode 100644 index 0000000000..d4255e44f8 --- /dev/null +++ b/crates/rest_api/src/service.rs @@ -0,0 +1,476 @@ +use crate::{cache::MarketSnapshotCache, provider::RaindexProvider}; +use async_trait::async_trait; +use raindex_common::raindex_client::{ + local_db::LocalDbSyncSnapshot, markets::RaindexMarketSnapshot, +}; +use std::{sync::Arc, time::Duration}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct IndexerHealth { + pub configured: bool, + pub healthy: bool, + pub sync_healthy: bool, + pub network_count: usize, + pub orderbook_count: usize, + pub snapshot_ready: bool, + pub snapshot_last_success_at: Option, + pub snapshot_refresh_healthy: bool, +} + +#[derive(Debug, thiserror::Error)] +pub enum MarketDataError { + #[error("market data source unavailable: {0}")] + Unavailable(String), +} + +#[async_trait] +pub trait MarketDataSource: Send + Sync { + async fn snapshots(&self) -> Result>, MarketDataError>; + async fn market(&self, ticker_id: &str) -> Result, MarketDataError>; + async fn health(&self) -> Result; +} + +#[derive(Debug, Clone)] +pub struct MarketDetail { + pub snapshot: Arc, + pub orderbook_observed_at: u64, +} + +pub struct CachedMarketData { + provider: Arc, + cache: MarketSnapshotCache, +} + +impl CachedMarketData { + pub fn new(provider: Arc, cache: MarketSnapshotCache) -> Self { + Self { provider, cache } + } + + pub async fn warm(&self) -> Result<(), MarketDataError> { + self.cache + .refresh_overview(&self.provider) + .await + .map(|_| ()) + .map_err(|error| MarketDataError::Unavailable(error.to_string())) + } + + pub async fn wait_for_local_index(&self, timeout: Duration) -> Result<(), MarketDataError> { + let wait = async { + loop { + let snapshot = self + .provider + .client() + .get_local_db_sync_snapshot() + .await + .map_err(|error| MarketDataError::Unavailable(error.to_string()))?; + match local_index_readiness(&snapshot) { + LocalIndexReadiness::Ready => return Ok(()), + LocalIndexReadiness::Failed(message) => { + return Err(MarketDataError::Unavailable(message)); + } + LocalIndexReadiness::Waiting => { + tokio::time::sleep(Duration::from_millis(500)).await; + } + } + } + }; + tokio::time::timeout(timeout, wait).await.map_err(|_| { + MarketDataError::Unavailable(format!( + "local index did not become ready within {} seconds", + timeout.as_secs() + )) + })? + } + + pub fn start_background_refresh(self: Arc, interval: Duration) { + tokio::spawn(async move { + let mut timer = tokio::time::interval(interval); + timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + timer.tick().await; + loop { + timer.tick().await; + match self.cache.refresh_overview(&self.provider).await { + Ok(snapshots) => { + tracing::info!(market_count = snapshots.len(), "market cache refreshed") + } + Err(error) => tracing::error!( + error = %error, + "market cache refresh failed; retaining the previous snapshot" + ), + } + } + }); + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum LocalIndexReadiness { + Ready, + Waiting, + Failed(String), +} + +fn local_index_readiness(snapshot: &LocalDbSyncSnapshot) -> LocalIndexReadiness { + if !snapshot.configured { + return LocalIndexReadiness::Ready; + } + if !snapshot.healthy { + let errors = snapshot + .networks + .iter() + .filter_map(|network| network.error.as_deref()) + .chain( + snapshot + .raindexes + .iter() + .filter_map(|raindex| raindex.error.as_deref()), + ) + .collect::>() + .join("; "); + return LocalIndexReadiness::Failed(if errors.is_empty() { + "local index synchronization failed".into() + } else { + format!("local index synchronization failed: {errors}") + }); + } + if !snapshot.networks.is_empty() && snapshot.networks.iter().all(|network| network.ready) { + LocalIndexReadiness::Ready + } else { + LocalIndexReadiness::Waiting + } +} + +#[async_trait] +impl MarketDataSource for CachedMarketData { + async fn snapshots(&self) -> Result>, MarketDataError> { + self.cache + .get_overview(&self.provider) + .await + .map_err(|error| MarketDataError::Unavailable(error.to_string())) + } + + async fn market(&self, ticker_id: &str) -> Result, MarketDataError> { + let overview = self.snapshots().await?; + let Some(overview_snapshot) = overview + .iter() + .find(|snapshot| snapshot.market.ticker_id.eq_ignore_ascii_case(ticker_id)) + else { + return Ok(None); + }; + let details = self + .cache + .get_market(&self.provider, ticker_id) + .await + .map_err(|error| MarketDataError::Unavailable(error.to_string()))?; + Ok(details.first().map(|detail| MarketDetail { + snapshot: Arc::new(merge_detail(overview_snapshot, detail)), + orderbook_observed_at: detail.observed_at, + })) + } + + async fn health(&self) -> Result { + let sync = self + .provider + .client() + .get_local_db_sync_snapshot() + .await + .map_err(|error| MarketDataError::Unavailable(error.to_string()))?; + let cache = self.cache.overview_health().await; + Ok(IndexerHealth { + configured: sync.configured, + healthy: sync.healthy && cache.ready && cache.refresh_healthy, + sync_healthy: sync.healthy, + network_count: sync.networks.len(), + orderbook_count: sync.raindexes.len(), + snapshot_ready: cache.ready, + snapshot_last_success_at: cache.last_success_at, + snapshot_refresh_healthy: cache.refresh_healthy, + }) + } +} + +fn merge_detail( + overview: &RaindexMarketSnapshot, + detail: &RaindexMarketSnapshot, +) -> RaindexMarketSnapshot { + let mut merged = overview.clone(); + merged.orderbook = detail.orderbook.clone(); + merged.block_number = detail.block_number.or(merged.block_number); + merged.errors.extend( + detail + .errors + .iter() + .filter(|error| error.source == "orderbook") + .cloned(), + ); + merged +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn sync_snapshot(healthy: bool, ready: bool, error: Option<&str>) -> LocalDbSyncSnapshot { + serde_json::from_value(json!({ + "configured": true, + "healthy": healthy, + "status": if healthy && ready { "active" } else if healthy { "syncing" } else { "failure" }, + "schedulerState": "leader", + "networks": [{ + "chainId": 8453, + "networkKey": "base", + "status": if healthy && ready { "active" } else if healthy { "syncing" } else { "failure" }, + "schedulerState": "leader", + "raindexCount": 1, + "ready": ready, + "error": error + }], + "raindexes": [] + })) + .expect("valid sync snapshot") + } + + #[test] + fn market_warmup_allows_an_unconfigured_local_index() { + assert_eq!( + local_index_readiness(&LocalDbSyncSnapshot::not_configured()), + LocalIndexReadiness::Ready + ); + } + + #[test] + fn market_warmup_waits_until_every_configured_network_is_ready() { + assert_eq!( + local_index_readiness(&sync_snapshot(true, false, None)), + LocalIndexReadiness::Waiting + ); + assert_eq!( + local_index_readiness(&sync_snapshot(true, true, None)), + LocalIndexReadiness::Ready + ); + } + + #[test] + fn market_warmup_surfaces_local_index_failures() { + assert_eq!( + local_index_readiness(&sync_snapshot(false, false, Some("RPCs unavailable"))), + LocalIndexReadiness::Failed( + "local index synchronization failed: RPCs unavailable".into() + ) + ); + } + + fn snapshot( + last_price: &str, + best_bid: Option<&str>, + errors: serde_json::Value, + ) -> RaindexMarketSnapshot { + serde_json::from_value(json!({ + "market": { + "id": "8453:base_quote", + "tickerId": "0x0000000000000000000000000000000000000001_0x0000000000000000000000000000000000000002", + "chainId": 8453, + "base": { + "chainId": 8453, + "address": "0x0000000000000000000000000000000000000001", + "name": "Base", + "symbol": "BASE", + "decimals": 18, + "variants": [] + }, + "quote": { + "chainId": 8453, + "address": "0x0000000000000000000000000000000000000002", + "name": "Quote", + "symbol": "QUOTE", + "decimals": 6, + "variants": [] + }, + "raindexAddresses": [] + }, + "orderbook": { + "bestBid": best_bid, + "bids": [], + "asks": [] + }, + "stats": { + "lastPrice": last_price, + "baseVolume24h": "10", + "targetVolume24h": "20", + "tradeCount24h": 1 + }, + "recentTrades": [], + "observedAt": 100, + "errors": errors + })) + .expect("valid snapshot fixture") + } + + #[test] + fn detail_merge_keeps_overview_stats_and_adds_only_orderbook_results() { + let overview = snapshot( + "2", + None, + json!([{"source": "ratios", "message": "unavailable"}]), + ); + let detail = snapshot( + "0", + Some("1.9"), + json!([ + {"source": "ratios", "message": "duplicate"}, + {"source": "orderbook", "message": "partial quote"} + ]), + ); + let merged = merge_detail(&overview, &detail); + assert_eq!(merged.stats.last_price.as_deref(), Some("2")); + assert_eq!(merged.orderbook.best_bid.as_deref(), Some("1.9")); + assert_eq!(merged.observed_at, overview.observed_at); + assert_eq!(merged.errors.len(), 2); + assert_eq!(merged.errors[1].source, "orderbook"); + } + + #[test] + fn overview_validation_rejects_ratio_errors() { + let snapshots = vec![snapshot( + "2", + None, + json!([{"source": "ratios", "message": "unavailable"}]), + )]; + assert!(matches!( + crate::cache::validate_overview(snapshots), + Err(crate::cache::MarketCacheError::InvalidSnapshot(_)) + )); + } + + #[test] + fn overview_validation_rejects_orderbook_errors() { + let snapshots = vec![snapshot( + "2", + None, + json!([{"source": "orderbook", "message": "crossed book"}]), + )]; + assert!(matches!( + crate::cache::validate_overview(snapshots), + Err(crate::cache::MarketCacheError::InvalidSnapshot(_)) + )); + } + + #[test] + fn overview_validation_rejects_trade_errors() { + let snapshots = vec![snapshot( + "0", + None, + json!([{"source": "trades", "message": "timed out"}]), + )]; + assert!(matches!( + crate::cache::validate_overview(snapshots), + Err(crate::cache::MarketCacheError::InvalidSnapshot(_)) + )); + } + + #[test] + fn overview_validation_omits_failed_markets_and_keeps_healthy_markets() { + let failed = snapshot( + "2", + None, + json!([{"source": "ratios", "message": "timed out"}]), + ); + let mut healthy = snapshot("3", None, json!([])); + healthy.market.id = "8453:healthy_quote".into(); + healthy.market.ticker_id = + "0x0000000000000000000000000000000000000003_0x0000000000000000000000000000000000000002" + .into(); + + let validated = crate::cache::validate_overview(vec![failed, healthy.clone()]) + .expect("healthy markets remain serviceable"); + + assert_eq!(validated, vec![healthy]); + } + + #[test] + fn overview_validation_allows_trade_ordering_warnings() { + let snapshots = vec![snapshot( + "2", + None, + json!([{ + "source": "trades", + "severity": "warning", + "message": "same-block ordering is unavailable" + }]), + )]; + assert!(crate::cache::validate_overview(snapshots).is_ok()); + } + + #[test] + fn overview_validation_rejects_duplicate_cross_chain_tickers() { + let first = snapshot("2", None, json!([])); + let mut second = first.clone(); + second.market.chain_id = 1; + second.market.id = "1:base_quote".into(); + assert!(matches!( + crate::cache::validate_overview(vec![first, second]), + Err(crate::cache::MarketCacheError::InvalidSnapshot(_)) + )); + } + + #[test] + fn detail_validation_rejects_orderbook_errors() { + let snapshots = vec![snapshot( + "2", + None, + json!([{"source": "orderbook", "message": "timed out"}]), + )]; + assert!(matches!( + crate::cache::validate_detail(snapshots), + Err(crate::cache::MarketCacheError::InvalidSnapshot(_)) + )); + } + + #[test] + fn detail_validation_rejects_ratio_normalization_errors() { + let snapshots = vec![snapshot( + "2", + Some("1.9"), + json!([{"source": "ratios", "message": "timed out"}]), + )]; + assert!(matches!( + crate::cache::validate_detail(snapshots), + Err(crate::cache::MarketCacheError::InvalidSnapshot(_)) + )); + } + + #[test] + fn detail_validation_rejects_duplicate_cross_chain_tickers() { + let first = snapshot("2", None, json!([])); + let mut second = first.clone(); + second.market.chain_id = 1; + second.market.id = "1:base_quote".into(); + + assert!(matches!( + crate::cache::validate_detail(vec![first, second]), + Err(crate::cache::MarketCacheError::InvalidSnapshot(_)) + )); + } + + #[test] + fn detail_validation_keeps_levels_when_a_variant_ratio_is_unavailable() { + let snapshots = vec![snapshot( + "2", + Some("1.9"), + json!([{ + "source": "ratios", + "severity": "warning", + "message": "legacy ERC4626 ratio is unavailable" + }]), + )]; + + let validated = crate::cache::validate_detail(snapshots).expect("partial orderbook"); + + assert_eq!(validated[0].orderbook.best_bid.as_deref(), Some("1.9")); + assert_eq!( + validated[0].errors[0].severity, + raindex_common::raindex_client::markets::RaindexMarketDataErrorSeverity::Warning + ); + } +} diff --git a/crates/rest_api/src/telemetry.rs b/crates/rest_api/src/telemetry.rs new file mode 100644 index 0000000000..8a07e96ef9 --- /dev/null +++ b/crates/rest_api/src/telemetry.rs @@ -0,0 +1,81 @@ +use std::path::Path; + +use tracing_appender::{ + non_blocking::WorkerGuard, + rolling::{InitError, RollingFileAppender, Rotation}, +}; +use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; + +const LOG_FILE_PREFIX: &str = "raindex-api.log"; +const MAX_LOG_FILES: usize = 14; +const DEFAULT_ENV_FILTER: &str = "raindex_rest_api=info,raindex_common=info,rocket=warn,warn"; + +fn file_appender(log_dir: &Path) -> Result { + RollingFileAppender::builder() + .rotation(Rotation::DAILY) + .filename_prefix(LOG_FILE_PREFIX) + .max_log_files(MAX_LOG_FILES) + .build(log_dir) +} + +pub struct LoggingGuard { + _file_guard: WorkerGuard, +} + +pub fn init(log_dir: &Path) -> Result> { + let appender = file_appender(log_dir)?; + let (file_writer, file_guard) = tracing_appender::non_blocking(appender); + let env_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(DEFAULT_ENV_FILTER)); + + tracing_subscriber::registry() + .with(env_filter) + .with(fmt::layer().json().with_current_span(false)) + .with( + fmt::layer() + .json() + .with_current_span(false) + .with_writer(file_writer), + ) + .try_init()?; + + Ok(LoggingGuard { + _file_guard: file_guard, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{fs, fs::File}; + + #[test] + fn file_appender_retains_fourteen_daily_logs() { + let directory = tempfile::tempdir().expect("temporary log directory"); + for day in 1..=15 { + File::create( + directory + .path() + .join(format!("{LOG_FILE_PREFIX}.2000-01-{day:02}")), + ) + .expect("seed daily log"); + } + let unrelated = directory.path().join("unrelated.log"); + File::create(&unrelated).expect("seed unrelated file"); + + drop(file_appender(directory.path()).expect("build file appender")); + + let retained = fs::read_dir(directory.path()) + .expect("read log directory") + .filter_map(Result::ok) + .filter(|entry| { + entry + .file_name() + .to_string_lossy() + .starts_with(LOG_FILE_PREFIX) + }) + .count(); + assert_eq!(retained, MAX_LOG_FILES); + assert!(unrelated.exists()); + } +} diff --git a/deploy.nix b/deploy.nix new file mode 100644 index 0000000000..24507f8bf0 --- /dev/null +++ b/deploy.nix @@ -0,0 +1,97 @@ +{ deploy-rs, self }: + +let + system = "x86_64-linux"; + inherit (deploy-rs.lib.${system}) activate; + profileBase = "/nix/var/nix/profiles/per-service"; + apiPackage = self.packages.${system}.raindex-api; + serviceDefinitions = import ./services.nix; + enabledServices = builtins.attrNames ( + builtins.removeAttrs serviceDefinitions ( + builtins.filter (name: !serviceDefinitions.${name}.enabled) (builtins.attrNames serviceDefinitions) + ) + ); + + mkServiceProfile = + name: + activate.custom apiPackage ( + builtins.concatStringsSep " && " [ + "systemctl stop ${name} || true" + "systemctl restart ${name}" + ] + ); + + mkProfile = name: { + path = mkServiceProfile name; + profilePath = "${profileBase}/${name}"; + }; + + serviceProfiles = builtins.listToAttrs ( + map (name: { + inherit name; + value = mkProfile name; + }) enabledServices + ); +in +{ + config.nodes.raindex-api = { + hostname = builtins.getEnv "DEPLOY_HOST"; + sshUser = "root"; + user = "root"; + profilesOrder = [ "system" ] ++ enabledServices; + profiles = { + system.path = activate.nixos self.nixosConfigurations.raindex-api; + } + // serviceProfiles; + }; + + wrappers = + { + pkgs, + infraPkgs, + localSystem, + }: + let + deployInputs = infraPkgs.buildInputs ++ [ deploy-rs.packages.${localSystem}.deploy-rs ]; + preamble = '' + ${infraPkgs.resolveIp} + export DEPLOY_HOST="$host_ip" + export NIX_SSHOPTS="-o IgnoreUnknown=UseKeychain -i $identity" + ssh_flag="--ssh-opts=-o IgnoreUnknown=UseKeychain -i $identity" + ''; + deployFlags = if localSystem == "x86_64-linux" then "" else "--skip-checks --remote-build"; + in + { + deployNixos = pkgs.writeShellApplication { + name = "deploy-nixos"; + runtimeInputs = deployInputs; + text = '' + ${preamble} + deploy ${deployFlags} ''${ssh_flag:+"$ssh_flag"} .#raindex-api.system \ + -- --impure "$@" + ''; + }; + + deployService = pkgs.writeShellApplication { + name = "deploy-service"; + runtimeInputs = deployInputs; + text = '' + ${preamble} + profile="''${1:?usage: deploy-service }" + shift + deploy ${deployFlags} ''${ssh_flag:+"$ssh_flag"} ".#raindex-api.$profile" \ + -- --impure "$@" + ''; + }; + + deployAll = pkgs.writeShellApplication { + name = "deploy-all"; + runtimeInputs = deployInputs; + text = '' + ${preamble} + deploy ${deployFlags} ''${ssh_flag:+"$ssh_flag"} .#raindex-api \ + -- --impure "$@" + ''; + }; + }; +} diff --git a/disko.nix b/disko.nix new file mode 100644 index 0000000000..3eb5ee254d --- /dev/null +++ b/disko.nix @@ -0,0 +1,34 @@ +{ lib, ... }: + +{ + disko.devices.disk.primary = { + device = lib.mkDefault "/dev/vda"; + type = "disk"; + content = { + type = "gpt"; + partitions = { + boot = { + size = "1M"; + type = "EF02"; + }; + esp = { + size = "500M"; + type = "EF00"; + content = { + type = "filesystem"; + format = "vfat"; + mountpoint = "/boot"; + }; + }; + root = { + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + }; + }; + }; + }; + }; +} diff --git a/flake.lock b/flake.lock index f915110b04..c8e3743aa6 100644 --- a/flake.lock +++ b/flake.lock @@ -1,6 +1,162 @@ { "nodes": { + "agenix": { + "inputs": { + "darwin": "darwin", + "home-manager": "home-manager", + "nixpkgs": [ + "ragenix", + "nixpkgs" + ], + "systems": "systems_3" + }, + "locked": { + "lastModified": 1761656077, + "narHash": "sha256-lsNWuj4Z+pE7s0bd2OKicOFq9bK86JE0ZGeKJbNqb94=", + "owner": "ryantm", + "repo": "agenix", + "rev": "9ba0d85de3eaa7afeab493fed622008b6e4924f5", + "type": "github" + }, + "original": { + "owner": "ryantm", + "repo": "agenix", + "type": "github" + } + }, + "crane": { + "locked": { + "lastModified": 1787326676, + "narHash": "sha256-lWhBbBvC05/xwivKBBiM2YNizpmgqCgyOIzomvRuwxs=", + "owner": "ipetkov", + "repo": "crane", + "rev": "692f7e9ef2ece8125b466f66f2af532b3edaed0d", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "crane_2": { + "locked": { + "lastModified": 1760924934, + "narHash": "sha256-tuuqY5aU7cUkR71sO2TraVKK2boYrdW3gCSXUkF4i44=", + "owner": "ipetkov", + "repo": "crane", + "rev": "c6b4d5308293d0d04fcfeee92705017537cad02f", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, + "darwin": { + "inputs": { + "nixpkgs": [ + "ragenix", + "agenix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1744478979, + "narHash": "sha256-dyN+teG9G82G+m+PX/aSAagkC+vUv0SgUw3XkPhQodQ=", + "owner": "lnl7", + "repo": "nix-darwin", + "rev": "43975d782b418ebf4969e9ccba82466728c2851b", + "type": "github" + }, + "original": { + "owner": "lnl7", + "ref": "master", + "repo": "nix-darwin", + "type": "github" + } + }, + "deploy-rs": { + "inputs": { + "flake-compat": "flake-compat", + "nixpkgs": "nixpkgs", + "utils": "utils" + }, + "locked": { + "lastModified": 1786361680, + "narHash": "sha256-IxaZkb9rCGEZ+yGndxKXONeIEcKMzoFUsvLTB5G/caw=", + "owner": "serokell", + "repo": "deploy-rs", + "rev": "16901271e5b30b591e56f7a84f25f186fb20f3e1", + "type": "github" + }, + "original": { + "owner": "serokell", + "repo": "deploy-rs", + "type": "github" + } + }, + "disko": { + "inputs": { + "nixpkgs": [ + "rainix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1781152676, + "narHash": "sha256-RxWs5ND31KzTG7wvMM+PMfUjyNpmIEr999lqNARaM5o=", + "owner": "nix-community", + "repo": "disko", + "rev": "ff8702b4de27f72b4c78573dfb89ec74e36abdf1", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "disko", + "type": "github" + } + }, + "disko_2": { + "inputs": { + "nixpkgs": [ + "nixos-anywhere", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1781152676, + "narHash": "sha256-RxWs5ND31KzTG7wvMM+PMfUjyNpmIEr999lqNARaM5o=", + "owner": "nix-community", + "repo": "disko", + "rev": "ff8702b4de27f72b4c78573dfb89ec74e36abdf1", + "type": "github" + }, + "original": { + "owner": "nix-community", + "ref": "master", + "repo": "disko", + "type": "github" + } + }, "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1733328505, + "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "flake-compat_2": { "flake": false, "locked": { "lastModified": 1767039857, @@ -16,7 +172,7 @@ "type": "github" } }, - "flake-compat_2": { + "flake-compat_3": { "flake": false, "locked": { "lastModified": 1767039857, @@ -34,7 +190,7 @@ }, "flake-utils": { "inputs": { - "systems": "systems" + "systems": "systems_2" }, "locked": { "lastModified": 1731533236, @@ -52,7 +208,7 @@ }, "flake-utils_2": { "inputs": { - "systems": "systems_2" + "systems": "systems_4" }, "locked": { "lastModified": 1731533236, @@ -70,7 +226,7 @@ }, "flake-utils_3": { "inputs": { - "systems": "systems_3" + "systems": "systems_5" }, "locked": { "lastModified": 1731533236, @@ -87,6 +243,24 @@ } }, "flake-utils_4": { + "inputs": { + "systems": "systems_6" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "flake-utils_5": { "locked": { "lastModified": 1644229661, "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", @@ -101,9 +275,9 @@ "type": "github" } }, - "flake-utils_5": { + "flake-utils_6": { "inputs": { - "systems": "systems_4" + "systems": "systems_7" }, "locked": { "lastModified": 1731533236, @@ -119,9 +293,9 @@ "type": "github" } }, - "flake-utils_6": { + "flake-utils_7": { "inputs": { - "systems": "systems_5" + "systems": "systems_8" }, "locked": { "lastModified": 1731533236, @@ -137,7 +311,7 @@ "type": "github" } }, - "flake-utils_7": { + "flake-utils_8": { "locked": { "lastModified": 1644229661, "narHash": "sha256-1YdnJAsNy69bpcjuoKdOYQX0YxZBiCYZo4Twxerqv7k=", @@ -152,9 +326,9 @@ "type": "github" } }, - "flake-utils_8": { + "flake-utils_9": { "inputs": { - "systems": "systems_6" + "systems": "systems_9" }, "locked": { "lastModified": 1731533236, @@ -172,8 +346,8 @@ }, "foundry": { "inputs": { - "flake-utils": "flake-utils_4", - "nixpkgs": "nixpkgs" + "flake-utils": "flake-utils_5", + "nixpkgs": "nixpkgs_3" }, "locked": { "lastModified": 1778486972, @@ -191,8 +365,8 @@ }, "foundry_2": { "inputs": { - "flake-utils": "flake-utils_7", - "nixpkgs": "nixpkgs_6" + "flake-utils": "flake-utils_8", + "nixpkgs": "nixpkgs_8" }, "locked": { "lastModified": 1778486972, @@ -210,9 +384,9 @@ }, "git-hooks-nix": { "inputs": { - "flake-compat": "flake-compat", + "flake-compat": "flake-compat_2", "gitignore": "gitignore", - "nixpkgs": "nixpkgs_2" + "nixpkgs": "nixpkgs_4" }, "locked": { "lastModified": 1778507602, @@ -230,9 +404,9 @@ }, "git-hooks-nix_2": { "inputs": { - "flake-compat": "flake-compat_2", + "flake-compat": "flake-compat_3", "gitignore": "gitignore_2", - "nixpkgs": "nixpkgs_7" + "nixpkgs": "nixpkgs_9" }, "locked": { "lastModified": 1778507602, @@ -293,43 +467,123 @@ "type": "github" } }, - "nixpkgs": { + "home-manager": { + "inputs": { + "nixpkgs": [ + "ragenix", + "agenix", + "nixpkgs" + ] + }, "locked": { - "lastModified": 1666753130, - "narHash": "sha256-Wff1dGPFSneXJLI2c0kkdWTgxnQ416KE6X4KnFkgPYQ=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "f540aeda6f677354f1e7144ab04352f61aaa0118", + "lastModified": 1745494811, + "narHash": "sha256-YZCh2o9Ua1n9uCvrvi5pRxtuVNml8X2a03qIFfRKpFs=", + "owner": "nix-community", + "repo": "home-manager", + "rev": "abfad3d2958c9e6300a883bd443512c55dfeb1be", "type": "github" }, "original": { - "id": "nixpkgs", - "type": "indirect" + "owner": "nix-community", + "repo": "home-manager", + "type": "github" } }, - "nixpkgs_10": { + "nix-vm-test": { + "inputs": { + "nixpkgs": [ + "nixos-anywhere", + "nixpkgs" + ] + }, "locked": { - "lastModified": 1777641297, - "narHash": "sha256-WNGcmeOZ8Tr9dq6ztCspYbzWFswr2mPebM9LpsfGxPk=", + "lastModified": 1786747096, + "narHash": "sha256-9QqhmaLVsPhKdMSBaWKjDqeGRn8G4ov4cVuZ6JFwXbo=", + "owner": "numtide", + "repo": "nix-vm-test", + "rev": "c8781a0ea2d8417506fff7722eae5a6316461212", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "nix-vm-test", + "type": "github" + } + }, + "nixos-anywhere": { + "inputs": { + "disko": "disko_2", + "nix-vm-test": "nix-vm-test", + "nixos-images": "nixos-images", + "nixos-stable": "nixos-stable", + "nixpkgs": [ + "rainix", + "nixpkgs" + ], + "treefmt-nix": "treefmt-nix" + }, + "locked": { + "lastModified": 1787124618, + "narHash": "sha256-aKf1k2hvYgaxP9oxDPRiv9npEJLODC9eKxk7nR69lzQ=", + "owner": "nix-community", + "repo": "nixos-anywhere", + "rev": "ad8fa24e11eef167fd72d49fafefa3f840312d71", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixos-anywhere", + "type": "github" + } + }, + "nixos-images": { + "inputs": { + "nixos-stable": [ + "nixos-anywhere", + "nixos-stable" + ], + "nixos-unstable": [ + "nixos-anywhere", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1786617555, + "narHash": "sha256-r3brKkBWm34wltszIeoZw2VCFUEoS2nNAe3W2Zgzc9k=", + "owner": "nix-community", + "repo": "nixos-images", + "rev": "f01878e82602ad3d0a2e74a14440521924e57bc5", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixos-images", + "type": "github" + } + }, + "nixos-stable": { + "locked": { + "lastModified": 1787042014, + "narHash": "sha256-lgnV/xeEatfEnPZG0RrXS0fhcr63p3tsVvL1GdaVkJQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "c6d65881c5624c9cae5ea6cedef24699b0c0a4c0", + "rev": "c69ae8fb8faeb3472fd11234ba55a70ac3601f9b", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixpkgs-unstable", + "ref": "nixos-26.05", "repo": "nixpkgs", "type": "github" } }, - "nixpkgs_2": { + "nixpkgs": { "locked": { - "lastModified": 1770073757, - "narHash": "sha256-Vy+G+F+3E/Tl+GMNgiHl9Pah2DgShmIUBJXmbiQPHbI=", + "lastModified": 1743014863, + "narHash": "sha256-jAIUqsiN2r3hCuHji80U7NNEafpIMBXiwKlSrjWMlpg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "47472570b1e607482890801aeaf29bfb749884f6", + "rev": "bd3bac8bfb542dbde7ffffb6987a1a1f9d41699f", "type": "github" }, "original": { @@ -339,7 +593,7 @@ "type": "github" } }, - "nixpkgs_3": { + "nixpkgs_10": { "locked": { "lastModified": 1778656924, "narHash": "sha256-lKVrom9wOmpC3i7m+uBoGaBdW0PfH3QbLRG1XmuC6YA=", @@ -354,7 +608,7 @@ "type": "github" } }, - "nixpkgs_4": { + "nixpkgs_11": { "locked": { "lastModified": 1744536153, "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", @@ -370,7 +624,7 @@ "type": "github" } }, - "nixpkgs_5": { + "nixpkgs_12": { "locked": { "lastModified": 1777641297, "narHash": "sha256-WNGcmeOZ8Tr9dq6ztCspYbzWFswr2mPebM9LpsfGxPk=", @@ -386,7 +640,23 @@ "type": "github" } }, - "nixpkgs_6": { + "nixpkgs_2": { + "locked": { + "lastModified": 1761672384, + "narHash": "sha256-o9KF3DJL7g7iYMZq9SWgfS1BFlNbsm6xplRjVlOCkXI=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "08dacfca559e1d7da38f3cf05f1f45ee9bfd213c", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_3": { "locked": { "lastModified": 1666753130, "narHash": "sha256-Wff1dGPFSneXJLI2c0kkdWTgxnQ416KE6X4KnFkgPYQ=", @@ -400,7 +670,7 @@ "type": "indirect" } }, - "nixpkgs_7": { + "nixpkgs_4": { "locked": { "lastModified": 1770073757, "narHash": "sha256-Vy+G+F+3E/Tl+GMNgiHl9Pah2DgShmIUBJXmbiQPHbI=", @@ -416,7 +686,7 @@ "type": "github" } }, - "nixpkgs_8": { + "nixpkgs_5": { "locked": { "lastModified": 1778656924, "narHash": "sha256-lKVrom9wOmpC3i7m+uBoGaBdW0PfH3QbLRG1XmuC6YA=", @@ -431,7 +701,7 @@ "type": "github" } }, - "nixpkgs_9": { + "nixpkgs_6": { "locked": { "lastModified": 1744536153, "narHash": "sha256-awS2zRgF4uTwrOKwwiJcByDzDOdo3Q1rPZbiHQg/N38=", @@ -447,9 +717,77 @@ "type": "github" } }, - "rain": { + "nixpkgs_7": { + "locked": { + "lastModified": 1777641297, + "narHash": "sha256-WNGcmeOZ8Tr9dq6ztCspYbzWFswr2mPebM9LpsfGxPk=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "c6d65881c5624c9cae5ea6cedef24699b0c0a4c0", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_8": { + "locked": { + "lastModified": 1666753130, + "narHash": "sha256-Wff1dGPFSneXJLI2c0kkdWTgxnQ416KE6X4KnFkgPYQ=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "f540aeda6f677354f1e7144ab04352f61aaa0118", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "nixpkgs_9": { + "locked": { + "lastModified": 1770073757, + "narHash": "sha256-Vy+G+F+3E/Tl+GMNgiHl9Pah2DgShmIUBJXmbiQPHbI=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "47472570b1e607482890801aeaf29bfb749884f6", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "ragenix": { "inputs": { + "agenix": "agenix", + "crane": "crane_2", "flake-utils": "flake-utils_2", + "nixpkgs": "nixpkgs_2", + "rust-overlay": "rust-overlay" + }, + "locked": { + "lastModified": 1761832913, + "narHash": "sha256-VCNVjjuRvrKPiYYwqhE3BAKIaReiKXGpxGp27lZ0MFM=", + "owner": "yaxitech", + "repo": "ragenix", + "rev": "83bccfdea758241999f32869fb6b36f7ac72f1ac", + "type": "github" + }, + "original": { + "owner": "yaxitech", + "repo": "ragenix", + "type": "github" + } + }, + "rain": { + "inputs": { + "flake-utils": "flake-utils_3", "rainix": "rainix" }, "locked": { @@ -468,11 +806,11 @@ }, "rainix": { "inputs": { - "flake-utils": "flake-utils_3", + "flake-utils": "flake-utils_4", "foundry": "foundry", "git-hooks-nix": "git-hooks-nix", - "nixpkgs": "nixpkgs_3", - "rust-overlay": "rust-overlay", + "nixpkgs": "nixpkgs_5", + "rust-overlay": "rust-overlay_2", "solc": "solc" }, "locked": { @@ -491,11 +829,11 @@ }, "rainix_2": { "inputs": { - "flake-utils": "flake-utils_6", + "flake-utils": "flake-utils_7", "foundry": "foundry_2", "git-hooks-nix": "git-hooks-nix_2", - "nixpkgs": "nixpkgs_8", - "rust-overlay": "rust-overlay_2", + "nixpkgs": "nixpkgs_10", + "rust-overlay": "rust-overlay_3", "solc": "solc_2" }, "locked": { @@ -514,14 +852,40 @@ }, "root": { "inputs": { + "crane": "crane", + "deploy-rs": "deploy-rs", + "disko": "disko", "flake-utils": "flake-utils", + "nixos-anywhere": "nixos-anywhere", + "ragenix": "ragenix", "rain": "rain", "rainix": "rainix_2" } }, "rust-overlay": { "inputs": { - "nixpkgs": "nixpkgs_4" + "nixpkgs": [ + "ragenix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1761791894, + "narHash": "sha256-myRIDh+PxaREz+z9LzbqBJF+SnTFJwkthKDX9zMyddY=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "59c45eb69d9222a4362673141e00ff77842cd219", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + }, + "rust-overlay_2": { + "inputs": { + "nixpkgs": "nixpkgs_6" }, "locked": { "lastModified": 1778642276, @@ -537,9 +901,9 @@ "type": "github" } }, - "rust-overlay_2": { + "rust-overlay_3": { "inputs": { - "nixpkgs": "nixpkgs_9" + "nixpkgs": "nixpkgs_11" }, "locked": { "lastModified": 1778642276, @@ -557,8 +921,8 @@ }, "solc": { "inputs": { - "flake-utils": "flake-utils_5", - "nixpkgs": "nixpkgs_5", + "flake-utils": "flake-utils_6", + "nixpkgs": "nixpkgs_7", "solc-macos-amd64-list-json": "solc-macos-amd64-list-json" }, "locked": { @@ -601,8 +965,8 @@ }, "solc_2": { "inputs": { - "flake-utils": "flake-utils_8", - "nixpkgs": "nixpkgs_10", + "flake-utils": "flake-utils_9", + "nixpkgs": "nixpkgs_12", "solc-macos-amd64-list-json": "solc-macos-amd64-list-json_2" }, "locked": { @@ -708,6 +1072,90 @@ "repo": "default", "type": "github" } + }, + "systems_7": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_8": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "systems_9": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": [ + "nixos-anywhere", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1786901030, + "narHash": "sha256-WSFCsDSE5ffgD2MqzkM2CYjeFiKhRF/dJUN8uedb6YE=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "27b3b12a8e6375f28ebe122f07d230ca5459bbfa", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + }, + "utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index 7b4ff9f052..b3b9145af0 100644 --- a/flake.nix +++ b/flake.nix @@ -5,19 +5,71 @@ rainix.url = "github:rainlanguage/rainix"; rain.url = "github:rainlanguage/rain.cli"; flake-utils.url = "github:numtide/flake-utils"; + ragenix.url = "github:yaxitech/ragenix"; + deploy-rs.url = "github:serokell/deploy-rs"; + crane.url = "github:ipetkov/crane"; + + disko.url = "github:nix-community/disko"; + disko.inputs.nixpkgs.follows = "rainix/nixpkgs"; + + nixos-anywhere.url = "github:nix-community/nixos-anywhere"; + nixos-anywhere.inputs.nixpkgs.follows = "rainix/nixpkgs"; }; outputs = { + self, flake-utils, rainix, rain, + ragenix, + deploy-rs, + crane, + disko, + nixos-anywhere, ... }: - flake-utils.lib.eachDefaultSystem ( + let + configuredHostname = builtins.getEnv "RAINDEX_API_HOSTNAME"; + apiHostname = if configuredHostname == "" then "api.raindex.finance" else configuredHostname; + in + { + nixosConfigurations.raindex-api = rainix.inputs.nixpkgs.lib.nixosSystem { + system = "x86_64-linux"; + specialArgs.raindexEnv = { + name = "prod"; + virtualHost = apiHostname; + dataDir = "/mnt/data/raindex-api"; + dataVolumeName = "raindex-api-data"; + }; + modules = [ + disko.nixosModules.disko + ragenix.nixosModules.default + ./os.nix + ]; + }; + + deploy = (import ./deploy.nix { inherit deploy-rs self; }).config; + checks.x86_64-linux = deploy-rs.lib.x86_64-linux.deployChecks self.deploy; + } + // flake-utils.lib.eachDefaultSystem ( system: let pkgs = rainix.pkgs.${system}; + craneLib = (crane.mkLib pkgs).overrideToolchain rainix.rust-toolchain.${system}; + infraPkgs = import ./infra { + inherit + pkgs + ragenix + rainix + system + ; + }; + deployPkgs = (import ./deploy.nix { inherit deploy-rs self; }).wrappers { + inherit pkgs infraPkgs; + localSystem = system; + }; + apiRust = pkgs.callPackage ./rust.nix { inherit craneLib; }; in rec { packages = rec { @@ -166,7 +218,83 @@ # so a rain.cli main move can't race the cache push. rain-cli = rain.defaultPackage.${system}; + # Terraform-compatible infrastructure tooling from the flake-pinned + # nixpkgs used by rainix. + inherit (pkgs) opentofu actionlint; + + raindex-api = apiRust.package; + raindex-api-clippy = apiRust.clippy; + + inherit (infraPkgs) + tfInit + tfPlan + tfApply + tfImport + tfDestroy + tfEditVars + ; + + bootstrap = rainix.mkTask.${system} { + name = "bootstrap-raindex-api-nixos"; + additionalBuildInputs = infraPkgs.buildInputs ++ [ + nixos-anywhere.packages.${system}.default + pkgs.openssh + pkgs.gnused + ]; + body = '' + ${infraPkgs.resolveIp} + ssh_opts="-o IgnoreUnknown=UseKeychain -o StrictHostKeyChecking=no -o ConnectTimeout=5 -i $identity" + + nixos-anywhere --flake ".#raindex-api" \ + --option pure-eval false \ + --ssh-option "IgnoreUnknown=UseKeychain" \ + --ssh-option "IdentityFile=$identity" \ + --target-host "root@$host_ip" "$@" + + echo "Waiting for the NixOS host to return..." + retries=0 + until ssh $ssh_opts "root@$host_ip" true 2>/dev/null; do + retries=$((retries + 1)) + if [ "$retries" -ge 60 ]; then + echo "Host did not return after 5 minutes" >&2 + exit 1 + fi + sleep 5 + done + + new_key=$(ssh $ssh_opts "root@$host_ip" \ + cat /etc/ssh/ssh_host_ed25519_key.pub | awk '{print $1 " " $2}') + if ! echo "$new_key" | grep -qE '^ssh-ed25519 [A-Za-z0-9+/=]+$'; then + echo "Invalid SSH host key returned by provisioned host" >&2 + exit 1 + fi + + sed -i.bak -E '/host =/s|"ssh-ed25519 [A-Za-z0-9+/=_-]+"|"'"$new_key"'"|' keys.nix + rm -f keys.nix.bak + echo "Updated keys.nix with the provisioned host key; commit it before deploying." + ''; + }; + + resolveIp = pkgs.writeShellApplication { + name = "resolve-ip"; + runtimeInputs = infraPkgs.buildInputs; + text = '' + ${infraPkgs.resolveIp} + echo "$host_ip" + ''; + }; + + remote = pkgs.writeShellApplication { + name = "raindex-api-remote"; + runtimeInputs = infraPkgs.buildInputs ++ [ pkgs.openssh ]; + text = '' + ${infraPkgs.resolveIp} + exec ssh -i "$identity" "root@$host_ip" "$@" + ''; + }; + } + // deployPkgs // rainix.packages.${system}; devShells.default = pkgs.mkShell { @@ -182,6 +310,8 @@ rain.defaultPackage.${system} packages.raindex-ui-components-prelude packages.raindex-cli-artifact + packages.opentofu + packages.actionlint ]; inherit (rainix.devShells.${system}.default) shellHook buildInputs nativeBuildInputs; diff --git a/infra/.gitignore b/infra/.gitignore new file mode 100644 index 0000000000..299bf51ef4 --- /dev/null +++ b/infra/.gitignore @@ -0,0 +1,8 @@ +.terraform/ +*.tfstate +*.tfstate.* +!terraform.tfstate.age +*.tfplan +tfplan +terraform.tfvars +zz_secret.auto.tfvars diff --git a/infra/.terraform.lock.hcl b/infra/.terraform.lock.hcl new file mode 100644 index 0000000000..da29c454f9 --- /dev/null +++ b/infra/.terraform.lock.hcl @@ -0,0 +1,26 @@ +# This file is maintained automatically by "tofu init". +# Manual edits may be lost in future updates. + +provider "registry.opentofu.org/digitalocean/digitalocean" { + version = "2.100.0" + constraints = "~> 2.40" + hashes = [ + "h1:9VsRnY7rmGiMFVG4ix3ry2fx8B8d7l9PyEpmGjjiW3I=", + "zh:021dbdacf7dd636cb66990964737bc4a6402160b5301e06b6fdab03a814f7e96", + "zh:083ff822b22738eedc9eae29aa5d4c369b2de921e49fefa9858a4e7c6274b0c1", + "zh:10695b5f7dbbe8ba1a20ac6bea1ab8613b43d2b1f6f9b53369bed296b2dff4d2", + "zh:110124b9d07cd1f0f2831ad27dfeb5a010a070952ea9a9e675ca3cfe158a0ffa", + "zh:409a8b22ffb9a53caf57d0a16f5f8ac0a2fcd61b6b0bcd512a685ea26cd06047", + "zh:488e4b494c198c0696c85d2c73c66c40567ca2ad5230bbd95d8ffdead86e646d", + "zh:4dd5b6e8c1be5fb619812fe738d0b49e0d24731fb290c0ab5190b7bc60c70e66", + "zh:6717fa7f623ad4c9b4d99acceffd1372d272ff02d7c98259dd17ec7ef11b836c", + "zh:7a25445caebd2e7f6ab2381578ecaf31744159a7e4a16476f19112d2e50d8519", + "zh:86b3d7050ba04ec7e0a14dbee8c0540fca8f4377fb072f87c88a6fcd26388714", + "zh:c23f30a38930c283055b6f04acbcad06cb0dd3c71c172eb23bb239f50dd12823", + "zh:d1f9c5be7f7d5c71830357bdd9c0fb529185e9380092e1b792f969495c943790", + "zh:d222573e7de8f9ad1c5b87a6c44c2dc59947c3968b3e70b7d4c76263d6c8fe5c", + "zh:e1f2d606661435416c4d2b48db80244d0acef88a5ce81b5dcbd65ff2642b1b4d", + "zh:ea13fad19414586864557601c59e85188601e150c815ec1e71f8ed83f1942a50", + "zh:f795be71dc933aaec7145f93068da17f227783964d33a79de591a042aac7c3ca", + ] +} diff --git a/infra/README.md b/infra/README.md new file mode 100644 index 0000000000..18328c5dba --- /dev/null +++ b/infra/README.md @@ -0,0 +1,73 @@ +# Raindex API infrastructure + +This is the production deployment path for the Raindex market API: DigitalOcean +resources managed by OpenTofu, encrypted local state, NixOS installed with +nixos-anywhere, separate deploy-rs system and service profiles, and a persistent +block volume. + +The NixOS configuration provides: + +- nginx TLS termination, security headers, and a 10 request/second per-IP + ingress limit with a burst of 20; +- a systemd-managed API that starts after reboot and restarts on failure; +- SQLite and daily JSON logs under `/mnt/data/raindex-api` so deployments do not + discard indexed data; +- 14 retained application log files, 14-day/512 MiB journald bounds, a host + firewall, automatic ACME renewal, and weekly Nix garbage collection. + +## One-time production provisioning + +The default hostname is `api.raindex.finance`. Export `RAINDEX_API_HOSTNAME` for +every Nix command if production uses another name. + +OpenTofu creates the DigitalOcean SSH key named `raindex-op` from +`infra/raindex-op.pub` and installs it on the new host. `keys.nix` reads the +same file, keeping provisioning and NixOS authorization in sync. The private key +is kept outside the repository at `~/.ssh/raindex-op`; pass +`-i ~/.ssh/raindex-op` to the commands below. + +### Operator access + +The production host accepts only keys listed in `roles.ssh` in `keys.nix`. To +grant another person access: + +1. Add their public key as a named entry under `keys`. +2. Add that name to `roles.ssh`. +3. Run `nix run .#deployNixos -- -i ~/.ssh/raindex-op`. + +Removing the name from `roles.ssh` and deploying NixOS revokes access. Add the +name to `roles.state` when that person must deploy, and to `roles.vars` only +when they must operate the DigitalOcean infrastructure. + +1. Run `nix run .#tfEditVars`, set the DigitalOcean token, and commit the + generated `infra/terraform.tfvars.age`. +2. Run `nix run .#tfInit`, `nix run .#tfPlan`, and `nix run .#tfApply`. The plan + command is a preview; apply creates and consumes a fresh ephemeral plan. + Commit the generated `infra/terraform.tfstate.age`; plaintext variables and + state are deleted by the wrappers and ignored by Git. +3. Point the API hostname at `nix run .#resolveIp`. +4. Run `nix run .#bootstrap`. This converts the temporary Ubuntu image to the + declared NixOS system and replaces the fail-closed placeholder in `keys.nix` + with the host's Ed25519 key. Verify and commit that change. +5. Run `nix run .#deployAll`. Subsequent API-only releases use + `nix run .#deployService -- rest-api`. + +The operator commands accept `-i /path/to/private-key` before other arguments; +otherwise they use `~/.ssh/id_ed25519`. + +## GitHub deployment + +Configure the production environment with: + +- `RAINDEX_API_SSH_KEY`: the private CI key already listed in `keys.nix` and + able to decrypt the committed encrypted Terraform state; +- optional `RAINDEX_API_HOSTNAME`: only when the hostname differs from + `api.raindex.finance`; +- optional `CACHIX_AUTH_TOKEN`. + +The manual `Deploy Raindex API` workflow validates the flake and OpenTofu, +resolves the reserved IP from encrypted state, checks the committed SSH host +key, deploys through deploy-rs, and requires `/health/detailed` to succeed. + +DNS is deliberately managed outside this stack. Neither a DigitalOcean API token +nor plaintext Terraform state is needed by the deployment workflow. diff --git a/infra/backend.tf b/infra/backend.tf new file mode 100644 index 0000000000..35e0ffc488 --- /dev/null +++ b/infra/backend.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.5" + + required_providers { + digitalocean = { + source = "digitalocean/digitalocean" + version = "~> 2.40" + } + } +} + +provider "digitalocean" { + token = var.do_token +} diff --git a/infra/default.nix b/infra/default.nix new file mode 100644 index 0000000000..5b6c6bb7a1 --- /dev/null +++ b/infra/default.nix @@ -0,0 +1,170 @@ +{ + pkgs, + ragenix, + rainix, + system, +}: + +let + buildInputs = [ + pkgs.opentofu + pkgs.rage + pkgs.jq + ragenix.packages.${system}.default + ]; + + tfState = "infra/terraform.tfstate"; + tfVars = "infra/terraform.tfvars"; + tfSecretVars = "infra/zz_secret.auto.tfvars"; + tfPlanFile = "infra/tfplan"; + + parseIdentity = '' + set -eo pipefail + umask 077 + + identity=~/.ssh/id_ed25519 + if [ "''${1:-}" = "-i" ]; then + identity="$2" + shift 2 + fi + ''; + + decryptState = '' + if [ -f ${tfState}.age ]; then + rage -d -i "$identity" ${tfState}.age > ${tfState} + fi + ''; + + encryptState = '' + if [ -f ${tfState} ]; then + nix eval --raw --file ${../.}/keys.nix roles.state \ + --apply 'builtins.concatStringsSep "\n"' \ + | rage -e -R /dev/stdin -o ${tfState}.age ${tfState} + fi + ''; + + decryptVars = '' + if [ -f ${tfVars}.age ]; then + rage -d -i "$identity" ${tfVars}.age > ${tfVars} + elif [ ! -f ${tfVars} ]; then + echo "${tfVars}.age is missing; run nix run .#tfEditVars first" >&2 + exit 1 + fi + if [ -n "''${TF_VAR_do_token:-}" ]; then + printf 'do_token = "%s"\n' "$TF_VAR_do_token" > ${tfSecretVars} + fi + ''; + + encryptVars = '' + nix eval --raw --file ${../.}/keys.nix roles.vars \ + --apply 'builtins.concatStringsSep "\n"' \ + | rage -e -R /dev/stdin -o ${tfVars}.age ${tfVars} + ''; + + cleanup = "rm -f ${tfState} ${tfState}.backup ${tfVars} ${tfSecretVars}"; + cleanupWithPlan = "${cleanup} ${tfPlanFile}"; + + preamble = '' + ${parseIdentity} + on_exit() { ${cleanupWithPlan}; } + trap on_exit EXIT + ${decryptVars} + ''; + + preambleWithEncrypt = '' + ${parseIdentity} + on_exit() { + ${encryptState} + ${cleanupWithPlan} + } + trap on_exit EXIT + ${decryptVars} + ''; + + resolveIp = '' + ${parseIdentity} + trap 'rm -f ${tfState}' EXIT + ${decryptState} + if [ ! -s ${tfState} ]; then + echo "encrypted Terraform state is missing; provision infrastructure first" >&2 + exit 1 + fi + host_ip=$(jq -r '.outputs.reserved_ip.value // empty' ${tfState}) + if [ -z "$host_ip" ] || [ "$host_ip" = "null" ]; then + echo "production infrastructure is not present in Terraform state" >&2 + exit 1 + fi + rm -f ${tfState} + ''; +in +{ + inherit buildInputs parseIdentity resolveIp; + + tfInit = rainix.mkTask.${system} { + name = "tf-init"; + additionalBuildInputs = buildInputs; + body = '' + ${preamble} + tofu -chdir=infra init "$@" + ''; + }; + + tfPlan = rainix.mkTask.${system} { + name = "tf-plan"; + additionalBuildInputs = buildInputs; + body = '' + ${preamble} + ${decryptState} + tofu -chdir=infra plan "$@" + ''; + }; + + tfApply = rainix.mkTask.${system} { + name = "tf-apply"; + additionalBuildInputs = buildInputs; + body = '' + ${preambleWithEncrypt} + ${decryptState} + tofu -chdir=infra plan -out=tfplan + tofu -chdir=infra apply "$@" tfplan + ''; + }; + + tfImport = rainix.mkTask.${system} { + name = "tf-import"; + additionalBuildInputs = buildInputs; + body = '' + ${preambleWithEncrypt} + ${decryptState} + tofu -chdir=infra import "$@" + ''; + }; + + tfDestroy = rainix.mkTask.${system} { + name = "tf-destroy"; + additionalBuildInputs = buildInputs; + body = '' + ${preambleWithEncrypt} + ${decryptState} + tofu -chdir=infra destroy "$@" + ''; + }; + + tfEditVars = rainix.mkTask.${system} { + name = "tf-edit-vars"; + additionalBuildInputs = buildInputs; + body = '' + ${parseIdentity} + on_exit() { rm -f ${tfVars}; } + trap on_exit EXIT + + if [ -f ${tfVars}.age ]; then + rage -d -i "$identity" ${tfVars}.age > ${tfVars} + else + cp ${tfVars}.example ${tfVars} + fi + ''${EDITOR:-vi} ${tfVars} + ${encryptVars} + ''; + }; +} diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 0000000000..54816a6141 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,36 @@ +resource "digitalocean_ssh_key" "operator" { + name = var.ssh_key_name + public_key = trimspace(file("${path.module}/raindex-op.pub")) +} + +resource "digitalocean_volume" "data" { + region = var.region + name = "raindex-api-data" + size = var.volume_size_gb + initial_filesystem_type = "ext4" + description = "Persistent Raindex indexer database and rotated logs" +} + +resource "digitalocean_droplet" "nixos" { + image = "ubuntu-24-04-x64" + name = "raindex-api-nixos" + region = var.region + size = var.droplet_size + ssh_keys = [digitalocean_ssh_key.operator.id] +} + +resource "digitalocean_volume_attachment" "data" { + droplet_id = digitalocean_droplet.nixos.id + volume_id = digitalocean_volume.data.id +} + +resource "digitalocean_reserved_ip" "nixos" { + region = var.region +} + +resource "digitalocean_reserved_ip_assignment" "nixos" { + ip_address = digitalocean_reserved_ip.nixos.ip_address + droplet_id = digitalocean_droplet.nixos.id + + depends_on = [digitalocean_volume_attachment.data] +} diff --git a/infra/outputs.tf b/infra/outputs.tf new file mode 100644 index 0000000000..e47045c3da --- /dev/null +++ b/infra/outputs.tf @@ -0,0 +1,19 @@ +output "droplet_id" { + description = "ID of the Raindex API droplet" + value = digitalocean_droplet.nixos.id +} + +output "droplet_ipv4" { + description = "Ephemeral IPv4 address of the droplet" + value = digitalocean_droplet.nixos.ipv4_address +} + +output "reserved_ip" { + description = "Stable public IP to use for DNS and deployment" + value = digitalocean_reserved_ip.nixos.ip_address +} + +output "volume_id" { + description = "Persistent Raindex API data volume" + value = digitalocean_volume.data.id +} diff --git a/infra/raindex-op.pub b/infra/raindex-op.pub new file mode 100644 index 0000000000..773e77783a --- /dev/null +++ b/infra/raindex-op.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGltf4G+jKIijoTywcoT8gqKd5gY5T+xhXHMBTZPzoUC diff --git a/infra/terraform.tfstate.age b/infra/terraform.tfstate.age new file mode 100644 index 0000000000..55990f8deb Binary files /dev/null and b/infra/terraform.tfstate.age differ diff --git a/infra/terraform.tfvars.age b/infra/terraform.tfvars.age new file mode 100644 index 0000000000..c1d590d616 Binary files /dev/null and b/infra/terraform.tfvars.age differ diff --git a/infra/terraform.tfvars.example b/infra/terraform.tfvars.example new file mode 100644 index 0000000000..14c77f736a --- /dev/null +++ b/infra/terraform.tfvars.example @@ -0,0 +1,8 @@ +# Copy to terraform.tfvars. Never commit terraform.tfvars or OpenTofu state. +do_token = "" + +# Optional capacity overrides: +# ssh_key_name = "raindex-op" +# region = "nyc3" +# droplet_size = "s-2vcpu-4gb" +# volume_size_gb = 10 diff --git a/infra/variables.tf b/infra/variables.tf new file mode 100644 index 0000000000..fc0646da80 --- /dev/null +++ b/infra/variables.tf @@ -0,0 +1,29 @@ +variable "do_token" { + description = "DigitalOcean API token" + type = string + sensitive = true +} + +variable "ssh_key_name" { + description = "DigitalOcean SSH key installed on the API host" + type = string + default = "raindex-op" +} + +variable "region" { + description = "DigitalOcean region" + type = string + default = "nyc3" +} + +variable "droplet_size" { + description = "DigitalOcean droplet size slug" + type = string + default = "s-2vcpu-4gb" +} + +variable "volume_size_gb" { + description = "Persistent indexer and log volume size" + type = number + default = 10 +} diff --git a/keys.nix b/keys.nix new file mode 100644 index 0000000000..9b428be6ff --- /dev/null +++ b/keys.nix @@ -0,0 +1,25 @@ +rec { + keys = { + raindex-op = builtins.replaceStrings [ "\n" ] [ "" ] (builtins.readFile ./infra/raindex-op.pub); + ci = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPTd2zKSwHgWegi290EiK5nYp1Wp4+x2fDYqFxbd0WLN"; + + # Replaced by `nix run .#bootstrap` after nixos-anywhere provisions the + # production host. Deploys trust this committed key and fail before then. + host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBiL05XFbmXMcwoNBhndBHb69QVDMIhAGJkLALzsblCM"; + }; + + roles = with keys; { + state = [ + raindex-op + ci + ]; + vars = [ raindex-op ]; + # Add another person's public key above, then add its name here to grant + # root SSH access. Add it to `state` only if they also need to deploy, and + # to `vars` only if they must operate the DigitalOcean infrastructure. + ssh = [ + raindex-op + ci + ]; + }; +} diff --git a/os.nix b/os.nix new file mode 100644 index 0000000000..7d7718c3e1 --- /dev/null +++ b/os.nix @@ -0,0 +1,251 @@ +{ + pkgs, + lib, + modulesPath, + raindexEnv ? { }, + ... +}: + +let + inherit (import ./keys.nix) roles; + + env = { + name = "prod"; + virtualHost = "api.raindex.finance"; + dataDir = "/mnt/data/raindex-api"; + dataVolumeName = "raindex-api-data"; + } + // raindexEnv; + + serviceDefinitions = import ./services.nix; + enabledServices = lib.filterAttrs (_: value: value.enabled) serviceDefinitions; + + mkService = + name: cfg: + let + executable = "/nix/var/nix/profiles/per-service/${name}/bin/${cfg.bin}"; + in + { + description = "Raindex ${cfg.bin} (${env.name}/${name})"; + wantedBy = [ "multi-user.target" ]; + wants = [ "network-online.target" ]; + after = [ "network-online.target" ]; + restartIfChanged = false; + stopIfChanged = false; + + unitConfig = { + "X-OnlyManualStart" = true; + ConditionPathExists = executable; + RequiresMountsFor = env.dataDir; + }; + + environment = { + ROCKET_ADDRESS = "127.0.0.1"; + ROCKET_PORT = "8000"; + RAINDEX_LOCAL_DB_PATH = "${env.dataDir}/market-data.sqlite"; + RAINDEX_LOG_DIR = "${env.dataDir}/logs"; + RAINDEX_TRUSTED_PROXY_IP_HEADER = "X-Real-IP"; + RUST_LOG = "raindex_rest_api=info,raindex_common=info,raindex_quote=info,rocket=warn,warn"; + }; + + serviceConfig = { + User = "raindex-api"; + Group = "raindex"; + ExecStart = executable; + Restart = "always"; + RestartSec = 5; + NoNewPrivileges = true; + PrivateTmp = true; + ProtectHome = true; + ProtectSystem = "strict"; + ReadWritePaths = [ env.dataDir ]; + }; + }; +in +{ + imports = [ + (modulesPath + "/virtualisation/digital-ocean-config.nix") + (modulesPath + "/profiles/qemu-guest.nix") + ./disko.nix + ]; + + boot.loader.grub = { + efiSupport = true; + efiInstallAsRemovable = true; + }; + + networking.useDHCP = lib.mkForce false; + + services = { + cloud-init = { + enable = true; + network.enable = true; + settings = { + datasource_list = [ + "ConfigDrive" + "Digitalocean" + ]; + datasource.ConfigDrive = { }; + datasource.Digitalocean = { }; + cloud_init_modules = [ + "seed_random" + "bootcmd" + "write_files" + "growpart" + "resizefs" + "set_hostname" + "update_hostname" + "set_password" + ]; + cloud_config_modules = [ + "ssh-import-id" + "keyboard" + "runcmd" + "disable_ec2_metadata" + ]; + cloud_final_modules = [ + "write_files_deferred" + "puppet" + "chef" + "ansible" + "mcollective" + "salt_minion" + "reset_rmc" + "scripts_per_once" + "scripts_per_boot" + "scripts_user" + "ssh_authkey_fingerprints" + "keys_to_console" + "install_hotplug" + "phone_home" + "final_message" + ]; + }; + }; + + openssh = { + enable = true; + settings = { + PasswordAuthentication = false; + PermitRootLogin = "prohibit-password"; + }; + }; + + nginx = { + enable = true; + recommendedTlsSettings = true; + recommendedProxySettings = true; + recommendedOptimisation = true; + recommendedGzipSettings = true; + + appendHttpConfig = '' + limit_req_zone $binary_remote_addr zone=api:10m rate=10r/s; + ''; + + virtualHosts.${env.virtualHost} = { + enableACME = true; + forceSSL = true; + + extraConfig = '' + add_header X-Content-Type-Options "nosniff" always; + add_header X-Frame-Options "DENY" always; + add_header Referrer-Policy "strict-origin-when-cross-origin" always; + client_max_body_size 1m; + ''; + + locations = { + "~* \\.(php|asp|aspx|jsp|cgi)$".return = "444"; + "~* ^/(containers|_ignition|vendor|public/index)".return = "444"; + "/" = { + proxyPass = "http://127.0.0.1:8000"; + extraConfig = '' + limit_req zone=api burst=20 nodelay; + limit_req_status 429; + ''; + }; + }; + }; + }; + + journald.extraConfig = '' + MaxRetentionSec=14day + SystemMaxUse=512M + ''; + }; + + security.acme = { + acceptTerms = true; + defaults.email = "ops@raindex.finance"; + }; + + networking.firewall = { + enable = true; + allowedTCPPorts = [ + 22 + 80 + 443 + ]; + }; + + fileSystems."/mnt/data" = { + device = "/dev/disk/by-id/scsi-0DO_Volume_${env.dataVolumeName}"; + fsType = "ext4"; + }; + + # Remote Rust builds can briefly exceed the production VM's physical RAM. + # Keep swap on the replaceable root disk; application data remains on the + # persistent volume above. + swapDevices = [ + { + device = "/var/lib/swapfile"; + size = 8192; + } + ]; + + nix = { + settings = { + experimental-features = [ + "nix-command" + "flakes" + ]; + auto-optimise-store = true; + download-buffer-size = 268435456; + }; + gc = { + automatic = true; + dates = "weekly"; + options = "--delete-older-than 30d"; + }; + }; + + users = { + groups.raindex = { }; + users = { + root.openssh.authorizedKeys.keys = roles.ssh; + raindex-api = { + isSystemUser = true; + group = "raindex"; + }; + }; + }; + + programs.bash.interactiveShellInit = "set -o vi"; + + systemd.tmpfiles.rules = [ + "d ${env.dataDir} 0775 root raindex -" + "d ${env.dataDir}/logs 0775 raindex-api raindex -" + ]; + systemd.services = lib.mapAttrs mkService enabledServices; + + environment.systemPackages = with pkgs; [ + bat + curl + htop + sqlite + zellij + ]; + + system.activationScripts.per-service-profiles.text = "mkdir -p /nix/var/nix/profiles/per-service"; + + system.stateVersion = "24.11"; +} diff --git a/rust.nix b/rust.nix new file mode 100644 index 0000000000..8c126c711d --- /dev/null +++ b/rust.nix @@ -0,0 +1,59 @@ +{ pkgs, craneLib }: + +let + sourceRoot = toString ./.; + source = pkgs.lib.cleanSourceWith { + src = ./.; + filter = + path: type: + let + relative = pkgs.lib.removePrefix "${sourceRoot}/" (toString path); + in + pkgs.lib.cleanSourceFilter path type + && relative != ".raindex" + && !(pkgs.lib.hasPrefix ".raindex/" relative) + && relative != "infra" + && !(pkgs.lib.hasPrefix "infra/" relative); + }; + commonArgs = { + pname = "raindex-api"; + version = "0.0.0-alpha.0"; + src = source; + strictDeps = true; + cargoExtraArgs = "-p raindex_rest_api"; + nativeBuildInputs = [ + pkgs.pkg-config + ] + ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isDarwin [ pkgs.darwin.DarwinTools ]; + buildInputs = [ + pkgs.libusb1 + pkgs.openssl + pkgs.sqlite + ] + ++ pkgs.lib.optionals pkgs.stdenv.hostPlatform.isDarwin [ pkgs.apple-sdk_15 ]; + }; + + cargoArtifacts = craneLib.buildDepsOnly commonArgs; +in +{ + package = craneLib.buildPackage ( + commonArgs + // { + inherit cargoArtifacts; + doCheck = true; + cargoTestExtraArgs = "-p raindex_rest_api"; + meta = { + description = "Raindex public market data API"; + homepage = "https://github.com/rainlanguage/rain.orderbook"; + }; + } + ); + + clippy = craneLib.cargoClippy ( + commonArgs + // { + inherit cargoArtifacts; + cargoClippyExtraArgs = "-p raindex_rest_api --all-targets -- -D warnings"; + } + ); +} diff --git a/services.nix b/services.nix new file mode 100644 index 0000000000..6b21efd4a9 --- /dev/null +++ b/services.nix @@ -0,0 +1,4 @@ +{ + rest-api.enabled = true; + rest-api.bin = "raindex-api"; +}