diff --git a/.github/workflows/testsuite.yml b/.github/workflows/testsuite.yml index 590723ff..514cbe31 100644 --- a/.github/workflows/testsuite.yml +++ b/.github/workflows/testsuite.yml @@ -39,3 +39,35 @@ jobs: - run: meteor npm ci - run: meteor npm run test:mocha + + browser-tests: + name: Meteor ${{ matrix.meteor-release }} browser tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + meteor-release: ['3.2.2', '3.5.2'] + steps: + - name: checkout + uses: actions/checkout@v6 + + - name: cache dependencies + uses: actions/cache@v4 + with: + path: | + ~/.npm + ~/.cache/ms-playwright + key: ${{ runner.os }}-meteor-browser-${{ matrix.meteor-release }}-${{ hashFiles('**/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-meteor-browser-${{ matrix.meteor-release }}- + + - name: Setup meteor + uses: meteorengineer/setup-meteor@v3 + with: + meteor-release: ${{ matrix.meteor-release }} + + - run: meteor npm ci + - run: meteor npx playwright install --with-deps chromium + - run: meteor npm run test:browser + env: + TEST_SERVER: '0' diff --git a/.versions b/.versions index ff6ef314..73292fe7 100644 --- a/.versions +++ b/.versions @@ -1,7 +1,7 @@ allow-deny@2.1.0 babel-compiler@7.15.1 babel-runtime@1.5.2 -base64@1.0.14 +base64@1.0.16 binary-heap@1.0.13 boilerplate-generator@2.1.0 callback-hook@1.8.0 @@ -23,7 +23,7 @@ fetch@0.2.0 geojson-utils@1.0.12 id-map@1.2.0 inter-process-messaging@0.1.3 -local-test:ostrio:files@3.1.0 +local-test:ostrio:files@4.0.0 logging@1.3.6 meteor@2.3.1 meteortesting:browser-tests@1.8.0 @@ -40,7 +40,7 @@ mongo-id@1.0.9 npm-mongo@6.16.3 ordered-dict@1.2.0 ostrio:cookies@3.0.0 -ostrio:files@3.1.0 +ostrio:files@4.0.0 promise@1.0.0 random@1.2.2 react-fast-refresh@0.3.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 7447c2cd..cfdc1dac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,64 @@ +# 4.0.0 + +## What's new + +This release adds signed download links, storage adapters with a built-in GridFS adapter, content-based file type detection, and uploads that resume after a server restart. It also tightens defaults: clients can not remove files, responses carry `nosniff`, risky types download as attachments, and only the server names files. Read "Major changes" before you upgrade, and follow the [migration guide to v4](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/migration-to-v4.md). + +## Major changes + +- ⚠️ Remove `FilesCursor#hasNext()` and `FilesCursor#countAsync()`. Use `hasNextAsync()` and `countDocuments()`. +- ⚠️ Move `findOne()` from `FilesCollectionCore` to the client class. The server class still throws `Meteor.Error(404)`. Use `findOneAsync()`. +- ⚠️ Support only a `Map` in `Meteor.server.sessions` (Meteor 3) in the default `x_mtok` lookup. A plain object throws. +- ⚠️ Default `allowClientCode` to `false`. Clients can not call `remove()` unless `allowClientCode: true` is set on the server and the client. Set `onBeforeRemove` when you enable it. +- ⚠️ Accept only a String `_id` in client `remove()` and `removeAsync()`. Use `find(selector).removeAsync()` to remove several files. It removes one `_id` per server call and is not atomic. +- ⚠️ Send `Cache-Control: private, max-age=31536000` by default for `protected` collections, so shared caches do not keep protected files. Set `cacheControl` to restore the 3.x value. +- ⚠️ Send `X-Content-Type-Options: nosniff` by default. Set `nosniff: false` to turn it off. +- ⚠️ Serve files `inline` only for `image/*` (not SVG), `video/*`, `audio/*`, `application/pdf`, and `text/plain`. Other files get `Content-Disposition: attachment`. Set `Content-Disposition` in `responseHeaders` to change it. +- ⚠️ Run `FileUpload#pipe()` functions in the order they were added. The first `pipe()` call runs first. Reverse chained `pipe()` calls written for 3.x. +- ⚠️ Name files on the server only. The client `namingFunction` option and the `FSName` field are ignored. Set `namingFunction` on the server. It receives `{ file, fileId, userId }` in upload Start, `writeAsync()`, and `loadAsync()`. +- ⚠️ Keep only `name`, `type`, `size`, and `meta` from the client `file` object. Send custom data in `meta`. +- ⚠️ Store `type`, `mime`, `versions.original.type`, and the `is*` flags from the file content instead of the uploader's claim. Text keeps the uploader's type only for a short list of passive types (`text/plain`, `text/csv`, `text/markdown`, `text/tab-separated-values`, `text/calendar`, `text/vtt`, `application/json`), other text is stored as `text/plain`. Set `trustClientMimeType: true` to store the uploader's type as in 3.x. +- ⚠️ Make `serve()` async. Await it when code runs after it. `unlinkAsync()` and `removeAsync()` remove files through the storage adapter (`FSStorage` by default, same files as before). Custom adapters receive `{ source }` as the 4th `put()` argument and must keep the caller's file when `source` is `'addFile'`. +- ⚠️ Restart uploads that were in progress during the upgrade from 3.x. They get `410` on their next chunk. Start rejects more than 100000 chunks with `400`, and the client raises `chunkSize` to stay below. + +## Other Changes + +### Added + +- ✨ Add signed download links. Set `downloadTokenSecret`, call `createDownloadToken()`, and pass the token to `link(fileRef, version, uriBase, { token })`. Tokens work without the `x_mtok` cookie and on any server instance. A token opens one file version in one collection, and token downloads get `Cache-Control: private` until the token expires. +- ✨ Add storage adapters with the `storage` option. `FSStorage` is the default and `GridFSStorage` keeps files in MongoDB GridFS. +- ✨ Add content-based type detection from a built-in signature table, and the `trustClientMimeType` option. +- ✨ Resume uploads after a server restart. The server records every written chunk in the upload record. +- ✨ Add a browser test suite to CI on Meteor 3.2.2 and 3.5.2. +- ✨ Accept an async `responseHeaders` function. Thanks to @ToyboxZach, #861. + +### Fixed + +- 🔧 Record written chunks instead of guessing them from the file size after a restart. A hole before the last written chunk no longer passes as complete. +- 🔧 Make one database read per protected download. +- 🔧 Finish an upload whose chunks reached several server instances: EOF reads the recorded chunks once before it gives up. Another instance finishing the upload no longer deletes the file or logs a failed write. + +### Changed + +- 👨‍💻 Deprecate `protected: true`. It logs a warning at startup and will be removed in v5. Pass a function. + +### Docs + +- 📔 Add the [migration guide to v4](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/migration-to-v4.md) with an upgrade checklist. +- 📔 Rewrite the S3 recipe as a storage adapter and lead the GridFS guide with the built-in adapter. Thanks to @ThaumRystra, #874. +- 📔 Update the security guide for the new defaults, signed links, and content-based types. +- 📔 Await `serve()` in the GridFS and Google Cloud Storage recipes. +- 📔 Show S3-compatible services such as MinIO and Wasabi in the S3 recipe. Thanks to @xet7 and @dhana-exe, #862, #799. +- 📔 Point to upload scanning and the OWASP File Upload Cheat Sheet in the security guide. Thanks to @jankapunkt, #753. + +### Tests + +- 🧪 Cover the client upload state machine in a browser with the `playwright` driver of `meteortesting:mocha`. + +### Dependencies + +- 📦 Add `playwright` as a dev dependency for browser tests. The runtime dependency list is unchanged. + # 3.1.0 ## What's new @@ -10,7 +71,7 @@ This release closes several upload and download security holes, fixes the client - ⚠️ Restart uploads that were in progress during the upgrade from 3.0.x. The server answers `403` or `410` for upload records created by 3.0.x. - ⚠️ Return HTTP upload errors as `{ error: , reason, isClientSafe? }` instead of `{ error: "" }`. `5xx` responses have a generic `reason`. - ⚠️ Limit upload sizes. `chunkSize` is at most 16 MiB. HTTP Start bodies, including `meta`, are at most 1 MiB. HTTP EOF bodies are at most 64 KiB. HTTP chunk bodies are at most the Base64 size of `chunkSize` plus 4 KiB. -- ⚠️ Reject `FileUpload` pipes that grow a chunk. A pipe must return Base64 that decodes to the same number of bytes as its input, so per-chunk encryption with an IV, a tag, or padding fails on the first chunk. Move such transforms to `onAfterUpload` on the server. +- ⚠️ Reject `FileUpload` pipes that grow a chunk. A pipe must return Base64 that decodes to the same number of bytes as its input, so per-chunk encryption with an IV, a tag, or padding fails on the first chunk. Move such transforms to `onAfterUpload` on the server. Thanks to @sylido, #505. - ⚠️ Emit only `error` and `end` when an upload fails. `pause`, `abort`, and `onAbort` now fire only when `abort()` is called. - ⚠️ Throw `Meteor.Error 404` from the synchronous `FilesCursor` methods (`get`, `fetch`, `first`, `last`, `next`, `previous`, `hasNext`, `count`, `forEach`, `each`, `map`, `current`, `remove`) and from `FileCursor#with()` and `FileCursor#remove()` on the server. They returned Promises or wrong values before. Use the `*Async` methods. - ⚠️ Reject `writeAsync()` and `loadAsync()` with `409` when `opts.fileId` already exists. Before, they overwrote the existing file. @@ -112,7 +173,7 @@ This release closes several upload and download security holes, fixes the client - 📔 Add a security guide. - 📔 Rename the migration doc to `migration-to-v3.md`. -- 📔 Rewrite the AWS S3 (`@aws-sdk/client-s3` v3), Dropbox, Google Cloud Storage, GridFS, and `sharp` thumbnail guides with async APIs and correct Range handling. +- 📔 Rewrite the AWS S3 (`@aws-sdk/client-s3` v3), Dropbox, Google Cloud Storage, GridFS, and `sharp` thumbnail guides with async APIs and correct Range handling. Thanks to @ThaumRystra, #874. - 📔 Document `updateAsync`, `countDocuments`, `estimatedDocumentCount`, `serve`, `download`, `allow`/`deny`, the exported helpers, and the `progress` event arguments. - 📔 Document upload rules, limits, and the HTTP error body in `about-transports.md`. - 📔 Document the event order: `abort()` emits `pause`, then `abort`, and no `end`. A failed upload emits `error`, then `end`. Document that `abort()` does not cancel an EOF or HTTP Start request that is already sent. diff --git a/CLAUDE.md b/CLAUDE.md index 9ba7b2e1..55113f19 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What this is -`ostrio:files` is an Atmosphere package for Meteor 3.2 or newer (`ostrio:cookies` needs `fetch@0.1.6`, first shipped in 3.2) that uploads files from the client to the server and serves them back. It is published from this repo root through `package.js`, not through npm. `package.json` holds only dev tooling. Runtime npm deps go in `Npm.depends` inside `package.js`, and both the `onUse` and `onTest` blocks must declare them. +`ostrio:files` 4.x is an Atmosphere package for Meteor 3.2 or newer (`ostrio:cookies` needs `fetch@0.1.6`, first shipped in 3.2) that uploads files from the client to the server and serves them back. It is published from this repo root through `package.js`, not through npm. `package.json` holds only dev tooling. Runtime npm deps go in `Npm.depends` inside `package.js`, and both the `onUse` and `onTest` blocks must declare them. ## Commands @@ -14,18 +14,26 @@ npm run lint # ESLint 9 flat config (eslint.config.mjs) npm run typecheck # tsc --noEmit -p . && tsd (index.test-d.ts) npm run test:mocha # meteor test-packages ./ --driver-package meteortesting:mocha --once npm run test:mocha:watch # same, re-runs on change +npm run test:browser # server + browser suite (playwright), TEST_SERVER=0 skips server tests +npx playwright install chromium # once, downloads the browser test:browser uses npm run test:mocha -- --port 3456 # use another port when 3000 is busy MOCHA_GREP="_checkAccess" npm run test:mocha # run tests whose name matches (works with meteortesting:mocha) meteor test-packages ./ --driver-package meteortesting:mocha --once --release 3.5.2 # pin a Meteor release ``` -Tests are server-only mocha (`chai`, `sinon`). `tests/server.js` is the entry point and imports these files. Add a new test file there. +Tests are mocha (`chai`, `sinon`). `tests/server.js` is the server entry point and imports these files. Add a new server test file there. - `core.test.js`: `FilesCollectionCore`, `link()`, `formatFileURL` - `cursor.test.js`: `FileCursor` and `FilesCursor`, including the server-side sync method errors - `helpers.test.js`: `lib.js` helpers - `server.test.js`: server API, `_checkAccess`, `loadAsync`, `writeAsync`, `serve()`, `WriteStream` - `security.test.js`: upload ownership, path and file identity checks, size limits, Range, HTTP errors, idempotent EOF +- `mime.test.js`: `mime.js` type detection +- `download-token.test.js`: `download-token.js` signed tokens +- `storage.test.js`: storage adapters (`FSStorage`, `GridFSStorage`) +- `browser-fixtures.js`: server half of the browser suite (`mfBrowserTests` collection, `mfTest.*` methods) + +`tests/client.js` is the browser entry point (server half in `browser-fixtures.js`). It runs only under `npm run test:browser`, which drives headless Chromium through `meteortesting:browser-tests` and the `playwright` devDependency. Run all three of lint, typecheck, and tests before finishing. `.versions` is not updated by `test-packages`. When dependencies change, regenerate it from a throwaway app (`meteor create --bare /tmp/x`, add `ostrio:files` and `meteortesting:mocha` with `METEOR_PACKAGE_DIRS` pointing at this repo's parent, copy the resolved versions, drop the `ostrio:files` line). `meteor publish` also rewrites it. @@ -38,16 +46,18 @@ Upload flow (client → server): 2. There are two transports, picked by `transport: 'ddp' | 'http'`: - DDP: Meteor methods named in `_methodNames` (`_FilesCollectionStart_`, `_Write_`, `_Abort_`, `_Remove_`). - HTTP: POST to `${downloadRoute}/${collectionName}/__upload` with `x-start`, `x-eof`, `x-chunkid`, `x-fileid`, and `x-mtok` headers, handled in the `WebApp` middleware in `server.js`. -3. Both transports converge in `_startUpload` (which calls `_prepareUpload`) and `_writeUpload` (server). An in-progress upload lives in `_preCollection` (TTL index, `continueUploadTTL`) and in `_currentUploads[fileId]`, a `WriteStream` (`write-stream.js`) that writes chunks at offset `(chunkId-1)*chunkSize` into one file handle. On EOF, `_finishUpload` inserts the final document into `this.collection` and emits `afterUpload`. +3. Both transports converge in `_startUpload` (which calls `_prepareUpload`) and `_writeUpload` (server). An in-progress upload lives in `_preCollection` (TTL index, `continueUploadTTL`) and in `_currentUploads[fileId]`, a `WriteStream` (`write-stream.js`) that writes chunks at offset `(chunkId-1)*chunkSize` into one file handle and records each written chunk in `chunkBits` of the `_preCollection` record (`_recordChunk`), so a restarted server resumes from the record. On EOF, `_finishUpload` detects the type (`mime.js`), calls `put()` of the storage adapter, inserts the final document into `this.collection`, and emits `afterUpload`. -Download flow: the same `WebApp` middleware matches `${downloadRoute}/${collectionName}/${_id}/${version}/${name}` (or the `public` variant). Then `_checkAccess` runs (the `protected` option), then `download()`, then `serve()`, which handles Range/206/416, `responseHeaders`, and `Content-Disposition`, and streams with `fs.createReadStream`. Hooks along this path: `interceptRequest`, `interceptDownload`, `downloadCallback`. +Download flow: the same `WebApp` middleware matches `${downloadRoute}/${collectionName}/${_id}/${version}/${name}` (or the `public` variant). Then `_checkAccess` runs (the `protected` option) and returns `false | { fileRef }`. A signed token (`downloadTokenSecret`, `download-token.js`, read by `_readDownloadToken`) replaces the `x_mtok` cookie. Then `download()` runs, then the async `serve()`, which handles Range/206/416, `responseHeaders`, and `Content-Disposition`, and streams through `this.storage`. Hooks along this path: `interceptRequest`, `interceptDownload`, `downloadCallback`. Auth on HTTP routes: by default the client sets an `x_mtok` cookie (via `ostrio:cookies`) to `Meteor.connection._lastSessionId`. The server maps it to a userId through `Meteor.server.sessions` (`_getUserId`, `_getUserDefault`). That lookup is in-process memory, so multi-instance deployments need sticky sessions. `config.getUser` replaces this lookup. -Storage is the filesystem only. Files live under `storagePath` (default `assets/app/uploads/`). Each document's `versions..path` points at a file on disk. Integrations with 3rd-party storage (S3, GridFS, and others) are recipes in `docs/` that move files in `onAfterUpload` and serve them through `interceptDownload`. +Uploads always land on the filesystem under `storagePath` (default `assets/app/uploads/`). The `storage` adapter (`storage.js`: `FSStorage` default, `GridFSStorage`) decides where finished files live. S3 is a docs recipe. Each document's `versions..path` points at the local file, and `versions..meta.storage` holds the adapter's result. `cursor.js`: `FileCursor` wraps one document, `FilesCursor` wraps a Mongo cursor. On the server only the `*Async` methods work, because Meteor 3 server cursors are async. The sync methods throw there. +`mime.js`: type detection at finish. `download-token.js`: HMAC helpers for signed download links. + `lib.js`: shared `helpers` (type checks, `clone`, `sanitize`), `fixJSONParse`/`fixJSONStringify` (Date round-trip for `meta` over HTTP), and `formatFileURL`. Types are published in `index.d.ts` through `zodern:types`. Keep the file in sync with any option or method change, including the `check()` patterns in the server and client constructors. @@ -56,10 +66,12 @@ Types are published in `index.d.ts` through `zodern:types`. Keep the file in syn - The server API is async only. Use `*Async` collection methods. The sync `findOne` on the server throws on purpose. - Every constructor option is validated with `check()` in the server and client constructors. A new option goes in the `check()` block, in the client `allowedParams` list (`client.js`) if the client accepts it, in the JSDoc, in `index.d.ts`, and in `docs/constructor.md`. -- Never trust client-supplied `file.*` fields (strip `RESERVED_FILE_KEYS`), `FSName`, `chunkId`, `chunkSize`, or `fileId`. Chunk size is 1 byte to 16 MiB. HTTP bodies are capped: Start 1 MiB, EOF 64 KiB, chunk is base64 of `chunkSize` plus slack. +- Keep only `CLIENT_FILE_KEYS` (`name`, `type`, `size`, `meta`) from client `file` objects. Never trust client `FSName` (ignored), `chunkSize`, `chunkId`, or `fileId`. Chunk size is 1 byte to 16 MiB. HTTP bodies are capped: Start 1 MiB, EOF 64 KiB, chunk is base64 of `chunkSize` plus slack. +- Only the server names files: `namingFunction({ file, fileId, userId })`. +- An upload has at most `MAX_UPLOAD_CHUNKS` (100000) chunks. - Ownership: Write, EOF, and `_Abort` must go through `_getUploadSession(fileId, userId)`. The stored `userId` must match the caller (`403`). `_Abort` on an unknown or foreign id is `404`. A repeated EOF is answered only for an authenticated owner (`_findFinishedUpload`). - File identity: Start creates the file with `O_CREAT|O_EXCL` and stores `dev` and `ino` in `_preCollection`. Resume and idle reopen use `O_RDWR` only and compare the identity (`410` missing, `409` replaced). `WriteStream#abort()` unlinks only its own file. -- Path containment is enforced at upload time only: the final path must resolve inside `storagePath(result)` (`_isPathInside`). `serve()` and `unlinkAsync()` trust stored paths, so never let clients write `path` or `versions` (`allowClient()` is unsafe). +- Path containment is enforced at upload time only: the final path must resolve inside `storagePath(result)` (`_isPathInside`). `serve()` and `unlinkAsync()` trust stored paths through `FSStorage`, so never let clients write `path` or `versions` (`allowClient()` is unsafe). - Never send server paths to the client (`_toClientFileObj`). HTTP errors use `{error, reason, isClientSafe?}`, with a generic reason for `5xx`. - Sync cursor methods throw `Meteor.Error(404)` on the server (`clientOnly()` in `cursor.js`). `link()` uses stored `_downloadRoute`/`_collectionName` only when safe and encodes ids. - Logging goes through `this._debug(...)`. It is gated by the `debug` option. diff --git a/README.md b/README.md index ff80f2c7..8d910808 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ Stable, fast, robust, and well-maintained Meteor.js package for files management - [✨ Key features](https://github.com/veliovgroup/Meteor-Files#key-features) - [📔 API Documentation](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/readme.md) - [🔒 Security guide](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/security.md) +- [🚚 Migration to v4](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/migration-to-v4.md) - [🚚 Migration to v3](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/migration-to-v3.md) - __⚡️ Quick start__: - [🔧 Installation](https://github.com/veliovgroup/Meteor-Files#installation) @@ -33,6 +34,10 @@ Stable, fast, robust, and well-maintained Meteor.js package for files management - Compatible with all front-end frameworks from Blaze to [React](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/react-example.md) - Upload via `HTTP` and `DDP` transports, [read about difference](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/about-transports.md) - Sustainable and "resumable" uploads will auto-resume when connection interrupted or server rebooted +- Signed download links with `createDownloadToken()`, they work without a cookie and on any server instance, see the [security guide](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/security.md#signed-download-links) +- Built-in [GridFS storage adapter](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/gridfs-bucket-integration.md#use-gridfs-with-gridfsbucket-as-a-storage) (`GridFSStorage`) and a `storage` option for your own adapter +- Content-based MIME type detection, the server does not trust the type sent by the client +- Uploads resume after a server restart, the server records every written chunk - Upload files through computing cloud without persistent File System, like Heroku (*"resumable" uploads are not supported on Heroku and alike*) - Use *[GridFS](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/gridfs-bucket-integration.md#use-gridfs-with-gridfsbucket-as-a-storage)*, *[AWS S3](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/aws-s3-integration.md)*, *[Google Storage](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/google-cloud-storage-integration.md)* or *[DropBox](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/dropbox-integration.md)* and other (*[3rd-party storage](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/3rd-party-storage.md)*) - APIs for checking file mime-type, size, extension, and other file's properties before upload using *[`onBeforeUpload` hook](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/constructor.md)* @@ -48,6 +53,10 @@ meteor add ostrio:files Requires Meteor 3.2 or newer (v3.1.0 and later). +### Upgrade + +Upgrading from 3.x to 4.0.0 changes defaults and removes APIs. Read the [migration guide to v4](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/migration-to-v4.md) and its upgrade checklist first. Coming from 2.x, read [migration to v3](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/migration-to-v3.md) as well. + ### ES6 Import: Import in isomorphic location (e.g. on server and client) diff --git a/client.js b/client.js index 5d71b3e9..cc34a413 100644 --- a/client.js +++ b/client.js @@ -4,7 +4,8 @@ import { Tracker } from 'meteor/tracker'; import { Cookies } from 'meteor/ostrio:cookies'; import { check, Match } from 'meteor/check'; import { UploadInstance } from './upload.js'; -import FilesCollectionCore from './core.js'; +import FilesCollectionCore, { SELECTOR_PATTERN } from './core.js'; +import { FileCursor } from './cursor.js'; import { formatFileURL, helpers } from './lib.js'; const NOOP = () => { }; @@ -102,7 +103,7 @@ const watchTokenCookie = (connection, setCookie, accounts) => { } }; -const allowedParams = ['allowClientCode', 'allowedCordovaOrigins', 'allowQueryStringCookies', 'chunkSize', 'collection', 'collectionName', 'ddp', 'debug', 'disableSetTokenCookie', 'disableUpload', 'downloadRoute', 'namingFunction', 'onBeforeUpload', 'onbeforeunloadMessage', 'public', 'sanitize', 'schema']; +const allowedParams = ['allowClientCode', 'allowedCordovaOrigins', 'allowQueryStringCookies', 'chunkSize', 'collection', 'collectionName', 'ddp', 'debug', 'disableSetTokenCookie', 'disableUpload', 'downloadRoute', 'onBeforeUpload', 'onbeforeunloadMessage', 'public', 'sanitize', 'schema']; /** * @locus Client @@ -116,9 +117,8 @@ const allowedParams = ['allowClientCode', 'allowedCordovaOrigins', 'allowQuerySt * @param config.downloadRoute {string} - [anywhere] server route used to retrieve files * @param config.collection {Mongo.Collection} - [anywhere] mongo collection instance * @param config.collectionName {string} - [anywhere] collection name - * @param config.namingFunction {function} - [anywhere] function that returns a string * @param config.onBeforeUpload {function} - [anywhere] function executed on server after receiving each chunk and on client before starting upload; return `true` to continue, `false` or `string` (error message) to abort - * @param config.allowClientCode {boolean} - [anywhere] allow to run remove from client + * @param config.allowClientCode {boolean} - [anywhere] allow to run remove from client; default: false * @param config.onbeforeunloadMessage {string|function} - [client] message shown to user when closing window/tab during upload * @param config.disableUpload {boolean} - disable file upload; useful for server-only solutions * @param config.disableSetTokenCookie {boolean} - disable cookie setting; useful when using multiple file collections or custom authorization @@ -171,6 +171,11 @@ class FilesCollection extends FilesCollectionCore { this.collection.filesCollection = this; check(this.collectionName, String); + if (config && config.namingFunction !== undefined) { + // eslint-disable-next-line no-console + console.warn(`[FilesCollection.${this.collectionName}] "namingFunction" is server-only since v4 and is ignored on the client. Set it in the server constructor.`); + } + if (this.public && !this.downloadRoute) { throw new Meteor.Error(500, `[FilesCollection.${this.collectionName}]: "downloadRoute" must be precisely provided on "public" collections! Note: "downloadRoute" must be equal or be inside of your web/proxy-server (relative) root.`); } @@ -189,16 +194,12 @@ class FilesCollection extends FilesCollectionCore { this.downloadRoute = this.downloadRoute.replace(/\/$/, ''); - if (!helpers.isFunction(this.namingFunction)) { - this.namingFunction = false; - } - if (!helpers.isFunction(this.onBeforeUpload)) { this.onBeforeUpload = false; } if (!helpers.isBoolean(this.allowClientCode)) { - this.allowClientCode = true; + this.allowClientCode = false; } if (!this.ddp) { @@ -244,7 +245,6 @@ class FilesCollection extends FilesCollectionCore { check(this.downloadRoute, String); check(this.disableUpload, Boolean); /* eslint-disable new-cap */ - check(this.namingFunction, Match.OneOf(false, Function)); check(this.onBeforeUpload, Match.OneOf(false, Function)); /* eslint-enable new-cap */ check(this.allowClientCode, Boolean); @@ -306,6 +306,26 @@ class FilesCollection extends FilesCollectionCore { return result; } + /** + * Finds and returns a FileCursor for a matching document. + * @locus Client + * @memberOf FilesCollection + * @name findOne + * @param {MeteorFilesSelector} [selector={}] - Mongo-style selector + * @param {MeteorFilesOptions} [options] - Mongo query options + * @returns {FileCursor|null} A FileCursor instance, or null if not found + */ + findOne(selector = {}, options) { + this._debug(`[FilesCollection] [findOne(${JSON.stringify(selector)}, ${JSON.stringify(options)})]`); + /* eslint-disable new-cap */ + check(selector, SELECTOR_PATTERN); + check(options, Match.Optional(Object)); + /* eslint-enable new-cap */ + + const doc = this.collection.findOne(selector, options); + return doc ? new FileCursor(doc, this) : null; + } + /** * Uploads a file to the server over DDP or HTTP. * @locus Client @@ -389,20 +409,19 @@ class FilesCollection extends FilesCollectionCore { * @locus Client * @memberOf FilesCollection * @name remove - * @param {MeteorFilesSelector} selector - mongo-style selector (see http://docs.meteor.com/api/collections.html#selectors) + * @param {string} _id - `_id` of the file to remove * @param {function(error, number): void} callback - callback with (error, number) arguments - * @summary Removes documents from the collection + * @summary Removes one file from the collection * @returns {FilesCollection} Instance */ - remove(selector = {}, callback) { - this._debug(`[FilesCollection] [remove(${JSON.stringify(selector)})]`); - /* eslint-disable new-cap */ - check(selector, Match.OneOf(Object, String)); + remove(_id, callback) { + this._debug(`[FilesCollection] [remove(${JSON.stringify(_id)})]`); + check(_id, String); + // eslint-disable-next-line new-cap check(callback, Match.Optional(Function)); - /* eslint-enable new-cap */ if (this.allowClientCode) { - this.ddp.call(this._methodNames._Remove, selector, (callback || NOOP)); + this.ddp.call(this._methodNames._Remove, _id, (callback || NOOP)); } else { callback && callback(new Meteor.Error(401, '[FilesCollection] [remove] Run code from client is not allowed!')); this._debug('[FilesCollection] [remove] Run code from client is not allowed!'); @@ -416,18 +435,17 @@ class FilesCollection extends FilesCollectionCore { * @locus Anywhere * @memberOf FilesCollection * @name removeAsync - * @param {MeteorFilesSelector} selector - mongo-style selector (see http://docs.meteor.com/api/collections.html#selectors) - * @summary Removes documents from the collection + * @param {string} _id - `_id` of the file to remove + * @summary Removes one file from the collection * @throws {Meteor.Error} 401 when `allowClientCode` is `false` - * @returns {Promise} number of matched and removed files/records + * @returns {Promise} number of removed files, `0` or `1` */ - async removeAsync(selector = {}) { - this._debug(`[FilesCollection] [removeAsync(${JSON.stringify(selector)})]`); - // eslint-disable-next-line new-cap - check(selector, Match.OneOf(Object, String)); + async removeAsync(_id) { + this._debug(`[FilesCollection] [removeAsync(${JSON.stringify(_id)})]`); + check(_id, String); if (this.allowClientCode) { - return await this.ddp.callAsync(this._methodNames._Remove, selector); + return await this.ddp.callAsync(this._methodNames._Remove, _id); } this._debug('[FilesCollection] [removeAsync] Run code from client is not allowed!'); diff --git a/core.js b/core.js index 83620b18..d2cb9ec7 100644 --- a/core.js +++ b/core.js @@ -9,7 +9,7 @@ import { FilesCursor, FileCursor } from './cursor.js'; /** * @const {Match.Pattern} SELECTOR_PATTERN - Selectors accepted by `find`, `findOne`, `findOneAsync`, and `countDocuments` */ -const SELECTOR_PATTERN = Match.Optional(Match.OneOf(Object, String, Boolean, Number, null, Mongo.ObjectID)); +export const SELECTOR_PATTERN = Match.Optional(Match.OneOf(Object, String, Boolean, Number, null, Mongo.ObjectID)); /* eslint-enable new-cap */ export default class FilesCollectionCore extends EventEmitter { @@ -247,32 +247,6 @@ export default class FilesCollectionCore extends EventEmitter { return null; } - /** - * Finds and returns a FileCursor for a matching document (client only). - * @locus Client - * @memberOf FilesCollectionCore - * @param {MeteorFilesSelector} [selector={}] - Mongo-style selector - * @param {MeteorFilesOptions} [options] - Mongo query options - * @returns {FileCursor|null} A FileCursor instance, or null if not found - * @throws {Meteor.Error} If called on the server - */ - findOne(selector = {}, options) { - this._debug(`[FilesCollection] [findOne(${JSON.stringify(selector)}, ${JSON.stringify(options)})]`, Meteor.isServer); - if (Meteor.isServer) { - throw new Meteor.Error(404, 'FilesCollection#findOne() not available in server! Use .findOneAsync instead'); - } - /* eslint-disable new-cap */ - check(selector, SELECTOR_PATTERN); - check(options, Match.Optional(Object)); - /* eslint-enable new-cap */ - - const doc = this.collection.findOne(selector, options); - if (doc) { - return new FileCursor(doc, this); - } - return null; - } - /** * Finds and returns a FilesCursor for matching documents. * @locus Anywhere @@ -362,18 +336,25 @@ export default class FilesCollectionCore extends EventEmitter { * @param {Partial|FileCursor|null} fileObj - A file object reference, or a FileCursor * @param {string} [version='original'] - The file version * @param {string} [uriBase] - Optional URI base + * @param {{token?: string}} [opts] - `token` from the server `createDownloadToken()`, appended as `?token=` * @summary Uses the document's stored `_downloadRoute` and `_collectionName` when they are safe, otherwise this collection's values * @returns {string} The download URL, or an empty string if the file is not found */ - link(fileObj, version = 'original', uriBase) { + link(fileObj, version = 'original', uriBase, opts) { if (!fileObj) { return ''; } const fileRef = (fileObj instanceof FileCursor) ? fileObj._fileRef : fileObj; this._debug(`[FilesCollection] [link(${helpers.isObject(fileRef) ? fileRef._id : undefined}, ${version})]`); - // eslint-disable-next-line new-cap + /* eslint-disable new-cap */ check(fileRef, Match.Where((obj) => helpers.isObject(obj))); - return formatFileURL(fileRef, version, uriBase, this); + check(opts, Match.Optional(Match.ObjectIncluding({ token: Match.Optional(String) }))); + /* eslint-enable new-cap */ + const url = formatFileURL(fileRef, version, uriBase, this); + if (!url || !helpers.isString(opts?.token) || !opts.token) { + return url; + } + return `${url}${url.includes('?') ? '&' : '?'}token=${encodeURIComponent(opts.token)}`; } } diff --git a/cursor.js b/cursor.js index ce9fa106..c472d900 100644 --- a/cursor.js +++ b/cursor.js @@ -72,12 +72,13 @@ export class FileCursor { * @locus Anywhere * @param {string} [version='original'] - Name of the file’s subversion. * @param {string} [uriBase] - Optional URI base. + * @param {{token?: string}} [opts] - `token` from the server `createDownloadToken()` * @returns {string} */ - link(version = 'original', uriBase) { + link(version = 'original', uriBase, opts) { this._collection._debug(`[FilesCollection] [FileCursor] [link(${version})]`); if (this._fileRef && this._fileRef._id) { - return this._collection.link(this._fileRef, version, uriBase); + return this._collection.link(this._fileRef, version, uriBase, opts); } return ''; } @@ -187,20 +188,6 @@ export class FilesCursor { return await this.fetchAsync(); } - /** - * Returns `true` if there is a next item available. - * @locus Client - * @deprecated since v3.0.0. use {@link FilesCursor#hasNextAsync} instead. - * @throws {Meteor.Error} If called on the server - * @returns {boolean} - */ - hasNext() { - this._collection._debug('[FilesCollection] [FilesCursor] [hasNext()]'); - clientOnly('FilesCursor', 'hasNext'); - Meteor.deprecate('FilesCursor#hasNext() is deprecated! Use `hasNextAsync` instead'); - return this._current < this.count() - 1; - } - /** * Asynchronously returns `true` if there is a next item available. * @locus Anywhere @@ -375,18 +362,6 @@ export class FilesCursor { return this.cursor.count(); } - /** - * Asynchronously returns the number of file documents that match the query. - * @locus Anywhere - * @deprecated since v3.0.0. use {@link FilesCursor#countDocuments} instead. - * @returns {Promise} - */ - async countAsync() { - this._collection._debug('[FilesCollection] [FilesCursor] [countAsync()]'); - Meteor.deprecate('FilesCursor#countAsync() is deprecated! Use `countDocuments` instead'); - return await this.cursor.countAsync(); - } - /** * Asynchronously returns the number of file documents that match the query. * @locus Anywhere @@ -408,18 +383,51 @@ export class FilesCursor { remove(callback = () => {}) { this._collection._debug('[FilesCollection] [FilesCursor] [remove()]'); clientOnly('FilesCursor', 'remove'); - this._collection.remove(this._selector, callback); + // The client removes one `_id` per call + const ids = this.cursor.map((doc) => doc._id); + if (!ids.length) { + callback(null, 0); + return this; + } + + let pending = ids.length; + let removed = 0; + let failed = false; + ids.forEach((_id) => { + this._collection.remove(_id, (error, count) => { + if (failed) { + return; + } + if (error) { + failed = true; + callback(error); + return; + } + removed += count || 0; + if (--pending === 0) { + callback(null, removed); + } + }); + }); return this; } /** - * Asynchronously removes all file documents that match the query. + * Asynchronously removes all file documents that match the query. The client removes them one `_id` at a time * @locus Anywhere * @returns {Promise} */ async removeAsync() { this._collection._debug('[FilesCollection] [FilesCursor] [removeAsync()]'); - return await this._collection.removeAsync(this._selector); + if (Meteor.isServer) { + return await this._collection.removeAsync(this._selector); + } + + let removed = 0; + for (const { _id } of await this.cursor.fetchAsync()) { + removed += await this._collection.removeAsync(_id); + } + return removed; } /** diff --git a/docs/3rd-party-storage.md b/docs/3rd-party-storage.md index 5513a0ae..138ace62 100644 --- a/docs/3rd-party-storage.md +++ b/docs/3rd-party-storage.md @@ -3,6 +3,11 @@ Meteor-Files package has flexible API, so it can be integrated with any 3rd party storage. Any 3rd party storage with REST API or Node.js SDK can be easily integrated. +Since v4, pass a storage adapter as the [`storage` option](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/constructor.md). An adapter is an object with `put()`, `createReadStream()`, `remove()`, and optional `stat()`. The package exports `FSStorage` (default) and `GridFSStorage` on the server. Recipes: + +- [AWS S3 adapter](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/aws-s3-integration.md) +- [Built-in GridFS adapter](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/gridfs-bucket-integration.md) + __Integration examples:__ - [AWS S3 Bucket Integration](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/aws-s3-integration.md) diff --git a/docs/FilesCursor.md b/docs/FilesCursor.md index bb4f57c9..069d5f72 100644 --- a/docs/FilesCursor.md +++ b/docs/FilesCursor.md @@ -15,11 +15,10 @@ const cursor = imagesCollection.find(); // <-- Returns FilesCursor Instance #### Methods: -On the Server the synchronous methods `get()`, `hasNext()`, `next()`, `previous()`, `fetch()`, `first()`, `last()`, `count()`, `remove()`, `forEach()`, `each()`, `map()`, and `current()` throw `Meteor.Error 404` with a message that names the `*Async` method to use. They work on the Client only. `hasPrevious()`, `observe()`, and `observeChanges()` work on both sides. +On the Server the synchronous methods `get()`, `next()`, `previous()`, `fetch()`, `first()`, `last()`, `count()`, `remove()`, `forEach()`, `each()`, `map()`, and `current()` throw `Meteor.Error 404` with a message that names the `*Async` method to use. They work on the Client only. `hasPrevious()`, `observe()`, and `observeChanges()` work on both sides. - [*Client*] `get()` - {*object[]*} - Returns all matching document(s) as an Array. Alias of `.fetch()` - `getAsync()` - {*Promise*} - Resolves to matching document(s) as an Array. Alias of `.fetchAsync()` -- __Deprecated__ [*Client*] `hasNext()` - {*boolean*} - Returns `true` if there is next item available on Cursor - `hasNextAsync()` - {*Promise*} - Resolves to `true` if there is next item available on Cursor - [*Client*] `next()` - {*object*|*undefined*} - Returns next available object on Cursor - `nextAsync()` - {*Promise*} - Resolves to next available object on Cursor @@ -36,10 +35,9 @@ On the Server the synchronous methods `get()`, `hasNext()`, `next()`, `previous( - [*Client*] `current()` - {*object*|*undefined*} - Returns current item on Cursor, if available - `currentAsync()` - {*Promise*} - Resolves to current item on Cursor, if available - __Deprecated__ [*Client*] `count()` - {*number*} - Returns the number of documents that match a query -- __Deprecated__ `countAsync()` - {*Promise*} - Resolves to the number of documents that match a query. Use `countDocuments()` - `countDocuments(opts: Mongo.CountDocumentsOptions)` - {*Promise*} - Resolves to the number of documents that match a query -- [*Client*] `remove(callback)` - {*FilesCursor*} - Removes all documents that match a query, [*Client*] only. Callback has `error` argument -- `removeAsync()` - {*Promise*} - Removes all documents that match a query. Resolves into number of removed records +- [*Client*] `remove(callback)` - {*FilesCursor*} - Removes all documents that match a query, [*Client*] only. Callback has `error` argument. The client sends one `_id` per server call, so N files take N calls, and the removal is not atomic: if one call fails, files already removed stay removed +- `removeAsync()` - {*Promise*} - Removes all documents that match a query. Resolves into number of removed records. On the Client it sends one `_id` per server call, so N files take N calls, and the removal is not atomic: if one call fails, files already removed stay removed - [*Client*] `forEach(callback, context)` - {*FilesCursor*} - *Same as `forEachAsync` in arguments and context* - `forEachAsync(callback, context)` - {*Promise*} - Call `callback` once for each matching document, sequentially. The callback can be `async` - `callback` - {*Function*} - Function to call. It will be called with three arguments: the `file`, a 0-based index, and cursor itself diff --git a/docs/about-transports.md b/docs/about-transports.md index 212b4f59..b90701c7 100644 --- a/docs/about-transports.md +++ b/docs/about-transports.md @@ -40,9 +40,11 @@ An upload is one Start request, chunks `1..N` in order, and one EOF request. The - Write: each chunk needs a `chunkId` in `1..N` and a length of at most `chunkSize` (the last chunk is limited by the declared size). The total can not exceed the declared size. HTTP chunk bodies are limited to the base64 size of `chunkSize` plus 4 KiB, over that the server replies `413` - EOF: HTTP EOF bodies are limited to 64 KiB. The server stores the real size found on disk, not the one the client declared - Owner only: Write, EOF, and `_Abort` from a different user than the one who started the upload get `403`. `_Abort` on an unknown or foreign upload id gets `404` -- Lost uploads: `408` means `_preCollection` has no record of the upload (it expired after `continueUploadTTL`, it finished, or the id is unknown). `410` means the record exists but the server can not resume it, because its file was removed or the record was created by 3.0.x. A replaced file gets `409` +- Lost uploads: `408` means `_preCollection` has no record of the upload (it expired after `continueUploadTTL`, it finished, or the id is unknown). `410` means the record exists but the server can not resume it, because its file was removed or the record was created before 4.0. A replaced file gets `409` - Repeated EOF: when the response to EOF was lost, an authenticated owner can send EOF again and receives the stored file record. Anonymous uploads get `408` - Idle uploads: the server closes the file handle after `uploadIdleTimeout` and reopens it with the next chunk +- Chunk limit: an upload has at most 100000 chunks. The client raises `chunkSize` for larger files (up to 16 MiB). The server rejects Start with more chunks with `400` +- Resume: the server records each written chunk in the upload record. After a server restart, an unfinished upload continues within `continueUploadTTL`, and EOF succeeds once every chunk is recorded. When the server can not record a chunk, it replies `503` and the client sends the chunk again HTTP error responses have a JSON body `{ "error": , "reason": "", "isClientSafe": true }`. `isClientSafe` is `true` only for `4xx` errors that are safe to show to users. The server replaces the reason of every `5xx` error with a generic text (`503` says to try again) and logs details when `debug` is on. A malformed Start request gets `400`. diff --git a/docs/aws-s3-integration.md b/docs/aws-s3-integration.md index 21e67496..3f23d4e6 100644 --- a/docs/aws-s3-integration.md +++ b/docs/aws-s3-integration.md @@ -1,6 +1,6 @@ # Use AWS:S3 as storage -The example below shows how to store and serve uploaded files via S3. It also removes the files from S3 when a record is removed from *FilesCollection*. +The example below shows how to store and serve uploaded files via S3 with a `storage` adapter. The adapter also removes the files from S3 when a record is removed from *FilesCollection*. The example uses the modular [AWS SDK for JavaScript v3](https://docs.aws.amazon.com/AWSJavaScriptSDK/v3/latest/client/s3/) (`@aws-sdk/client-s3`). The old `aws-sdk` v2 package is in maintenance mode, do not use it in new code. @@ -59,22 +59,20 @@ if (process.env.S3) { } ``` -## Move a file to AWS:S3 after upload +## Store files in AWS:S3 with a storage adapter File: `Server-side-file-store.js`. Use this in Meteor's `imports/server` directory, __NOT__ on the client. +The adapter is passed as the [`storage` option](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/constructor.md). Uploads are written to `storagePath` first. When a file is complete, `put()` copies it to S3 and deletes the local copy. Its result is stored at `versions..meta.storage`, before the document insert and before `onAfterUpload`. `serve()` streams through `createReadStream()` with the requested byte range, so `Range`, `responseHeaders`, and `Content-Disposition` work as with local files. `removeAsync()` and `unlinkAsync()` call `remove()`. + +`put()` gets `{ source }` as its 4th argument. When `source` is `'addFile'`, the local file belongs to the caller of `addFile()`, so the adapter keeps it. + ```js +import fs from 'node:fs'; import { Meteor } from 'meteor/meteor'; -import { Random } from 'meteor/random'; import { FilesCollection } from 'meteor/ostrio:files'; -import fs from 'node:fs'; -import { - S3Client, - PutObjectCommand, - DeleteObjectCommand, - GetObjectCommand, -} from '@aws-sdk/client-s3'; +import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3'; /* Example: S3='{"s3":{"key": "xxx", "secret": "xxx", "bucket": "xxx", "region": "xxx"}}' meteor */ if (process.env.S3) { @@ -82,225 +80,89 @@ if (process.env.S3) { } const s3Conf = Meteor.settings.s3 || {}; - -/* Check settings existence in `Meteor.settings` */ -/* This is the best practice for app security */ if (!s3Conf.key || !s3Conf.secret || !s3Conf.bucket || !s3Conf.region) { throw new Meteor.Error(401, 'Missing Meteor file settings'); } -// Create a new S3 client -const s3Client = new S3Client({ - region: s3Conf.region, - credentials: { - accessKeyId: s3Conf.key, - secretAccessKey: s3Conf.secret, - }, -}); - -/** - * Resolve a `Range` header into an explicit, inclusive byte range. - * Returns `null` when the full content must be sent (no header, a malformed header, - * or a multi-range request), `false` when the range is not satisfiable, and `{ start, end }` otherwise. - */ -const resolveRange = (header, size) => { - if (!header) { - return null; +class S3Storage { + constructor({ client, bucket, prefix = '' }) { + this.client = client; + this.bucket = bucket; + this.prefix = prefix; } - if (header.includes(',')) { - // Multi-range requests are answered with full content - return null; + async put(fileRef, versionName, localPath, { source } = {}) { + const vRef = fileRef.versions[versionName]; + const key = `${this.prefix}${fileRef._id}/${versionName}${fileRef.extensionWithDot || ''}`; + const { size } = await fs.promises.stat(localPath); + await this.client.send(new PutObjectCommand({ + Bucket: this.bucket, + Key: key, + Body: fs.createReadStream(localPath), + ContentLength: size, + ContentType: vRef.type || fileRef.type, + })); + // Files passed to `addFile()` belong to the caller + if (source !== 'addFile') { + await fs.promises.unlink(localPath); + } + return { name: 's3', bucket: this.bucket, key }; } - const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim()); - if (!match || (!match[1] && !match[2])) { - // Malformed header: ignored, same as `.serve()` does - return null; + async stat(fileRef, versionName) { + return fileRef.versions[versionName]?.meta?.storage?.key ? { size: fileRef.versions[versionName].size } : null; } - let start; - let end; - if (!match[1]) { - // Suffix range: the last N bytes - const suffix = parseInt(match[2], 10); - if (suffix === 0) { - return false; + async createReadStream(fileRef, versionName, { start, end } = {}) { + const key = fileRef.versions[versionName]?.meta?.storage?.key; + if (!key) { + throw new Meteor.Error(404, 'File not found'); } - start = Math.max(size - suffix, 0); - end = size - 1; - } else { - start = parseInt(match[1], 10); - end = match[2] ? Math.min(parseInt(match[2], 10), size - 1) : size - 1; + const res = await this.client.send(new GetObjectCommand({ + Bucket: this.bucket, + Key: key, + Range: Number.isInteger(start) ? `bytes=${start}-${end}` : undefined, + })); + return res.Body; } - if (start >= size || end < start) { - return false; + async remove(fileRef, versionName) { + const key = fileRef.versions[versionName]?.meta?.storage?.key; + if (key) { + await this.client.send(new DeleteObjectCommand({ Bucket: this.bucket, Key: key })); + } } +} - return { start, end }; -}; +const s3 = new S3Client({ + region: s3Conf.region, + credentials: { accessKeyId: s3Conf.key, secretAccessKey: s3Conf.secret }, +}); -// Declare the Meteor file collection on the Server -const UserFiles = new FilesCollection({ - debug: false, // Change to `true` for debugging - storagePath: 'assets/app/uploads/uploadedFiles', +export const UserFiles = new FilesCollection({ collectionName: 'userFiles', - // Disallow Client to execute remove, use a Meteor method - allowClientCode: false, - - // Start moving files to AWS:S3 - // after fully received by the Meteor server - async onAfterUpload(fileRef) { - // Run through each of the uploaded file - for (const version of Object.keys(fileRef.versions)) { - const vRef = fileRef.versions[version]; - if (!vRef) { - continue; - } - - // We use Random.id() instead of real file's _id - // to secure files from reverse engineering - // As after viewing this code it will be easy - // to get access to unlisted and protected files - const filePath = `files/${Random.id()}-${version}.${fileRef.extension}`; - const fileStream = fs.createReadStream(vRef.path); - fileStream.on('error', (error) => { - console.error('[onAfterUpload] [createReadStream] [ERROR:] File was not uploaded to S3', fileRef._id, error); - }); - - try { - await s3Client.send(new PutObjectCommand({ - StorageClass: 'STANDARD', - Bucket: s3Conf.bucket, - Key: filePath, - Body: fileStream, - ContentLength: vRef.size, - ContentType: vRef.type, - })); - } catch (error) { - console.error('[onAfterUpload] [PutObjectCommand] Error:', fileRef._id, error); - continue; - } - - try { - await this.collection.updateAsync({ _id: fileRef._id }, { - $set: { - [`versions.${version}.meta.pipePath`]: filePath - } - }); - // Unlink original file from FS - // after successful upload to AWS:S3 - await this.unlinkAsync(fileRef, version); - } catch (_unlinkError) { - // If file was removed before it was fully moved to S3 - // `.updateAsync()` or `.unlinkAsync()` will throw an error - // Then we will need to remove that file from S3 - try { - await s3Client.send(new DeleteObjectCommand({ - Bucket: s3Conf.bucket, - Key: filePath, - })); - console.info('[onAfterUpload] [DeleteObjectCommand] unlinked file successfully removed from S3', fileRef._id); - } catch (deleteError) { - console.error('[onAfterUpload] [DeleteObjectCommand] Error:', fileRef._id, deleteError); - } - } - } - }, - - // Intercept calls to `.removeAsync()` to remove file from S3 - // onAfterRemove is called right after the record is removed from the MongoDB Collection - // and before calling `.unlinkAsync()`, - // return `true` to prevent calling `.unlinkAsync()` - async onAfterRemove(docs) { - for (const doc of docs) { - for (const version of Object.keys(doc.versions || {})) { - const vRef = doc.versions[version]; - if (vRef?.meta?.pipePath) { - try { - await s3Client.send(new DeleteObjectCommand({ - Bucket: s3Conf.bucket, - Key: vRef.meta.pipePath, - })); - console.info('[onAfterRemove] [DeleteObjectCommand] Successfully removed from S3', vRef.path, vRef.meta.pipePath); - } catch (error) { - console.error('[onAfterRemove] [DeleteObjectCommand] Error:', error); - } - } - } - } - - // Return `true` only if every record was moved to S3 - // and its files were already removed from FS after upload - return docs.length > 0 && docs.every((doc) => doc.versions?.original?.meta?.pipePath); - }, + storagePath: 'assets/app/uploads/uploadedFiles', + storage: new S3Storage({ client: s3, bucket: s3Conf.bucket, prefix: 'files/' }), +}); +``` - // Intercept access to the file - // And serve the file from AWS:S3 - async interceptDownload(http, fileRef, version) { - const vRef = fileRef?.versions?.[version]; - const path = vRef?.meta?.pipePath; +### S3-compatible storage (MinIO, Wasabi, and others) - if (!path) { - // While file is not yet uploaded to AWS:S3 - // it will be served from FS - return false; - } +`S3Storage` works with any S3-compatible service. Point the client at the service with `endpoint`. MinIO and most self-hosted services also need `forcePathStyle: true`: - // If file is successfully moved to AWS:S3 - // we will pipe the S3 response to the client - // So, original link will always stay secure - - // To keep ?play and ?download parameters, original file name, - // content-type, content-disposition, Range responses, - // and cache-control we use the low-level .serve() method - const opts = { - Bucket: s3Conf.bucket, - Key: path, - }; - - const range = resolveRange(http.request.headers.range, vRef.size); - // With `strict: false` an unsatisfiable range gets the full content with `200`, as in `.serve()` - if (range === false && this.strict !== false) { - http.response.writeHead(416, { 'Content-Range': `bytes */${vRef.size}` }); - http.response.end(); - return true; - } +```js +const s3 = new S3Client({ + region: s3Conf.region, // Any value for MinIO, for example 'us-east-1'. Wasabi uses its own regions, like 'eu-central-1' + endpoint: s3Conf.endpoint, // For example 'https://minio.example.com' or 'https://s3.eu-central-1.wasabisys.com' + forcePathStyle: true, + credentials: { accessKeyId: s3Conf.key, secretAccessKey: s3Conf.secret }, +}); +``` - if (range) { - // Same explicit range for S3 and for .serve() - opts.Range = `bytes=${range.start}-${range.end}`; - http.request.headers.range = opts.Range; - } else { - // Full content requested: .serve() answers `200` - delete http.request.headers.range; - } - try { - const { Body } = await s3Client.send(new GetObjectCommand(opts)); - Body.on('error', (error) => { - console.error('[interceptDownload] [GetObject stream]', error); - if (!http.response.writableEnded) { - http.response.end(); - } - }); - - this.serve(http, fileRef, vRef, version, Body); - } catch (error) { - console.error('[interceptDownload] [GetObjectCommand]', error); - if (!http.response.headersSent) { - http.response.writeHead(404); - } - if (!http.response.writableEnded) { - http.response.end(); - } - } +`stat()` is optional. Without it `download()` can not answer `404` or apply `integrityCheck` before streaming. The `stat()` above trusts the size stored in the document. Call `HeadObjectCommand` instead to check that the object exists. - return true; - } -}); -``` +Files created by `onAfterUpload` subversions must be passed to `this.storage.put(fileRef, versionName, localPath)` and the result saved at `versions..meta.storage`. Pass `{ source: 'write' }` as the 4th argument to let the adapter delete the local file after the copy, or `{ source: 'addFile' }` to keep it. With the adapter above, `onAfterUpload` no longer finds the original file on local disk. Create subversions from the S3 object, or with the Lambda function below. ## Further image (JPEG, PNG) processing with AWS Lambda diff --git a/docs/collection.md b/docs/collection.md index 7fb8b99f..41fa8d9a 100644 --- a/docs/collection.md +++ b/docs/collection.md @@ -112,7 +112,7 @@ __[*Isomorphic*]__ `await filesCollection.estimatedDocumentCount(options?)`. Res __[*Server*]__ Use these two methods to answer download requests from your own routes. Most apps never call them, as the package registers the download route itself. - `await filesCollection.download(http, version = 'original', fileRef)` - Runs `downloadCallback`, then `interceptDownload`, then checks the file on disk and calls `serve()`. Replies with `404` when `fileRef` or the requested version is missing. `http` is `{ request, response, params }`. -- `filesCollection.serve(http, fileRef, vRef, version = 'original', readableStream = null, responseType = '200', force200 = false)` - Writes the response: sets headers (including `responseHeaders`), handles `Range` requests (`206`/`416`) and streams the file. Pass your own `readableStream` to serve content from a non-filesystem source, as in the [GridFS](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/gridfs-streaming.md) and [S3](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/aws-s3-integration.md) guides. For a `Range` request, the stream should contain exactly the requested bytes, as `serve()` answers `206` with the parsed `Content-Range`. Otherwise delete `http.request.headers.range` or pass `force200`. On `200`, `serve()` sets `Content-Length` to the stored size, so a custom stream must send the whole file. On `206`, `serve()` sets `Content-Length` only when it reads the file from disk itself, and a custom stream is sent chunked. Set `force200` to `true` to send the full content even if the request has a `Range` header. +- `await filesCollection.serve(http, fileRef, vRef, version = 'original', readableStream = null, responseType = '200', force200 = false)` - Writes the response: sets headers (including `responseHeaders`), handles `Range` requests (`206`/`416`) and streams the file. Pass your own `readableStream` to serve content from a non-filesystem source, as in the [GridFS streaming](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/gridfs-streaming.md) guide. The [`storage` option](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/constructor.md) does this for you: `serve()` reads through the adapter's `createReadStream()`, as in the [S3 adapter](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/aws-s3-integration.md). For a `Range` request, the stream should contain exactly the requested bytes, as `serve()` answers `206` with the parsed `Content-Range`. Otherwise delete `http.request.headers.range` or pass `force200`. On `200`, `serve()` sets `Content-Length` to the stored size, so a custom stream must send the whole file. On `206`, `serve()` sets `Content-Length` only when it reads the file from disk itself, and a custom stream is sent chunked. Set `force200` to `true` to send the full content even if the request has a `Range` header. For range rules see [custom response headers](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/custom-response-headers.md). diff --git a/docs/constructor.md b/docs/constructor.md index 6e01ac2b..a04dd5c5 100644 --- a/docs/constructor.md +++ b/docs/constructor.md @@ -112,7 +112,7 @@ Server - Time in seconds, during upload may be continued, default 3 hours (10800 seconds) + Time in seconds, during upload may be continued, default 3 hours (10800 seconds). Unfinished uploads resume after a server restart within this time: the server records each written chunk 10800 (3 hours) @@ -147,7 +147,7 @@ Set Cache-Control header - public, max-age=31536000, s-maxage=31536000 + private, max-age=31536000 when protected is set, otherwise public, max-age=31536000, s-maxage=31536000 @@ -165,7 +165,7 @@ Default Function - An Object sets the same headers on every response. A Function receives (responseCode, fileRef, versionRef, version, http) and returns an Object of headers. We recommend to keep original function structure, with your modifications, see example altering default headers + An Object sets the same headers on every response. A Function receives (responseCode, fileRef, versionRef, version, http) and returns an Object of headers, or a Promise that resolves to one. We recommend to keep original function structure, with your modifications, see example altering default headers @@ -214,7 +214,7 @@ 524288 (512 KB) - The constructor rounds this option down to a multiple of 8 (at least 8 on the client). The server accepts upload chunk sizes from 1 byte to 16777216 (16 MiB) and rejects larger values with 400. The client reduces larger values to the server maximum. Over HTTP, a chunk request body (base64 of chunkSize plus 4 KiB) larger than the limit gets 413. Start requests (including meta) are limited to 1 MiB and EOF requests to 64 KiB + The constructor rounds this option down to a multiple of 8 (at least 8 on the client). The server accepts upload chunk sizes from 1 byte to 16777216 (16 MiB) and rejects larger values with 400. The client reduces larger values to the server maximum. Over HTTP, a chunk request body (base64 of chunkSize plus 4 KiB) larger than the limit gets 413. Start requests (including meta) are limited to 1 MiB and EOF requests to 64 KiB. An upload has at most 100000 chunks. The client raises the chunk size of larger files to fit (up to 16 MiB), and the server rejects Start with more chunks with 400 @@ -222,18 +222,16 @@ config.namingFunction {Function} - Isomorphic + Server - Function which returns String. Use it to create nested directories in the storage folder. Note: file extension appended to returned value. The server sanitizes each path segment (/ separates segments, . and .. are dropped) and the result must stay inside storagePath. Upload start returns 409 if a file already exists at that path + Returns the file name on disk, without extension. Called as namingFunction({ file, fileId, userId }) with this set to the collection, on upload Start, in writeAsync(), and in loadAsync(). file differs per call. On upload Start it holds name, type, size, and meta as sent by the uploader (not verified), plus the server-computed extension, ext, _id, and userId. In writeAsync() and loadAsync() it holds name, type, and meta from the call options, and only writeAsync() adds size. May return a Promise. Note: file extension appended to returned value. The server sanitizes each path segment (/ separates segments, . and .. are dropped) and the result must stay inside storagePath. Upload start returns 409 if a file already exists at that path false - Primarily sets file name on FS
- if namingFunction is not set
- FS-name is equal to file's record _id + Without it, or when it returns an empty value, the name on disk is the file _id. The client ignores this option and logs a warning @@ -371,7 +369,7 @@ false - If true - files will be served to any signed-in user, the check does not compare the user with the file's owner (see security guide), if function() - you're able to check visitor's permissions in your own way.
+ true is deprecated since v4 and will be removed in v5. It allows any signed-in user to download any file, the check does not compare the user with the file's owner (see security guide). Pass a function to check visitor's permissions in your own way.
  • return true to continue @@ -463,6 +461,7 @@ return false to abort or {String} to abort upload with message
+

file.type is the type the uploader sent and is not verified

note: Because sending meta data as part of every chunk would hit the performance, meta is always empty ({}) except on the first chunk (chunkId=1 or chunkId=-1) and on eof (eof=true or chunkId=-1) (Fixed. Since v1.6.0 full file object is available in onBeforeUpload callback)

@@ -616,10 +615,10 @@ Isomorphic - Allow use remove() method on client + Allow clients to call remove() and removeAsync() with one file _id. Set onBeforeRemove when you turn it on - true + false @@ -832,11 +831,62 @@ Adds the X-Content-Type-Options: nosniff header to file responses + + true + + + Set false only when a proxy in front of the app adds this header + + + + + config.trustClientMimeType {Boolean} + + + Server + + + Store the mime type the uploader sent. When false the server reads the first 4100 bytes of the file and stores the detected type, the uploader's type for UTF-8 text when it is text/plain, text/csv, text/markdown, text/tab-separated-values, text/calendar, text/vtt, or application/json (otherwise text/plain), or application/octet-stream. Also applies to writeAsync(), loadAsync(), and addFile() without an explicit type + false - Will default to true in v4. Enable it now, as Content-Type comes from the type the uploader sent + onBeforeUpload sees the type the uploader sent, which is not verified. isImage, isVideo, and the other flags follow the stored type + + + + + config.downloadTokenSecret {String} + + + Server + + + Secret for signed download links, at least 32 characters. createDownloadToken(fileRef, { version, userId, expiresIn }) returns a token for link(fileRef, version, uriBase, { token }). The token opens one file version in this collection only. A valid token sets the request user to its userId for protected and downloadCallback. An invalid or expired token gets 403 + + + Not set: ?token= is ignored + + + Load it from Meteor.settings. Every instance with the same secret accepts the token, so it works without sticky sessions + + + + + config.storage {Object} + + + Server + + + Storage adapter with put(fileRef, versionName, localPath, { source }), createReadStream(fileRef, versionName, { start, end }) (inclusive end), remove(fileRef, versionName), and optional stat(fileRef, versionName). Uploads always write chunks to storagePath first. put() runs before the document insert and onAfterUpload, and its result is stored at versions.<name>.meta.storage. source is 'upload', 'write' (writeAsync()), 'load' (loadAsync()), or 'addFile'. An addFile file belongs to the caller, so an adapter must not delete it. serve() streams through createReadStream(), removeAsync() and unlinkAsync() call remove(). interceptDownload still runs first. Without stat(), download() can not answer 404 or apply integrityCheck before streaming + + + new FSStorage() + + + Built in: FSStorage and new GridFSStorage({ bucketName }), both exported from meteor/ostrio:files on the server only. In shared code, use storage: Meteor.isServer ? new GridFSStorage() : undefined. GridFSStorage deletes the local file after put() for uploads, writeAsync(), and loadAsync(), and keeps the file passed to addFile(). S3: see AWS S3 integration @@ -1078,10 +1128,9 @@ const imagesCollection = new FilesCollection({ } return false; }, - namingFunction(file) { - // MAKE SURE namingFunction IS SET ON Server - // OVERWRITE Client's namingFunction FOR SECURITY REASONS AGAINST REVERSE-ENGINEERING ACTIONS - return helpers.sanitize(file.fileId); + // Server only: the client ignores this option + namingFunction({ fileId, userId }) { + return `${helpers.sanitize(userId || 'anonymous')}/${fileId}`; }, }); diff --git a/docs/custom-response-headers.md b/docs/custom-response-headers.md index fc4f3477..f9b43213 100644 --- a/docs/custom-response-headers.md +++ b/docs/custom-response-headers.md @@ -101,4 +101,8 @@ The server answers `Range` requests with `206 Partial Content` and `Content-Rang ## Security headers -Set `nosniff: true` in the constructor to add `X-Content-Type-Options: nosniff` to file responses. It will default to `true` in v4. To force browsers to download a file instead of rendering it, either request it with `?download=true` or return `Content-Disposition: attachment` from `responseHeaders`. See the [security guide](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/security.md). +File responses carry `X-Content-Type-Options: nosniff` by default since v4. Set `nosniff: false` in the constructor to turn it off. See the [security guide](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/security.md). + +## Default Content-Disposition + +Files are served `inline` only when their type is `image/*` (except `image/svg+xml`), `video/*`, `audio/*`, `application/pdf`, or `text/plain`. Everything else gets `Content-Disposition: attachment`. `?download=true` always forces `attachment`. Return `Content-Disposition` from `responseHeaders` to set your own value. diff --git a/docs/google-cloud-storage-integration.md b/docs/google-cloud-storage-integration.md index c0698674..7128a246 100644 --- a/docs/google-cloud-storage-integration.md +++ b/docs/google-cloud-storage-integration.md @@ -142,7 +142,7 @@ const Files = new FilesCollection({ return docs.length > 0 && docs.every((doc) => doc.versions?.original?.meta?.pipePath); }, - interceptDownload(http, fileRef, version) { + async interceptDownload(http, fileRef, version) { const vRef = fileRef.versions?.[version]; const path = vRef?.meta?.pipePath; @@ -173,7 +173,7 @@ const Files = new FilesCollection({ remoteReadStream = bucket.file(path).createReadStream(); } - this.serve(http, fileRef, vRef, version, remoteReadStream); + await this.serve(http, fileRef, vRef, version, remoteReadStream); return true; } }); diff --git a/docs/gridfs-bucket-integration.md b/docs/gridfs-bucket-integration.md index 5fb7997c..584b6f8c 100644 --- a/docs/gridfs-bucket-integration.md +++ b/docs/gridfs-bucket-integration.md @@ -1,6 +1,25 @@ # Use GridFS with `GridFSBucket` as a storage -This example shows how to handle (store, serve, remove) uploaded files via GridFS. +Since v4, use the built-in adapter. Create it on the server only, the client build does not export `GridFSStorage`: + +```js +import { Meteor } from 'meteor/meteor'; +import { FilesCollection, GridFSStorage } from 'meteor/ostrio:files'; + +// Shared code. The client build does not export `GridFSStorage`, so create it on the server only +export const Images = new FilesCollection({ + collectionName: 'images', + storage: Meteor.isServer ? new GridFSStorage({ bucketName: 'images' }) : undefined, +}); +``` + +`GridFSStorage` is server only. Uploads are written to `storagePath` first. When a file is complete, the adapter copies it into the bucket, deletes the local copy, and stores `{ name: 'gridfs', bucketName, id }` at `versions..meta.storage`. `addFile()` keeps the caller's file on disk. Downloads support `Range` (`206`), and `removeAsync()` deletes the bucket file. Options: `bucketName` (default `'fs'`), `chunkSizeBytes` (driver default when not set), and `db` (default: the app's database). See the [`storage` option](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/constructor.md). + +The adapter runs before `onAfterUpload`, and it deletes the local file after the copy. `onAfterUpload` can not read `fileRef.path` with `GridFSStorage`, because the local file is already gone. Read it with `await this.storage.createReadStream(fileRef, 'original')` instead. The [AWS S3 guide](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/aws-s3-integration.md) describes the same limit. + +## Custom GridFS handling + +The manual recipe below handles (stores, serves, removes) uploaded files via GridFS with hooks. Use it when the built-in adapter does not fit. The Javascript Mongo driver (the one that Meteor uses under the hood) allows to define [so called "Buckets"](https://mongodb.github.io/node-mongodb-native/6.0/classes/GridFSBucket.html). @@ -98,13 +117,13 @@ export const createOnAfterUpload = (bucket) => { ### 4. Create download handler We also need to handle to retrieve files from GridFS when a download is initiated. We will use the same -factory function as in step 3. The handler uses `serve()` to set `Content-Disposition`, `Content-Type`, and `Cache-Control` headers. It deletes the `Range` header so `serve()` replies with `200` and the whole file. For `206` partial content see [GridFS streaming](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/gridfs-streaming.md). +factory function as in step 3. The handler calls `await this.serve()`, which is async since v4. It sets the `Content-Disposition`, `Content-Type`, and `Cache-Control` headers. It deletes the `Range` header so `serve()` replies with `200` and the whole file. For `206` partial content see [GridFS streaming](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/gridfs-streaming.md). ```js import { createObjectId } from '../createObjectId'; export const createInterceptDownload = (bucket) => { - return function interceptDownload (http, file, versionName) { + return async function interceptDownload (http, file, versionName) { const vRef = file.versions[versionName]; const { gridFsFileId } = vRef.meta || {}; if (!gridFsFileId) { @@ -124,7 +143,7 @@ export const createInterceptDownload = (bucket) => { }); delete http.request.headers.range; - this.serve(http, file, vRef, versionName, readStream); + await this.serve(http, file, vRef, versionName, readStream); return true; }; }; diff --git a/docs/gridfs-streaming.md b/docs/gridfs-streaming.md index 513cf06a..8b6fafa8 100644 --- a/docs/gridfs-streaming.md +++ b/docs/gridfs-streaming.md @@ -49,7 +49,7 @@ const resolveRange = (header, size) => { }; export const createInterceptDownload = (bucket) => { - return function interceptDownload(http, fileRef, versionName) { + return async function interceptDownload(http, fileRef, versionName) { const vRef = fileRef.versions[versionName]; const gridFsFileId = (vRef.meta || {}).gridFsFileId; if (!gridFsFileId) { @@ -87,7 +87,7 @@ export const createInterceptDownload = (bucket) => { // `.serve()` sets Content-Disposition, Content-Type, Cache-Control, // `Content-Range`, and the `200` or `206` status - this.serve(http, fileRef, vRef, versionName, stream); + await this.serve(http, fileRef, vRef, versionName, stream); return true; }; }; diff --git a/docs/insert.md b/docs/insert.md index a1755565..2cdad973 100644 --- a/docs/insert.md +++ b/docs/insert.md @@ -345,7 +345,7 @@ The `FileUpload` instance is the `this` *context* in all callback functions (*se Pipe data before upload - All data must be in `data URI` scheme (*Base64*). A pipe must not change the decoded byte length of a chunk, see [Piping](#piping). Pipes run in reverse order of registration: the pipe added last runs first. This order changes in v4 (first added runs first) + All data must be in `data URI` scheme (*Base64*). A pipe must not change the decoded byte length of a chunk, see [Piping](#piping). Pipes run in the order they were added: the first added pipe runs first @@ -755,7 +755,7 @@ Note: data flow in `ddp` and `http` uses dataURI (e.g. *Base64*) Each pipe gets a chunk as a Base64 string and must return a Base64 string that decodes to the same number of bytes. The server writes chunk `N` at offset `(N - 1) * chunkSize` and rejects a chunk that is larger than `chunkSize` with `400 Invalid chunk size`. Over HTTP the server may close the connection instead, and the upload fails after 10 retries of that chunk. A shorter chunk leaves zero-filled gaps in the stored file. So per-chunk compression, and encryption that adds an IV, a tag, or padding, do not work as pipes. Run those on the server in `onAfterUpload`, or transform the whole file before calling `insert()`. -Pipes run in reverse order of registration. In the example below `count` runs first, then `mask`. This order changes in v4, where the first registered pipe runs first. +Pipes run in the order they were added. In the example below `mask` runs first, then `count`. Before v4 the last added pipe ran first, see [migration to v4](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/migration-to-v4.md). ```js import { Template } from 'meteor/templating'; diff --git a/docs/insertAsync.md b/docs/insertAsync.md index b0383df3..bc5c6b54 100644 --- a/docs/insertAsync.md +++ b/docs/insertAsync.md @@ -356,7 +356,7 @@ The `FileUpload` instance is the `this` *context* in all callback functions (*se Pipe data before upload - All data must be in `data URI` scheme (*Base64*). A pipe must not change the decoded byte length of a chunk, see [Piping](#piping). Pipes run in reverse order of registration: the pipe added last runs first. This order changes in v4 (first added runs first) + All data must be in `data URI` scheme (*Base64*). A pipe must not change the decoded byte length of a chunk, see [Piping](#piping). Pipes run in the order they were added: the first added pipe runs first @@ -766,7 +766,7 @@ Note: data flow in `ddp` and `http` uses dataURI (e.g. *Base64*) Each pipe gets a chunk as a Base64 string and must return a Base64 string that decodes to the same number of bytes. The server writes chunk `N` at offset `(N - 1) * chunkSize` and rejects a chunk that is larger than `chunkSize` with `400 Invalid chunk size`. Over HTTP the server may close the connection instead, and the upload fails after 10 retries of that chunk. A shorter chunk leaves zero-filled gaps in the stored file. So per-chunk compression, and encryption that adds an IV, a tag, or padding, do not work as pipes. Run those on the server in `onAfterUpload`, or transform the whole file before calling `insertAsync()`. -Pipes run in reverse order of registration. In the example below `count` runs first, then `mask`. This order changes in v4, where the first registered pipe runs first. +Pipes run in the order they were added. In the example below `mask` runs first, then `count`. Before v4 the last added pipe ran first, see [migration to v4](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/migration-to-v4.md). ```js import { Template } from 'meteor/templating'; diff --git a/docs/link.md b/docs/link.md index a0dc1726..f0b2180e 100644 --- a/docs/link.md +++ b/docs/link.md @@ -12,12 +12,14 @@ There are two options to get *downloadable* URL to the uploaded file using `.lin Use `.link()` method of *FilesCollection* instance to create *downloadable* link from *file's* plain object. To get an *Object* use `await FilesCollection#collection.findOneAsync({})`, for example inside [`end` event](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/insert.md) on the *Client* and [`onAfterUpload`](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/constructor.md) on the *Server* ```js -FilesCollection#link(fileRef, version, URIBase); // [*Isomorphic*] +FilesCollection#link(fileRef, version, URIBase, opts); // [*Isomorphic*] ``` - `fileRef` {*Object*} - Object returned from MongoDB collection or [after upload](https://github.com/veliovgroup/meteor-files-website/blob/master/imports/client/upload/upload-form.js#L194-L205) - `version` {*String*|*void 0*} - [OPTIONAL] File's subversion name, default: `original`. If requested subversion isn't found, `original` will be returned - `URIBase` {*String*} - [OPTIONAL] base URI (domain), default: `ROOT_URL` or `MOBILE_ROOT_URL` on *Cordova*. +- `opts` {*Object*} - [OPTIONAL] + - `opts.token` {*String*} - Token from the server `createDownloadToken()`, appended as `?token=`. See [Signed download links](#signed-download-links) - Returns {*String*} - Absolute URL to file. Returns an empty string for `null`/`undefined`, for a file object without `_id` (for example the file of a rejected upload), and when no safe route is available ## How the URL is built @@ -31,11 +33,12 @@ FilesCollection#link(fileRef, version, URIBase); // [*Isomorphic*] Use `.link()` method of [*FileCursor* instance](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/FileCursor.md) to create *downloadable* link from a cursor returned for example from [`FilesCollection#findOneAsync({})`](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/findOneAsync.md) ```js -FileCursor#link(version, URIBase); // [*Isomorphic*] +FileCursor#link(version, URIBase, opts); // [*Isomorphic*] ``` - `version` {*String*|*void 0*} - [OPTIONAL] File's subversion name, default: `original`. If requested subversion isn't found, `original` will be returned - `URIBase` {*String*} - [OPTIONAL] base URI (domain), default: `ROOT_URL` or `MOBILE_ROOT_URL` on *Cordova*. +- `opts` {*Object*} - [OPTIONAL] same as `FilesCollection#link`, `{ token }` - Returns {*String*} - Full URL to file ## Required fields: @@ -81,3 +84,31 @@ imagesCollection.link(fileRef, 'original', 'https://other-domain.com/'); // Relative path to domain: imagesCollection.link(fileRef, 'original', '/'); ``` + +## Signed download links + +With `downloadTokenSecret` set on the server, `createDownloadToken()` returns a token that opens one `_id` and one version in this collection until it expires. Pass it as `{ token }` to get a link that works without the `x_mtok` cookie and on any server instance with the same secret. The token's `userId` becomes `this.userId` in `protected` and `http.userId` in `downloadCallback`. An invalid or expired token gets `403`. A token download gets `Cache-Control: private, max-age=` unless `responseHeaders` sets `Cache-Control`. See the [security guide](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/security.md#signed-download-links). + +```js +// Shared code. `Meteor.settings.private` is undefined on the client +const files = new FilesCollection({ + collectionName: 'files', + downloadTokenSecret: Meteor.isServer ? Meteor.settings.private.filesTokenSecret : undefined, + protected(fileObj) { + return !!fileObj && fileObj.userId === this.userId; + }, +}); + +// Server only: `createDownloadToken()` does not exist on the client +Meteor.methods({ + async 'files.downloadLink'(fileId) { + check(fileId, String); + const file = await files.findOneAsync({ _id: fileId, userId: this.userId }); + if (!file) { + throw new Meteor.Error(404, 'Not found'); + } + const token = files.createDownloadToken(file, { userId: this.userId, expiresIn: 600 }); + return files.link(file, 'original', undefined, { token }); + }, +}); +``` diff --git a/docs/migration-to-v4.md b/docs/migration-to-v4.md new file mode 100644 index 00000000..791b1fd6 --- /dev/null +++ b/docs/migration-to-v4.md @@ -0,0 +1,54 @@ +# Migration to v4 + +ostrio:files 4.0.0 requires Meteor 3.2 or newer, like 3.1.0. This page lists every breaking change and what to change in your app. The [upgrade checklist](#upgrade-checklist) at the end sums up the steps. + +## Breaking changes + +### Removed + +- `FilesCursor#hasNext()` is removed. Use `await cursor.hasNextAsync()`. +- `FilesCursor#countAsync()` is removed. Use `await cursor.countDocuments()`. +- `findOne()` moved from the isomorphic core to the client class. On the server it still throws `Meteor.Error(404)`; use `findOneAsync()`. +- The default `x_mtok` lookup supports only a `Map` in `Meteor.server.sessions` (Meteor 3). + +### Changed defaults + +- `allowClientCode` defaults to `false` on the server and the client. Set `allowClientCode: true` on both sides, together with a server `onBeforeRemove`, if clients remove files. +- `cacheControl` defaults to `private, max-age=31536000` on `protected` collections, so a CDN or shared proxy no longer caches protected files. Set `cacheControl` to restore the 3.x value `public, max-age=31536000, s-maxage=31536000`. +- `nosniff` defaults to `true`: responses carry `X-Content-Type-Options: nosniff`. Set `nosniff: false` to restore the 3.x behavior. +- `Content-Disposition` is `inline` only for `image/*` (except `image/svg+xml`), `video/*`, `audio/*`, `application/pdf`, and `text/plain`. Other files download as `attachment`. Return `Content-Disposition` from `responseHeaders` to change it. + +### Changed behavior + +- `FileUpload#pipe()` functions run in the order they were added (first `pipe()` call runs first). v3 ran the last added pipe first. Reverse your `pipe()` calls if you chain more than one. +- Client `remove()` and `removeAsync()` accept only a String `_id`, and the `_FilesCollectionRemove_` method rejects anything else. `FilesCursor#remove()` and `#removeAsync()` on the client remove the matching files one `_id` at a time. The removal is not atomic. +- Only the server names files. Remove `namingFunction` from client code (it is ignored with a warning). The server ignores `FSName` sent by v3 clients. +- `namingFunction` receives one object `{ file, fileId, userId }` in upload Start, `writeAsync()`, and `loadAsync()`. v3 passed the upload options in Start and the call options in `writeAsync()`/`loadAsync()`. +- The server keeps only `name`, `type`, `size`, and `meta` from the client `file` object. Other top-level keys are dropped. Move custom upload data into `meta`. +- The stored `type`, `mime`, `mime-type`, `versions.original.type`, and `is*` flags come from the file content (first 4100 bytes), not from the uploader. Text is stored as `text/plain`, except when the uploader sent one of `text/plain`, `text/csv`, `text/markdown`, `text/tab-separated-values`, `text/calendar`, `text/vtt`, or `application/json`. SVG, HTML, XML, CSS, and JavaScript are therefore stored as `text/plain`. Unknown binary formats are stored as `application/octet-stream`. Office files and other zip, OLE, and mp4 based formats keep the uploader's type when it names a known format of the detected container. Set `trustClientMimeType: true` to keep the 3.x behavior. `writeAsync()`, `loadAsync()`, and `addFile()` detect the type when `opts.type` is not set. Documents stored by 3.x keep their stored type, the server does not detect it again. The new `Content-Disposition` rule applies to them on download. +- `serve()` is `async` and returns a Promise. Without a `readableStream` it reads the file through the storage adapter (`FSStorage` by default, same files as before). Await it if your code runs after it, for example in an `interceptDownload` recipe. +- `unlinkAsync()` and `removeAsync()` remove files through the storage adapter. `unlink()` (deprecated) still unlinks local paths only. +- Uploads that were in progress during the upgrade from 3.x get `410` on their next chunk and must start again. 4.0 records written chunks in the upload record and does not guess from the file size. +- An upload has at most 100000 chunks. The client raises `chunkSize` for larger files. A Start with more chunks gets `400`. + +## Deprecations + +- `protected: true` logs a warning at startup and will be removed in v5. Pass a function, for example `protected(fileObj) { return !!fileObj && fileObj.userId === this.userId; }`. + +## New options + +- `trustClientMimeType` (server, default `false`): keep the type sent by the uploader instead of detecting it from content. +- `downloadTokenSecret` (server) and `createDownloadToken()`: sign download links that work without the `x_mtok` cookie. Pass the token to `link(fileRef, version, uriBase, { token })`. See the [security guide](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/security.md#signed-download-links). +- `storage` (server): the adapter that stores, serves, and removes finished files. `FSStorage` is the default, `GridFSStorage` keeps files in MongoDB GridFS. See the [`storage` option](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/constructor.md). + +## Upgrade checklist + +1. Replace `cursor.hasNext()` with `await cursor.hasNextAsync()` and `cursor.countAsync()` with `await cursor.countDocuments()`. +2. Move `namingFunction` to the server constructor and change it to `({ file, fileId, userId })`. +3. Set `allowClientCode: true` and `onBeforeRemove` only if clients remove files, and pass an `_id` to `remove()`. +4. Check code that reads `type` or `isImage` after upload: the server now detects them from content. If your app relies on uploads stored as `text/html` or `image/svg+xml`, set `trustClientMimeType: true`. Documents stored by 3.x keep their old `type`, the server does not detect it again. +5. Check links to files that are not images, video, audio, PDF, or plain text: they download as attachments. This also applies to documents stored by 3.x. +6. Add `await` to `serve()` calls in `interceptDownload` and in your own download handlers when code runs after them. +7. Reverse chained `pipe()` calls. +8. Replace `protected: true` with a function. +9. Ask users to restart uploads that were running during the deploy. diff --git a/docs/readme.md b/docs/readme.md index 746b94ec..f24434a0 100644 --- a/docs/readme.md +++ b/docs/readme.md @@ -8,6 +8,8 @@ Browse [documentation directory](https://github.com/veliovgroup/Meteor-Files/tre - [About Meteor-Files package](#about) - [Security guide](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/security.md) + - [Signed download links](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/security.md#signed-download-links) - `createDownloadToken()` and `link(fileRef, version, uriBase, { token })` +- [Migration to v4](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/migration-to-v4.md) - [Migration to v3](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/migration-to-v3.md) - [API](#api) - [Examples](#examples) @@ -76,7 +78,7 @@ Meteor-Files library features and highlights - [__See all *FileCursor* methods__](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/FileCursor.md) - [`FilesCursor` Class](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/FilesCursor.md) - Instance of this class is returned from [`.find()`](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/find.md) method - `fetchAsync()` - {*Promise*} - Returns all matching document(s) as an Array - - `countDocuments()` - {*Promise*} - Returns the number of documents that match a query (`countAsync()` is deprecated) + - `countDocuments()` - {*Promise*} - Returns the number of documents that match a query - `removeAsync()` - {*Promise*} - Removes all documents that match a query, resolves to a number of removed records - `forEachAsync(callback, context)` - {*Promise*} - Call `callback` once for each matching document - `eachAsync()` - {*Promise*} - Resolves to Array of `FileCursor` made for each document on current Cursor diff --git a/docs/remove.md b/docs/remove.md index a6c772a0..c6a5cdea 100644 --- a/docs/remove.md +++ b/docs/remove.md @@ -1,13 +1,15 @@ -### `remove(selector[, cb])` [*Client*] +### `remove(_id[, cb])` [*Client*] > __Deprecated.__ The callback API works on the Client only. There is no synchronous `remove()` on the Server. Use [`removeAsync()`](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/removeAsync.md) everywhere. -Remove records from FilesCollection and files from FS. Requires `allowClientCode: true` (default), and `onBeforeRemove` should authorize the user. +Remove records from FilesCollection and files from FS. Requires `allowClientCode: true` on the server and the client (default is `false` since v4), and `onBeforeRemove` must authorize the user. -- `selector` {*Object*|*String*} - See [Mongo Selectors](https://docs.meteor.com/api/collections.html#selectors) +- `_id` {*String*} - `_id` of the file to remove - `cb` {*Function*} - Callback, with `error` and the number of removed records - Returns {*FilesCollection*} - Current FilesCollection instance +Since v4 the client removes one file per call. Use `files.find(selector).removeAsync()` to remove several files; it calls the server once per `_id`. + ```js import { FilesCollection } from 'meteor/ostrio:files'; @@ -15,12 +17,12 @@ const imagesCollection = new FilesCollection({collectionName: 'images'}); // Usage (Client): // Remove particular file -imagesCollection.remove({_id: 'Rfy2HLutYK4XWkwhm'}); +imagesCollection.remove('Rfy2HLutYK4XWkwhm'); // Equals to above -imagesCollection.findOne({_id: 'Rfy2HLutYK4XWkwhm'}).remove(); +imagesCollection.findOne('Rfy2HLutYK4XWkwhm').remove(); // Using callback -imagesCollection.remove({_id: 'Rfy2HLutYK4XWkwhm'}, (error) => { +imagesCollection.remove('Rfy2HLutYK4XWkwhm', (error) => { if (error) { console.error(`File wasn't removed, error: ${error.reason}`); } else { diff --git a/docs/removeAsync.md b/docs/removeAsync.md index a09f2ce8..1f9b8b05 100644 --- a/docs/removeAsync.md +++ b/docs/removeAsync.md @@ -1,14 +1,18 @@ ### `removeAsync` [*Isomorphic*] ```ts -FilesCollection#removeAsync(selector: MeteorFilesSelector): Promise +FilesCollection#removeAsync(selector: MeteorFilesSelector): Promise // Server +FilesCollection#removeAsync(_id: string): Promise // Client ``` Remove records from FilesCollection and files from FS. -- `selector` {*Object*} - See [Mongo Selectors](https://docs.meteor.com/api/collections.html#selectors) +- `selector` {*Object*|*String*} - [*Server*] See [Mongo Selectors](https://docs.meteor.com/api/collections.html#selectors) +- `_id` {*String*} - [*Client*] `_id` of the file to remove - Returns {*Promise*} - Number of removed records +Since v4 the client removes one file per call. Use `files.find(selector).removeAsync()` to remove several files; it calls the server once per `_id`. + The `onAfterRemove` hook can return `true` to skip deleting files from FS, see [constructor options](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/constructor.md). ```js @@ -17,10 +21,10 @@ import { FilesCollection } from 'meteor/ostrio:files'; const imagesCollection = new FilesCollection({collectionName: 'images'}); // Usage: -// Drop collection's data and remove all associated files from FS +// [Server] Drop collection's data and remove all associated files from FS await imagesCollection.removeAsync({}); -// Remove particular file -await imagesCollection.removeAsync({_id: 'Rfy2HLutYK4XWkwhm'}); +// Remove particular file, Server and Client +await imagesCollection.removeAsync('Rfy2HLutYK4XWkwhm'); // Equals to above const file = await imagesCollection.findOneAsync({_id: 'Rfy2HLutYK4XWkwhm'}); await file.removeAsync(); diff --git a/docs/schema.md b/docs/schema.md index cacc2306..e66af666 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -150,3 +150,5 @@ const imagesCollection = new FilesCollection({ schema: mySchema }); ``` + +Custom top-level fields are set on the server, for example in `onAfterUpload`. The server keeps only `name`, `type`, `size`, and `meta` from the file object a client sends, so send custom upload data from the client in `meta`. diff --git a/docs/security.md b/docs/security.md index ec808a16..2483949a 100644 --- a/docs/security.md +++ b/docs/security.md @@ -1,10 +1,10 @@ # Security guide -Read this page before you ship a `FilesCollection` to production. Defaults favor getting started quickly, not locked-down access. +Read this page before you ship a `FilesCollection` to production. Since v4 the defaults are stricter: clients can not remove files, responses carry `nosniff`, the server detects file types, and only the server names files. Downloads and uploads still allow anonymous access unless you add the checks below. See [migration to v4](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/migration-to-v4.md) when you upgrade. ## Protect downloads -`protected: true` only checks that the visitor is logged in. Any signed-in user can download any file if they know or guess its URL (an IDOR risk). Pass a function to compare the file owner with the current user: +`protected: true` is deprecated since v4 and will be removed in v5. It only checks that the visitor is logged in. Any signed-in user can download any file if they know or guess its URL (an IDOR risk). Pass a function to compare the file owner with the current user: ```js import { FilesCollection } from 'meteor/ostrio:files'; @@ -49,33 +49,34 @@ const files = new FilesCollection({ The server enforces these rules for in-progress uploads: - Only the user who started an upload can send chunks, the end-of-file message, or abort it. Others get `403`. Aborting an unknown or foreign upload gets `404`. Uploads started without a logged-in user are anonymous and belong to any anonymous caller, so check `this.userId` in `onBeforeUpload` when that matters -- The server ignores client-supplied reserved file fields: `_id`, `fileId`, `path`, `_storagePath`, `_downloadRoute`, `_collectionName`, `versions`, `userId`, `public`, the extension and mime fields, and the type flags (`isVideo`, `isImage`, and others) +- The server keeps only `name`, `type`, `size`, and `meta` from the client `file` object and computes every other field - `chunkSize` must be an integer from 1 byte to 16 MiB. The declared `size` and the chunk count must agree, chunks must be in range, and the total written can not exceed the declared `size`. The stored `size` is the real size of the file on disk - HTTP request bodies are limited: 1 MiB for Start (including `meta`), 64 KiB for EOF, and the base64 size of `chunkSize` plus 4 KiB for a chunk. Larger bodies get `413` - Start returns `409` if the target file already exists, if the file id is already in use, or if another pending upload claims the same path. The server creates the file exclusively, so it never overwrites an existing file -- File names from `namingFunction` are sanitized per path segment, and the final path must resolve inside the `storagePath` of the collection. Otherwise Start returns `400` +- File names from `namingFunction` are sanitized per path segment, and the final path must resolve inside the `storagePath` of the collection. Otherwise Start returns `400`. Only the server names files: the client `namingFunction` option and the `FSName` field are ignored since v4 +- The server records every written chunk in the upload record, so a restarted server resumes from the record. An upload has at most 100000 chunks - The server remembers the device and inode of the file it created. If the file is removed or replaced before the upload ends, the upload fails with `410` or `409` and the other file is not touched - The abort call removes only the unfinished upload, never a finished file - A repeated EOF returns the stored file record only to the authenticated user who owns the upload. Anonymous callers get `408` - Responses to the client never contain server paths (`path`, `_storagePath`, `versions.*.path`), and HTTP errors with `5xx` status do not expose internal messages - An open file handle of an idle upload is closed after `uploadIdleTimeout` -Client-supplied values such as `type`, `name`, and `meta` are still untrusted. Verify content in `onAfterUpload`, for example with the `file-type` package. +The client-supplied `name` and `meta` are untrusted. The server detects `type` from the file content (see [Content-Type and downloads](#content-type-and-downloads)), but it knows only a built-in table of signatures. Verify content in `onAfterUpload` when you accept formats outside that table, for example with the `file-type` package. ## Protect removal -`allowClientCode` is `true` by default. Without `onBeforeRemove`, any client can call `removeAsync()` and delete any file. Do one of these: +`allowClientCode` defaults to `false` since v4, so clients can not remove files. If you set `allowClientCode: true` without `onBeforeRemove`, any client can call `removeAsync()` and delete any file. Pick one of these: ```js -// Option 1: no remove from client code at all +// Option 1: no remove from client code at all (the default) const files = new FilesCollection({ collectionName: 'files', - allowClientCode: false, }); // Option 2: allow removal and check the owner const files2 = new FilesCollection({ collectionName: 'files2', + allowClientCode: true, async onBeforeRemove(cursor) { if (!this.userId) { return false; @@ -86,13 +87,13 @@ const files2 = new FilesCollection({ }); ``` -When `allowClientCode` is `true` and `onBeforeRemove` is not set, the server prints a warning at start. The default changes to `false` in v4. +When `allowClientCode` is `true` and `onBeforeRemove` is not set, the server prints a warning at start. Set `allowClientCode: true` on the client constructor too. Also call `denyClient()` on the server, or define your own `allow`/`deny` rules, so clients can not write to the underlying `Mongo.Collection` directly. ### Do not call `allowClient()` in production -`serve()` and `unlinkAsync()` trust the file paths stored in documents (`versions.*.path`). The server verifies that paths stay inside `storagePath` only when it creates an upload, not later. `allowClient()` lets any client update documents, including `path`, so a client could point a document at another file on the server and then download or delete it. The same applies to any `allow` rule or method that lets users write `path` or `versions` fields. Paths you pass to `addFile()`, `writeAsync()`, and `loadAsync()` are trusted too. Never build them from user input. +`serve()` and `unlinkAsync()` trust the file paths stored in documents (`versions.*.path`) through `FSStorage`, the default `storage` adapter. The server verifies that paths stay inside `storagePath` only when it creates an upload, not later. `allowClient()` lets any client update documents, including `path`, so a client could point a document at another file on the server and then download or delete it. The same applies to any `allow` rule or method that lets users write `path` or `versions` fields. Paths you pass to `addFile()`, `writeAsync()`, and `loadAsync()` are trusted too. Never build them from user input. ## Authentication cookie @@ -103,14 +104,51 @@ HTTP routes identify the user with the `x_mtok` cookie. The client sets it to th - The client sets the cookie to the session id of `Meteor.connection`, also when a collection uses a custom `ddp` connection - `allowQueryStringCookies` puts the token in URLs, where it can leak to logs, proxies, and `Referer` headers. Enable it only for Cordova and Meteor-Desktop, and set `allowedCordovaOrigins` with it. Do not enable it for web apps +## Signed download links + +Set `downloadTokenSecret` on the server to hand out links that work without the `x_mtok` cookie, on any instance: + +```js +// Shared code. `Meteor.settings.private` is undefined on the client +const files = new FilesCollection({ + collectionName: 'files', + downloadTokenSecret: Meteor.isServer ? Meteor.settings.private.filesTokenSecret : undefined, + protected(fileObj) { + return !!fileObj && fileObj.userId === this.userId; + }, +}); + +// Server only: `createDownloadToken()` does not exist on the client +Meteor.methods({ + async 'files.downloadLink'(fileId) { + check(fileId, String); + const file = await files.findOneAsync({ _id: fileId, userId: this.userId }); + if (!file) { + throw new Meteor.Error(404, 'Not found'); + } + const token = files.createDownloadToken(file, { userId: this.userId, expiresIn: 600 }); + return files.link(file, 'original', undefined, { token }); + }, +}); +``` + +- The token opens one `_id` and one version in one collection until it expires. Anyone who has the URL can use it, so keep `expiresIn` short +- URLs end up in logs, proxies, and `Referer` headers +- The token carries its `userId` in base64url, readable by whoever holds the link +- An invalid or expired token gets `403`. Public collections ignore tokens +- A token download gets `Cache-Control: private, max-age=`, so a CDN or shared proxy does not keep it after the token expires. A `Cache-Control` header from `responseHeaders` replaces this value +- Downloads from `protected` collections without a token get `Cache-Control: private, max-age=31536000` by default, so a CDN or shared proxy does not keep them. Browsers still cache them for the signed-in user. Set `cacheControl: 'no-store'` to turn that off + ## Content-Type and downloads -The uploader supplies the file `type`, and the server uses it as the `Content-Type` of the response. A file labeled `text/html` is rendered by the browser, which can lead to stored XSS. Reduce the risk: +Since v4 the server detects the type from the first 4100 bytes of the file and stores it, unless `trustClientMimeType` is `true`. Text files are stored as `text/plain` unless the uploader said `text/plain`, `text/csv`, `text/markdown`, `text/tab-separated-values`, `text/calendar`, `text/vtt`, or `application/json`. An SVG, HTML, XML, CSS, or JavaScript file labeled as an image or as `text/html`, `text/xml`, `text/css`, or `text/javascript` is stored as `text/plain`, so the browser does not run it. `onBeforeUpload` still sees the uploader's type. Reduce the risk further: -- Set `nosniff: true` to add `X-Content-Type-Options: nosniff` to responses. It defaults to `false` in 3.x and will default to `true` in v4 +- `nosniff` is on by default since v4 and adds `X-Content-Type-Options: nosniff` to responses - Validate the type and extension in `onBeforeUpload`, and verify real content in `onAfterUpload` -- Serve untrusted files as downloads. Link with `?download=true`, or return `Content-Disposition: attachment` from [`responseHeaders`](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/custom-response-headers.md) +- Files are served `inline` only when their type is `image/*` (except `image/svg+xml`), `video/*`, `audio/*`, `application/pdf`, or `text/plain`. Everything else gets `Content-Disposition: attachment`. `?download=true` always forces `attachment`, and [`responseHeaders`](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/custom-response-headers.md) can set its own `Content-Disposition` - Serve user files from a separate domain when possible +- Scan uploads that other users open (antivirus, document sanitizers) in `onAfterUpload`, before you mark the file as available +- Follow the [OWASP File Upload Cheat Sheet](https://cheatsheetseries.owasp.org/cheatsheets/File_Upload_Cheat_Sheet.html) for the checks that depend on your app ## Other options diff --git a/docs/unlink.md b/docs/unlink.md index 32331f81..bff0ec3c 100644 --- a/docs/unlink.md +++ b/docs/unlink.md @@ -1,3 +1,3 @@ ### `unlink(fileRef [, version, callback])` [*Server*] -DEPRECATED SINCE `v3.0.0`, use [`unlinkAsync()`](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/unlinkAsync.md) instead +DEPRECATED SINCE `v3.0.0`, use [`unlinkAsync()`](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/unlinkAsync.md) instead. Works with the default `FSStorage` adapter only: it unlinks local paths and does not call the adapter's `remove()` diff --git a/docs/unlinkAsync.md b/docs/unlinkAsync.md index 6cd64671..2f730efc 100644 --- a/docs/unlinkAsync.md +++ b/docs/unlinkAsync.md @@ -1,6 +1,6 @@ ### `unlinkAsync` [*Server*] -Unlink file and its subversions from FS. +Remove file and its subversions through the [storage adapter](https://github.com/veliovgroup/Meteor-Files/blob/master/docs/constructor.md) (`remove()`). With the default `FSStorage` it unlinks the files from FS. A file that is already gone counts as removed. ```ts FilesCollection#unlinkAsync(fileRef: FileObj, version?: string): Promise diff --git a/download-token.js b/download-token.js new file mode 100644 index 00000000..23c61217 --- /dev/null +++ b/download-token.js @@ -0,0 +1,83 @@ +import crypto from 'node:crypto'; + +/** + * @const {number} MIN_SECRET_LENGTH - Shortest accepted `downloadTokenSecret` + */ +export const MIN_SECRET_LENGTH = 32; + +/** + * @const {number} MAX_TOKEN_LENGTH - Longer tokens are rejected before any work + */ +const MAX_TOKEN_LENGTH = 512; + +/** + * @private + * @summary HMAC-SHA256 over `${collectionName}\n${_id}\n${version}\n${userId}\n${exp}` + * @returns {Buffer} + */ +const sign = (secret, collectionName, _id, version, userId, exp) => crypto + .createHmac('sha256', secret) + .update(`${collectionName}\n${_id}\n${version}\n${userId}\n${exp}`) + .digest(); + +/** + * @function createDownloadToken + * @param {string} secret - `downloadTokenSecret` + * @param {Object} opts + * @param {string} opts.collectionName - Collection the token opens + * @param {string} opts._id - File `_id` + * @param {string} [opts.version='original'] - File version + * @param {string|null} [opts.userId=null] - User the request acts as + * @param {number} opts.exp - Expiry, Unix time in seconds + * @summary Returns `..` + * @returns {string} + */ +export const createDownloadToken = (secret, { collectionName, _id, version = 'original', userId = null, exp }) => { + const uid = userId ?? ''; + return `${exp}.${Buffer.from(uid, 'utf8').toString('base64url')}.${sign(secret, collectionName, _id, version, uid, exp).toString('base64url')}`; +}; + +/** + * @function verifyDownloadToken + * @param {string} secret - `downloadTokenSecret` + * @param {*} token - Value of the `token` query parameter + * @param {Object} target + * @param {string} target.collectionName - Collection that serves the request + * @param {string} target._id - Requested file `_id` + * @param {string} target.version - Requested version + * @param {number} [target.now=Date.now()] - Current time in milliseconds + * @summary Checks format, expiry, and HMAC with `timingSafeEqual` + * @returns {{userId: string|null, exp: number}|null} `null` for any invalid token + */ +export const verifyDownloadToken = (secret, token, { collectionName, _id, version, now = Date.now() }) => { + if (typeof token !== 'string' || token.length > MAX_TOKEN_LENGTH) { + return null; + } + + const parts = token.split('.'); + // Canonical form only: no leading zeros, an unpadded 43-character signature + if (parts.length !== 3 || !/^[1-9]\d{0,11}$/.test(parts[0]) || !/^[A-Za-z0-9_-]{43}$/.test(parts[2])) { + return null; + } + + const exp = Number(parts[0]); + if (exp * 1000 <= now) { + return null; + } + + const userId = Buffer.from(parts[1], 'base64url').toString('utf8'); + if (Buffer.from(userId, 'utf8').toString('base64url') !== parts[1]) { + return null; + } + + const expected = sign(secret, collectionName, _id, version, userId, exp); + const given = Buffer.from(parts[2], 'base64url'); + // The last character has 2 unused bits, reject signatures that set them + if (given.toString('base64url') !== parts[2]) { + return null; + } + if (given.length !== expected.length || !crypto.timingSafeEqual(given, expected)) { + return null; + } + return { userId: userId || null, exp }; +}; diff --git a/eslint.config.mjs b/eslint.config.mjs index 1715800f..bcf885e7 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -220,7 +220,12 @@ export default [ }, // server code and tests { - files: ['server.js', 'write-stream.js', 'tests/**/*.js'], + files: ['server.js', 'write-stream.js', 'mime.js', 'download-token.js', 'storage.js', 'tests/**/*.js'], languageOptions: languageOptions({ ...globals.es2021, ...globals.node }), }, + // browser tests + { + files: ['tests/client.js'], + languageOptions: languageOptions({ ...globals.es2021, ...globals.browser }), + }, ]; diff --git a/index.d.ts b/index.d.ts index 1a628dbe..f3dcff99 100644 --- a/index.d.ts +++ b/index.d.ts @@ -5,6 +5,7 @@ import type { CountDocumentsOptions, EstimatedDocumentCountOptions } from 'mongo import type { ReactiveVar } from 'meteor/reactive-var'; import type SimpleSchema from 'simpl-schema'; import type * as http from 'node:http'; +import type { Readable } from 'node:stream'; import type { IncomingMessage } from 'connect'; import type { DDP } from 'meteor/ddp'; import type { Tracker } from 'meteor/tracker'; @@ -22,6 +23,8 @@ export interface ContextHTTP { request: IncomingMessage; response: http.ServerResponse; params: ParamsHTTP; + /** Result of the `?token=` check: the token data, `null` without a token or secret, `false` for an invalid token. */ + downloadToken?: { userId: string | null; exp: number } | null | false; } export interface ContextUser { @@ -106,7 +109,7 @@ export class WriteStream { * @param file - An object containing file properties such as `size` and `chunkSize`. * @param permissions - The file permissions (number, e.g. `0o644`) to use when creating the file. * @param parentDirPermissions - Permissions (number, e.g. `0o755`) of created parent directories. - * @param options - `exclusive` creates a new file and fails with 409 when it exists. `identity` is the expected `{dev, ino, birth}` of an existing file (`birth` is the birth time in nanoseconds, optional). `idleTimeout` closes the handle after this many ms without writes. `fileId` is part of the handle cache key. `onAbort` runs after `abort()`. + * @param options - `exclusive` creates a new file and fails with 409 when it exists. `identity` is the expected `{dev, ino, birth}` of an existing file (`birth` is the birth time in nanoseconds, optional). `idleTimeout` closes the handle after this many ms without writes. `fileId` is part of the handle cache key. `onAbort` runs after `abort()`. `writtenChunkIds` lists chunks already on disk when resuming. `onChunkWritten` runs after a chunk is on disk, the chunk counts as written only when it resolves `true`. */ constructor( path: string, @@ -120,6 +123,10 @@ export class WriteStream { idleTimeout?: number; fileId?: string; onAbort?: (stream: WriteStream) => unknown; + /** Chunk ids already on disk, used when resuming. */ + writtenChunkIds?: number[]; + /** Called after a chunk is on disk. The chunk counts as written only when it resolves `true`. */ + onChunkWritten?: (chunkId: number) => Promise; } ); @@ -164,6 +171,64 @@ export class WriteStream { stop(isAborted?: boolean): Promise; } +export interface LinkOptions { + /** Token from the server `createDownloadToken()`, appended as `?token=`. */ + token?: string; +} + +export interface DownloadTokenOptions { + version?: string; + userId?: string | null; + /** Seconds, a positive integer. Default: 3600. */ + expiresIn?: number; +} + +/** Byte range for `createReadStream()`, `end` inclusive. */ +export interface StorageRange { + start?: number; + end?: number; +} + +/** + * Origin of the local file passed to `put()`. The package created `upload`, `write` (`writeAsync()`), and `load` (`loadAsync()`) files. + * `addFile` is the caller's own file: an adapter must not delete it. + */ +export type StorageSource = 'upload' | 'write' | 'load' | 'addFile'; + +export interface StoragePutOptions { + source: StorageSource; +} + +/** Where finished files live. Methods get the file document and a version name, and read `versions[versionName]`. */ +export interface FilesStorageAdapter { + /** Called after a file is complete on local disk, before the insert and `onAfterUpload`. An object result is stored at `versions[versionName].meta.storage`. */ + put(fileRef: FileObj, versionName: string, localPath: string, opts?: StoragePutOptions): Promise | void>; + createReadStream(fileRef: FileObj, versionName: string, range?: StorageRange): Promise; + remove(fileRef: FileObj, versionName: string): Promise; + /** Optional. `null` when the stored file is missing. Enables the `404` and `integrityCheck` before streaming. */ + stat?(fileRef: FileObj, versionName: string): Promise<{ size: number } | null>; +} + +/** Default adapter: files stay at `versions[versionName].path`. Server only: the client build does not export it, so create it in server code or behind `Meteor.isServer`. */ +export class FSStorage implements FilesStorageAdapter { + readonly name: 'fs'; + put(fileRef: FileObj, versionName: string, localPath: string, opts?: StoragePutOptions): Promise; + createReadStream(fileRef: FileObj, versionName: string, range?: StorageRange): Promise; + remove(fileRef: FileObj, versionName: string): Promise; + stat(fileRef: FileObj, versionName: string): Promise<{ size: number } | null>; +} + +/** Copies finished files into a GridFS bucket of the app database and deletes the local copy, except when `source` is `'addFile'`. Server only: the client build does not export it, so create it in server code or behind `Meteor.isServer`. */ +export class GridFSStorage implements FilesStorageAdapter { + constructor(opts?: { bucketName?: string; chunkSizeBytes?: number; db?: unknown }); + readonly name: 'gridfs'; + readonly bucketName: string; + put(fileRef: FileObj, versionName: string, localPath: string, opts?: StoragePutOptions): Promise<{ name: 'gridfs'; bucketName: string; id: string }>; + createReadStream(fileRef: FileObj, versionName: string, range?: StorageRange): Promise; + remove(fileRef: FileObj, versionName: string): Promise; + stat(fileRef: FileObj, versionName: string): Promise<{ size: number } | null>; +} + /** * Core class for FilesCollection. Most other classes extend and build on this one. */ @@ -251,15 +316,6 @@ export class FilesCollectionCore extends EventEmitter { */ findOneAsync(selector?: MeteorFilesSelector, options?: MeteorFilesOptions): Promise<(FileCursor & FileObj) | null>; - /** - * Find and return a FileCursor for a matching document (client only). - * @param selector - Mongo-style selector. - * @param options - Mongo query options. - * @returns {FileCursor | null} The FileCursor instance or null if not found. - * @throws {Meteor.Error} If called on the server. - */ - findOne(selector?: MeteorFilesSelector, options?: MeteorFilesOptions): (FileCursor & FileObj) | null; - /** * Find and return a FilesCursor for matching documents. * @param selector - Mongo-style selector. @@ -304,7 +360,18 @@ export class FilesCollectionCore extends EventEmitter { * @param uriBase - Optional URI base. * @returns {string} The download URL, or an empty string if the file is invalid. */ - link(fileRef: Partial | FileCursor | null | undefined, version?: string, uriBase?: string): string; + link(fileRef: Partial | FileCursor | null | undefined, version?: string, uriBase?: string, opts?: LinkOptions): string; +} + +/** + * Argument of `namingFunction`. On upload Start `file` holds the uploader's `name`, `type`, `size`, and `meta` (not verified) + * plus the server-computed `extension`, `ext`, `_id`, and `userId`. In `writeAsync()` and `loadAsync()` it holds `name`, `type`, and `meta` + * from the call options, and only `writeAsync()` adds `size`. + */ +export interface NamingContext { + file: Partial & { name?: string; type?: string; size?: number; meta?: MetadataType }; + fileId: string; + userId: string | null; } export interface FilesCollectionConfig { @@ -315,22 +382,26 @@ export interface FilesCollectionConfig { continueUploadTTL?: number; /** [Client] custom DDP connection. */ ddp?: DDP.DDPStatic; + /** Default `Cache-Control` header. Default: `private, max-age=31536000` when `protected` is set, otherwise `public, max-age=31536000, s-maxage=31536000`. */ cacheControl?: string; - responseHeaders?: { [x: string]: string } | ((responseCode?: string, fileObj?: FileObj, versionRef?: Version, version?: string) => { [x: string]: string }); + responseHeaders?: { [x: string]: string } | ((responseCode?: string, fileObj?: FileObj, versionRef?: Version, version?: string) => { [x: string]: string } | Promise<{ [x: string]: string }>); /** @deprecated No effect. */ throttle?: number | boolean; downloadRoute?: string; schema?: SimpleSchema | Record; chunkSize?: number | 'dynamic'; - namingFunction?: (fileData: FileData) => MaybePromise; + /** [Server] Name on disk, without extension. The client ignores this option and warns. */ + namingFunction?: (this: FilesCollection, context: NamingContext) => MaybePromise; permissions?: number; parentDirPermissions?: number; integrityCheck?: boolean; strict?: boolean; /** [Server] Called before file download. Return `false` to deny. */ downloadCallback?: (this: FilesCollection, http: ContextHTTP & ContextUser, fileObj: FileObj) => MaybePromise; + /** [Server] A function that returns `true` to allow the download, or an HTTP status. `true` is deprecated: it allows any logged-in user. */ protected?: boolean | ((this: ContextHTTP & ContextUser, fileObj: FileObj) => MaybePromise); public?: boolean; + /** `fileData.type` is the type the uploader sent and is not verified. */ onBeforeUpload?: (this: ContextUpload & ContextUser, fileData: FileData) => MaybePromise; onBeforeRemove?: (this: ContextUser, cursor: FilesCursor) => MaybePromise; onInitiateUpload?: (this: ContextUpload & ContextUser, fileData: FileData) => MaybePromise; @@ -338,6 +409,7 @@ export interface FilesCollectionConfig { onAfterRemove?: (files: ReadonlyArray) => MaybePromise; /** [Client] Message shown when closing the tab during upload. */ onbeforeunloadMessage?: string | ((this: FileUpload, fileData: FileData) => string); + /** Allow clients to call `remove()` and `removeAsync()` with an `_id`. Set `onBeforeRemove` too. Default: `false`. */ allowClientCode?: boolean; debug?: boolean | ((...args: unknown[]) => void); /** [Server] Serve the file from a custom source. Return `true` when the request is handled. */ @@ -356,8 +428,14 @@ export interface FilesCollectionConfig { /** [Client] Do not set the `x_mtok` cookie. */ disableSetTokenCookie?: boolean; sanitize?: (str: string, max?: number, replacement?: string) => string; - /** [Server] Send `X-Content-Type-Options: nosniff`. Default: `false`. */ + /** [Server] Send `X-Content-Type-Options: nosniff`. Default: `true`. */ nosniff?: boolean; + /** [Server] Store the type the uploader sent instead of the type detected from the file content. Default: `false`. */ + trustClientMimeType?: boolean; + /** [Server] HMAC secret for signed download links, at least 32 characters. Without it `?token=` is ignored. */ + downloadTokenSecret?: string; + /** [Server] Storage adapter. Default: `new FSStorage()`. */ + storage?: FilesStorageAdapter; /** [Server] Milliseconds before an idle upload file handle is closed. Default: 900000. */ uploadIdleTimeout?: number; _preCollection?: Mongo.Collection<{ _id?: string }>; @@ -501,7 +579,7 @@ export class UploadInstance extends EventEmitter { /** @internal `true` when a Start request failed without a response, so the server may have the upload. */ startMaybeReceived: boolean; /** @internal */ - startOpts?: { file: FileData; fileId: string; chunkSize: number; fileLength: number; FSName?: string }; + startOpts?: { file: FileData; fileId: string; chunkSize: number; fileLength: number }; /** @internal The one request in flight. */ inFlight: { kind: 'start' | 'eof' } | { kind: 'chunk'; chunkId: number } | null; /** @internal */ @@ -513,7 +591,6 @@ export class UploadInstance extends EventEmitter { /** @internal */ hasTimers: boolean; fileId: string; - FSName: string; pipes: Array<(data: string) => string>; fileData: FileData; result: FileUpload; @@ -555,7 +632,7 @@ export class UploadInstance extends EventEmitter { _prepare(): Promise; /** @internal */ _setup(): void; - /** Pipes run in reverse order of registration. */ + /** Pipes run in the order they were added. */ pipe(func: (data: string) => string): this; start(): Promise; manual(): FileUpload; @@ -572,7 +649,7 @@ export class FileCursor { /** Client only, throws on server. */ remove(callback?: (error: Meteor.Error | null, count?: number) => void): FileCursor; removeAsync(): Promise; - link(version?: string, uriBase?: string): string; + link(version?: string, uriBase?: string, opts?: LinkOptions): string; get(): FileObj; get(property: K): FileObj[K]; get(property: string): unknown; @@ -599,8 +676,6 @@ export class FilesCursor { /** Synchronous methods (`get`, `fetch`, `next`, `each`, `map`, and others) work on client only and throw `Meteor.Error` on server; use `*Async` on server. */ get(): FileObj[]; getAsync(): Promise; - /** @deprecated Client only. Prefer `hasNextAsync()`. */ - hasNext(): boolean; hasNextAsync(): Promise; next(): FileObj | undefined; nextAsync(): Promise; @@ -616,8 +691,6 @@ export class FilesCursor { lastAsync(): Promise; /** @deprecated Use `countDocuments()`. */ count(): number; - /** @deprecated Use `countDocuments()`. */ - countAsync(): Promise; countDocuments(options?: CountDocumentsOptions): Promise; /** Client only. */ remove(callback?: (error: Meteor.Error | null, count?: number) => void): FilesCursor; @@ -644,6 +717,11 @@ export class FilesCollection extends FilesCollectionCore { // Client/Browser-specific overloads for FilesCollection // -------------------------------------------------------------------------- export interface FilesCollection { + /** + * Finds a document and wraps it in a FileCursor. Client only, throws `Meteor.Error(404)` on the server. + */ + findOne(selector?: MeteorFilesSelector, options?: MeteorFilesOptions): (FileCursor & FileObj) | null; + /** * Inserts a file into the collection and returns an instance of FileUpload/UploadInstance. * @param config - The insert options. @@ -663,16 +741,17 @@ export interface FilesCollection { insertAsync(config: InsertOptions, autoStart?: boolean): Promise; /** - * Removes files/documents from the collection. Client only, throws on server. - * @param selector - A Mongo-style selector. + * Removes one file from the collection. Client only, throws on server. + * @param _id - `_id` of the file to remove. * @param callback - Optional callback function. */ - remove(selector?: MeteorFilesSelector, callback?: (error: Meteor.Error | null, count?: number) => void): FilesCollection; + remove(_id: string, callback?: (error: Meteor.Error | null, count?: number) => void): FilesCollection; /** * Asynchronously removes files/documents from the collection. - * On client rejects with `Meteor.Error(401)` when `allowClientCode` is `false`. - * @param selector - A Mongo-style selector. + * On the client accepts only a String `_id` and rejects with `Meteor.Error(401)` when `allowClientCode` is `false`. + * On the server accepts any selector. + * @param selector - A Mongo-style selector. On the client, a String `_id`. */ removeAsync(selector?: MeteorFilesSelector): Promise; @@ -713,6 +792,11 @@ export interface LoadOpts { // Server-specific overloads for FilesCollection // -------------------------------------------------------------------------- export interface FilesCollection { + /** Storage adapter, `FSStorage` unless the `storage` option is set. */ + storage: FilesStorageAdapter; + + /** Signed token for `link(file, version, uriBase, { token })`, valid for one file version in this collection. Needs `downloadTokenSecret`. */ + createDownloadToken(fileRef: Partial | FileCursor | string, opts?: DownloadTokenOptions): string; /** * Downloads a file by preparing HTTP response and piping file data. @@ -740,7 +824,7 @@ export interface FilesCollection { readableStream?: NodeJS.ReadableStream | null, _responseType?: string, force200?: boolean - ): void; + ): Promise; /** * Adds an existing file on disk to the FilesCollection. diff --git a/index.test-d.ts b/index.test-d.ts index 1766e4ab..b1229cde 100644 --- a/index.test-d.ts +++ b/index.test-d.ts @@ -1,6 +1,7 @@ import { expectType, expectError, expectAssignable } from 'tsd'; import type { Meteor } from 'meteor/meteor'; import type { ReactiveVar } from 'meteor/reactive-var'; +import type { Readable } from 'node:stream'; import { FilesCollection, FileUpload, @@ -12,6 +13,9 @@ import { InsertOptions, ContextUser, ContextHTTP, + FSStorage, + GridFSStorage, + FilesStorageAdapter, } from 'meteor/ostrio:files'; // config options, including 3.1 additions @@ -22,6 +26,8 @@ const config: FilesCollectionConfig = { chunkSize: 'dynamic', allowedCordovaOrigins: /^https:\/\/localhost:12[0-9]{3}$/, nosniff: true, + trustClientMimeType: false, + downloadTokenSecret: 'x'.repeat(32), uploadIdleTimeout: 900000, allowedOrigins: false, disableUpload: false, @@ -37,8 +43,10 @@ const config: FilesCollectionConfig = { expectType(http.params._id); return false; }, - async namingFunction(fileData) { - return fileData.name; + async namingFunction({ file, fileId, userId }) { + expectType(fileId); + expectType(userId); + return file.name; }, async protected(fileObj) { expectType(fileObj); @@ -76,6 +84,8 @@ expectAssignable({ allowedCordovaOrigins: true }); expectAssignable({ allowedCordovaOrigins: 'https://example.com' }); expectError({ allowedCordovaOrigins: 1 }); expectError({ nosniff: 'yes' }); +expectError({ trustClientMimeType: 'yes' }); +expectError({ downloadTokenSecret: 32 }); expectError({ uploadIdleTimeout: '900000' }); const files = new FilesCollection(config); @@ -133,6 +143,8 @@ expectType(cursor.map((file) => file.name)); expectType>(cursor.eachAsync()); expectType>(cursor.hasNextAsync()); expectType>(cursor.lastAsync()); +expectError(cursor.countAsync()); +expectError(cursor.hasNext()); declare const fileCursor: FileCursor; expectType(fileCursor.link('original')); @@ -146,6 +158,8 @@ async function findFile() { expectType(file._id); expectType(file.link()); expectType(files.link(file)); + expectType(files.link(file, 'original', undefined, { token: files.createDownloadToken(file, { userId: null, expiresIn: 60 }) })); + expectType(file.link('original', undefined, { token: 't' })); } expectType>(files.find({})); expectType>(files.countDocuments({})); @@ -167,3 +181,18 @@ async function serverMethods() { files.deny({ remove: () => true }); } void serverMethods; + +// storage adapters +expectAssignable({ storage: new GridFSStorage({ bucketName: 'files' }) }); +expectAssignable({ storage: new FSStorage() }); +declare const someStream: Readable; +const customStorage: FilesStorageAdapter = { + async put(_fileRef, _versionName, _localPath, opts) { + expectType<'upload' | 'write' | 'load' | 'addFile' | undefined>(opts?.source); + return { key: 'k' }; + }, + async createReadStream() { return someStream; }, + async remove() {}, +}; +expectAssignable({ storage: customStorage }); +expectError({ storage: 'fs' }); diff --git a/lib.js b/lib.js index 04f3f14e..f0e2f79a 100644 --- a/lib.js +++ b/lib.js @@ -161,6 +161,37 @@ const helpers = { } }; +/** + * @const {number} MAX_UPLOAD_CHUNKS - Most chunks one upload may have. Bounds the chunk bit set stored in the upload record + */ +const MAX_UPLOAD_CHUNKS = 100000; + +/** + * @function fitChunkSize + * @param {number} total - File size in bytes, or base64 length for base64 uploads + * @param {number} chunkSize - Chunk size the upload would use + * @param {number} step - Chunk size must stay a multiple of it (8 for files, 4 for base64) + * @param {number} maxChunkSize - Largest accepted chunk size + * @param {number} [maxChunks=MAX_UPLOAD_CHUNKS] - Most chunks allowed + * @summary Raise `chunkSize` so the upload has at most `maxChunks` chunks + * @returns {number} + */ +const fitChunkSize = (total, chunkSize, step, maxChunkSize, maxChunks = MAX_UPLOAD_CHUNKS) => { + if (Math.ceil(total / chunkSize) <= maxChunks) { + return chunkSize; + } + return Math.min(maxChunkSize, Math.ceil(total / maxChunks / step) * step); +}; + +/** + * @function applyPipes + * @param {Array} pipes - Transform functions, in the order they were added + * @param {string} data - Base64 chunk + * @summary Run upload pipes in the order they were added: the first `pipe()` call runs first + * @returns {string} + */ +const applyPipes = (pipes, data) => pipes.reduce((value, pipe) => pipe(value), data); + /** * @const {function} fixJSONParse - Fix issue with Date parse * @summary Revive `=--JSON-DATE--=` strings into `Date` objects in place. Walks own keys only and skips `__proto__`, `constructor`, and `prototype` @@ -323,4 +354,4 @@ const formatFileURL = (_fileRef, version = 'original', _uriBase = (__meteor_runt return `${_root}${route}/${collectionName}/${_id}/${_version}/${_id}${ext}`; }; -export { fixJSONParse, fixJSONStringify, formatFileURL, helpers }; +export { MAX_UPLOAD_CHUNKS, applyPipes, fitChunkSize, fixJSONParse, fixJSONStringify, formatFileURL, helpers }; diff --git a/mime.js b/mime.js new file mode 100644 index 00000000..75ed472e --- /dev/null +++ b/mime.js @@ -0,0 +1,262 @@ +import fs from 'node:fs'; + +/** + * @const {number} SNIFF_BYTES - Bytes read from the start of a file to detect its type + */ +export const SNIFF_BYTES = 4100; + +/** + * @const {RegExp} MIME_RE - A lowercase `type/subtype` made of RFC 6838 token characters + */ +const MIME_RE = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/; + +/** + * @const {Object} FTYP_BRANDS - ISO base media `ftyp` major brands + */ +const FTYP_BRANDS = { + avif: 'image/avif', + avis: 'image/avif', + heic: 'image/heic', + heix: 'image/heic', + heim: 'image/heic', + heis: 'image/heic', + hevc: 'image/heic', + hevx: 'image/heic', + mif1: 'image/heif', + msf1: 'image/heif', + 'M4A ': 'audio/mp4', + 'M4B ': 'audio/mp4', + 'F4A ': 'audio/mp4', + 'qt ': 'video/quicktime', + isom: 'video/mp4', + iso2: 'video/mp4', + iso4: 'video/mp4', + iso5: 'video/mp4', + iso6: 'video/mp4', + mp41: 'video/mp4', + mp42: 'video/mp4', + avc1: 'video/mp4', + dash: 'video/mp4', + mmp4: 'video/mp4', + 'M4V ': 'video/mp4', + M4VH: 'video/mp4', + M4VP: 'video/mp4', + 'f4v ': 'video/mp4', + 'F4V ': 'video/mp4', +}; + +/** + * @const {Object} REFINEMENTS - Client types that name a specific format inside a detected container + */ +const REFINEMENTS = { + 'application/zip': /^application\/(?:vnd\.openxmlformats-officedocument\.[a-z0-9.+-]+|vnd\.oasis\.opendocument\.[a-z0-9.+-]+|epub\+zip|java-archive|vnd\.android\.package-archive)$/, + 'application/x-cfb': /^application\/(?:msword|vnd\.ms-[a-z0-9.+-]+|vnd\.visio)$/, + 'video/mp4': /^(?:audio\/mp4|audio\/x-m4a|video\/x-m4v)$/, +}; + +/** + * @const {Set} TEXT_TYPES - Client types kept for UTF-8 text. Active types such as `text/html`, `text/xml`, `text/css`, and `text/javascript` are not listed + */ +const TEXT_TYPES = new Set(['text/plain', 'text/csv', 'text/markdown', 'text/tab-separated-values', 'text/calendar', 'text/vtt', 'application/json']); + +/** + * @private + * @summary Accepts a Buffer or any typed array view, returns a Buffer over the same memory, or `null` + */ +const toBuffer = (input) => { + if (Buffer.isBuffer(input)) { + return input; + } + if (ArrayBuffer.isView(input)) { + return Buffer.from(input.buffer, input.byteOffset, input.byteLength); + } + return null; +}; + +const hasBytes = (buf, offset, list) => { + if (buf.length < offset + list.length) { + return false; + } + for (let i = 0; i < list.length; i++) { + if (buf[offset + i] !== list[i]) { + return false; + } + } + return true; +}; + +const hasText = (buf, offset, text) => hasBytes(buf, offset, Array.from(text, (char) => char.charCodeAt(0))); + +/** + * @private + * @summary MPEG audio frame header: sync bits set, version and layer not reserved (layer `00` is AAC ADTS), valid bitrate and sample rate + */ +const isMpegAudioFrame = (buf) => buf.length >= 4 + && buf[0] === 0xff + && (buf[1] & 0xe0) === 0xe0 + && ((buf[1] >> 3) & 0x03) !== 0x01 + && ((buf[1] >> 1) & 0x03) !== 0x00 + && (buf[2] >> 4) !== 0x0f + && ((buf[2] >> 2) & 0x03) !== 0x03; + +/** + * @function detectMimeType + * @param {Buffer|Uint8Array} input - First bytes of a file + * @summary Match the built-in signature table + * @returns {string|null} Detected type, or `null` when no signature matches + */ +export const detectMimeType = (input) => { + const buf = toBuffer(input); + if (!buf || buf.length < 2) { + return null; + } + + if (hasBytes(buf, 0, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) { + return 'image/png'; + } + if (hasBytes(buf, 0, [0xff, 0xd8, 0xff])) { + return 'image/jpeg'; + } + if (hasText(buf, 0, 'GIF87a') || hasText(buf, 0, 'GIF89a')) { + return 'image/gif'; + } + if (hasText(buf, 0, 'RIFF')) { + if (hasText(buf, 8, 'WEBP')) { + return 'image/webp'; + } + if (hasText(buf, 8, 'WAVE')) { + return 'audio/wav'; + } + if (hasText(buf, 8, 'AVI ')) { + return 'video/x-msvideo'; + } + } + if (hasText(buf, 0, 'BM') && hasBytes(buf, 6, [0, 0, 0, 0])) { + return 'image/bmp'; + } + if (hasBytes(buf, 0, [0x00, 0x00, 0x01, 0x00]) && buf.length >= 6 && (buf[4] | (buf[5] << 8)) > 0) { + return 'image/x-icon'; + } + if (hasBytes(buf, 0, [0x49, 0x49, 0x2a, 0x00]) || hasBytes(buf, 0, [0x4d, 0x4d, 0x00, 0x2a])) { + return 'image/tiff'; + } + if (buf.length >= 12 && hasText(buf, 4, 'ftyp')) { + return FTYP_BRANDS[buf.toString('latin1', 8, 12)] || null; + } + if (hasText(buf, 0, '%PDF-')) { + return 'application/pdf'; + } + if (hasBytes(buf, 0, [0x50, 0x4b]) && (hasBytes(buf, 2, [0x03, 0x04]) || hasBytes(buf, 2, [0x05, 0x06]) || hasBytes(buf, 2, [0x07, 0x08]))) { + return 'application/zip'; + } + if (hasBytes(buf, 0, [0x1f, 0x8b, 0x08])) { + return 'application/gzip'; + } + if (hasBytes(buf, 0, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c])) { + return 'application/x-7z-compressed'; + } + if (hasText(buf, 0, 'Rar!') && hasBytes(buf, 4, [0x1a, 0x07]) && (buf[6] === 0x00 || (buf[6] === 0x01 && buf[7] === 0x00))) { + return 'application/vnd.rar'; + } + if (hasBytes(buf, 0, [0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1])) { + return 'application/x-cfb'; + } + if (hasBytes(buf, 0, [0x1a, 0x45, 0xdf, 0xa3])) { + // The EBML DocType sits in the first bytes of the header + return buf.toString('latin1', 0, Math.min(buf.length, 64)).includes('webm') ? 'video/webm' : 'video/x-matroska'; + } + if (hasText(buf, 0, 'ID3') || isMpegAudioFrame(buf)) { + return 'audio/mpeg'; + } + if (hasText(buf, 0, 'OggS')) { + return buf.toString('latin1').includes('\x80theora') ? 'video/ogg' : 'audio/ogg'; + } + if (hasText(buf, 0, 'fLaC')) { + return 'audio/flac'; + } + if (hasBytes(buf, 0, [0x00, 0x61, 0x73, 0x6d])) { + return 'application/wasm'; + } + if (hasText(buf, 0, 'wOFF')) { + return 'font/woff'; + } + if (hasText(buf, 0, 'wOF2')) { + return 'font/woff2'; + } + return null; +}; + +/** + * @function baseMimeType + * @param {*} type - Mime type, possibly with parameters + * @summary Lowercase `type/subtype` without parameters, or an empty string when it is not a valid token + * @returns {string} + */ +export const baseMimeType = (type) => { + if (typeof type !== 'string') { + return ''; + } + const base = type.split(';')[0].trim().toLowerCase(); + return MIME_RE.test(base) ? base : ''; +}; + +/** + * @function isUtf8Text + * @param {Buffer|Uint8Array} input - First bytes of a file + * @param {boolean} isComplete - `false` when `input` is a prefix of a longer file, so a multibyte character cut at the end is allowed + * @summary Valid UTF-8 without NUL bytes. Empty input is not text + * @returns {boolean} + */ +export const isUtf8Text = (input, isComplete) => { + const buf = toBuffer(input); + if (!buf || !buf.length || buf.includes(0)) { + return false; + } + + try { + new TextDecoder('utf-8', { fatal: true }).decode(buf, { stream: !isComplete }); + return true; + } catch (_decodeError) { + return false; + } +}; + +/** + * @function resolveMimeType + * @param {Buffer|Uint8Array} input - First bytes of a file, up to `SNIFF_BYTES` + * @param {string} [clientType] - Type the uploader sent, not trusted + * @param {boolean} [isComplete=true] - `false` when `input` is a prefix of a longer file + * @summary Type to store: a detected signature (refined by the client type for known containers), `text/plain` or an allowlisted client type (`text/plain`, `text/csv`, `text/markdown`, `text/tab-separated-values`, `text/calendar`, `text/vtt`, `application/json`) for UTF-8 text, otherwise `application/octet-stream` + * @returns {string} + */ +export const resolveMimeType = (input, clientType, isComplete = true) => { + const client = baseMimeType(clientType); + const detected = detectMimeType(input); + if (detected) { + return (REFINEMENTS[detected] && !client.endsWith('+xml') && REFINEMENTS[detected].test(client)) ? client : detected; + } + + if (isUtf8Text(input, isComplete)) { + return TEXT_TYPES.has(client) ? client : 'text/plain'; + } + return 'application/octet-stream'; +}; + +/** + * @function sniffFile + * @param {string} path - File on disk + * @param {string} [clientType] - Type the uploader sent, not trusted + * @summary Read the first `SNIFF_BYTES` bytes of `path` and resolve its type + * @returns {Promise} + */ +export const sniffFile = async (path, clientType) => { + const fh = await fs.promises.open(path, 'r'); + try { + const buf = Buffer.alloc(SNIFF_BYTES); + const { bytesRead } = await fh.read(buf, 0, SNIFF_BYTES, 0); + const { size } = await fh.stat(); + return resolveMimeType(buf.subarray(0, bytesRead), clientType, size <= bytesRead); + } finally { + await fh.close(); + } +}; diff --git a/package-lock.json b/package-lock.json index 7801444e..d632eff8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ostrio-meteor-files", - "version": "3.1.0", + "version": "4.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ostrio-meteor-files", - "version": "3.1.0", + "version": "4.0.0", "devDependencies": { "@babel/core": "^7.29.0", "@babel/eslint-parser": "^7.28.0", @@ -18,6 +18,7 @@ "eventemitter3": "^5.0.4", "globals": "^17.0.0", "mongodb": "^6.21.0", + "playwright": "^1.63.0", "simpl-schema": "^3.4.7", "sinon": "^22.0.0", "tsd": "^0.33.0", @@ -3164,6 +3165,35 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz", + "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.63.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright-core": { + "version": "1.63.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.63.0.tgz", + "integrity": "sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/plur": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/plur/-/plur-4.0.0.tgz", diff --git a/package.js b/package.js index bc2aa6da..d39185ab 100755 --- a/package.js +++ b/package.js @@ -1,6 +1,6 @@ Package.describe({ name: 'ostrio:files', - version: '3.1.0', + version: '4.0.0', summary: 'Upload files to a server or 3rd party storage: AWS:S3, GridFS, DropBox, and other', git: 'https://github.com/veliovgroup/Meteor-Files', documentation: 'README.md' @@ -30,6 +30,7 @@ Package.onTest((api) => { api.use('meteortesting:mocha@3.4.0'); api.use(['ecmascript', 'ostrio:files'], ['client', 'server']); api.mainModule('tests/server.js', 'server'); + api.mainModule('tests/client.js', 'client'); Npm.depends({ eventemitter3: '5.0.4', diff --git a/package.json b/package.json index 8a19007f..93a611ed 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,12 @@ { "name": "ostrio-meteor-files", - "version": "3.1.0", + "version": "4.0.0", "private": true, "types": "index.d.ts", "scripts": { "test:mocha": "meteor test-packages ./ --driver-package meteortesting:mocha --once", "test:mocha:watch": "meteor test-packages ./ --driver-package meteortesting:mocha", + "test:browser": "TEST_BROWSER_DRIVER=playwright meteor test-packages ./ --driver-package meteortesting:mocha --once", "lint": "eslint .", "lint:fix": "eslint . --fix", "typecheck": "tsc --noEmit -p . && tsd" @@ -21,6 +22,7 @@ "eventemitter3": "^5.0.4", "globals": "^17.0.0", "mongodb": "^6.21.0", + "playwright": "^1.63.0", "simpl-schema": "^3.4.7", "sinon": "^22.0.0", "tsd": "^0.33.0", diff --git a/server.js b/server.js index 94788a32..d26225e9 100644 --- a/server.js +++ b/server.js @@ -6,9 +6,12 @@ import { Random } from 'meteor/random'; import { Cookies } from 'meteor/ostrio:cookies'; import { check, Match } from 'meteor/check'; -import WriteStream, { fileIdentity, isSameFile } from './write-stream.js'; +import WriteStream, { chunkIdsFromBits, fileIdentity, isSameFile } from './write-stream.js'; import FilesCollectionCore from './core.js'; -import { fixJSONParse, fixJSONStringify, helpers } from './lib.js'; +import { MAX_UPLOAD_CHUNKS, fixJSONParse, fixJSONStringify, helpers } from './lib.js'; +import { SNIFF_BYTES, resolveMimeType, sniffFile } from './mime.js'; +import { MIN_SECRET_LENGTH, createDownloadToken as signDownloadToken, verifyDownloadToken } from './download-token.js'; +import { FSStorage, GridFSStorage, isStorageAdapter } from './storage.js'; import fs from 'node:fs'; import nodeQs from 'node:querystring'; @@ -54,9 +57,28 @@ const withCharset = (type) => { }; /** - * @const {string[]} RESERVED_FILE_KEYS - Keys of client-supplied `file` object the server computes itself + * @const {RegExp} INLINE_TYPE_RE - Types served `inline` by default: images except SVG, video, audio, PDF, and plain text */ -const RESERVED_FILE_KEYS = ['_id', 'fileId', '_downloadRoute', '_collectionName', '_storagePath', 'path', 'versions', 'userId', 'public', 'extension', 'ext', 'extensionWithDot', 'isVideo', 'isAudio', 'isImage', 'isText', 'isJSON', 'isPDF', 'mime', 'mime-type', '__proto__', 'constructor', 'prototype']; +const INLINE_TYPE_RE = /^(?:image\/(?!svg\+xml$)[a-z0-9.+-]+|video\/[a-z0-9.+-]+|audio\/[a-z0-9.+-]+|application\/pdf|text\/plain)$/; + +/** + * @function dispositionType + * @param {string} [type] - Mime type of the served version, parameters such as `charset` are ignored + * @param {boolean} forceDownload - `true` for `?download=true` + * @summary Returns `inline` for types a browser shows safely, `attachment` for everything else + * @returns {'inline'|'attachment'} + */ +const dispositionType = (type, forceDownload) => { + if (forceDownload || !helpers.isString(type)) { + return 'attachment'; + } + return INLINE_TYPE_RE.test(type.split(';')[0].trim().toLowerCase()) ? 'inline' : 'attachment'; +}; + +/** + * @const {string[]} CLIENT_FILE_KEYS - The only keys of a client-supplied `file` object the server keeps. It computes everything else + */ +const CLIENT_FILE_KEYS = ['name', 'type', 'size', 'meta']; /** * Returns `code` if it is an HTTP error status (400-599), otherwise `fallback` @@ -126,7 +148,7 @@ const createIndex = async (_collection, keys, opts) => { * @param config.schema {Object} - [Both] Collection Schema * @param config.public {boolean} - [Both] Store files in folder accessible for proxy servers, for limits, and more - read docs * @param config.strict {boolean} - [Server] Strict mode for partial content. When `true` (default) the server responds `416` to a `Range` that starts outside of the file. When `false` it ignores such `Range` and responds `200` - * @param config.protected {function} - [Server] If `true` - files will be served only to authorized users, if `function()` - you're able to check visitor's permissions in your own way function's context has: + * @param config.protected {boolean|function} - [Server] A function checks access per file. `true` (deprecated, removed in v5) allows any logged-in user. Function context has: * - `request` * - `response` * - `userAsync()` @@ -135,15 +157,18 @@ const createIndex = async (_collection, keys, opts) => { * @param config.permissions {number} - [Server] Permissions which will be set to uploaded files (octal), like: `511` or `0o755`. Default: 0644 * @param config.parentDirPermissions {number} - [Server] Permissions which will be set to parent directory of uploaded files (octal), like: `0o611` or `0o777`. Default: 0755 * @param config.storagePath {string|function} - [Server] Storage path on file system. The function can be async - * @param config.cacheControl {string} - [Server] Default `Cache-Control` header - * @param config.responseHeaders {object|function} - [Server] Custom response headers, if function is passed, must return Object - * @param config.nosniff {boolean} - [Server] Send `X-Content-Type-Options: nosniff` header with served files. Default: `false` + * @param config.cacheControl {string} - [Server] Default `Cache-Control` header. Default: `private, max-age=31536000` for `protected` collections, `public, max-age=31536000, s-maxage=31536000` otherwise + * @param config.responseHeaders {object|function} - [Server] Custom response headers. A function returns an Object, or a Promise that resolves to one + * @param config.nosniff {boolean} - [Server] Send `X-Content-Type-Options: nosniff` header with served files. Default: `true` + * @param config.trustClientMimeType {boolean} - [Server] Store the type the uploader sent. When `false` (default) the stored type comes from the file content + * @param config.downloadTokenSecret {string} - [Server] HMAC secret for signed download links (`createDownloadToken()`), at least 32 characters. Without it `?token=` is ignored + * @param config.storage {FilesStorageAdapter} - [Server] Where finished files live: `put`, `createReadStream`, `remove`, optional `stat`. `put()` gets `{ source }` (`'upload'`, `'write'`, `'load'`, or `'addFile'`) as its 4th argument. Default: `new FSStorage()` * @param config.uploadIdleTimeout {number} - [Server] Close file handle of an upload after this many ms without new chunks, it is reopened on the next chunk. Default: 900000 (15 minutes) * @param config.throttle {number} - [Server] DEPRECATED bps throttle threshold * @param config.downloadRoute {string} - [Both] Server Route used to retrieve files * @param config.collection {Mongo.Collection} - [Both] Mongo Collection Instance * @param config.collectionName {string} - [Both] Collection name - * @param config.namingFunction {function}- [Both] Function which returns `String` + * @param config.namingFunction {function}- [Server] Returns the file name on disk. Called with `{ file, fileId, userId }` on upload Start, in `writeAsync` and `loadAsync`. `file` differs per call: on Start it has the uploader's `name`, `type`, `size`, `meta` plus server-computed `extension`, `ext`, `_id`, `userId`; `writeAsync` passes `name`, `type`, `size`, `meta`; `loadAsync` passes `name`, `type`, `meta` (no `size`) * @param config.integrityCheck {boolean} - [Server] Check file's integrity before serving to users * @param config.onAfterUpload {function}- [Server] Called right after file is ready on FS. Use to transfer file somewhere else, or do other thing with file directly * @param config.onAfterRemove {function(fileObj[]): boolean} - [Server] Called with single argument with array of removed `fileObj[]` right after file(s) is removed. Return `true` to intercept `.unlinkAsync` method; return `false` to continue default behavior @@ -154,7 +179,7 @@ const createIndex = async (_collection, keys, opts) => { * @param config.getUser {function} - [Server] Replace default way of recognizing user, useful when you want to auth user based on custom cookie (or other way). arguments {http: {request: {...}, response: {...}}}, need to return {userId: String, userAsync: Function} * @param config.onInitiateUpload {function} - [Server] Function which executes on server right before upload is begin and right after `onBeforeUpload` hook. This hook is fully asynchronous. * @param config.onBeforeRemove {function} - [Server] Executes before removing file on server, so you can check permissions. Return `true` to allow physical file removal and `false` to deny. - * @param config.allowClientCode {boolean} - [Both] Allow to run `remove` from client + * @param config.allowClientCode {boolean} - [Both] Allow to run `remove` from client. Default: `false` * @param config.downloadCallback {function} - [Server] Callback triggered each time file is requested, return truthy value to continue download, or falsy to abort * @param config.interceptRequest {function} - [Server] Intercept incoming HTTP request, so you can do whatever you want, no checks or preprocessing, argument: http {request, response, params} * @param config.interceptDownload {function} - [Server] Intercept download request, so you can serve file from third-party resource, arguments {http: {request: {...}, response: {...}}, fileRef: {...}} @@ -172,6 +197,7 @@ class FilesCollection extends FilesCollectionCore { constructor(config) { super(); let storagePath; + let downloadTokenSecret; if (config) { ({ _preCollection: this._preCollection, @@ -190,6 +216,7 @@ class FilesCollection extends FilesCollectionCore { disableUpload: this.disableUpload, downloadCallback: this.downloadCallback, downloadRoute: this.downloadRoute, + downloadTokenSecret, getUser: this.getUser, integrityCheck: this.integrityCheck, interceptDownload: this.interceptDownload, @@ -208,8 +235,10 @@ class FilesCollection extends FilesCollectionCore { responseHeaders: this.responseHeaders, sanitize: this.sanitize, schema: this.schema, + storage: this.storage, storagePath, strict: this.strict, + trustClientMimeType: this.trustClientMimeType, uploadIdleTimeout: this.uploadIdleTimeout, } = config); } @@ -270,7 +299,7 @@ class FilesCollection extends FilesCollectionCore { } if (!helpers.isBoolean(this.allowClientCode)) { - this.allowClientCode = true; + this.allowClientCode = false; } if (!helpers.isFunction(this.onInitiateUpload)) { @@ -302,7 +331,8 @@ class FilesCollection extends FilesCollectionCore { } if (!helpers.isString(this.cacheControl)) { - this.cacheControl = 'public, max-age=31536000, s-maxage=31536000'; + // A shared cache must not hand one user's protected file to another user + this.cacheControl = this.protected ? 'private, max-age=31536000' : 'public, max-age=31536000, s-maxage=31536000'; } if (!helpers.isFunction(this.onAfterUpload)) { @@ -353,7 +383,15 @@ class FilesCollection extends FilesCollectionCore { } if (this.nosniff === void 0) { - this.nosniff = false; + this.nosniff = true; + } + + if (this.trustClientMimeType === void 0) { + this.trustClientMimeType = false; + } + + if (this.storage === void 0) { + this.storage = new FSStorage(); } if (this.uploadIdleTimeout === void 0) { @@ -447,6 +485,9 @@ class FilesCollection extends FilesCollectionCore { check(this.continueUploadTTL, Number); check(this.allowQueryStringCookies, Boolean); check(this.nosniff, Boolean); + check(this.trustClientMimeType, Boolean); + // eslint-disable-next-line new-cap + check(this.storage, Match.Where(isStorageAdapter)); check(this.uploadIdleTimeout, Number); /* eslint-disable new-cap */ check(this.onAfterRemove, Match.OneOf(false, Function)); @@ -460,6 +501,12 @@ class FilesCollection extends FilesCollectionCore { check(this.allowedCordovaOrigins, Match.Optional(Match.OneOf(Boolean, RegExp, String))); /* eslint-enable new-cap */ + /* eslint-disable new-cap */ + check(downloadTokenSecret, Match.Optional(Match.Where((secret) => helpers.isString(secret) && secret.length >= MIN_SECRET_LENGTH))); + /* eslint-enable new-cap */ + // Not enumerable: debug mode logs the collection object + Object.defineProperty(this, 'downloadTokenSecret', { value: downloadTokenSecret || null, enumerable: false, writable: false, configurable: false }); + this._cookies = new Cookies({ allowQueryStringCookies: this.allowQueryStringCookies, allowedCordovaOrigins: this.allowedCordovaOrigins ?? this.allowedOrigins @@ -510,7 +557,8 @@ class FilesCollection extends FilesCollectionCore { self._debug(`[FilesCollection] [_preCollectionCursor.observe] [removeUnfinishedUpload]: ${doc._id}`); await upload.abort(); } else { - await upload.end(); + // Another process finished this upload, its chunks are not in this stream. Close the handle, keep the file + await upload.stop(false); } } delete self._currentUploads[doc._id]; @@ -525,6 +573,12 @@ class FilesCollection extends FilesCollectionCore { fileId: _id, idleTimeout: this.uploadIdleTimeout, identity: opts.fileIdentity, + onChunkWritten: (chunkId) => this._recordChunk(_id, chunkId), + // Chunks of this upload can land on other instances, read their records before giving up at EOF + loadRecordedChunkIds: async () => { + const record = await this._preCollection.findOneAsync({ _id }); + return helpers.isArray(record?.chunkBits) ? chunkIdsFromBits(record.chunkBits, opts.fileLength) : []; + }, onAbort: async () => { // Aborted upload can not be continued, drop its record await this._preCollection.removeAsync({ _id }); @@ -563,14 +617,14 @@ class FilesCollection extends FilesCollectionCore { } checkOwner(contUpld); - if (!helpers.isObject(contUpld.fileIdentity)) { - // Record from an older version, the file can not be verified + if (!helpers.isObject(contUpld.fileIdentity) || !helpers.isArray(contUpld.chunkBits)) { + // Record from an older version: its file or its written chunks can not be verified throw new Meteor.Error(410, 'Upload can not be resumed. Start upload again.'); } let stream; try { - stream = await this._createStream(_id, contUpld.path, contUpld, { exclusive: false }); + stream = await this._createStream(_id, contUpld.path, contUpld, { exclusive: false, writtenChunkIds: chunkIdsFromBits(contUpld.chunkBits, contUpld.fileLength) }); } catch (streamError) { if (streamError?.error === 409 || streamError?.error === 410) { // File is gone or replaced: drop the upload, do not touch the file @@ -626,14 +680,26 @@ class FilesCollection extends FilesCollectionCore { throw new Meteor.Error(500, `[FilesCollection.${this.collectionName}]: Files can not be public and protected at the same time!`); } + if (this.protected === true) { + // eslint-disable-next-line no-console + console.warn(`[FilesCollection.${this.collectionName}] "protected: true" is deprecated and will be removed in v5. It lets any logged-in user download any file. Pass a function that checks access, for example: protected(fileObj) { return !!fileObj && fileObj.userId === this.userId; }`); + } + if (!this.disableUpload && this.allowClientCode && !this.onBeforeRemove) { // eslint-disable-next-line no-console - console.warn(`[FilesCollection.${this.collectionName}] "allowClientCode" is on and "onBeforeRemove" is not set: any client can remove files. Set "onBeforeRemove" or "allowClientCode: false" (the default changes to false in v4).`); + console.warn(`[FilesCollection.${this.collectionName}] "allowClientCode" is on and "onBeforeRemove" is not set: any client can remove files. Set "onBeforeRemove" or remove "allowClientCode: true".`); } + // Returns `false` after it sent the denial, otherwise `{ fileRef }`: + // the document the protected function received (`null` when not found), or `undefined` when nothing was read this._checkAccess = async (http) => { + if (helpers.isObject(http) && this._readDownloadToken(http) === false) { + this._denyAccess(http, 403); + return false; + } + if (!this.protected) { - return true; + return { fileRef: undefined }; } if (!helpers.isObject(http)) { @@ -642,37 +708,25 @@ class FilesCollection extends FilesCollectionCore { } let result; - const {userAsync, userId} = this._getUser(http); + let fileRef; + const { userAsync, userId } = this._getHttpUser(http); if (helpers.isFunction(this.protected)) { - let fileObj; + fileRef = null; if (helpers.isObject(http.params) && http.params._id) { - fileObj = await this.collection.findOneAsync(http.params._id); + fileRef = (await this.collection.findOneAsync(http.params._id)) || null; } - result = await this.protected.call(Object.assign(http, {userAsync, userId}), (fileObj || null)); + result = await this.protected.call(Object.assign(http, { userAsync, userId }), fileRef); } else { result = !!userId; } if (result === true) { - return true; - } - - const rc = toHttpErrorCode(result, 401); - this._debug('[FilesCollection._checkAccess] WARN: Access denied!'); - const text = 'Access denied!'; - if (!http.response.headersSent) { - http.response.writeHead(rc, { - 'Content-Type': 'text/plain', - 'Content-Length': text.length - }); - } - - if (!http.response.finished) { - http.response.end(text); + return { fileRef }; } + this._denyAccess(http, toHttpErrorCode(result, 401)); return false; }; @@ -825,8 +879,11 @@ class FilesCollection extends FilesCollectionCore { return; } - if (await this._checkAccess(http)) { - await this.download(http, uris[1], await this.collection.findOneAsync(uris[0])); + const access = await this._checkAccess(http); + if (access) { + // The protected function already read the document, do not read it again + const fileRef = access.fileRef === undefined ? await this.collection.findOneAsync(uris[0]) : access.fileRef; + await this.download(http, uris[1], fileRef); } } catch (downloadError) { handleDownloadError(downloadError); @@ -844,8 +901,8 @@ class FilesCollection extends FilesCollectionCore { // Method used to remove file // from Client side _methods[this._methodNames._Remove] = async function (selector) { - // eslint-disable-next-line new-cap - check(selector, Match.OneOf(String, Object)); + // One file per call: a client can not pass a query selector + check(selector, String); self._debug(`[FilesCollection] [Unlink Method] [.removeAsync(${selector})]`); if (self.allowClientCode) { @@ -888,6 +945,7 @@ class FilesCollection extends FilesCollectionCore { check(opts, { file: Object, fileId: String, + // Accepted from v3 clients and ignored: only the server names files FSName: Match.Optional(String), chunkSize: Number, fileLength: Number @@ -971,6 +1029,17 @@ class FilesCollection extends FilesCollectionCore { } + /** + * @locus Server + * @memberOf FilesCollection + * @name findOne + * @summary Not available on the server, where collections are async only + * @throws {Meteor.Error} 404, always. Use `findOneAsync()` + */ + findOne() { + throw new Meteor.Error(404, 'FilesCollection#findOne() is not available on the server! Use .findOneAsync() instead'); + } + /** * @locus Server * @memberOf FilesCollection @@ -1035,12 +1104,26 @@ class FilesCollection extends FilesCollectionCore { return segments.length ? segments.join(nodePath.sep) : fallback; } + /** + * @locus Server + * @memberOf FilesCollection + * @name _namingContext + * @param {Object} file - File data known so far + * @param {string} fileId - `_id` of the future document + * @param {string|null} [userId] - Uploader + * @summary Internal method. Argument of `namingFunction`, the same wrapper shape in upload Start, `writeAsync()`, and `loadAsync()` + * @returns {{file: Object, fileId: string, userId: string|null}} + */ + _namingContext(file, fileId, userId) { + return { file: helpers.cloneDeep(file), fileId, userId: userId || null }; + } + /** * @locus Server * @memberOf FilesCollection * @name _pickClientFileFields * @param {Object} file - Client-supplied `file` object - * @summary Internal method. Deep copy of client `file` object without keys the server computes itself + * @summary Internal method. Deep copy of the allowed keys of a client `file` object: `name`, `type`, `size`, and `meta` * @returns {Object} */ _pickClientFileFields(file) { @@ -1049,8 +1132,8 @@ class FilesCollection extends FilesCollectionCore { return picked; } - for (const key of Object.keys(file)) { - if (!RESERVED_FILE_KEYS.includes(key)) { + for (const key of CLIENT_FILE_KEYS) { + if (Object.hasOwn(file, key)) { picked[key] = helpers.cloneDeep(file[key]); } } @@ -1116,9 +1199,10 @@ class FilesCollection extends FilesCollectionCore { result.userId = (isStart ? userId : opts.userId) || null; if (isStart) { - let FSName = this.sanitize(helpers.isString(opts.FSName) && opts.FSName ? opts.FSName : opts.fileId); + // Only the server names files: a client `FSName` is ignored + let FSName = this.sanitize(opts.fileId); if (this.namingFunction) { - FSName = this._sanitizeFSName(await this.namingFunction(Object.assign({}, opts, { file: result, FSName })), FSName); + FSName = this._sanitizeFSName(await this.namingFunction(this._namingContext(result, opts.fileId, result.userId)), FSName); } opts.FSName = FSName; @@ -1165,7 +1249,7 @@ class FilesCollection extends FilesCollectionCore { * @locus Server * @memberOf FilesCollection * @name _startUpload - * @param {Object} opts - Start request: `{file, fileId, FSName, chunkSize, fileLength}` + * @param {Object} opts - Start request: `{file, fileId, chunkSize, fileLength}`. A client `FSName` is accepted and ignored * @param {string|null} userId - Caller's userId * @param {string} transport - Transport name, used in logs * @summary Internal method. Validate start request, create upload record in `_preCollection` and an empty file on FS @@ -1188,6 +1272,10 @@ class FilesCollection extends FilesCollectionCore { throw new Meteor.Error(400, 'Invalid fileLength'); } + if (fileLength > MAX_UPLOAD_CHUNKS) { + throw new Meteor.Error(400, `Too many chunks, at most ${MAX_UPLOAD_CHUNKS}. Use a larger chunkSize`); + } + const fileId = this.sanitize(opts.fileId, 20, 'a'); if (!fileId) { throw new Meteor.Error(400, 'Invalid fileId'); @@ -1204,7 +1292,6 @@ class FilesCollection extends FilesCollectionCore { const { result, opts: prepared, ctx } = await this._prepareUpload({ file: opts.file, fileId, - FSName: opts.FSName, chunkSize, fileLength, ___s: true, @@ -1237,6 +1324,8 @@ class FilesCollection extends FilesCollectionCore { chunkSize, fileLength, maxLength: fileLength, + // Bit set of written chunk ids, see `_recordChunk()` + chunkBits: new Array(Math.ceil(fileLength / 32)).fill(0), userId: userId || null, path: result.path, _storagePath: result._storagePath, @@ -1320,6 +1409,29 @@ class FilesCollection extends FilesCollectionCore { return session; } + /** + * @locus Server + * @memberOf FilesCollection + * @name _recordChunk + * @param {string} fileId - Upload id + * @param {number} chunkId - Chunk on disk, 1-based + * @summary Internal method. Set the chunk bit in `chunkBits` of the upload record, so the upload can resume after a restart. Atomic, so concurrent chunks and instances do not lose bits + * @returns {Promise} `false` when the record is gone, finished, or the update failed + */ + async _recordChunk(fileId, chunkId) { + const index = chunkId - 1; + try { + const { matchedCount } = await this._preCollection.rawCollection().updateOne( + { _id: fileId, isFinished: { $ne: true } }, + { $bit: { [`chunkBits.${Math.floor(index / 32)}`]: { or: (1 << (index % 32)) | 0 } } } + ); + return matchedCount === 1; + } catch (recordError) { + this._debug(`[FilesCollection] [_recordChunk] Can not record chunk #${chunkId} of ${fileId}:`, recordError); + return false; + } + } + /** * @locus Server * @memberOf FilesCollection @@ -1705,6 +1817,7 @@ class FilesCollection extends FilesCollectionCore { check(opts, { file: Object, fileId: String, + // Accepted from v3 clients and ignored: only the server names files FSName: Match.Optional(String), chunkSize: Number, fileLength: Number, @@ -1770,20 +1883,29 @@ class FilesCollection extends FilesCollectionCore { if (helpers.isObject(result.versions) && helpers.isObject(result.versions.original)) { result.versions.original.size = size; } - result.type = this._getMimeType(opts.file); + const clientType = this._getMimeType(opts.file); + // The uploader's type is not verified: by default the stored type comes from the file content + this._setFileType(result, this.trustClientMimeType ? clientType : await sniffFile(result.path, clientType)); result.public = this.public; - this._updateFileTypes(result); + + let storageMeta; + try { + storageMeta = await this._putVersion(result, 'original', result.path, 'upload'); + } catch (putError) { + this._debug('[FilesCollection] [_finishUpload] [storage.put] Error:', putError); + await this._unlinkFile(result.path, '[_finishUpload]'); + throw putError; + } let _id; try { _id = await this.collection.insertAsync(helpers.cloneDeep(result)); } catch (colInsertError) { this._debug('[FilesCollection] [_finishUpload] [insert] Error:', colInsertError); - try { - await fs.promises.unlink(result.path); - } catch (unlinkError) { - this._debug('[FilesCollection] [_finishUpload] [unlink] Error:', unlinkError); + if (storageMeta) { + await this._removeVersion(result, 'original', '[_finishUpload]'); } + await this._unlinkFile(result.path, '[_finishUpload]'); throw colInsertError; } @@ -1855,6 +1977,25 @@ class FilesCollection extends FilesCollectionCore { return mime; } + /** + * @locus Server + * @memberOf FilesCollection + * @name _setFileType + * @param {Partial} fileObj - File object to update in place + * @param {string} type - Mime type to store + * @summary Internal method. Set `type`, `mime`, `mime-type`, `versions.original.type`, and the `is*` flags + * @returns {void} + */ + _setFileType(fileObj, type) { + fileObj.type = type; + fileObj.mime = type; + fileObj['mime-type'] = type; + if (helpers.isObject(fileObj.versions) && helpers.isObject(fileObj.versions.original)) { + fileObj.versions.original.type = type; + } + this._updateFileTypes(fileObj); + } + /** * @locus Server * @memberOf FilesCollection @@ -1868,22 +2009,13 @@ class FilesCollection extends FilesCollectionCore { } const sessions = Meteor.server.sessions; - if (sessions instanceof Map) { - // to be used with >= Meteor 1.8.1 where Meteor.server.sessions is a Map - const session = sessions.get(xmtok); - return helpers.isObject(session) ? (session.userId || null) : null; + if (!(sessions instanceof Map)) { + // Meteor 3 keeps sessions in a Map. Fail loudly if that changes + throw new Error('Received incompatible type of Meteor.server.sessions'); } - if (helpers.isObject(sessions)) { - // to be used with < Meteor 1.8.1 where Meteor.server.sessions is an Object - if (Object.hasOwn(sessions, xmtok) && helpers.isObject(sessions[xmtok])) { - return sessions[xmtok].userId || null; - } - return null; - } - - // throw an error upon an unexpected type of Meteor.server.sessions in order to identify breaking changes - throw new Error('Received incompatible type of Meteor.server.sessions'); + const session = sessions.get(xmtok); + return helpers.isObject(session) ? (session.userId || null) : null; } /** @@ -1940,6 +2072,91 @@ class FilesCollection extends FilesCollectionCore { return result; } + /** + * @locus Server + * @memberOf FilesCollection + * @name createDownloadToken + * @param {FileObj|FileCursor|string} fileRef - File, or its `_id` + * @param {Object} [opts] + * @param {string} [opts.version='original'] - Version the token opens + * @param {string|null} [opts.userId=null] - User the download acts as, for `protected` and `downloadCallback` + * @param {number} [opts.expiresIn=3600] - Lifetime in seconds, a positive integer + * @summary Signed token for `link(fileRef, version, uriBase, { token })`. Needs `downloadTokenSecret`. Works on any instance with the same secret + * @throws {Meteor.Error} 500 without `downloadTokenSecret`, `Match.Error` on invalid input + * @returns {string} + */ + createDownloadToken(fileRef, { version = 'original', userId = null, expiresIn = 3600 } = {}) { + if (!this.downloadTokenSecret) { + throw new Meteor.Error(500, '[FilesCollection] [createDownloadToken] "downloadTokenSecret" is not set'); + } + + const _id = helpers.isString(fileRef) ? fileRef : fileRef?._id; + /* eslint-disable new-cap */ + check(_id, Match.Where((id) => helpers.isString(id) && id.length > 0)); + check(version, String); + check(userId, Match.OneOf(String, null)); + check(expiresIn, Match.Where((n) => Number.isInteger(n) && n > 0)); + /* eslint-enable new-cap */ + + // `\n` separates the signed fields + if ([_id, version, userId].some((value) => helpers.isString(value) && value.includes('\n'))) { + throw new Meteor.Error(400, '[FilesCollection] [createDownloadToken] "_id", "version", and "userId" can not contain line breaks'); + } + + return signDownloadToken(this.downloadTokenSecret, { collectionName: this.collectionName, _id, version, userId, exp: Math.floor(Date.now() / 1000) + expiresIn }); + } + + /** + * @locus Server + * @memberOf FilesCollection + * @name _readDownloadToken + * @param {ContextHTTP} http - Server HTTP object, `params` hold `_id`, `version`, and `query` + * @summary Internal method. Verify `?token=` once per request and keep the result in `http.downloadToken` + * @returns {{userId: string|null, exp: number}|null|false} `null` without token or secret, `false` for an invalid token + */ + _readDownloadToken(http) { + if (http.downloadToken !== undefined) { + return http.downloadToken; + } + + const token = http.params?.query?.token; + // Public collections never check tokens + if (this.public || !this.downloadTokenSecret || token === undefined) { + http.downloadToken = null; + return null; + } + + http.downloadToken = verifyDownloadToken(this.downloadTokenSecret, token, { collectionName: this.collectionName, _id: http.params._id, version: http.params.version }) || false; + return http.downloadToken; + } + + /** + * @locus Server + * @memberOf FilesCollection + * @name _getHttpUser + * @param {ContextHTTP} http - Server HTTP object + * @summary Internal method. The user of a download: the token user for a valid token, no user for an invalid one, otherwise `_getUser(http)` + * @returns {ContextUser} + */ + _getHttpUser(http) { + const token = helpers.isObject(http) ? this._readDownloadToken(http) : null; + if (token === false) { + // An invalid token never falls back to the cookie user + return { userId: null, async userAsync() { return null; } }; + } + if (!token) { + return this._getUser(http); + } + + const userId = token.userId; + return { + userId, + async userAsync() { + return (userId && Meteor.users) ? await Meteor.users.findOneAsync(userId) : null; + }, + }; + } + /** * @locus Server * @memberOf FilesCollection @@ -1976,14 +2193,18 @@ class FilesCollection extends FilesCollectionCore { throw new Meteor.Error(409, `[FilesCollection] [writeAsync] File with _id "${fileId}" already exists`); } - const fsName = this.namingFunction ? this._sanitizeFSName(await this.namingFunction(opts), fileId) : fileId; + const fsName = this.namingFunction + ? this._sanitizeFSName(await this.namingFunction(this._namingContext({ name: opts.name || opts.fileName, type: opts.type, size: buffer.length, meta: opts.meta }, fileId, opts.userId)), fileId) + : fileId; const fileName = (opts.name || opts.fileName) ? (opts.name || opts.fileName) : fsName; const {extension, extensionWithDot} = this._getExt(fileName); const storagePath = await this.storagePath(opts); opts.path = `${storagePath}${nodePath.sep}${fsName}${extensionWithDot}`; - opts.type = this._getMimeType(opts); + if (!helpers.isString(opts.type) || !opts.type) { + opts.type = this.trustClientMimeType ? 'application/octet-stream' : resolveMimeType(buffer.subarray(0, SNIFF_BYTES), void 0, buffer.length <= SNIFF_BYTES); + } if (!helpers.isObject(opts.meta)) { opts.meta = {}; } @@ -2023,8 +2244,19 @@ class FilesCollection extends FilesCollectionCore { } } + let storageMeta; + let isInserted = false; + try { + storageMeta = await this._putVersion(result, 'original', opts.path, 'write'); + } catch (putError) { + this._debug(`[FilesCollection] [writeAsync] [storage.put] Error: ${fileName} -> ${this.collectionName}`, putError); + await this._unlinkFile(opts.path, '[writeAsync]'); + throw new Meteor.Error('writeAsync', putError); + } + try { const _id = await this.collection.insertAsync(result); + isInserted = true; fileObj = await this.collection.findOneAsync(_id); if (proceedAfterUpload === true) { @@ -2036,6 +2268,10 @@ class FilesCollection extends FilesCollectionCore { this._debug(`[FilesCollection] [write]: ${fileName} -> ${this.collectionName}`); } catch (insertErr) { this._debug(`[FilesCollection] [write] [insert] Error: ${fileName} -> ${this.collectionName}`, insertErr); + // The stored copy belongs to the document once it is inserted, an `onAfterUpload` error must not remove it + if (storageMeta && !isInserted) { + await this._removeVersion(result, 'original', '[writeAsync]'); + } throw new Meteor.Error('writeAsync', insertErr); } @@ -2091,7 +2327,9 @@ class FilesCollection extends FilesCollectionCore { throw new Meteor.Error(409, `[FilesCollection] [loadAsync] File with _id "${fileId}" already exists`); } - const fsName = this.namingFunction ? this._sanitizeFSName(await this.namingFunction(opts), fileId) : fileId; + const fsName = this.namingFunction + ? this._sanitizeFSName(await this.namingFunction(this._namingContext({ name: opts.name || opts.fileName, type: opts.type, meta: opts.meta }, fileId, opts.userId)), fileId) + : fileId; const pathParts = url.split('/'); const fileName = (opts.name || opts.fileName) ? (opts.name || opts.fileName) : pathParts[pathParts.length - 1].split('?')[0] || fsName; @@ -2100,6 +2338,9 @@ class FilesCollection extends FilesCollectionCore { opts.path = `${storagePath}${nodePath.sep}${fsName}${extensionWithDot}`; let fileObj; + let result; + let storageMeta; + let isInserted = false; let isFileCreated = false; const controller = new AbortController(); let timer = null; @@ -2140,11 +2381,11 @@ class FilesCollection extends FilesCollectionCore { // Content-Length is wrong for compressed responses, use size on disk const { size } = await fs.promises.stat(opts.path); - const result = this._dataToSchema({ + result = this._dataToSchema({ name: fileName, path: opts.path, meta: opts.meta, - type: opts.type || res.headers.get('content-type') || this._getMimeType({path: opts.path}), + type: opts.type || (this.trustClientMimeType ? (res.headers.get('content-type') || 'application/octet-stream') : await sniffFile(opts.path, res.headers.get('content-type'))), size, userId: opts.userId, extension, @@ -2152,7 +2393,9 @@ class FilesCollection extends FilesCollectionCore { }); result._id = fileId; + storageMeta = await this._putVersion(result, 'original', opts.path, 'load'); const _id = await this.collection.insertAsync(result); + isInserted = true; fileObj = await this.collection.findOneAsync(_id); this._debug(`[FilesCollection] [load] [insert] ${fileName} -> ${this.collectionName}`); } catch (error) { @@ -2166,6 +2409,10 @@ class FilesCollection extends FilesCollectionCore { } } + if (storageMeta && !isInserted) { + await this._removeVersion(result, 'original', '[loadAsync]'); + } + throw error; } @@ -2235,8 +2482,8 @@ class FilesCollection extends FilesCollectionCore { const { extension } = this._getExt(opts.fileName); - if (!helpers.isString(opts.type)) { - opts.type = this._getMimeType(opts); + if (!helpers.isString(opts.type) || !opts.type) { + opts.type = this.trustClientMimeType ? 'application/octet-stream' : await sniffFile(path); } if (!helpers.isObject(opts.meta)) { @@ -2259,11 +2506,21 @@ class FilesCollection extends FilesCollectionCore { fileId: (opts.fileId && this.sanitize(opts.fileId, 20, 'a')) || null, }); + // Adapters key stored copies by `_id`, so it is known before `put()` + if (!result._id) { + result._id = Random.id(); + } + // The fs adapter keeps the file at `path`. Other adapters copy it and leave the caller's file in place + const storageMeta = await this._putVersion(result, 'original', path, 'addFile'); + let _id; try { _id = await this.collection.insertAsync(result); } catch (insertErr) { this._debug(`[FilesCollection] [addFile] [insertAsync] Error: ${result.name} -> ${this.collectionName}`, insertErr); + if (storageMeta) { + await this._removeVersion(result, 'original', '[addFile]'); + } throw new Meteor.Error(insertErr.code, insertErr.message); } @@ -2386,7 +2643,7 @@ class FilesCollection extends FilesCollectionCore { * @param {fileObj} fileRef - fileObj * @param {string} [version] - [Optional] file's version * @param {function} [callback] - [Optional] callback function - * @summary Unlink files and its versions from FS + * @summary Unlink files and its versions from FS. Works with the fs adapter only * @deprecated since v3.0.0. use {@link FilesCollection#unlinkAsync} instead. * @returns {FilesCollection} Instance */ @@ -2426,25 +2683,23 @@ class FilesCollection extends FilesCollectionCore { * @name unlinkAsync * @param {fileObj} fileRef - fileObj * @param {string} [version] - file's version - * @summary Remove files and all its versions from FS, or only particular version if `version` param is passed. Paths stored in the document are trusted, so do not let clients edit documents (see `allowClient()`) + * @summary Remove files and all versions through the storage adapter, or one version if `version` is passed. The fs adapter trusts paths stored in the document, so do not let clients edit documents (see `allowClient()`) * @returns {Promise} Instance */ async unlinkAsync(fileRef, version) { this._debug(`[FilesCollection] [unlinkAsync(${fileRef._id}, ${version})]`); if (version) { - if (helpers.isObject(fileRef.versions) && helpers.isObject(fileRef.versions[version]) && fileRef.versions[version].path) { - await this._unlinkFile(fileRef.versions[version].path, `[${version}]`); + if (helpers.isObject(fileRef.versions) && helpers.isObject(fileRef.versions[version])) { + await this._removeVersion(fileRef, version, `[${version}]`); } - } else { - if (helpers.isObject(fileRef.versions)) { - for(let vKey in fileRef.versions) { - if (fileRef.versions[vKey] && fileRef.versions[vKey].path) { - await this._unlinkFile(fileRef.versions[vKey].path, '[versions]'); - } + } else if (helpers.isObject(fileRef.versions)) { + for (const vKey of Object.keys(fileRef.versions)) { + if (helpers.isObject(fileRef.versions[vKey])) { + await this._removeVersion(fileRef, vKey, '[versions]'); } - } else { - await this._unlinkFile(fileRef.path, ''); } + } else { + await this._removeVersion(this._storageRef(fileRef, fileRef, 'original'), 'original', ''); } return this; } @@ -2470,6 +2725,68 @@ class FilesCollection extends FilesCollectionCore { } } + /** + * @locus Server + * @memberOf FilesCollection + * @name _storageRef + * @param {FileObj} fileRef - File document + * @param {Object} vRef - Version object to serve + * @param {string} version - Version name + * @summary Internal method. File object whose `versions[version]` is `vRef`, so adapters read the same data `serve()` got + * @returns {Object} + */ + _storageRef(fileRef, vRef, version) { + if (helpers.isObject(fileRef?.versions) && fileRef.versions[version] === vRef) { + return fileRef; + } + return Object.assign({}, fileRef, { + versions: Object.assign({}, helpers.isObject(fileRef?.versions) ? fileRef.versions : {}, { [version]: vRef }), + }); + } + + /** + * @locus Server + * @memberOf FilesCollection + * @name _putVersion + * @param {Partial} fileRef - File object, updated in place + * @param {string} versionName - Version + * @param {string} localPath - File on disk + * @param {'upload'|'write'|'load'|'addFile'} source - Origin of the local file, passed to `put()` as `{ source }` + * @summary Internal method. Hand a finished file to the storage adapter, keep its result at `versions[versionName].meta.storage` + * @returns {Promise} The adapter result + */ + async _putVersion(fileRef, versionName, localPath, source) { + const storageMeta = await this.storage.put(fileRef, versionName, localPath, { source }); + if (helpers.isObject(storageMeta)) { + const vRef = fileRef.versions[versionName]; + vRef.meta = Object.assign({}, helpers.isObject(vRef.meta) ? vRef.meta : {}, { storage: storageMeta }); + } + return storageMeta; + } + + /** + * @locus Server + * @memberOf FilesCollection + * @name _removeVersion + * @param {FileObj} fileRef - File document + * @param {string} versionName - Version + * @param {string} [label] - Log label + * @summary Internal method. Remove a version through the adapter. A file that is already gone counts as removed. Never throws + * @returns {Promise} + */ + async _removeVersion(fileRef, versionName, label = '') { + const prefix = `[FilesCollection] [unlinkAsync] ${label}${label ? ' ' : ''}`; + try { + await this.storage.remove(fileRef, versionName); + } catch (removeError) { + if (removeError?.code === 'ENOENT') { + this._debug(`${prefix}File is already removed: ${removeError.path}`); + return; + } + this._debug(`${prefix}Caught silent error`, removeError); + } + } + /** * @locus Server * @memberOf FilesCollection @@ -2493,6 +2810,30 @@ class FilesCollection extends FilesCollectionCore { } } + /** + * @locus Server + * @memberOf FilesCollection + * @name _denyAccess + * @param {ContextHTTP} http - Server HTTP object + * @param {number} code - HTTP status, 400-599 + * @summary Internal method. Answer a denied download with `code` and `Access denied!` + * @returns {void} + */ + _denyAccess(http, code) { + this._debug('[FilesCollection._checkAccess] WARN: Access denied!'); + const text = 'Access denied!'; + if (!http.response.headersSent) { + http.response.writeHead(code, { + 'Content-Type': 'text/plain', + 'Content-Length': text.length + }); + } + + if (!http.response.finished) { + http.response.end(text); + } + } + /** * @locus Server * @memberOf FilesCollection @@ -2511,6 +2852,9 @@ class FilesCollection extends FilesCollectionCore { if (helpers.has(fileRef, 'versions') && helpers.has(fileRef.versions, version)) { vRef = fileRef.versions[version]; vRef._id = fileRef._id; + } else if (http.downloadToken && version !== 'original') { + // A token opens one version, do not fall back to the original + vRef = false; } else { vRef = fileRef; } @@ -2521,7 +2865,7 @@ class FilesCollection extends FilesCollectionCore { if (!vRef || !helpers.isObject(vRef)) { return this._404(http); } else if (fileRef) { - if (helpers.isFunction(this.downloadCallback) && !(await this.downloadCallback(Object.assign(http, this._getUser(http)), fileRef))) { + if (helpers.isFunction(this.downloadCallback) && !(await this.downloadCallback(Object.assign(http, this._getHttpUser(http)), fileRef))) { return this._404(http); } @@ -2529,29 +2873,24 @@ class FilesCollection extends FilesCollectionCore { return void 0; } - let stats; - - try { - stats = await fs.promises.stat(vRef.path); - } catch (statErr){ - if (statErr) { + let responseType = '200'; + // Adapters without `stat()` skip the existence and integrity checks, a missing file fails while streaming + if (helpers.isFunction(this.storage.stat)) { + const stats = await this.storage.stat(this._storageRef(fileRef, vRef, version), version); + if (!stats) { return this._404(http); } - } - if (!stats.isFile()) { - return this._404(http); - } - let responseType; - if (stats.size !== vRef.size && !this.integrityCheck) { - vRef.size = stats.size; - } - - if (stats.size !== vRef.size && this.integrityCheck) { - responseType = '400'; + if (stats.size !== vRef.size) { + if (this.integrityCheck) { + responseType = '400'; + } else { + vRef.size = stats.size; + } + } } - return this.serve(http, fileRef, vRef, version, null, responseType || '200'); + return await this.serve(http, fileRef, vRef, version, null, responseType); } return this._404(http); } @@ -2607,14 +2946,14 @@ class FilesCollection extends FilesCollectionCore { * @param {stream.Readable|null} readableStream - Readable stream, which serves binary file data * @param {string} responseType - Response code * @param {boolean} force200 - Force 200 response code over 206 - * @summary Handle and reply to incoming request - * @returns {undefined} + * @summary Handle and reply to incoming request. Without `readableStream` the file is read through the storage adapter + * @returns {Promise} */ - serve(http, fileRef, vRef, version = 'original', readableStream = null, _responseType = '200', force200 = false) { + async serve(http, fileRef, vRef, version = 'original', readableStream = null, _responseType = '200', force200 = false) { let reqRange = false; let responseType = _responseType; - let disposition = (http.params?.query?.download === 'true') ? 'attachment' : 'inline'; + let disposition = dispositionType(vRef.type || fileRef?.type, http.params?.query?.download === 'true'); const name = vRef.name || fileRef.name; if (helpers.isString(name) && name.length) { // RFC 6266: ASCII fallback in `filename` (no `%`, some clients decode it), RFC 8187 encoded value in `filename*` @@ -2640,7 +2979,7 @@ class FilesCollection extends FilesCollectionCore { } } - const headers = (helpers.isFunction(this.responseHeaders) ? this.responseHeaders(responseType, fileRef, vRef, version, http) : this.responseHeaders) || {}; + const headers = (helpers.isFunction(this.responseHeaders) ? await this.responseHeaders(responseType, fileRef, vRef, version, http) : this.responseHeaders) || {}; if (this.nosniff && !http.response.headersSent) { http.response.setHeader('X-Content-Type-Options', 'nosniff'); @@ -2648,7 +2987,11 @@ class FilesCollection extends FilesCollectionCore { if (!headers['Cache-Control']) { if (!http.response.headersSent) { - http.response.setHeader('Cache-Control', this.cacheControl); + // A shared cache must not keep a token response past the token expiry + const cacheControl = http.downloadToken + ? `private, max-age=${Math.max(0, http.downloadToken.exp - Math.floor(Date.now() / 1000))}` + : this.cacheControl; + http.response.setHeader('Cache-Control', cacheControl); } } @@ -2658,17 +3001,54 @@ class FilesCollection extends FilesCollectionCore { } } - const respond = (stream, code) => { - const writeHead = () => { + const storageRef = readableStream ? null : this._storageRef(fileRef, vRef, version); + + const fail = (stream, error) => { + this._debug(`[FilesCollection] [serve(${vRef.path}, ${version})] [500]`, error); + if (stream && !stream.destroyed) { + stream.destroy(); + } + + if (!http.response.headersSent) { + const text = 'Internal Server Error'; + http.response.removeHeader('Content-Length'); + http.response.removeHeader('Content-Range'); + http.response.writeHead(500, { + 'Content-Type': 'text/plain', + 'Content-Length': text.length + }); + http.response.end(text); + } else if (!http.response.writableEnded) { + http.response.destroy(); + } + }; + + const respond = async (code, range) => { + let stream = readableStream; + if (stream) { if (!http.response.headersSent) { http.response.writeHead(code); } - }; - - if (readableStream) { - writeHead(); } else { - stream.once('open', writeHead); + try { + stream = await this.storage.createReadStream(storageRef, version, range ? { start: range.start, end: range.end } : {}); + } catch (openError) { + if (openError?.error === 404) { + http.response.removeHeader('Content-Range'); + this._404(http); + return; + } + fail(null, openError); + return; + } + + if (http.response.destroyed) { + // The client left while the adapter opened the file + stream.destroy(); + return; + } + // Headers go out with the first byte, so an error before it still answers 500 + http.response.statusCode = code; } // Free the file descriptor when the client goes away @@ -2677,27 +3057,16 @@ class FilesCollection extends FilesCollectionCore { stream.destroy(); } }); - - stream.once('error', (error) => { - this._debug(`[FilesCollection] [serve(${vRef.path}, ${version})] [500]`, error); - if (!stream.destroyed) { - stream.destroy(); - } - - if (!http.response.headersSent) { - const text = 'Internal Server Error'; - http.response.removeHeader('Content-Length'); - http.response.removeHeader('Content-Range'); - http.response.writeHead(500, { - 'Content-Type': 'text/plain', - 'Content-Length': text.length - }); - http.response.end(text); - } else if (!http.response.writableEnded) { - http.response.destroy(); + // `on`, not `once`: a second error from an adapter stream without a listener would crash the process + let failed = false; + stream.on('error', (error) => { + if (failed) { + this._debug(`[FilesCollection] [serve(${vRef.path}, ${version})] [500] another stream error`, error); + return; } + failed = true; + fail(stream, error); }); - stream.pipe(http.response); }; @@ -2741,7 +3110,7 @@ class FilesCollection extends FilesCollectionCore { http.response.removeHeader('Transfer-Encoding'); } } - respond(readableStream || fs.createReadStream(vRef.path, { start: reqRange.start, end: reqRange.end }), 206); + await respond(206, reqRange); break; default: if (!http.response.headersSent && Number.isInteger(vRef.size) && vRef.size >= 0) { @@ -2750,10 +3119,10 @@ class FilesCollection extends FilesCollectionCore { http.response.removeHeader('Transfer-Encoding'); } this._debug(`[FilesCollection] [serve(${vRef.path}, ${version})] [200]`); - respond(readableStream || fs.createReadStream(vRef.path), 200); + await respond(200, null); break; } } } -export { FilesCollection, WriteStream, helpers }; +export { FilesCollection, FSStorage, GridFSStorage, WriteStream, helpers }; diff --git a/storage.js b/storage.js new file mode 100644 index 00000000..5d5621c7 --- /dev/null +++ b/storage.js @@ -0,0 +1,214 @@ +import fs from 'node:fs'; +import { pipeline } from 'node:stream/promises'; +import { Meteor } from 'meteor/meteor'; +import { MongoInternals } from 'meteor/mongo'; +import { helpers } from './lib.js'; + +const { GridFSBucket, ObjectId } = MongoInternals.NpmModules.mongodb.module; + +/** + * @typedef {'upload'|'write'|'load'|'addFile'} StorageSource + * @summary Where the local file passed to `put()` comes from. `upload`, `write` (`writeAsync()`), and `load` (`loadAsync()`) files were created by the package. `addFile` is the caller's own file + */ + +/** + * @function isStorageAdapter + * @param {*} adapter - Value of the `storage` option + * @summary An object with `put`, `createReadStream`, and `remove` functions + * @returns {boolean} + */ +export const isStorageAdapter = (adapter) => helpers.isObject(adapter) + && helpers.isFunction(adapter.put) + && helpers.isFunction(adapter.createReadStream) + && helpers.isFunction(adapter.remove); + +/** + * @private + * @summary `fileRef.versions[versionName]` when it is an object, otherwise `null` + */ +const versionOf = (fileRef, versionName) => { + return (helpers.isObject(fileRef?.versions) && helpers.isObject(fileRef.versions[versionName])) ? fileRef.versions[versionName] : null; +}; + +/** + * @locus Server + * @class FSStorage + * @summary Default adapter: files stay where the upload wrote them, at `versions..path` + */ +export class FSStorage { + constructor() { + this.name = 'fs'; + } + + /** + * @summary Nothing to move, the file is already in place + * @param {FileObj} _fileRef - File object + * @param {string} _versionName - Version + * @param {string} _localPath - File on disk + * @param {{source: StorageSource}} [_opts] - Origin of the local file + * @returns {Promise} + */ + async put() { + return void 0; + } + + /** + * @param {FileObj} fileRef - File document + * @param {string} versionName - Version + * @returns {Promise<{size: number}|null>} `null` when the file is missing or not a regular file + */ + async stat(fileRef, versionName) { + const vRef = versionOf(fileRef, versionName); + if (!helpers.isString(vRef?.path)) { + return null; + } + + try { + const stats = await fs.promises.stat(vRef.path); + return stats.isFile() ? { size: stats.size } : null; + } catch (_statError) { + return null; + } + } + + /** + * @param {FileObj} fileRef - File document + * @param {string} versionName - Version + * @param {{start?: number, end?: number}} [range] - Byte range, `end` inclusive + * @throws {Meteor.Error} 404 when the version has no path + * @returns {Promise} + */ + async createReadStream(fileRef, versionName, range = {}) { + const vRef = versionOf(fileRef, versionName); + if (!helpers.isString(vRef?.path)) { + throw new Meteor.Error(404, 'File not found'); + } + return fs.createReadStream(vRef.path, Number.isInteger(range.start) ? { start: range.start, end: range.end } : {}); + } + + /** + * @param {FileObj} fileRef - File document + * @param {string} versionName - Version + * @summary Unlink the version file. fs errors, `ENOENT` included, are rethrown for the caller to log + * @returns {Promise} + */ + async remove(fileRef, versionName) { + const vRef = versionOf(fileRef, versionName); + if (helpers.isString(vRef?.path)) { + await fs.promises.unlink(vRef.path); + } + } +} + +/** + * @locus Server + * @class GridFSStorage + * @param {Object} [opts] + * @param {string} [opts.bucketName='fs'] - GridFS bucket + * @param {number} [opts.chunkSizeBytes] - GridFS chunk size, driver default when not set + * @param {Db} [opts.db] - Database, default: the app's default database + * @summary Copies finished files into a GridFS bucket and deletes the local copy, except for `addFile()` where the file belongs to the caller. Stores `{ name: 'gridfs', bucketName, id }` at `versions..meta.storage` + */ +export class GridFSStorage { + constructor({ bucketName = 'fs', chunkSizeBytes, db } = {}) { + if (!helpers.isString(bucketName) || !bucketName) { + throw new Meteor.Error(500, '[GridFSStorage] "bucketName" must be a non-empty String'); + } + this.name = 'gridfs'; + this.bucketName = bucketName; + this.chunkSizeBytes = chunkSizeBytes; + this.db = db || null; + this._bucket = null; + } + + /** + * @summary Created on first use, after Meteor connected to MongoDB + * @returns {GridFSBucket} + */ + get bucket() { + if (!this._bucket) { + const options = { bucketName: this.bucketName }; + if (Number.isInteger(this.chunkSizeBytes) && this.chunkSizeBytes > 0) { + options.chunkSizeBytes = this.chunkSizeBytes; + } + this._bucket = new GridFSBucket(this.db || MongoInternals.defaultRemoteCollectionDriver().mongo.db, options); + } + return this._bucket; + } + + /** + * @private + * @returns {ObjectId|null} Bucket file id stored at `versions[versionName].meta.storage.id` + */ + _fileId(fileRef, versionName) { + const id = versionOf(fileRef, versionName)?.meta?.storage?.id; + return (helpers.isString(id) && /^[a-f0-9]{24}$/.test(id)) ? new ObjectId(id) : null; + } + + /** + * @param {FileObj} fileRef - File object + * @param {string} versionName - Version + * @param {string} localPath - File on disk + * @param {{source: StorageSource}} [opts] - Origin of the local file. The local file is kept when `source` is `'addFile'` + * @returns {Promise<{name: 'gridfs', bucketName: string, id: string}>} + */ + async put(fileRef, versionName, localPath, { source } = {}) { + const id = new ObjectId(); + const vRef = versionOf(fileRef, versionName) || {}; + await pipeline( + fs.createReadStream(localPath), + this.bucket.openUploadStreamWithId(id, fileRef.name || `${fileRef._id}`, { + metadata: { fileId: fileRef._id, versionName, type: vRef.type || fileRef.type }, + }) + ); + // The bucket holds the file now. `addFile()` files belong to the caller, keep them + if (source !== 'addFile') { + try { + await fs.promises.unlink(localPath); + } catch (unlinkError) { + // Failing here would orphan the bucket file, a leftover local file is the smaller loss + Meteor._debug(`[GridFSStorage] [put] Can not delete the local file after the upload to "${this.bucketName}"`, localPath, unlinkError); + } + } + return { name: this.name, bucketName: this.bucketName, id: id.toHexString() }; + } + + async stat(fileRef, versionName) { + const id = this._fileId(fileRef, versionName); + if (!id) { + return null; + } + const doc = await this.bucket.find({ _id: id }).next(); + return doc ? { size: doc.length } : null; + } + + async createReadStream(fileRef, versionName, range = {}) { + const id = this._fileId(fileRef, versionName); + if (!id) { + throw new Meteor.Error(404, 'File not found'); + } + + const options = {}; + if (Number.isInteger(range.start)) { + options.start = range.start; + // GridFS `end` is exclusive + options.end = range.end + 1; + } + return this.bucket.openDownloadStream(id, options); + } + + async remove(fileRef, versionName) { + const id = this._fileId(fileRef, versionName); + if (!id) { + return; + } + + try { + await this.bucket.delete(id); + } catch (deleteError) { + if (!/not found/i.test(`${deleteError?.message}`)) { + throw deleteError; + } + } + } +} diff --git a/tests/browser-constants.js b/tests/browser-constants.js new file mode 100644 index 00000000..211b707b --- /dev/null +++ b/tests/browser-constants.js @@ -0,0 +1,4 @@ +/** + * @const {string} BROWSER_COLLECTION - Collection used by the browser suite. The server half lives in browser-fixtures.js + */ +export const BROWSER_COLLECTION = 'mfBrowserTests'; diff --git a/tests/browser-fixtures.js b/tests/browser-fixtures.js new file mode 100644 index 00000000..9f0dae37 --- /dev/null +++ b/tests/browser-fixtures.js @@ -0,0 +1,86 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import nodePath from 'node:path'; +import { Meteor } from 'meteor/meteor'; +import { check } from 'meteor/check'; +import { FilesCollection } from '../server.js'; +import { BROWSER_COLLECTION } from './browser-constants.js'; + +const storagePath = fs.mkdtempSync(nodePath.join(os.tmpdir(), 'mf-browser-')); +const startAttempts = new Map(); +const chunkLog = new Map(); + +/** + * Server half of the browser suite. File names select the behavior: + * - `reject.txt`: onBeforeUpload rejects with 403 + * - `always-503.txt`: every Start fails with 503, attempts are counted + */ +export const browserFiles = new FilesCollection({ + collectionName: BROWSER_COLLECTION, + storagePath, + allowClientCode: true, + onBeforeRemove: () => true, + onBeforeUpload(file) { + if (this.chunkId > 0) { + chunkLog.set(file._id, [...(chunkLog.get(file._id) || []), this.chunkId]); + } + + if (file.name === 'reject.txt') { + return 'Rejected by test server'; + } + + if (file.name === 'always-503.txt') { + startAttempts.set(file.name, (startAttempts.get(file.name) || 0) + 1); + throw new Meteor.Error(503, 'Busy, try again'); + } + return true; + }, +}); + +Meteor.methods({ + 'mfTest.reset'() { + startAttempts.clear(); + }, + 'mfTest.attempts'(name) { + check(name, String); + return startAttempts.get(name) || 0; + }, + 'mfTest.chunks'(fileId) { + check(fileId, String); + return chunkLog.get(fileId) || []; + }, + async 'mfTest.simulateRestart'(fileId) { + check(fileId, String); + const stream = browserFiles._currentUploads[fileId]; + if (!stream) { + return false; + } + // Let a chunk that is still being written finish, closing the handle under it would abort the upload + while (stream.pendingWrites > 0) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + // Same as a restart for this upload: the stream and its file handle are gone, the record stays + clearTimeout(stream.idleTimer); + stream.idleTimer = null; + await stream.fh?.close(); + stream.fh = null; + delete browserFiles._currentUploads[fileId]; + return true; + }, + async 'mfTest.pending'(fileId) { + check(fileId, String); + return !!(await browserFiles._preCollection.findOneAsync({ _id: fileId })); + }, + async 'mfTest.stored'(_id) { + check(_id, String); + const doc = await browserFiles.collection.findOneAsync({ _id }); + if (!doc) { + return null; + } + return { size: doc.size, type: doc.type, content: await fs.promises.readFile(doc.path, 'utf8') }; + }, +}); + +Meteor.publish('mfTest.files', function () { + return browserFiles.collection.find({}); +}); diff --git a/tests/client.js b/tests/client.js new file mode 100644 index 00000000..f0d0af90 --- /dev/null +++ b/tests/client.js @@ -0,0 +1,226 @@ +/* global describe, it, beforeEach */ +import { expect } from 'chai'; +import sinon from 'sinon'; +import { Meteor } from 'meteor/meteor'; +import { FilesCollection } from '../client.js'; +import { BROWSER_COLLECTION } from './browser-constants.js'; + +export const files = new FilesCollection({ collectionName: BROWSER_COLLECTION, allowClientCode: true }); + +export const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const waitUntil = async (condition, timeout = 10000) => { + const started = Date.now(); + while (!(await condition())) { + if (Date.now() - started > timeout) { + throw new Error('waitUntil timed out'); + } + await sleep(50); + } +}; + +export const makeFile = (name, size, fill = 'a') => new File([fill.repeat(size)], name, { type: 'text/plain' }); + +/** + * Resolves on the `end` event: `{ error, fileObj }`. `end` fires on success and on failure, not on `abort()` + */ +export const settle = (upload) => new Promise((resolve) => { + upload.once('end', (error, fileObj) => resolve({ error, fileObj })); +}); + +['ddp', 'http'].forEach((transport) => { + describe(`FileUpload over ${transport.toUpperCase()}`, function () { + this.timeout(30000); + + beforeEach(async function () { + await Meteor.callAsync('mfTest.reset'); + }); + + it('uploads: start, progress, completed, content stored', async function () { + const upload = files.insert({ file: makeFile('ok.txt', 4096, 'b'), chunkSize: 1024, transport }, false); + const events = []; + upload.on('start', () => events.push('start')); + upload.on('progress', (progress) => events.push(progress)); + const done = settle(upload); + await upload.start(); + const { error, fileObj } = await done; + + expect(error).to.not.exist; + expect(events[0]).to.equal('start'); + expect(events.filter((e) => typeof e === 'number').length).to.be.greaterThan(0); + expect(upload.state.get()).to.equal('completed'); + expect(upload.progress.get()).to.equal(100); + expect(fileObj._id).to.equal(upload.config.fileId); + expect(fileObj).to.not.have.property('path'); + const stored = await Meteor.callAsync('mfTest.stored', fileObj._id); + expect(stored.size).to.equal(4096); + expect(stored.content).to.equal('b'.repeat(4096)); + }); + + it('pauses and continues', async function () { + const upload = files.insert({ file: makeFile('pause.txt', 8 * 1024, 'c'), chunkSize: 1024, transport }, false); + const done = settle(upload); + let paused = false; + upload.on('progress', () => { + if (!paused) { + paused = true; + upload.pause(); + } + }); + await upload.start(); + await waitUntil(() => upload.state.get() === 'paused'); + await sleep(500); + expect(upload.state.get()).to.equal('paused'); + expect(upload.progress.get()).to.be.below(100); + + upload.continue(); + const { error, fileObj } = await done; + expect(error).to.not.exist; + expect(upload.state.get()).to.equal('completed'); + expect((await Meteor.callAsync('mfTest.stored', fileObj._id)).content).to.equal('c'.repeat(8 * 1024)); + }); + + it('resumes after a server restart without sending acknowledged chunks again', async function () { + const content = 'f'.repeat(8 * 1024); + const upload = files.insert({ file: new File([content], 'restart.txt', { type: 'text/plain' }), chunkSize: 1024, transport }, false); + const fileId = upload.config.fileId; + const done = settle(upload); + let paused = false; + upload.on('progress', () => { + if (!paused) { + paused = true; + upload.pause(); + } + }); + await upload.start(); + await waitUntil(() => upload.state.get() === 'paused'); + await sleep(300); + const before = await Meteor.callAsync('mfTest.chunks', fileId); + // Some chunks, not all: otherwise the check below passes without a resume + expect(before.length).to.be.greaterThan(0); + expect(before.length).to.be.lessThan(8); + expect(await Meteor.callAsync('mfTest.simulateRestart', fileId)).to.equal(true); + + upload.continue(); + const { error, fileObj } = await done; + expect(error).to.not.exist; + expect((await Meteor.callAsync('mfTest.stored', fileObj._id)).content).to.equal(content); + const after = (await Meteor.callAsync('mfTest.chunks', fileId)).slice(before.length); + // A chunk cancelled by pause() may be sent again, older ones never are + expect(after.every((id) => id >= Math.max(...before))).to.equal(true); + expect([...new Set([...before, ...after])].sort((a, b) => a - b)).to.deep.equal([1, 2, 3, 4, 5, 6, 7, 8]); + }); + + it('aborts and removes the pending upload on the server', async function () { + const upload = files.insert({ file: makeFile('abort.txt', 16 * 1024, 'd'), chunkSize: 1024, transport }, false); + const fileId = upload.config.fileId; + const aborted = new Promise((resolve) => upload.once('abort', resolve)); + upload.on('progress', () => { + upload.abort(); + }); + await upload.start(); + await aborted; + expect(upload.state.get()).to.equal('aborted'); + await waitUntil(async () => !(await Meteor.callAsync('mfTest.pending', fileId))); + }); + + it('ends with the server error when onBeforeUpload rejects', async function () { + const upload = files.insert({ file: makeFile('reject.txt', 16), chunkSize: 1024, transport }, false); + const done = settle(upload); + await upload.start(); + const { error } = await done; + expect(error.error).to.equal(403); + expect(error.reason).to.equal('Rejected by test server'); + expect(upload.state.get()).to.equal('aborted'); + }); + + it('runs pipes in the order they were added', async function () { + const order = []; + const upload = files.insert({ file: makeFile('pipes.txt', 1024, 'e'), chunkSize: 1024, transport }, false); + upload + .pipe((data) => { order.push('first'); return data; }) + .pipe((data) => { order.push('second'); return data; }); + const done = settle(upload); + await upload.start(); + const { error } = await done; + expect(error).to.not.exist; + expect(order).to.deep.equal(['first', 'second']); + }); + + it('gives up after 5 attempts when the server answers 503', async function () { + const upload = files.insert({ file: makeFile('always-503.txt', 16), chunkSize: 1024, transport }, false); + const done = settle(upload); + await upload.start(); + const { error } = await done; + expect(error.error).to.equal(503); + expect(upload.state.get()).to.equal('aborted'); + expect(await Meteor.callAsync('mfTest.attempts', 'always-503.txt')).to.equal(5); + }); + }); +}); + +describe('remove from the client', function () { + this.timeout(30000); + + const uploadOne = async (name) => { + const upload = files.insert({ file: makeFile(name, 16, 'r'), chunkSize: 1024, transport: 'ddp' }, false); + const done = settle(upload); + await upload.start(); + const { error, fileObj } = await done; + expect(error).to.not.exist; + return fileObj._id; + }; + + it('rejects an object selector before calling the server', async function () { + let caught; + try { + await files.removeAsync({ _id: 'x' }); + } catch (e) { + caught = e; + } + expect(caught?.errorType).to.equal('Match.Error'); + }); + + it('removes by _id', async function () { + const _id = await uploadOne('remove-me.txt'); + expect(await files.removeAsync(_id)).to.equal(1); + expect(await Meteor.callAsync('mfTest.stored', _id)).to.equal(null); + }); + + it('FilesCursor#removeAsync() removes each matching file by _id', async function () { + const a = await uploadOne('cursor-a.txt'); + const b = await uploadOne('cursor-b.txt'); + const handle = Meteor.subscribe('mfTest.files'); + await waitUntil(() => handle.ready() && files.collection.find({ _id: { $in: [a, b] } }).count() === 2); + expect(await files.find({ _id: { $in: [a, b] } }).removeAsync()).to.equal(2); + expect(await Meteor.callAsync('mfTest.stored', a)).to.equal(null); + expect(await Meteor.callAsync('mfTest.stored', b)).to.equal(null); + handle.stop(); + }); +}); + +describe('client options', function () { + it('allowClientCode defaults to false and removeAsync rejects with 401', async function () { + const local = new FilesCollection({ collection: files.collection }); + expect(local.allowClientCode).to.equal(false); + let caught; + try { + await local.removeAsync('anyId'); + } catch (e) { + caught = e; + } + expect(caught.error).to.equal(401); + }); + + it('ignores namingFunction on the client and warns', function () { + const warn = sinon.stub(console, 'warn'); + try { + const local = new FilesCollection({ collection: files.collection, namingFunction: () => 'x' }); + expect(local.namingFunction).to.equal(undefined); + expect(warn.calledOnce).to.equal(true); + expect(String(warn.firstCall.args[0])).to.include('namingFunction'); + } finally { + warn.restore(); + } + }); +}); diff --git a/tests/core.test.js b/tests/core.test.js index 25c87470..ec7421ee 100644 --- a/tests/core.test.js +++ b/tests/core.test.js @@ -226,6 +226,12 @@ describe('FilesCollectionCore (3.1 fixes)', function() { expect(files.link(cursor, 'thumb')).to.equal(`${ROOT}/cdn/storage/${collectionName}/abc123/thumb/abc123.png`); }); + it('appends an encoded token', function () { + const url = files.link({ _id: 'abc', extension: 'txt', _downloadRoute: '/cdn/storage', _collectionName: 'c1' }, 'original', 'https://example.com', { token: 'a b+c' }); + expect(url).to.equal('https://example.com/cdn/storage/c1/abc/original/abc.txt?token=a%20b%2Bc'); + expect(files.link({ _id: 'abc', _downloadRoute: '/cdn/storage', _collectionName: 'c1' }, 'original', 'https://example.com', {})).to.not.include('token'); + }); + it('keeps URLs of normal files unchanged', function() { expect(files.link(doc())).to.equal(`${ROOT}/cdn/storage/${collectionName}/abc123/original/abc123.jpg`); expect(files.link(doc(), 'original', 'https://cdn.example.com/')).to.equal(`https://cdn.example.com/cdn/storage/${collectionName}/abc123/original/abc123.jpg`); @@ -326,5 +332,9 @@ describe('FilesCollectionCore (3.1 fixes)', function() { it('throws a Meteor.Error', function() { expect(() => files.findOne({})).to.throw(Meteor.Error); }); + + it('is not part of the isomorphic core', function() { + expect(Object.prototype.hasOwnProperty.call(FilesCollectionCore.prototype, 'findOne')).to.equal(false); + }); }); }); diff --git a/tests/cursor.test.js b/tests/cursor.test.js index bc24cac7..128d12cb 100644 --- a/tests/cursor.test.js +++ b/tests/cursor.test.js @@ -117,7 +117,7 @@ describe('FilesCursor', function() { }); describe('M1: synchronous methods on server', function() { - const syncMethods = ['get', 'hasNext', 'next', 'previous', 'fetch', 'first', 'last', 'count', 'forEach', 'each', 'map', 'current', 'remove']; + const syncMethods = ['get', 'next', 'previous', 'fetch', 'first', 'last', 'count', 'forEach', 'each', 'map', 'current', 'remove']; syncMethods.forEach((method) => { it(`#${method}() throws a Meteor.Error pointing to the async method`, function() { const cursor = new FilesCursor({}, {}, filesCollection); @@ -263,12 +263,11 @@ describe('FilesCursor', function() { }); }); - describe('#countAsync()', function() { - it('should return the number of documents that match a query', async function() { + describe('v4 removals', function() { + it('has no hasNext() and no countAsync()', function() { const cursor = new FilesCursor({}, {}, filesCollection); - sandbox.stub(cursor.cursor, 'countAsync').returns(Promise.resolve(2)); - const count = await cursor.countAsync(); - expect(count).to.equal(2); + expect(cursor.hasNext).to.equal(undefined); + expect(cursor.countAsync).to.equal(undefined); }); }); diff --git a/tests/download-token.test.js b/tests/download-token.test.js new file mode 100644 index 00000000..d6d85b20 --- /dev/null +++ b/tests/download-token.test.js @@ -0,0 +1,69 @@ +/* global describe, it */ +import { expect } from 'chai'; +import { createDownloadToken, verifyDownloadToken } from '../download-token.js'; + +const SECRET = 's'.repeat(32); +const NOW = Date.UTC(2026, 9, 1); +const EXP = Math.floor(NOW / 1000) + 60; +const TARGET = { collectionName: 'files', _id: 'file1', version: 'original', now: NOW }; +const make = (overrides = {}) => createDownloadToken(SECRET, { collectionName: 'files', _id: 'file1', version: 'original', userId: 'u1', exp: EXP, ...overrides }); + +describe('download-token.js', function () { + it('round-trips userId and exp for the same _id and version', function () { + const token = make(); + expect(token.split('.')).to.have.length(3); + expect(token.startsWith(`${EXP}.`)).to.equal(true); + expect(verifyDownloadToken(SECRET, token, TARGET)).to.deep.equal({ userId: 'u1', exp: EXP }); + }); + + it('carries a null userId', function () { + expect(verifyDownloadToken(SECRET, make({ userId: null }), TARGET)).to.deep.equal({ userId: null, exp: EXP }); + }); + + it('keeps userIds with dots and non-ASCII characters', function () { + expect(verifyDownloadToken(SECRET, make({ userId: 'a.b.ü' }), TARGET).userId).to.equal('a.b.ü'); + }); + + it('rejects another _id or version', function () { + expect(verifyDownloadToken(SECRET, make(), { ...TARGET, _id: 'file2' })).to.equal(null); + expect(verifyDownloadToken(SECRET, make(), { ...TARGET, version: 'thumbnail' })).to.equal(null); + }); + + it('rejects another collection with the same _id and version', function () { + expect(verifyDownloadToken(SECRET, make(), { ...TARGET, collectionName: 'images' })).to.equal(null); + expect(verifyDownloadToken(SECRET, make({ collectionName: 'images' }), TARGET)).to.equal(null); + }); + + it('rejects an expired token', function () { + expect(verifyDownloadToken(SECRET, make({ exp: Math.floor(NOW / 1000) - 1 }), TARGET)).to.equal(null); + expect(verifyDownloadToken(SECRET, make({ exp: Math.floor(NOW / 1000) }), TARGET)).to.equal(null); + }); + + it('rejects tampered tokens and another secret', function () { + const [exp, user, sig] = make().split('.'); + // The first base64url character holds 6 signature bits; the last one has unused padding bits + const flipped = `${sig[0] === 'A' ? 'B' : 'A'}${sig.slice(1)}`; + expect(verifyDownloadToken(SECRET, `${exp}.${user}.${flipped}`, TARGET)).to.equal(null); + expect(verifyDownloadToken(SECRET, `${exp}.${Buffer.from('admin').toString('base64url')}.${sig}`, TARGET)).to.equal(null); + expect(verifyDownloadToken(SECRET, `${Number(exp) + 3600}.${user}.${sig}`, TARGET)).to.equal(null); + expect(verifyDownloadToken('t'.repeat(32), make(), TARGET)).to.equal(null); + }); + + it('accepts only the canonical form of exp and signature', function () { + const [exp, user, sig] = make().split('.'); + const ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_'; + // Flip the lowest bit of the last character: an unused bit, the decoded bytes stay the same + const lastSwapped = `${sig.slice(0, -1)}${ALPHABET[ALPHABET.indexOf(sig.at(-1)) ^ 1]}`; + expect(Buffer.from(lastSwapped, 'base64url').equals(Buffer.from(sig, 'base64url'))).to.equal(true); + expect(verifyDownloadToken(SECRET, `${exp}.${user}.${lastSwapped}`, TARGET)).to.equal(null); + expect(verifyDownloadToken(SECRET, `${exp}.${user}.${sig}=`, TARGET)).to.equal(null); + expect(verifyDownloadToken(SECRET, `0${exp}.${user}.${sig}`, TARGET)).to.equal(null); + expect(verifyDownloadToken(SECRET, `${exp}.${user}.${sig}`, TARGET)).to.deep.equal({ userId: 'u1', exp: EXP }); + }); + + it('rejects malformed input', function () { + [undefined, null, 42, ['a'], '', 'a.b', 'a.b.c.d', `${EXP}..`, `x${EXP}.AA.AA`, '9'.repeat(20) + '.AA.AA', 'x'.repeat(600)].forEach((token) => { + expect(verifyDownloadToken(SECRET, token, TARGET), String(token)).to.equal(null); + }); + }); +}); diff --git a/tests/helpers.test.js b/tests/helpers.test.js index 75d026b0..a6aec9f9 100644 --- a/tests/helpers.test.js +++ b/tests/helpers.test.js @@ -1,10 +1,22 @@ /* global describe, it */ import { expect } from 'chai'; import { FilesCollection } from '../server.js'; +import { MAX_UPLOAD_CHUNKS, applyPipes, fitChunkSize } from '../lib.js'; const helpers = FilesCollection.__helpers; describe('Helpers', function () { + it('applyPipes runs pipes in the order they were added', function () { + const order = []; + const result = applyPipes([ + (data) => { order.push('first'); return `${data}1`; }, + (data) => { order.push('second'); return `${data}2`; }, + ], 'x'); + expect(order).to.deep.equal(['first', 'second']); + expect(result).to.equal('x12'); + expect(applyPipes([], 'same')).to.equal('same'); + }); + it('isUndefined', function () { expect(helpers.isUndefined(null), 'isUndefined - null false').to.equal(false); expect(helpers.isUndefined(true), 'isUndefined - true false').to.equal(false); @@ -212,6 +224,18 @@ describe('Helpers', function () { expect(helpers.isUndefined(test2.hey)).to.equal(true); }); + it('fitChunkSize keeps the chunk count at or below MAX_UPLOAD_CHUNKS', function () { + const MiB = 1024 * 1024; + expect(MAX_UPLOAD_CHUNKS).to.equal(100000); + expect(fitChunkSize(10, 4, 8, 16 * MiB)).to.equal(4); + const total = (100000 * 512 * 1024) + 1; + const size = fitChunkSize(total, 512 * 1024, 8, 16 * MiB); + expect(size % 8).to.equal(0); + expect(Math.ceil(total / size)).to.be.at.most(100000); + expect(fitChunkSize(100000 * 20 * MiB, 512 * 1024, 8, 16 * MiB)).to.equal(16 * MiB); + expect(fitChunkSize(1000001, 4, 4, 1024, 1000) % 4).to.equal(0); + }); + it('now', function () { expect(helpers.now(), 'helpers.now() ~ +new Date()').to.be.closeTo(+new Date(), 50); }); diff --git a/tests/mime.test.js b/tests/mime.test.js new file mode 100644 index 00000000..cbd50e25 --- /dev/null +++ b/tests/mime.test.js @@ -0,0 +1,158 @@ +/* global describe, it, after */ +import { expect } from 'chai'; +import fs from 'node:fs'; +import os from 'node:os'; +import nodePath from 'node:path'; +import { SNIFF_BYTES, baseMimeType, detectMimeType, isUtf8Text, resolveMimeType, sniffFile } from '../mime.js'; + +const bytes = (...parts) => Buffer.concat(parts.map((part) => (typeof part === 'string' ? Buffer.from(part, 'latin1') : Buffer.from(part)))); +const ftyp = (brand) => bytes([0, 0, 0, 0x18], 'ftyp', brand, [0, 0, 0, 0], 'isommp42'); +const TMP_ROOT = fs.mkdtempSync(nodePath.join(os.tmpdir(), 'mf-mime-')); + +describe('mime.js', function () { + after(function () { + fs.rmSync(TMP_ROOT, { recursive: true, force: true }); + }); + + describe('detectMimeType', function () { + [ + ['png', bytes([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0x0d]), 'image/png'], + ['jpeg', bytes([0xff, 0xd8, 0xff, 0xe0]), 'image/jpeg'], + ['gif87a', bytes('GIF87a'), 'image/gif'], + ['gif89a', bytes('GIF89a'), 'image/gif'], + ['webp', bytes('RIFF', [0x24, 0, 0, 0], 'WEBPVP8 '), 'image/webp'], + ['bmp', bytes('BM', [0x36, 0, 0, 0], [0, 0, 0, 0], [0x36, 0, 0, 0]), 'image/bmp'], + ['ico', bytes([0, 0, 1, 0, 1, 0]), 'image/x-icon'], + ['tiff little-endian', bytes('II*', [0]), 'image/tiff'], + ['tiff big-endian', bytes('MM', [0], '*'), 'image/tiff'], + ['avif', ftyp('avif'), 'image/avif'], + ['heic', ftyp('heic'), 'image/heic'], + ['heif', ftyp('mif1'), 'image/heif'], + ['pdf', bytes('%PDF-1.7\n'), 'application/pdf'], + ['zip', bytes([0x50, 0x4b, 0x03, 0x04]), 'application/zip'], + ['gzip', bytes([0x1f, 0x8b, 0x08]), 'application/gzip'], + ['7z', bytes([0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]), 'application/x-7z-compressed'], + ['rar 4', bytes('Rar!', [0x1a, 0x07, 0x00]), 'application/vnd.rar'], + ['rar 5', bytes('Rar!', [0x1a, 0x07, 0x01, 0x00]), 'application/vnd.rar'], + ['cfb', bytes([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]), 'application/x-cfb'], + ['mp4', ftyp('isom'), 'video/mp4'], + ['m4a', ftyp('M4A '), 'audio/mp4'], + ['mov', ftyp('qt '), 'video/quicktime'], + ['webm', bytes([0x1a, 0x45, 0xdf, 0xa3, 0x9f, 0x42, 0x82, 0x84], 'webm'), 'video/webm'], + ['mkv', bytes([0x1a, 0x45, 0xdf, 0xa3, 0x9f, 0x42, 0x82, 0x88], 'matroska'), 'video/x-matroska'], + ['mp3 with ID3', bytes('ID3', [0x04, 0, 0, 0, 0, 0, 0]), 'audio/mpeg'], + ['mp3 frame', bytes([0xff, 0xfb, 0x90, 0x64]), 'audio/mpeg'], + ['wav', bytes('RIFF', [0x24, 0, 0, 0], 'WAVEfmt '), 'audio/wav'], + ['ogg vorbis', bytes('OggS', new Array(24).fill(0), [0x01], 'vorbis'), 'audio/ogg'], + ['ogg theora', bytes('OggS', new Array(24).fill(0), [0x80], 'theora'), 'video/ogg'], + ['flac', bytes('fLaC', [0, 0, 0, 0x22]), 'audio/flac'], + ['avi', bytes('RIFF', [0x24, 0, 0, 0], 'AVI LIST'), 'video/x-msvideo'], + ['wasm', bytes([0x00, 0x61, 0x73, 0x6d, 0x01, 0, 0, 0]), 'application/wasm'], + ['woff', bytes('wOFF', [0, 1, 0, 0]), 'font/woff'], + ['woff2', bytes('wOF2', [0, 1, 0, 0]), 'font/woff2'], + ].forEach(([name, input, expected]) => { + it(`detects ${name}`, function () { + expect(detectMimeType(input)).to.equal(expected); + expect(detectMimeType(new Uint8Array(input))).to.equal(expected); + }); + }); + + it('returns null for text, AAC frames, unknown ftyp brands, and short input', function () { + expect(detectMimeType(bytes('BMW is a car brand'))).to.equal(null); + expect(detectMimeType(bytes([0xff, 0xf1, 0x50, 0x80]))).to.equal(null); + expect(detectMimeType(ftyp('zzzz'))).to.equal(null); + expect(detectMimeType(bytes([0x89]))).to.equal(null); + expect(detectMimeType(Buffer.alloc(0))).to.equal(null); + expect(detectMimeType('not bytes')).to.equal(null); + }); + }); + + describe('baseMimeType', function () { + it('lowercases, drops parameters, and rejects invalid tokens', function () { + expect(baseMimeType('Text/Markdown; charset=utf-8')).to.equal('text/markdown'); + expect(baseMimeType('text/plain\r\nX-Evil: 1')).to.equal(''); + expect(baseMimeType('nonsense')).to.equal(''); + expect(baseMimeType(undefined)).to.equal(''); + }); + }); + + describe('isUtf8Text', function () { + it('accepts UTF-8 without NUL and rejects the rest', function () { + expect(isUtf8Text(Buffer.from('héllo ✓'), true)).to.equal(true); + expect(isUtf8Text(bytes([0x41, 0x00, 0x42]), true)).to.equal(false); + expect(isUtf8Text(bytes([0xc3, 0x28]), true)).to.equal(false); + expect(isUtf8Text(Buffer.alloc(0), true)).to.equal(false); + }); + + it('accepts a multibyte character cut at the end of a partial read', function () { + const cut = bytes('a'.repeat(10), [0xe2, 0x82]); + expect(isUtf8Text(cut, false)).to.equal(true); + expect(isUtf8Text(cut, true)).to.equal(false); + }); + }); + + describe('resolveMimeType', function () { + const png = bytes([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]); + const zip = bytes([0x50, 0x4b, 0x03, 0x04]); + const cfb = bytes([0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1]); + + it('prefers the detected type over the client type', function () { + expect(resolveMimeType(png, 'text/html')).to.equal('image/png'); + expect(resolveMimeType(zip, 'image/png')).to.equal('application/zip'); + }); + + it('keeps a client type that refines a container', function () { + expect(resolveMimeType(zip, 'application/vnd.openxmlformats-officedocument.wordprocessingml.document')).to.equal('application/vnd.openxmlformats-officedocument.wordprocessingml.document'); + expect(resolveMimeType(zip, 'application/vnd.oasis.opendocument.text')).to.equal('application/vnd.oasis.opendocument.text'); + expect(resolveMimeType(zip, 'application/epub+zip')).to.equal('application/epub+zip'); + expect(resolveMimeType(cfb, 'application/msword')).to.equal('application/msword'); + expect(resolveMimeType(cfb, 'application/vnd.ms-excel')).to.equal('application/vnd.ms-excel'); + expect(resolveMimeType(cfb, 'image/png')).to.equal('application/x-cfb'); + expect(resolveMimeType(ftyp('isom'), 'audio/mp4')).to.equal('audio/mp4'); + }); + + it('stores text as text/plain unless the client said an allowlisted text type', function () { + expect(resolveMimeType(Buffer.from(''), 'image/svg+xml')).to.equal('text/plain'); + expect(resolveMimeType(Buffer.from(''), 'image/png')).to.equal('text/plain'); + expect(resolveMimeType(Buffer.from('{"a":1}'), 'application/json')).to.equal('application/json'); + expect(resolveMimeType(Buffer.from('a,b\n'), 'text/csv')).to.equal('text/csv'); + expect(resolveMimeType(Buffer.from('# hi'), 'TEXT/Markdown; charset=utf-8')).to.equal('text/markdown'); + expect(resolveMimeType(Buffer.from('plain'), undefined)).to.equal('text/plain'); + expect(resolveMimeType(Buffer.from('plain'), 'text/plain\r\nX: y')).to.equal('text/plain'); + ['text/tab-separated-values', 'text/calendar', 'text/vtt'].forEach((type) => { + expect(resolveMimeType(Buffer.from('a'), type)).to.equal(type); + }); + }); + + it('stores UTF-8 text labeled with an active text type as text/plain', function () { + ['text/html', 'text/xml', 'text/css', 'text/javascript', 'text/xsl', 'application/xhtml+xml'].forEach((type) => { + expect(resolveMimeType(Buffer.from(''), type)).to.equal('text/plain'); + }); + }); + + it('does not keep a +xml client type for a container', function () { + expect(resolveMimeType(zip, 'application/vnd.ms-foo+xml')).to.equal('application/zip'); + expect(resolveMimeType(zip, 'application/vnd.openxmlformats-officedocument.foo+xml')).to.equal('application/zip'); + expect(resolveMimeType(zip, 'application/vnd.oasis.opendocument.foo+xml')).to.equal('application/zip'); + expect(resolveMimeType(cfb, 'application/vnd.ms-foo+xml')).to.equal('application/x-cfb'); + }); + + it('stores other binary data and empty files as application/octet-stream', function () { + expect(resolveMimeType(bytes([0x00, 0x01, 0x02, 0x03]), 'image/png')).to.equal('application/octet-stream'); + expect(resolveMimeType(Buffer.alloc(0), 'text/plain')).to.equal('application/octet-stream'); + }); + }); + + describe('sniffFile', function () { + it(`reads only the first ${SNIFF_BYTES} bytes`, async function () { + const textPath = nodePath.join(TMP_ROOT, 'long.txt'); + // Binary after the sniffed window does not change the result + fs.writeFileSync(textPath, Buffer.concat([Buffer.alloc(SNIFF_BYTES, 0x61), Buffer.from([0x00, 0xff])])); + expect(await sniffFile(textPath, 'text/plain')).to.equal('text/plain'); + + const pngPath = nodePath.join(TMP_ROOT, 'image.bin'); + fs.writeFileSync(pngPath, Buffer.concat([bytes([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), Buffer.alloc(10000)])); + expect(await sniffFile(pngPath, 'application/octet-stream')).to.equal('image/png'); + }); + }); +}); diff --git a/tests/security.test.js b/tests/security.test.js index 2dd60ef2..8a2015ae 100644 --- a/tests/security.test.js +++ b/tests/security.test.js @@ -10,6 +10,8 @@ import { Meteor } from 'meteor/meteor'; import { Random } from 'meteor/random'; import { FilesCollection } from '../server.js'; import { fixJSONParse } from '../lib.js'; +import { createDownloadToken as signToken } from '../download-token.js'; +import { chunkIdsFromBits } from '../write-stream.js'; const TMP_ROOT = fs.mkdtempSync(nodePath.join(os.tmpdir(), 'mf-security-')); let counter = 0; @@ -30,6 +32,8 @@ const createCollection = (config = {}) => { }, // Keeps the S5 warning out of the test output, S5 tests pass `onBeforeRemove: undefined` onBeforeRemove: () => true, + // Tests pick the file name on disk through `meta.fsName`: since v4 only the server names files + namingFunction: ({ file }) => file?.meta?.fsName, ...config, }); }; @@ -49,11 +53,10 @@ const startOpts = (overrides = {}) => { const size = overrides.size ?? 8; const chunkSize = overrides.chunkSize ?? 1024; return { - file: { name: 'file.txt', type: 'text/plain', size, meta: {}, ...(overrides.file || {}) }, + file: { name: 'file.txt', type: 'text/plain', size, meta: overrides.FSName ? { fsName: overrides.FSName } : {}, ...(overrides.file || {}) }, fileId: overrides.fileId || Random.id(), chunkSize, fileLength: overrides.fileLength ?? Math.max(1, Math.ceil(size / chunkSize)), - ...(overrides.FSName ? { FSName: overrides.FSName } : {}), }; }; @@ -314,6 +317,92 @@ describe('Security', function () { expect(res.size).to.equal(2048); }); + it('keeps the file when another process finishes the upload', async function () { + const opts = startOpts({ size: 2048, chunkSize: 1024 }); + await call(fc, '_Start', 'userA', opts); + await call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId: 1, binData: chunk(97) }); + const stream = fc._currentUploads[opts.fileId]; + // This process never sees chunk 2, so waiting for it would end in abort() + stream.maxEndRetries = 2; + // Another process wrote chunk 2, inserted the document, and marked the record finished + await fc.collection.insertAsync({ _id: opts.fileId, name: 'file.txt', path: stream.path, userId: 'userA' }); + await fc._preCollection.updateAsync({ _id: opts.fileId }, { $set: { isFinished: true } }); + for (let i = 0; i < 200 && fc._currentUploads[opts.fileId]; i++) { + await new Promise((resolve) => setTimeout(resolve, 10)); + } + expect(fc._currentUploads[opts.fileId]).to.equal(undefined); + expect(stream.ended).to.equal(true); + expect(stream.aborted).to.equal(false); + expect(fs.existsSync(stream.path)).to.equal(true); + }); + + it('persists written chunk ids as a bit set', async function () { + const opts = startOpts({ size: 33 * 8, chunkSize: 8 }); + await call(fc, '_Start', 'userA', opts); + expect((await fc._preCollection.findOneAsync(opts.fileId)).chunkBits).to.deep.equal([0, 0]); + for (const chunkId of [1, 32, 33]) { + await call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId, binData: Buffer.alloc(8, chunkId).toString('base64') }); + } + const record = await fc._preCollection.findOneAsync(opts.fileId); + expect(record.chunkBits).to.deep.equal([1 | (1 << 31), 1]); + expect(chunkIdsFromBits(record.chunkBits, 33)).to.deep.equal([1, 32, 33]); + }); + + it('records every bit when chunks of one word are written concurrently', async function () { + const opts = startOpts({ size: 40 * 8, chunkSize: 8 }); + await call(fc, '_Start', 'userA', opts); + const ids = Array.from({ length: 40 }, (_, i) => i + 1); + await Promise.all(ids.map((chunkId) => call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId, binData: Buffer.alloc(8, chunkId).toString('base64') }))); + const record = await fc._preCollection.findOneAsync(opts.fileId); + expect(chunkIdsFromBits(record.chunkBits, 40)).to.deep.equal(ids); + await simulateRestart(opts.fileId); + const res = await call(fc, '_Write', 'userA', { fileId: opts.fileId, eof: true }); + expect(res.size).to.equal(40 * 8); + }); + + it('resumes from recorded chunk ids, not from the file size', async function () { + const opts = startOpts({ size: 3072, chunkSize: 1024 }); + await call(fc, '_Start', 'userA', opts); + // Only the last chunk: the file is 3072 bytes long, chunks 1 and 2 are holes + await call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId: 3, binData: chunk(99) }); + await simulateRestart(opts.fileId); + await call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId: 2, binData: chunk(98) }); + const stream = fc._currentUploads[opts.fileId]; + expect([...stream.chunkIds].sort()).to.deep.equal([2, 3]); + stream.maxEndRetries = 2; + await expectMeteorError(call(fc, '_Write', 'userA', { fileId: opts.fileId, eof: true }), 503); + }); + + it('completes after restart once the missing chunks arrive', async function () { + const opts = startOpts({ size: 3072, chunkSize: 1024 }); + await call(fc, '_Start', 'userA', opts); + await call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId: 3, binData: chunk(99) }); + await simulateRestart(opts.fileId); + await call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId: 1, binData: chunk(97) }); + await call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId: 2, binData: chunk(98) }); + const res = await call(fc, '_Write', 'userA', { fileId: opts.fileId, eof: true }); + const doc = await fc.collection.findOneAsync(res._id); + expect(fs.readFileSync(doc.path, 'latin1')).to.equal(`${'a'.repeat(1024)}${'b'.repeat(1024)}${'c'.repeat(1024)}`); + }); + + it('does not count a chunk whose record failed, and keeps the upload', async function () { + const opts = startOpts({ size: 2048, chunkSize: 1024 }); + await call(fc, '_Start', 'userA', opts); + sinon.stub(fc, '_recordChunk').resolves(false); + await expectMeteorError(call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId: 1, binData: chunk(97) }), 503); + const stream = fc._currentUploads[opts.fileId]; + expect(stream.chunkIds.size).to.equal(0); + expect(stream.aborted).to.equal(false); + }); + + it('rejects resume of records without chunkBits (created before 4.0) with 410', async function () { + const opts = startOpts({ size: 2048, chunkSize: 1024 }); + await call(fc, '_Start', 'userA', opts); + await simulateRestart(opts.fileId); + await fc._preCollection.updateAsync({ _id: opts.fileId }, { $unset: { chunkBits: '' } }); + await expectMeteorError(call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId: 1, binData: chunk(97) }), 410); + }); + it('creates one WriteStream for concurrent resume requests', async function () { const opts = startOpts({ size: 2048, chunkSize: 1024 }); await call(fc, '_Start', 'userA', opts); @@ -504,6 +593,33 @@ describe('Security', function () { }); }); + describe('protected downloads read the file once', function () { + const insertFile = async (fc, _id) => { + const path = nodePath.join(fc.storagePath({}), `${_id}.txt`); + fs.writeFileSync(path, 'once'); + await fc.collection.insertAsync({ _id, name: `${_id}.txt`, size: 4, type: 'text/plain', path, versions: { original: { path, size: 4, type: 'text/plain', extension: 'txt' } } }); + }; + + it('reuses the document fetched for the protected function', async function () { + const fc = createCollection({ protected(fileObj) { return !!fileObj; } }); + await insertFile(fc, 'onceFn1'); + const findOne = sinon.spy(fc.collection, 'findOneAsync'); + const res = await httpRequest(`${fc.downloadRoute}/${fc.collectionName}/onceFn1/original/onceFn1.txt`); + expect(res.status).to.equal(200); + expect(res.body).to.equal('once'); + expect(findOne.callCount).to.equal(1); + }); + + it('reads once for protected: true', async function () { + const fc = createCollection({ protected: true }); + await insertFile(fc, 'onceBool1'); + const findOne = sinon.spy(fc.collection, 'findOneAsync'); + const res = await httpRequest(`${fc.downloadRoute}/${fc.collectionName}/onceBool1/original/onceBool1.txt`, { headers: { 'x-test-user': 'userA' } }); + expect(res.status).to.equal(200); + expect(findOne.callCount).to.equal(1); + }); + }); + describe('A2: idempotent EOF', function () { let fc; before(function () { @@ -685,6 +801,19 @@ describe('Security', function () { }); }); + describe('EOF reads chunks recorded by other instances', function () { + it('gives the stream the chunk ids stored in the upload record', async function () { + const fc = createCollection(); + const _id = `rec${Random.id()}`; + const path = nodePath.join(fc.storagePath({}), `${_id}.bin`); + const stream = await fc._createStream(_id, path, { fileLength: 2, chunkSize: 4 }, { exclusive: true }); + expect(await stream.loadRecordedChunkIds()).to.deep.equal([]); + await fc._preCollection.insertAsync({ _id, fileLength: 2, chunkBits: [0b11] }); + expect(await stream.loadRecordedChunkIds()).to.deep.equal([1, 2]); + await stream.abort(); + }); + }); + describe('A2: Start registration and path index', function () { it('registers the stream before the upload record is saved', async function () { const fc = createCollection(); @@ -835,6 +964,13 @@ describe('Security', function () { fc = createCollection(); }); + it('rejects Start with more than 100000 chunks', async function () { + const error = await expectMeteorError(call(fc, '_Start', 'userA', startOpts({ size: 100001, chunkSize: 1 })), 400); + expect(error.reason).to.include('Too many chunks'); + const ok = await call(fc, '_Start', 'userA', startOpts({ size: 100000, chunkSize: 1 })); + expect(ok).to.deep.equal({ status: 204 }); + }); + it('rejects Start with invalid chunkSize (0, negative, fractional, NaN, > 16 MiB)', async function () { for (const chunkSize of [0, -1, 1.5, NaN, 16 * 1024 * 1024 + 1]) { const opts = startOpts({ size: 4 }); @@ -890,18 +1026,25 @@ describe('Security', function () { }); }); - describe('S5: allowClientCode warning', function () { - it('warns once when allowClientCode is on and onBeforeRemove is missing', function () { + describe('S5: allowClientCode', function () { + it('defaults to false and the remove method answers 405', async function () { + const fc = createCollection({ allowClientCode: undefined }); + expect(fc.allowClientCode).to.equal(false); + await expectMeteorError(call(fc, '_Remove', 'userA', 'someId'), 405); + }); + + it('warns once when allowClientCode is true and onBeforeRemove is missing', function () { const warn = sinon.stub(console, 'warn'); - createCollection({ onBeforeRemove: undefined }); + createCollection({ allowClientCode: true, onBeforeRemove: undefined }); expect(warn.calledOnce).to.equal(true); expect(String(warn.firstCall.args[0])).to.include('onBeforeRemove'); + expect(String(warn.firstCall.args[0])).to.not.include('v4'); }); - it('does not warn when onBeforeRemove is set or allowClientCode is false', function () { + it('does not warn when onBeforeRemove is set or allowClientCode is not true', function () { const warn = sinon.stub(console, 'warn'); - createCollection({ onBeforeRemove: () => true }); - createCollection({ onBeforeRemove: undefined, allowClientCode: false }); + createCollection({ allowClientCode: true, onBeforeRemove: () => true }); + createCollection({ onBeforeRemove: undefined }); expect(warn.called).to.equal(false); }); }); @@ -968,8 +1111,8 @@ describe('Security', function () { }); }); - describe('S8: reserved client fields', function () { - it('ignores reserved keys sent in opts.file and keeps user keys', async function () { + describe('S8: client file fields allow-list', function () { + it('keeps only name, type, size, and meta from opts.file', async function () { const fc = createCollection(); const opts = startOpts({ size: 4, @@ -985,10 +1128,13 @@ describe('Security', function () { isImage: true, mime: 'text/html', 'mime-type': 'text/html', - custom: 'keep-me', + custom: 'dropped', + meta: { custom: 'kept' }, }, }); await call(fc, '_Start', 'userA', opts); + const record = await fc._preCollection.findOneAsync(opts.fileId); + expect(Object.keys(record.file).sort()).to.deep.equal(['meta', 'name', 'size', 'type']); await call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId: 1, binData: b64('data') }); await call(fc, '_Write', 'userA', { fileId: opts.fileId, eof: true }); const doc = await fc.collection.findOneAsync(opts.fileId); @@ -999,7 +1145,8 @@ describe('Security', function () { expect(doc.extension).to.equal('txt'); expect(doc.isImage).to.equal(false); expect(doc.mime).to.equal('text/plain'); - expect(doc.custom).to.equal('keep-me'); + expect(doc).to.not.have.property('custom'); + expect(doc.meta).to.deep.equal({ custom: 'kept' }); expect(fc._isPathInside(fc.storagePath({}), doc.path)).to.equal(true); expect(doc.versions.original.path).to.equal(doc.path); }); @@ -1257,7 +1404,7 @@ describe('Security', function () { const fc = createCollection(); const path = nodePath.join(fc.storagePath({}), 'cd.txt'); fs.writeFileSync(path, 'x'); - const res = await serveRequest(fc, { vRef: { name: 'naïve (1)\'s *"f".txt', size: 1, path } }); + const res = await serveRequest(fc, { vRef: { name: 'naïve (1)\'s *"f".txt', size: 1, path, type: 'text/plain' } }); expect(res.headers['content-disposition']).to.equal('inline; filename="na_ve (1)\'s *_f_.txt"; filename*=UTF-8\'\'na%C3%AFve%20%281%29%27s%20%2A%22f%22.txt'); }); @@ -1265,7 +1412,7 @@ describe('Security', function () { const fc = createCollection(); const path = nodePath.join(fc.storagePath({}), 'cd3.txt'); fs.writeFileSync(path, 'x'); - const res = await serveRequest(fc, { vRef: { name: '100%25 done.txt', size: 1, path } }); + const res = await serveRequest(fc, { vRef: { name: '100%25 done.txt', size: 1, path, type: 'text/plain' } }); expect(res.headers['content-disposition']).to.equal('inline; filename="100_25 done.txt"; filename*=UTF-8\'\'100%2525%20done.txt'); }); @@ -1278,6 +1425,53 @@ describe('Security', function () { }); }); + describe('Content-Disposition by type', function () { + let fc; + let path; + before(function () { + fc = createCollection(); + path = nodePath.join(fc.storagePath({}), 'cd-type.bin'); + fs.writeFileSync(path, 'x'); + }); + + const dispositionOf = async (type, extra = {}) => { + const res = await serveRequest(fc, { vRef: { name: 'f.bin', size: 1, path, type }, ...extra }); + return res.headers['content-disposition'].split(';')[0]; + }; + + [ + ['image/png', 'inline'], + ['IMAGE/JPEG', 'inline'], + ['image/svg+xml', 'attachment'], + ['video/mp4', 'inline'], + ['audio/mpeg', 'inline'], + ['application/pdf', 'inline'], + ['text/plain', 'inline'], + ['text/plain; charset=utf-8', 'inline'], + ['text/html', 'attachment'], + ['application/json', 'attachment'], + ['application/javascript', 'attachment'], + ['application/octet-stream', 'attachment'], + [undefined, 'attachment'], + ].forEach(([type, expected]) => { + it(`serves ${type} as ${expected}`, async function () { + expect(await dispositionOf(type)).to.equal(expected); + }); + }); + + it('forces attachment with ?download=true', async function () { + expect(await dispositionOf('image/png', { query: { download: 'true' } })).to.equal('attachment'); + }); + + it('lets responseHeaders override it', async function () { + const custom = createCollection({ responseHeaders: { 'Content-Disposition': 'inline' } }); + const p = nodePath.join(custom.storagePath({}), 'cd-override.html'); + fs.writeFileSync(p, 'x'); + const res = await serveRequest(custom, { vRef: { name: 'o.html', size: 1, path: p, type: 'text/html' } }); + expect(res.headers['content-disposition']).to.equal('inline'); + }); + }); + describe('Default Content-Type charset', function () { const serveType = async (fc, type) => { const path = nodePath.join(fc.storagePath({}), 'ct.txt'); @@ -1312,13 +1506,319 @@ describe('Security', function () { expect(res.headers['x-content-type-options']).to.equal('nosniff'); }); - it('is off by default and validated', async function () { + it('is on by default, can be turned off, and is validated', async function () { const fc = createCollection(); - expect(fc.nosniff).to.equal(false); + expect(fc.nosniff).to.equal(true); + const path = nodePath.join(fc.storagePath({}), 'ns-default.txt'); + fs.writeFileSync(path, 'x'); + const on = await serveRequest(fc, { vRef: { name: 'ns-default.txt', size: 1, path } }); + expect(on.headers['x-content-type-options']).to.equal('nosniff'); + + const off = createCollection({ nosniff: false }); + const offPath = nodePath.join(off.storagePath({}), 'ns-off.txt'); + fs.writeFileSync(offPath, 'x'); + const res = await serveRequest(off, { vRef: { name: 'ns-off.txt', size: 1, path: offPath } }); + expect(res.headers).to.not.have.property('x-content-type-options'); expect(() => createCollection({ nosniff: 'yes' })).to.throw(); }); }); + describe('Cache-Control default', function () { + const cacheHeader = async (fc, name) => { + const path = nodePath.join(fc.storagePath({}), name); + fs.writeFileSync(path, 'x'); + return (await serveRequest(fc, { vRef: { name, size: 1, path } })).headers['cache-control']; + }; + + it('is public for collections without protected', async function () { + expect(await cacheHeader(createCollection(), 'cc-public.txt')).to.equal('public, max-age=31536000, s-maxage=31536000'); + }); + + it('is private for protected collections', async function () { + expect(await cacheHeader(createCollection({ protected: () => true }), 'cc-fn.txt')).to.equal('private, max-age=31536000'); + sinon.stub(console, 'warn'); + expect(await cacheHeader(createCollection({ protected: true }), 'cc-true.txt')).to.equal('private, max-age=31536000'); + }); + + it('keeps an explicit cacheControl on protected collections', async function () { + expect(await cacheHeader(createCollection({ protected: () => true, cacheControl: 'no-store' }), 'cc-own.txt')).to.equal('no-store'); + }); + }); + + describe('_Remove accepts only a String _id', function () { + it('rejects an object selector with a Match error', async function () { + const fc = createCollection({ allowClientCode: true }); + let caught; + try { + await call(fc, '_Remove', 'userA', { _id: { $ne: null } }); + } catch (e) { + caught = e; + } + expect(caught?.errorType).to.equal('Match.Error'); + }); + + it('removes one file by String _id', async function () { + const fc = createCollection({ allowClientCode: true }); + const fileObj = await fc.writeAsync(Buffer.from('bye'), { name: 'bye.txt', type: 'text/plain' }); + expect(await call(fc, '_Remove', 'userA', fileObj._id)).to.equal(1); + expect(await fc.collection.findOneAsync(fileObj._id)).to.equal(undefined); + expect(fs.existsSync(fileObj.path)).to.equal(false); + }); + }); + + describe('protected: true deprecation', function () { + it('warns once per collection for protected: true', function () { + const warn = sinon.stub(console, 'warn'); + createCollection({ protected: true }); + const calls = warn.getCalls().filter((c) => String(c.args[0]).includes('"protected: true" is deprecated')); + expect(calls).to.have.length(1); + }); + + it('does not warn for a protected function', function () { + const warn = sinon.stub(console, 'warn'); + createCollection({ protected: () => true }); + expect(warn.called).to.equal(false); + }); + }); + + describe('naming is server-only', function () { + it('ignores FSName sent over DDP', async function () { + const fc = createCollection(); + const opts = { ...startOpts({ size: 4 }), FSName: 'client-chosen' }; + await call(fc, '_Start', 'userA', opts); + expect(nodePath.basename(fc._currentUploads[opts.fileId].path)).to.equal(`${opts.fileId}.txt`); + }); + + it('ignores FSName sent over HTTP', async function () { + const fc = createCollection(); + const opts = { ...startOpts({ size: 4 }), FSName: 'client-chosen-http' }; + const res = await httpRequest(`${fc.downloadRoute}/${fc.collectionName}/__upload`, { + method: 'POST', + headers: { 'x-start': '1', 'x-test-user': 'userA', 'content-type': 'application/json' }, + body: JSON.stringify(opts), + }); + expect(res.status).to.equal(204); + expect(nodePath.basename(fc._currentUploads[opts.fileId].path)).to.equal(`${opts.fileId}.txt`); + }); + + it('calls namingFunction with { file, fileId, userId } on Start', async function () { + const naming = sinon.spy(() => 'named'); + const fc = createCollection({ namingFunction: naming }); + const opts = startOpts({ size: 4, file: { meta: { a: 1 } } }); + await call(fc, '_Start', 'userA', opts); + const [ctx] = naming.firstCall.args; + expect(Object.keys(ctx).sort()).to.deep.equal(['file', 'fileId', 'userId']); + expect(ctx.fileId).to.equal(opts.fileId); + expect(ctx.userId).to.equal('userA'); + expect(ctx.file.name).to.equal('file.txt'); + expect(ctx.file.type).to.equal('text/plain'); + expect(ctx.file.size).to.equal(4); + expect(ctx.file.meta).to.deep.equal({ a: 1 }); + expect(ctx.file).to.not.have.property('path'); + expect(naming.firstCall.thisValue).to.equal(fc); + expect(nodePath.basename(fc._currentUploads[opts.fileId].path)).to.equal('named.txt'); + }); + }); + + describe('stored type comes from the file content', function () { + const PNG = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0x0d]); + + const upload = async (fc, data, file) => { + const opts = startOpts({ size: data.length, file }); + await call(fc, '_Start', 'userA', opts); + await call(fc, '_Write', 'userA', { fileId: opts.fileId, chunkId: 1, binData: data.toString('base64') }); + return call(fc, '_Write', 'userA', { fileId: opts.fileId, eof: true }); + }; + + it('stores the detected type and flags, not the client type', async function () { + const fc = createCollection(); + const res = await upload(fc, PNG, { name: 'x.html', type: 'text/html' }); + expect(res.type).to.equal('image/png'); + expect(res.isImage).to.equal(true); + expect(res.isText).to.equal(false); + const doc = await fc.collection.findOneAsync(res._id); + expect(doc.mime).to.equal('image/png'); + expect(doc['mime-type']).to.equal('image/png'); + expect(doc.versions.original.type).to.equal('image/png'); + }); + + it('stores text/plain for text sent as an image', async function () { + const fc = createCollection(); + const res = await upload(fc, Buffer.from(''), { name: 'x.svg', type: 'image/svg+xml' }); + expect(res.type).to.equal('text/plain'); + expect(res.isImage).to.equal(false); + expect(res.isText).to.equal(true); + }); + + it('stores text/plain for a script labeled text/html', async function () { + const fc = createCollection(); + const res = await upload(fc, Buffer.from(''), { name: 'x.html', type: 'text/html' }); + expect(res.type).to.equal('text/plain'); + expect(res.isText).to.equal(true); + }); + + it('stores application/octet-stream for unknown binary data', async function () { + const fc = createCollection(); + const res = await upload(fc, Buffer.from([0, 1, 2, 3]), { name: 'x.png', type: 'image/png' }); + expect(res.type).to.equal('application/octet-stream'); + expect(res.isImage).to.equal(false); + }); + + it('keeps the client type with trustClientMimeType: true', async function () { + const fc = createCollection({ trustClientMimeType: true }); + const res = await upload(fc, PNG, { name: 'x.html', type: 'text/html' }); + expect(res.type).to.equal('text/html'); + }); + + it('passes the client type to onBeforeUpload', async function () { + const seen = []; + const fc = createCollection({ onBeforeUpload(file) { seen.push(file.type); return true; } }); + await upload(fc, PNG, { name: 'x.html', type: 'text/html' }); + expect(seen.length).to.be.greaterThan(0); + expect(seen.every((type) => type === 'text/html')).to.equal(true); + }); + + it('validates trustClientMimeType', function () { + expect(createCollection().trustClientMimeType).to.equal(false); + expect(() => createCollection({ trustClientMimeType: 'yes' })).to.throw(); + }); + }); + + describe('signed download tokens', function () { + const SECRET = 'k'.repeat(40); + const ownerOnly = function (fileObj) { + return !!fileObj && fileObj.userId === this.userId; + }; + + const setup = async (config = {}) => { + const fc = createCollection({ downloadTokenSecret: SECRET, protected: ownerOnly, ...config }); + const _id = `tok${Random.id(6)}`; + const path = nodePath.join(fc.storagePath({}), `${_id}.txt`); + fs.writeFileSync(path, 'secret'); + await fc.collection.insertAsync({ _id, name: `${_id}.txt`, size: 6, type: 'text/plain', extension: 'txt', userId: 'owner', path, _downloadRoute: fc.downloadRoute, _collectionName: fc.collectionName, versions: { original: { path, size: 6, type: 'text/plain', extension: 'txt' } } }); + const doc = await fc.collection.findOneAsync(_id); + return { fc, doc, url: (token) => fc.link(doc, 'original', '/', { token }) }; + }; + + it('serves the file to the token user without a session', async function () { + const { fc, doc, url } = await setup(); + const res = await httpRequest(url(fc.createDownloadToken(doc, { userId: 'owner', expiresIn: 60 }))); + expect(res.status).to.equal(200); + expect(res.body).to.equal('secret'); + }); + + it('passes the token user to protected as this.userId and this.downloadToken', async function () { + const seen = []; + const { fc, doc, url } = await setup({ + protected(fileObj) { + seen.push({ userId: this.userId, token: this.downloadToken }); + return !!fileObj && fileObj.userId === this.userId; + }, + }); + const res = await httpRequest(url(fc.createDownloadToken(doc._id, { userId: 'intruder' }))); + expect(res.status).to.equal(401); + expect(seen[0].userId).to.equal('intruder'); + expect(seen[0].token.userId).to.equal('intruder'); + }); + + it('lets protected: true accept a token with a userId', async function () { + const { fc, doc, url } = await setup({ protected: true }); + expect((await httpRequest(url(fc.createDownloadToken(doc, { userId: 'someone' })))).status).to.equal(200); + expect((await httpRequest(url(fc.createDownloadToken(doc)))).status).to.equal(401); + }); + + it('answers 403 for a tampered, expired, or foreign token', async function () { + const { fc, doc, url } = await setup(); + const good = fc.createDownloadToken(doc, { userId: 'owner' }); + const [goodExp, goodUser, goodSig] = good.split('.'); + const tampered = `${goodExp}.${goodUser}.${goodSig[0] === 'A' ? 'B' : 'A'}${goodSig.slice(1)}`; + const expired = signToken(SECRET, { collectionName: fc.collectionName, _id: doc._id, version: 'original', userId: 'owner', exp: Math.floor(Date.now() / 1000) - 5 }); + const otherFile = fc.createDownloadToken('another1', { userId: 'owner' }); + const otherVersion = fc.createDownloadToken(doc, { userId: 'owner', version: 'thumbnail' }); + for (const token of [tampered, expired, otherFile, otherVersion, 'garbage']) { + const res = await httpRequest(url(token)); + expect(res.status, token).to.equal(403); + expect(res.body).to.equal('Access denied!'); + } + }); + + it('rejects a token minted by another collection for the same _id and version', async function () { + const { fc: fcA, doc } = await setup(); + const fcB = createCollection({ downloadTokenSecret: SECRET, protected: ownerOnly }); + await fcB.collection.insertAsync({ ...doc, _downloadRoute: fcB.downloadRoute, _collectionName: fcB.collectionName }); + const docB = await fcB.collection.findOneAsync(doc._id); + const urlB = (token) => fcB.link(docB, 'original', '/', { token }); + expect((await httpRequest(urlB(fcA.createDownloadToken(doc, { userId: 'owner' })))).status).to.equal(403); + expect((await httpRequest(urlB(fcB.createDownloadToken(doc, { userId: 'owner' })))).status).to.equal(200); + }); + + it('sends a private Cache-Control that ends with the token', async function () { + const { fc, doc, url } = await setup(); + const res = await httpRequest(url(fc.createDownloadToken(doc, { userId: 'owner', expiresIn: 120 }))); + expect(res.status).to.equal(200); + const match = /^private, max-age=(\d+)$/.exec(res.headers['cache-control']); + expect(match, res.headers['cache-control']).to.not.equal(null); + expect(Number(match[1])).to.be.within(118, 120); + }); + + it('keeps a Cache-Control set by responseHeaders on token downloads', async function () { + const { fc, doc, url } = await setup({ responseHeaders: { 'Cache-Control': 'no-store' } }); + const res = await httpRequest(url(fc.createDownloadToken(doc, { userId: 'owner' }))); + expect(res.status).to.equal(200); + expect(res.headers['cache-control']).to.equal('no-store'); + }); + + it('answers 404 for a token version the file does not have', async function () { + const { fc, doc } = await setup(); + const token = fc.createDownloadToken(doc, { userId: 'owner', version: 'thumbnail' }); + const res = await httpRequest(fc.link(doc, 'thumbnail', '/', { token })); + expect(res.status).to.equal(404); + expect(res.body).to.equal('File Not Found :('); + }); + + it('gives an invalid token no user, not the cookie user', function () { + const fc = createCollection({ downloadTokenSecret: SECRET }); + const httpObj = { request: { headers: { 'x-test-user': 'u1' } }, params: { _id: 'id1', version: 'original', query: { token: 'garbage' } } }; + expect(fc._getHttpUser(httpObj).userId).to.equal(null); + expect(fc._getHttpUser({ ...httpObj, downloadToken: undefined, params: { ...httpObj.params, query: {} } }).userId).to.equal('u1'); + }); + + it('ignores the token when no secret is set', async function () { + const { doc, url } = await setup({ downloadTokenSecret: undefined }); + expect(doc).to.be.an('object'); + expect((await httpRequest(url('garbage'))).status).to.equal(401); + }); + + it('ignores the token on public collections', function () { + const fc = createCollection({ downloadTokenSecret: SECRET, public: true, downloadRoute: `/pub${Random.id(6)}` }); + const httpObj = { params: { _id: 'id1', version: 'original', query: { token: 'garbage' } } }; + expect(fc._readDownloadToken(httpObj)).to.equal(null); + expect(fc._getHttpUser({ ...httpObj, request: { headers: { 'x-test-user': 'u1' } } }).userId).to.equal('u1'); + }); + + it('validates the secret and createDownloadToken() input', function () { + expect(() => createCollection({ downloadTokenSecret: 'short' })).to.throw(); + expect(() => createCollection({ downloadTokenSecret: 42 })).to.throw(); + const noSecret = createCollection(); + expect(() => noSecret.createDownloadToken('id1')).to.throw(Meteor.Error); + const fc = createCollection({ downloadTokenSecret: SECRET }); + expect(() => fc.createDownloadToken('id1', { expiresIn: 0 })).to.throw(); + expect(() => fc.createDownloadToken('id1', { expiresIn: 1.5 })).to.throw(); + expect(() => fc.createDownloadToken({})).to.throw(); + for (const [ref, opts] of [['id\n1', {}], ['id1', { version: 'original\nx' }], ['id1', { userId: 'u\n1' }]]) { + expect(() => fc.createDownloadToken(ref, opts)).to.throw(Meteor.Error).with.property('error', 400); + } + expect(fc.createDownloadToken('id1')).to.match(/^\d+\.\.[A-Za-z0-9_-]+$/); + }); + + it('keeps the secret out of enumerable properties', function () { + const fc = createCollection({ downloadTokenSecret: SECRET }); + expect(Object.keys(fc)).to.not.include('downloadTokenSecret'); + expect(Object.values(fc).includes(SECRET)).to.equal(false); + expect(fc.downloadTokenSecret).to.equal(SECRET); + }); + }); + describe('Misc hardening', function () { it('_getUserId: Map sessions resolve userId', function () { const fc = createCollection(); @@ -1332,14 +1832,12 @@ describe('Security', function () { } }); - it('_getUserId: object sessions ignore inherited keys', function () { + it('_getUserId: throws on plain object sessions (Map only since v4)', function () { const fc = createCollection(); const original = Meteor.server.sessions; try { Meteor.server.sessions = { tok: { userId: 'u2' } }; - expect(fc._getUserId('tok')).to.equal('u2'); - expect(fc._getUserId('constructor')).to.equal(null); - expect(fc._getUserId('__proto__')).to.equal(null); + expect(() => fc._getUserId('tok')).to.throw('incompatible'); } finally { Meteor.server.sessions = original; } diff --git a/tests/server.js b/tests/server.js index 8d4696d1..63028973 100644 --- a/tests/server.js +++ b/tests/server.js @@ -4,13 +4,17 @@ import './core.test'; import './cursor.test'; import './server.test'; import './helpers.test'; +import './mime.test'; import './security.test'; +import './download-token.test'; +import './storage.test'; +import './browser-fixtures'; -// Collections created in tests do not set `onBeforeRemove`, drop the S5 startup warning from the output +// Collections created in tests trigger the allowClientCode and protected: true startup warnings, drop them from the output const originalWarn = console.warn; before(function () { console.warn = function (...args) { - if (typeof args[0] === 'string' && args[0].includes('"allowClientCode" is on')) { + if (typeof args[0] === 'string' && (args[0].includes('"allowClientCode" is on') || args[0].includes('"protected: true" is deprecated'))) { return; } originalWarn.apply(console, args); diff --git a/tests/server.test.js b/tests/server.test.js index ca06155d..cbbdaa8a 100644 --- a/tests/server.test.js +++ b/tests/server.test.js @@ -11,6 +11,7 @@ import { Readable } from 'node:stream'; import { Meteor } from 'meteor/meteor'; import { Random } from 'meteor/random'; import { FilesCollection, WriteStream, helpers } from '../server.js'; +import { chunkIdsFromBits } from '../write-stream.js'; const TMP_ROOT = fs.mkdtempSync(nodePath.join(os.tmpdir(), 'mf-server-')); const tmpDir = (name) => { @@ -18,6 +19,7 @@ const tmpDir = (name) => { fs.mkdirSync(dir, { recursive: true }); return dir; }; +const PNG_BYTES = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0x0d]); /** * Assert that a promise rejects, and return the error @@ -246,6 +248,8 @@ describe('FilesCollection', function() { expect(newOpts).to.be.an('object'); expect(result.path).to.equal(nodePath.join(filesCollection.storagePath({}), 'newName')); expect(namingFunctionStub.calledOnce).to.be.true; + expect(namingFunctionStub.firstCall.args[0]).to.deep.include({ fileId: '123', userId: 'user1' }); + expect(namingFunctionStub.firstCall.args[0].file.name).to.equal('testFile'); expect(onBeforeUploadStub.calledOnce).to.be.true; // onInitiateUpload runs in _startUpload, after the upload record is saved expect(onInitiateUploadStub.called).to.be.false; @@ -475,6 +479,23 @@ describe('FilesCollection', function() { const fileObj = await fc.writeAsync(Buffer.from('n'), { name: 'n.txt' }); expect(fileObj.path).to.equal(nodePath.join(fc.storagePath({}), 'sub', 'x-y.txt')); }); + + it('passes { file, fileId, userId } to namingFunction', async function() { + collectionMock.restore(); + const naming = sinon.spy(() => 'ctx-name'); + const fc = new FilesCollection({ collectionName: `testserver-naming-${Random.id(4)}`, storagePath: tmpDir('naming-ctx'), namingFunction: naming }); + const fileObj = await fc.writeAsync(Buffer.from('abc'), { name: 'c.txt', type: 'text/plain', meta: { k: 1 }, userId: 'u9', fileId: 'ctx1' }); + expect(naming.firstCall.args[0]).to.deep.equal({ file: { name: 'c.txt', type: 'text/plain', size: 3, meta: { k: 1 } }, fileId: 'ctx1', userId: 'u9' }); + expect(nodePath.basename(fileObj.path)).to.equal('ctx-name.txt'); + }); + + it('detects the type of a buffer written without opts.type', async function() { + collectionMock.restore(); + const fileObj = await filesCollection.writeAsync(PNG_BYTES, { name: 'x.bin', fileId: 'sniff1' }); + expect(fileObj.type).to.equal('image/png'); + expect(fileObj.isImage).to.equal(true); + expect(fileObj.versions.original.type).to.equal('image/png'); + }); }); describe('#loadAsync()', function() { @@ -506,6 +527,12 @@ describe('FilesCollection', function() { return; } + if (req.url === '/png') { + res.writeHead(200, { 'Content-Type': 'text/html' }); + res.end(PNG_BYTES); + return; + } + res.statusCode = 200; res.setHeader('Content-Type', 'text/plain'); res.end(testdata); @@ -538,6 +565,11 @@ describe('FilesCollection', function() { expect(fs.readFileSync(file.path, 'utf8')).to.equal(testdata); }); + it('detects the type instead of trusting the response Content-Type', async function() { + const fileObj = await filesCollection.loadAsync(`http://127.0.0.1:${port}/png`, { name: 'p.html', fileId: 'png1' }); + expect(fileObj.type).to.equal('image/png'); + }); + it('C3: rejects with 408 on timeout instead of throwing inside a timer', async function() { const error = await expectRejects(filesCollection.loadAsync(`http://127.0.0.1:${port}/hang`, { name: 'hang.txt', fileId: 'hang1', timeout: 100 })); expect(error.error).to.equal(408); @@ -569,6 +601,14 @@ describe('FilesCollection', function() { const logged = debugSpy.getCalls().map((c) => c.args.map((a) => (typeof a === 'string' ? a : '')).join(' ')).join('\n'); expect(logged).to.not.include('secret-token'); }); + + it('passes { file, fileId, userId } to namingFunction', async function() { + const naming = sinon.spy(() => 'load-name'); + const fc = new FilesCollection({ collectionName: `testserver-loadnaming-${Random.id(4)}`, storagePath: tmpDir('load-naming'), namingFunction: naming }); + const fileObj = await fc.loadAsync(`http://127.0.0.1:${port}`, { name: 'l.txt', fileId: 'lctx1', userId: 'u8' }); + expect(naming.firstCall.args[0]).to.deep.equal({ file: { name: 'l.txt', type: undefined, meta: undefined }, fileId: 'lctx1', userId: 'u8' }); + expect(nodePath.basename(fileObj.path)).to.equal('load-name.txt'); + }); }); describe('#addFile', () => { @@ -590,6 +630,14 @@ describe('FilesCollection', function() { sinon.restore(); }); + it('detects the type of a file added without opts.type', async () => { + const pngPath = nodePath.join(nodePath.dirname(path), 'image.data'); + fs.writeFileSync(pngPath, PNG_BYTES); + const result = await filesCollection.addFile(pngPath, {}); + expect(result.type).to.equal('image/png'); + expect(result.isImage).to.equal(true); + }); + it('should add a file successfully', async () => { const result = await filesCollection.addFile(path, opts, proceedAfterUpload); @@ -856,6 +904,13 @@ describe('FilesCollection', function() { expect(res.body).to.equal('testfile'); }); + it('awaits an async responseHeaders function', async function() { + sinon.stub(filesCollection, 'responseHeaders').value(async () => ({ 'X-Async-Header': 'yes' })); + const res = await get(`http://127.0.0.1:${port}`); + expect(res.headers['x-async-header']).to.equal('yes'); + expect(res.body).to.equal('testfile'); + }); + it('C2: sends object responseHeaders', async function() { sinon.stub(filesCollection, 'responseHeaders').value({ 'X-Custom-Header': 'yes' }); const res = await get(`http://127.0.0.1:${port}`); @@ -903,14 +958,14 @@ describe('FilesCollection', function() { it('protected: true allows requests with a valid x-mtok', async function() { fc = new FilesCollection({ collectionName: `testserver-access-${Random.id(4)}`, storagePath: tmpDir('access'), protected: true }); - expect(await fc._checkAccess(makeHttp({ 'x-mtok': token }))).to.equal(true); + expect(await fc._checkAccess(makeHttp({ 'x-mtok': token }))).to.deep.equal({ fileRef: undefined }); }); it('protected: true allows requests with a valid x_mtok cookie', async function() { fc = new FilesCollection({ collectionName: `testserver-access-${Random.id(4)}`, storagePath: tmpDir('access'), protected: true }); const httpObj = makeHttp(); httpObj.request.Cookies = { has: (name) => name === 'x_mtok', get: () => token }; - expect(await fc._checkAccess(httpObj)).to.equal(true); + expect(await fc._checkAccess(httpObj)).to.deep.equal({ fileRef: undefined }); }); it('protected: true denies malformed or unknown x-mtok', async function() { @@ -925,7 +980,7 @@ describe('FilesCollection', function() { fc = new FilesCollection({ collectionName: `testserver-access-${Random.id(4)}`, storagePath: tmpDir('access'), protected: () => answer }); answer = true; - expect(await fc._checkAccess(makeHttp())).to.equal(true); + expect(await fc._checkAccess(makeHttp())).to.deep.equal({ fileRef: null }); answer = false; let httpObj = makeHttp(); @@ -943,7 +998,7 @@ describe('FilesCollection', function() { expect(httpObj.response.codes).to.deep.equal([401]); answer = Promise.resolve(true); - expect(await fc._checkAccess(makeHttp())).to.equal(true); + expect(await fc._checkAccess(makeHttp())).to.deep.equal({ fileRef: null }); }); it('protected function gets null fileObj for unknown _id, the doc for known _id, and userId', async function() { @@ -1025,12 +1080,64 @@ describe('FilesCollection', function() { dir = tmpDir('write-stream'); }); + afterEach(function() { + sinon.restore(); + }); + const create = async (name, maxLength = 2, options = {}) => { const stream = new WriteStream(nodePath.join(dir, name), maxLength, { chunkSize: 4 }, 0o644, 0o755, { fileId: name, exclusive: true, ...options }); await stream.init(); return stream; }; + it('chunkIdsFromBits reads ids from int32 words, including bit 31', function() { + expect(chunkIdsFromBits([0b101, -2147483648], 64)).to.deep.equal([1, 3, 64]); + expect(chunkIdsFromBits([0xff], 4)).to.deep.equal([1, 2, 3, 4]); + expect(chunkIdsFromBits(undefined, 4)).to.deep.equal([]); + }); + + it('end() re-reads recorded chunks once before it gives up', async function() { + const reload = sinon.stub().resolves([1, 2]); + const stream = await create('reload-ok.txt', 2, { loadRecordedChunkIds: reload }); + stream.maxEndRetries = 2; + // Another instance wrote chunk 2 into the same file and recorded it + expect(await stream.write(1, Buffer.from('abcd'))).to.equal(true); + fs.writeFileSync(stream.path, 'abcdefgh'); + expect(await stream.end()).to.equal(true); + expect(reload.calledOnce).to.equal(true); + expect(fs.readFileSync(stream.path, 'utf8')).to.equal('abcdefgh'); + }); + + it('end() aborts when the recorded chunks are still incomplete', async function() { + const stream = await create('reload-partial.txt', 2, { loadRecordedChunkIds: async () => [1, 99, 'x'] }); + stream.maxEndRetries = 2; + expect(await stream.write(1, Buffer.from('abcd'))).to.equal(true); + expect(await stream.end()).to.equal(false); + expect(stream.aborted).to.equal(true); + expect(fs.existsSync(stream.path)).to.equal(false); + }); + + it('end() aborts when re-reading recorded chunks fails', async function() { + const stream = await create('reload-error.txt', 2, { loadRecordedChunkIds: async () => { throw new Error('db down'); } }); + stream.maxEndRetries = 2; + sinon.stub(Meteor, '_debug'); + expect(await stream.end()).to.equal(false); + expect(stream.aborted).to.equal(true); + }); + + it('a write cut off by stop(false) returns false quietly and keeps the file', async function() { + const stream = await create('stopped-write.txt', 2); + sinon.stub(stream.fh, 'write').callsFake(async () => { + await stream.stop(false); + throw Object.assign(new Error('EBADF'), { code: 'EBADF' }); + }); + const debug = sinon.stub(Meteor, '_debug'); + expect(await stream.write(1, Buffer.from('abcd'))).to.equal(false); + expect(debug.called).to.equal(false); + expect(stream.aborted).to.equal(false); + expect(fs.existsSync(stream.path)).to.equal(true); + }); + it('C6: end() returns false after abort', async function() { const stream = await create('c6.txt'); await stream.abort(); diff --git a/tests/storage.test.js b/tests/storage.test.js new file mode 100644 index 00000000..e8f13863 --- /dev/null +++ b/tests/storage.test.js @@ -0,0 +1,342 @@ +/* global describe, it, before, after, afterEach */ +import { expect } from 'chai'; +import sinon from 'sinon'; +import fs from 'node:fs'; +import os from 'node:os'; +import nodePath from 'node:path'; +import http from 'node:http'; +import { Readable } from 'node:stream'; +import { Meteor } from 'meteor/meteor'; +import { Random } from 'meteor/random'; +import { MongoInternals } from 'meteor/mongo'; +import { FilesCollection, FSStorage, GridFSStorage } from '../server.js'; + +const { GridFSBucket, ObjectId } = MongoInternals.NpmModules.mongodb.module; +const TMP_ROOT = fs.mkdtempSync(nodePath.join(os.tmpdir(), 'mf-storage-')); + +const get = (path, headers = {}) => new Promise((resolve, reject) => { + http.get(new URL(path, Meteor.absoluteUrl()), { headers }, (res) => { + let data = ''; + res.on('data', (chunk) => { data += chunk; }); + res.on('end', () => resolve({ status: res.statusCode, headers: res.headers, body: data })); + }).on('error', reject); +}); + +const call = (fc, kind, userId, ...args) => { + return Meteor.server.method_handlers[fc._methodNames[kind]].apply({ userId, connection: null, unblock() {} }, args); +}; + +const create = (config = {}) => { + const collectionName = `st${Random.id(8)}`; + return new FilesCollection({ collectionName, storagePath: nodePath.join(TMP_ROOT, collectionName), ...config }); +}; + +const uploadDDP = async (fc, content) => { + const fileId = Random.id(); + await call(fc, '_Start', 'u1', { file: { name: 'up.txt', type: 'text/plain', size: content.length, meta: {} }, fileId, chunkSize: 1024, fileLength: 1 }); + await call(fc, '_Write', 'u1', { fileId, chunkId: 1, binData: Buffer.from(content).toString('base64') }); + return call(fc, '_Write', 'u1', { fileId, eof: true }); +}; + +const listen = (handler) => new Promise((resolve) => { + const server = http.createServer(handler); + server.listen(0, '127.0.0.1', () => resolve(server)); +}); + +const memoryAdapter = () => { + const files = new Map(); + const calls = []; + const sources = []; + return { + calls, + files, + sources, + async put(fileRef, versionName, localPath, opts) { + calls.push(['put', fileRef._id, versionName]); + sources.push(opts?.source); + files.set(`${fileRef._id}/${versionName}`, await fs.promises.readFile(localPath)); + return { name: 'memory', key: `${fileRef._id}/${versionName}` }; + }, + async createReadStream(fileRef, versionName, { start, end } = {}) { + calls.push(['read', fileRef._id, versionName, start, end]); + const data = files.get(fileRef.versions[versionName].meta.storage.key); + return Readable.from([Number.isInteger(start) ? data.subarray(start, end + 1) : data]); + }, + async remove(fileRef, versionName) { + calls.push(['remove', fileRef._id, versionName]); + files.delete(`${fileRef._id}/${versionName}`); + }, + }; +}; + +describe('Storage adapters', function () { + this.timeout(15000); + + after(function () { + fs.rmSync(TMP_ROOT, { recursive: true, force: true }); + }); + + afterEach(function () { + sinon.restore(); + }); + + describe('FSStorage', function () { + it('is the default adapter and invalid adapters are rejected', function () { + expect(create().storage).to.be.instanceOf(FSStorage); + expect(() => create({ storage: { put() {} } })).to.throw(); + expect(() => create({ storage: 'fs' })).to.throw(); + }); + + it('streams a range with an inclusive end', async function () { + const path = nodePath.join(TMP_ROOT, 'fs-range.txt'); + fs.writeFileSync(path, '0123456789'); + const stream = await new FSStorage().createReadStream({ versions: { original: { path } } }, 'original', { start: 2, end: 5 }); + let data = ''; + for await (const chunk of stream) { + data += chunk; + } + expect(data).to.equal('2345'); + }); + + it('rethrows ENOENT from remove() so unlinkAsync can log it', async function () { + let caught; + try { + await new FSStorage().remove({ versions: { original: { path: nodePath.join(TMP_ROOT, 'missing.txt') } } }, 'original'); + } catch (e) { + caught = e; + } + expect(caught?.code).to.equal('ENOENT'); + }); + }); + + describe('custom adapter', function () { + it('runs put() at upload EOF, before onAfterUpload, and stores its result', async function () { + const adapter = memoryAdapter(); + const seen = []; + const fc = create({ storage: adapter, onAfterUpload(fileRef) { seen.push(fileRef.versions.original.meta?.storage?.name); } }); + const res = await uploadDDP(fc, 'data'); + expect(seen).to.deep.equal(['memory']); + expect(adapter.files.get(`${res._id}/original`).toString()).to.equal('data'); + const doc = await fc.collection.findOneAsync(res._id); + expect(doc.versions.original.meta.storage).to.deep.equal({ name: 'memory', key: `${res._id}/original` }); + }); + + it('runs put() in writeAsync() before onAfterUpload', async function () { + const adapter = memoryAdapter(); + const order = []; + const fc = create({ storage: adapter, onAfterUpload() { order.push('onAfterUpload'); } }); + const put = adapter.put; + adapter.put = async (...args) => { + order.push('put'); + return put(...args); + }; + const doc = await fc.writeAsync(Buffer.from('hello'), { name: 'h.txt', type: 'text/plain' }, true); + expect(order).to.deep.equal(['put', 'onAfterUpload']); + expect(doc.versions.original.meta.storage.key).to.equal(`${doc._id}/original`); + }); + + it('serves through createReadStream(), with Range', async function () { + const adapter = memoryAdapter(); + const fc = create({ storage: adapter }); + const doc = await fc.writeAsync(Buffer.from('0123456789'), { name: 'r.txt', type: 'text/plain' }); + const full = await get(fc.link(doc, 'original', '/')); + expect(full.status).to.equal(200); + expect(full.body).to.equal('0123456789'); + const part = await get(fc.link(doc, 'original', '/'), { range: 'bytes=2-5' }); + expect(part.status).to.equal(206); + expect(part.body).to.equal('2345'); + expect(part.headers['content-length']).to.equal('4'); + expect(adapter.calls.filter((c) => c[0] === 'read').pop()).to.deep.equal(['read', doc._id, 'original', 2, 5]); + }); + + it('runs interceptDownload before the adapter', async function () { + const adapter = memoryAdapter(); + const fc = create({ + storage: adapter, + interceptDownload(httpObj) { + httpObj.response.writeHead(200); + httpObj.response.end('intercepted'); + return true; + }, + }); + const doc = await fc.writeAsync(Buffer.from('x'), { name: 'i.txt', type: 'text/plain' }); + expect((await get(fc.link(doc, 'original', '/'))).body).to.equal('intercepted'); + expect(adapter.calls.some((c) => c[0] === 'read')).to.equal(false); + }); + + it('answers 500 when createReadStream() rejects', async function () { + const adapter = memoryAdapter(); + const fc = create({ storage: adapter }); + const doc = await fc.writeAsync(Buffer.from('x'), { name: 'e.txt', type: 'text/plain' }); + adapter.createReadStream = async () => { + throw new Error('backend down'); + }; + const res = await get(fc.link(doc, 'original', '/')); + expect(res.status).to.equal(500); + expect(res.body).to.equal('Internal Server Error'); + }); + + it('answers 500 and survives an adapter stream that emits two errors', async function () { + const adapter = memoryAdapter(); + const fc = create({ storage: adapter }); + const doc = await fc.writeAsync(Buffer.from('x'), { name: 'e2.txt', type: 'text/plain' }); + let emitted = 0; + adapter.createReadStream = async () => { + const stream = new Readable({ read() {} }); + setTimeout(() => { + stream.emit('error', new Error('first')); + stream.emit('error', new Error('second')); + emitted = 2; + }, 5); + return stream; + }; + const res = await get(fc.link(doc, 'original', '/')); + expect(res.status).to.equal(500); + expect(res.body).to.equal('Internal Server Error'); + expect(emitted).to.equal(2); + }); + + it('removeAsync() calls remove() for each version', async function () { + const adapter = memoryAdapter(); + const fc = create({ storage: adapter }); + const doc = await fc.writeAsync(Buffer.from('x'), { name: 'd.txt', type: 'text/plain' }); + await fc.collection.updateAsync(doc._id, { $set: { 'versions.thumb': { path: '/nowhere', size: 1, type: 'text/plain', extension: 'txt' } } }); + expect(await fc.removeAsync({ _id: doc._id })).to.equal(1); + expect(adapter.calls.filter((c) => c[0] === 'remove')).to.deep.equal([['remove', doc._id, 'original'], ['remove', doc._id, 'thumb']]); + }); + + it('keeps the stored copy when onAfterUpload throws in writeAsync()', async function () { + const adapter = memoryAdapter(); + const fc = create({ storage: adapter, onAfterUpload() { throw new Error('hook failed'); } }); + let caught; + try { + await fc.writeAsync(Buffer.from('x'), { name: 'k.txt', type: 'text/plain', fileId: 'hookFails1' }, true); + } catch (e) { + caught = e; + } + expect(caught).to.be.instanceOf(Error); + expect(await fc.collection.findOneAsync('hookFails1')).to.be.an('object'); + expect(adapter.calls.map((c) => c[0])).to.deep.equal(['put']); + expect(adapter.files.has('hookFails1/original')).to.equal(true); + }); + + it('tells put() where the local file comes from', async function () { + const adapter = memoryAdapter(); + const fc = create({ storage: adapter }); + await uploadDDP(fc, 'up'); + await fc.writeAsync(Buffer.from('w'), { name: 'w.txt', type: 'text/plain' }); + const server = await listen((req, res) => res.end('loaded')); + try { + await fc.loadAsync(`http://127.0.0.1:${server.address().port}/l.txt`, { fileName: 'l.txt' }); + } finally { + server.close(); + } + const path = nodePath.join(TMP_ROOT, `add-${Random.id(6)}.txt`); + fs.writeFileSync(path, 'added'); + const added = await fc.addFile(path, { type: 'text/plain' }); + expect(adapter.sources).to.deep.equal(['upload', 'write', 'load', 'addFile']); + expect(adapter.files.get(`${added._id}/original`).toString()).to.equal('added'); + }); + + it('removes the stored copy when the insert fails after put()', async function () { + const adapter = memoryAdapter(); + const fc = create({ storage: adapter }); + sinon.stub(fc.collection, 'insertAsync').rejects(new Error('db down')); + let caught; + try { + await fc.writeAsync(Buffer.from('x'), { name: 'f.txt', type: 'text/plain', fileId: 'insertFails1' }); + } catch (e) { + caught = e; + } + expect(caught).to.be.instanceOf(Error); + expect(adapter.calls.map((c) => c[0])).to.deep.equal(['put', 'remove']); + expect(adapter.files.size).to.equal(0); + }); + }); + + describe('GridFSStorage', function () { + let fc; + let bucketName; + const bucket = () => new GridFSBucket(MongoInternals.defaultRemoteCollectionDriver().mongo.db, { bucketName }); + + before(function () { + bucketName = `mfgfs${Random.id(6)}`; + fc = create({ storage: new GridFSStorage({ bucketName }) }); + }); + + it('moves the file into the bucket and removes the local copy', async function () { + const doc = await fc.writeAsync(Buffer.from('gridfs data'), { name: 'g.txt', type: 'text/plain' }); + const { storage } = doc.versions.original.meta; + expect(storage.name).to.equal('gridfs'); + expect(storage.bucketName).to.equal(bucketName); + expect(storage.id).to.match(/^[a-f0-9]{24}$/); + expect(fs.existsSync(doc.path)).to.equal(false); + const stored = await bucket().find({ _id: new ObjectId(storage.id) }).toArray(); + expect(stored).to.have.length(1); + expect(stored[0].length).to.equal(11); + }); + + it('returns the bucket id when the local file can not be deleted', async function () { + const storage = new GridFSStorage({ bucketName }); + const localPath = nodePath.join(TMP_ROOT, `unlink-${Random.id(6)}.txt`); + fs.writeFileSync(localPath, 'kept'); + sinon.stub(fs.promises, 'unlink').rejects(Object.assign(new Error('EBUSY'), { code: 'EBUSY' })); + sinon.stub(Meteor, '_debug'); + const ref = await storage.put({ _id: 'unlinkFails1', name: 'u.txt', type: 'text/plain' }, 'original', localPath, { source: 'upload' }); + expect(ref.id).to.match(/^[a-f0-9]{24}$/); + expect(await bucket().find({ _id: new ObjectId(ref.id) }).toArray()).to.have.length(1); + expect(Meteor._debug.calledOnce).to.equal(true); + }); + + it('serves the whole file and the last byte', async function () { + const doc = await fc.writeAsync(Buffer.from('0123456789'), { name: 'g2.txt', type: 'text/plain' }); + expect((await get(fc.link(doc, 'original', '/'))).body).to.equal('0123456789'); + const last = await get(fc.link(doc, 'original', '/'), { range: 'bytes=9-9' }); + expect(last.status).to.equal(206); + expect(last.body).to.equal('9'); + }); + + it('answers 404 when the bucket file is gone', async function () { + const doc = await fc.writeAsync(Buffer.from('gone'), { name: 'g3.txt', type: 'text/plain' }); + await bucket().delete(new ObjectId(doc.versions.original.meta.storage.id)); + expect((await get(fc.link(doc, 'original', '/'))).status).to.equal(404); + }); + + it('removeAsync() deletes the bucket file', async function () { + const doc = await fc.writeAsync(Buffer.from('bye'), { name: 'g4.txt', type: 'text/plain' }); + const id = new ObjectId(doc.versions.original.meta.storage.id); + expect(await fc.removeAsync({ _id: doc._id })).to.equal(1); + expect(await bucket().find({ _id: id }).toArray()).to.have.length(0); + }); + + it('keeps the caller\'s file passed to addFile()', async function () { + const path = nodePath.join(TMP_ROOT, `gfs-add-${Random.id(6)}.txt`); + fs.writeFileSync(path, 'mine'); + const doc = await fc.addFile(path, { type: 'text/plain' }); + expect(doc.versions.original.meta.storage.name).to.equal('gridfs'); + expect(fs.readFileSync(path, 'utf8')).to.equal('mine'); + expect((await get(fc.link(doc, 'original', '/'))).body).to.equal('mine'); + }); + + it('removes the local copy of loadAsync() files', async function () { + const server = await listen((req, res) => res.end('remote')); + let doc; + try { + doc = await fc.loadAsync(`http://127.0.0.1:${server.address().port}/r.txt`, { fileName: 'r.txt' }); + } finally { + server.close(); + } + expect(fs.existsSync(doc.path)).to.equal(false); + expect((await get(fc.link(doc, 'original', '/'))).body).to.equal('remote'); + }); + + it('stores DDP uploads in the bucket', async function () { + const res = await uploadDDP(fc, 'data'); + expect(res.versions.original.meta.storage.name).to.equal('gridfs'); + const doc = await fc.collection.findOneAsync(res._id); + expect(doc.path).to.be.a('string'); + expect(fs.existsSync(doc.path)).to.equal(false); + expect((await get(fc.link(res, 'original', '/'))).body).to.equal('data'); + }); + }); +}); diff --git a/upload.js b/upload.js index 711e9044..69549588 100644 --- a/upload.js +++ b/upload.js @@ -5,7 +5,7 @@ import { Tracker } from 'meteor/tracker'; import { ReactiveVar } from 'meteor/reactive-var'; import { EventEmitter } from 'eventemitter3'; import { check, Match } from 'meteor/check'; -import { fixJSONParse, fixJSONStringify, helpers } from './lib.js'; +import { applyPipes, fitChunkSize, fixJSONParse, fixJSONStringify, helpers } from './lib.js'; const _rootUrl = (window.__meteor_runtime_config__.MOBILE_ROOT_URL || window.__meteor_runtime_config__.ROOT_URL).replace(/\/+$/, ''); const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); @@ -709,10 +709,7 @@ export class UploadInstance extends EventEmitter { } this.result.emit('data', evt.data.bin); - // Pipes run in reverse order of registration: the last added pipe runs first - for (let i = this.pipes.length - 1; i >= 0; i--) { - opts.binData = this.pipes[i](opts.binData); - } + opts.binData = applyPipes(this.pipes, opts.binData); } catch (pipeError) { this.inFlight = null; this.emit('error', toMeteorError(pipeError)); @@ -1025,7 +1022,7 @@ export class UploadInstance extends EventEmitter { if (!this.isStarted) { if (!this.startOpts) { - // `_prepare()` is still running (async `namingFunction`), it sends Start when ready + // `_prepare()` has not built the Start payload yet, it sends Start when ready return this; } await this._sendStart(); @@ -1083,9 +1080,12 @@ export class UploadInstance extends EventEmitter { // 4 base64 characters are 3 bytes const maxBase64ChunkSize = Math.floor((MAX_CHUNK_SIZE / 3)) * 4; this.config.chunkSize = Math.min(Math.max(4, Math.floor(this.config.chunkSize / 4) * 4), maxBase64ChunkSize); + // The server accepts at most MAX_UPLOAD_CHUNKS chunks + this.config.chunkSize = fitChunkSize(this.config.file.length, this.config.chunkSize, 4, maxBase64ChunkSize); _len = Math.ceil(this.config.file.length / this.config.chunkSize); } else { this.config.chunkSize = Math.min(Math.max(8, Math.floor(this.config.chunkSize / 8) * 8), MAX_CHUNK_SIZE); + this.config.chunkSize = fitChunkSize(this.fileData.size, this.config.chunkSize, 8, MAX_CHUNK_SIZE); _len = Math.ceil(this.fileData.size / this.config.chunkSize); } @@ -1099,18 +1099,13 @@ export class UploadInstance extends EventEmitter { fileLength: this.fileLength }; - this.FSName = this.collection.namingFunction ? (await this.collection.namingFunction(this.fileData)) : this.fileId; - if (this.FSName !== this.fileId) { - opts.FSName = this.FSName; - } - this.startOpts = opts; await this._upload(); } /** * Adds a transformation function to the upload pipeline. - * Pipes run in reverse order of registration: the last added pipe runs first. + * Pipes run in the order they were added: the first added pipe runs first. * @param {function(string): string} func - A function to process the binary data. * @returns {UploadInstance} Returns the current UploadInstance for chaining. */ diff --git a/write-stream.js b/write-stream.js index 15848ef9..fcdeec42 100644 --- a/write-stream.js +++ b/write-stream.js @@ -44,6 +44,28 @@ const isSameFile = (identity, stats) => { return !identity.birth || `${stats.birthtimeNs}` === identity.birth; }; +/** + * @function chunkIdsFromBits + * @param {number[]} bits - `chunkBits` of an upload record: int32 words, bit `(id - 1) % 32` of word `Math.floor((id - 1) / 32)` marks chunk `id` + * @param {number} maxLength - Number of chunks of the upload + * @summary Chunk ids recorded as written + * @returns {number[]} + */ +const chunkIdsFromBits = (bits, maxLength) => { + const ids = []; + if (!helpers.isArray(bits)) { + return ids; + } + + for (let id = 1; id <= maxLength; id++) { + const word = bits[Math.floor((id - 1) / 32)]; + if (Number.isInteger(word) && (word & (1 << ((id - 1) % 32))) !== 0) { + ids.push(id); + } + } + return ids; +}; + /** * @private * @locus Server @@ -59,6 +81,9 @@ const isSameFile = (identity, stats) => { * @param [options.idleTimeout=0] {number} - Close the file handle after this many ms without writes, reopen on the next write. `0` disables * @param [options.fileId] {string} - Upload id, used as part of the file handle cache key * @param [options.onAbort] {function} - Called after the stream is aborted + * @param [options.writtenChunkIds] {number[]} - Chunk ids already on disk, used when resuming. A resumed stream knows nothing else about the file content + * @param [options.loadRecordedChunkIds] {function} - Async, returns chunk ids recorded by any server instance. Read once before an incomplete `end()` gives up + * @param [options.onChunkWritten] {function(number): Promise} - Called after a chunk is on disk. The chunk counts as written only when it resolves `true` * @summary Writes chunks at their offsets into one file and tracks which chunks are written */ export default class WriteStream { @@ -76,6 +101,9 @@ export default class WriteStream { this.cacheKey = `${options.fileId || file?.fileId || file?._id || ''}:${this.path}`; this.identity = (helpers.isObject(options.identity) && options.identity.dev && options.identity.ino) ? fileIdentity(options.identity) : null; this.onAbort = helpers.isFunction(options.onAbort) ? options.onAbort : null; + this.onChunkWritten = helpers.isFunction(options.onChunkWritten) ? options.onChunkWritten : null; + this.loadRecordedChunkIds = helpers.isFunction(options.loadRecordedChunkIds) ? options.loadRecordedChunkIds : null; + this.writtenChunkIds = helpers.isArray(options.writtenChunkIds) ? options.writtenChunkIds : []; this.opening = null; this.fh = null; @@ -127,15 +155,13 @@ export default class WriteStream { throw openError; } - const stats = await fh.stat(); - if (stats.size > 0) { - // Chunks are written in order, so the existing size tells how many chunks are on disk - const written = Math.min(this.maxLength, Math.ceil(stats.size / this.file.chunkSize)); - for (let i = 1; i <= written; i++) { - this.chunkIds.add(i); + // Only recorded chunks count: the file size says nothing about holes + for (const id of this.writtenChunkIds) { + if (Number.isInteger(id) && id >= 1 && id <= this.maxLength) { + this.chunkIds.add(id); } - this.writtenChunks = this.chunkIds.size; } + this.writtenChunks = this.chunkIds.size; } this.fh = fh; @@ -304,10 +330,20 @@ export default class WriteStream { return false; } + if (this.onChunkWritten && !(await this.onChunkWritten(num))) { + // Not recorded: the client sends this chunk again + return false; + } + this.chunkIds.add(num); this.writtenChunks = this.chunkIds.size; return true; } catch (error) { + if (this.ended && !this.aborted) { + // `stop(false)` closed the handle mid-write: the upload finished elsewhere, nothing to report + return false; + } + const isFileLost = error?.error === 409 || error?.error === 410; if (!isFileLost) { Meteor._debug('[FilesCollection] [writeStream] [write] [Error:]', error); @@ -359,10 +395,46 @@ export default class WriteStream { return await this.stop(false); } + if (!this.ended && await this._mergeRecordedChunkIds() && this.isComplete()) { + return await this.stop(false); + } + await this.abort(); return false; } + /** + * @memberOf WriteStream + * @name _mergeRecordedChunkIds + * @summary Adds chunk ids that other server instances recorded for this upload + * @returns {Promise} - `true` if the recorded ids were read + */ + async _mergeRecordedChunkIds() { + if (!this.loadRecordedChunkIds) { + return false; + } + + let ids; + try { + ids = await this.loadRecordedChunkIds(); + } catch (loadError) { + Meteor._debug('[FilesCollection] [writeStream] [loadRecordedChunkIds] [ERROR:]', this.path, loadError); + return false; + } + + if (!helpers.isArray(ids) || this.aborted || this.ended) { + return false; + } + + for (const id of ids) { + if (Number.isInteger(id) && id >= 1 && id <= this.maxLength) { + this.chunkIds.add(id); + } + } + this.writtenChunks = this.chunkIds.size; + return true; + } + /** * @memberOf WriteStream * @name end @@ -475,4 +547,4 @@ export default class WriteStream { } } -export { fileIdentity, isSameFile }; +export { chunkIdsFromBits, fileIdentity, isSameFile };