From 79e36313418839046cd9e1129ab07a5aea99afe7 Mon Sep 17 00:00:00 2001 From: KIRTAN Date: Wed, 17 Jun 2026 22:01:10 +0530 Subject: [PATCH 1/2] fix(dashboard-api): optimize getGlobalStats database connections concurrency Fetch per-project user counts in parallel using Promise.all instead of sequentially to prevent linear slowdown with project counts. --- .../__tests__/analytics.controller.test.js | 369 ++++++++++++++++++ .../src/controllers/analytics.controller.js | 11 +- package-lock.json | 26 ++ 3 files changed, 401 insertions(+), 5 deletions(-) create mode 100644 apps/dashboard-api/src/__tests__/analytics.controller.test.js diff --git a/apps/dashboard-api/src/__tests__/analytics.controller.test.js b/apps/dashboard-api/src/__tests__/analytics.controller.test.js new file mode 100644 index 00000000..cb51014f --- /dev/null +++ b/apps/dashboard-api/src/__tests__/analytics.controller.test.js @@ -0,0 +1,369 @@ +'use strict'; + +const mongoose = require('mongoose'); + +class mockAppError extends Error { + constructor(statusCode, message) { + super(message); + this.statusCode = statusCode; + } +} + +class mockApiResponse { + constructor(data, message = '') { + this.data = data; + this.message = message; + } + send(res) { + return res.json({ + success: true, + data: this.data, + message: this.message + }); + } +} + +const mockProjectAggregate = jest.fn(); +const mockProjectFind = jest.fn(); +const mockDeveloperFindById = jest.fn(); +const mockLogCountDocuments = jest.fn(); +const mockLogFind = jest.fn(); +const mockLogDistinct = jest.fn(); +const mockWebhookCountDocuments = jest.fn(); +const mockGetConnection = jest.fn(); +const mockPlatformEventFind = jest.fn(); +const mockPlatformEventFindOne = jest.fn(); +const mockDeveloperActivityFindOne = jest.fn(); +const mockDeveloperActivityAggregate = jest.fn(); + +jest.mock('@urbackend/common', () => ({ + Project: { + aggregate: mockProjectAggregate, + find: mockProjectFind, + }, + Developer: { + findById: mockDeveloperFindById, + }, + Log: { + countDocuments: mockLogCountDocuments, + find: mockLogFind, + distinct: mockLogDistinct, + }, + Webhook: { + countDocuments: mockWebhookCountDocuments, + }, + PlatformEvent: { + find: mockPlatformEventFind, + findOne: mockPlatformEventFindOne, + }, + DeveloperActivity: { + findOne: mockDeveloperActivityFindOne, + aggregate: mockDeveloperActivityAggregate, + }, + getConnection: mockGetConnection, + resolveEffectivePlan: jest.fn(dev => dev?.plan || 'free'), + getPlanLimits: jest.fn(() => ({ maxProjects: 5, maxCollections: 10 })), + getProjectAccessQuery: jest.fn(userId => ({ owner: userId })), + AppError: mockAppError, + ApiResponse: mockApiResponse, +})); + +const controller = require('../controllers/analytics.controller'); + +describe('Analytics Controller', () => { + let req, res, next; + + beforeEach(() => { + jest.clearAllMocks(); + req = { + user: { _id: new mongoose.Types.ObjectId().toString() }, + }; + res = { + json: jest.fn().mockReturnThis(), + }; + next = jest.fn(); + }); + + describe('getGlobalStats', () => { + it('should return aggregated global stats with user counts fetched in parallel', async () => { + const userId = req.user._id; + + mockProjectAggregate.mockResolvedValueOnce([ + { + _id: null, + totalProjects: 2, + totalDatabaseUsed: 100, + totalStorageUsed: 200, + totalCollections: 5, + } + ]); + + mockDeveloperFindById.mockReturnValue({ + select: jest.fn().mockResolvedValue({ + plan: 'pro', + maxProjects: 10, + maxCollections: 20, + planExpiresAt: null, + }), + }); + + mockProjectFind.mockReturnValue({ + select: jest.fn().mockReturnValue({ + lean: jest.fn().mockResolvedValue([ + { _id: new mongoose.Types.ObjectId('60c72b2f9b1d8b22fc5a87ba') }, + { _id: new mongoose.Types.ObjectId('60c72b2f9b1d8b22fc5a87bb') } + ]) + }) + }); + + mockLogCountDocuments.mockResolvedValueOnce(150); + mockWebhookCountDocuments.mockResolvedValueOnce(25); + + const mockDbConn1 = { + collection: jest.fn().mockReturnValue({ + countDocuments: jest.fn().mockResolvedValue(10), + }), + }; + const mockDbConn2 = { + collection: jest.fn().mockReturnValue({ + countDocuments: jest.fn().mockResolvedValue(20), + }), + }; + + mockGetConnection + .mockResolvedValueOnce(mockDbConn1) + .mockResolvedValueOnce(mockDbConn2); + + await controller.getGlobalStats(req, res, next); + + expect(mockGetConnection).toHaveBeenCalledTimes(2); + expect(mockGetConnection).toHaveBeenNthCalledWith(1, '60c72b2f9b1d8b22fc5a87ba'); + expect(mockGetConnection).toHaveBeenNthCalledWith(2, '60c72b2f9b1d8b22fc5a87bb'); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + expect(responseData.data.usage).toEqual({ + totalProjects: 2, + totalCollections: 5, + totalStorageUsed: 200, + totalDatabaseUsed: 100, + totalRequests: 150, + totalWebhooks: 25, + totalUsers: 30, // 10 + 20 + }); + }); + + it('should handle db connection errors gracefully without failing the entire request', async () => { + mockProjectAggregate.mockResolvedValueOnce([]); + mockDeveloperFindById.mockReturnValue({ + select: jest.fn().mockResolvedValue(null), + }); + mockProjectFind.mockReturnValue({ + select: jest.fn().mockReturnValue({ + lean: jest.fn().mockResolvedValue([ + { _id: new mongoose.Types.ObjectId('60c72b2f9b1d8b22fc5a87bc') }, + { _id: new mongoose.Types.ObjectId('60c72b2f9b1d8b22fc5a87bd') } + ]) + }) + }); + + mockLogCountDocuments.mockResolvedValueOnce(0); + mockWebhookCountDocuments.mockResolvedValueOnce(0); + + // First project connection fails, second succeeds + const mockDbConn = { + collection: jest.fn().mockReturnValue({ + countDocuments: jest.fn().mockResolvedValue(5), + }), + }; + mockGetConnection + .mockRejectedValueOnce(new Error('Connection timeout')) + .mockResolvedValueOnce(mockDbConn); + + await controller.getGlobalStats(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.data.usage.totalUsers).toBe(5); // 0 + 5 + }); + }); + + describe('getRecentActivity', () => { + it('should return formatted recent logs', async () => { + mockProjectFind.mockReturnValue({ + distinct: jest.fn().mockResolvedValue(['proj1']) + }); + + mockLogFind.mockReturnValue({ + sort: jest.fn().mockReturnThis(), + limit: jest.fn().mockReturnThis(), + populate: jest.fn().mockReturnThis(), + lean: jest.fn().mockResolvedValue([ + { + _id: 'log1', + projectId: { _id: 'proj1', name: 'My Project' }, + method: 'GET', + path: '/api/users', + status: 200, + timestamp: '2026-06-17T00:00:00Z', + } + ]) + }); + + await controller.getRecentActivity(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + expect(responseData.data).toEqual([ + { + id: 'log1', + projectName: 'My Project', + projectId: 'proj1', + method: 'GET', + path: '/api/users', + status: 200, + timestamp: '2026-06-17T00:00:00Z', + } + ]); + }); + }); + + describe('getActivationFunnel', () => { + it('should return status of activation funnel steps', async () => { + mockPlatformEventFind.mockReturnValue({ + sort: jest.fn().mockReturnThis(), + select: jest.fn().mockReturnThis(), + lean: jest.fn().mockResolvedValue([ + { event: 'signup_completed', timestamp: '2026-06-10T00:00:00Z' }, + { event: 'email_verified', timestamp: '2026-06-10T00:05:00Z' } + ]) + }); + + await controller.getActivationFunnel(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + const steps = responseData.data.steps; + expect(steps[0]).toEqual({ step: 'signup_completed', order: 1, completed: true, completedAt: '2026-06-10T00:00:00Z' }); + expect(steps[1]).toEqual({ step: 'email_verified', order: 2, completed: true, completedAt: '2026-06-10T00:05:00Z' }); + expect(steps[2]).toEqual({ step: 'project_created', order: 3, completed: false, completedAt: null }); + }); + }); + + describe('getRetention', () => { + it('should return d1/d7/d30 retention flags', async () => { + mockPlatformEventFindOne.mockReturnValue({ + sort: jest.fn().mockReturnThis(), + lean: jest.fn().mockResolvedValue({ + timestamp: '2026-06-10T12:00:00Z' + }) + }); + + mockDeveloperActivityFindOne + .mockImplementation(({ date }) => { + return { + lean: jest.fn().mockResolvedValue({ date }) + }; + }); + + await controller.getRetention(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + expect(responseData.data.d1).toBe(true); + expect(responseData.data.d7).toBe(true); + expect(responseData.data.d30).toBe(true); + }); + + it('should return false flags if no signup event exists', async () => { + mockPlatformEventFindOne.mockReturnValue({ + sort: jest.fn().mockReturnThis(), + lean: jest.fn().mockResolvedValue(null) + }); + + await controller.getRetention(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.data).toEqual({ d1: false, d7: false, d30: false, signupDate: null }); + }); + }); + + describe('getEngagement', () => { + it('should return aggregated engagement metrics', async () => { + mockDeveloperActivityAggregate.mockResolvedValueOnce([ + { + totalApiCalls: 500, + totalMailSent: 50, + totalStorageUploads: 10, + totalWebhooksFired: 5, + activeDays: 3, + allProjectIds: [['proj1'], ['proj2', 'proj1']] + } + ]); + + await controller.getEngagement(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + expect(responseData.data).toEqual({ + window: '30d', + totalApiCalls: 500, + totalMailSent: 50, + totalStorageUploads: 10, + totalWebhooksFired: 5, + activeDays: 3, + uniqueActiveProjects: 2, + }); + }); + }); + + describe('getNorthStar', () => { + it('should return north star metrics', async () => { + mockProjectFind.mockReturnValue({ + select: jest.fn().mockReturnValue({ + lean: jest.fn().mockResolvedValue([ + { _id: 'proj1', name: 'Proj 1' }, + { _id: 'proj2', name: 'Proj 2' } + ]) + }) + }); + + mockLogDistinct.mockResolvedValueOnce(['proj1']); + + await controller.getNorthStar(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + expect(responseData.data).toEqual({ + activeProjects: 1, + totalProjects: 2, + percentage: 50, + }); + }); + + it('should return 0 metrics if developer has no projects', async () => { + mockProjectFind.mockReturnValue({ + select: jest.fn().mockReturnValue({ + lean: jest.fn().mockResolvedValue([]) + }) + }); + + await controller.getNorthStar(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.data).toEqual({ + activeProjects: 0, + totalProjects: 0, + percentage: 0, + }); + }); + }); +}); diff --git a/apps/dashboard-api/src/controllers/analytics.controller.js b/apps/dashboard-api/src/controllers/analytics.controller.js index 2f388fe6..e7b5c7bc 100644 --- a/apps/dashboard-api/src/controllers/analytics.controller.js +++ b/apps/dashboard-api/src/controllers/analytics.controller.js @@ -40,16 +40,17 @@ module.exports.getGlobalStats = async (req, res, next) => { const totalRequests = await Log.countDocuments({ projectId: { $in: projectIds } }); const totalWebhooks = await Webhook.countDocuments({ projectId: { $in: projectIds } }); - let totalUsers = 0; - for (const project of projects) { + const userCountPromises = projects.map(async (project) => { try { const conn = await getConnection(project._id.toString()); - const userCount = await conn.collection('users').countDocuments(); - totalUsers += userCount; + return await conn.collection('users').countDocuments(); } catch (err) { console.error(`Failed to count users for project ${project._id}:`, err.message); + return 0; } - } + }); + const userCounts = await Promise.all(userCountPromises); + const totalUsers = userCounts.reduce((sum, count) => sum + count, 0); const effectivePlan = resolveEffectivePlan(dev); const limits = getPlanLimits({ diff --git a/package-lock.json b/package-lock.json index fb6c3540..987f5df7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2912,6 +2912,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2945,6 +2946,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -2978,6 +2980,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -3043,6 +3046,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -6875,6 +6879,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -6891,6 +6896,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -6907,6 +6913,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -6923,6 +6930,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -6939,6 +6947,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -6955,6 +6964,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -6971,6 +6981,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -6987,6 +6998,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7003,6 +7015,7 @@ "cpu": [ "arm" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7019,6 +7032,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7035,6 +7049,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7051,6 +7066,7 @@ "cpu": [ "loong64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7067,6 +7083,7 @@ "cpu": [ "mips64el" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7083,6 +7100,7 @@ "cpu": [ "ppc64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7099,6 +7117,7 @@ "cpu": [ "riscv64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7115,6 +7134,7 @@ "cpu": [ "s390x" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7131,6 +7151,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7147,6 +7168,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7163,6 +7185,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7179,6 +7202,7 @@ "cpu": [ "x64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7195,6 +7219,7 @@ "cpu": [ "arm64" ], + "dev": true, "license": "MIT", "optional": true, "os": [ @@ -7211,6 +7236,7 @@ "cpu": [ "ia32" ], + "dev": true, "license": "MIT", "optional": true, "os": [ From b293187cc716bd9e0c08a83b376b2ce605b93869 Mon Sep 17 00:00:00 2001 From: KIRTAN Date: Fri, 19 Jun 2026 17:44:40 +0530 Subject: [PATCH 2/2] fix: bounded concurrency for global stats user count and strengthen parallelism test --- .../__tests__/analytics.controller.test.js | 702 +++++++++--------- .../src/controllers/analytics.controller.js | 159 ++-- 2 files changed, 476 insertions(+), 385 deletions(-) diff --git a/apps/dashboard-api/src/__tests__/analytics.controller.test.js b/apps/dashboard-api/src/__tests__/analytics.controller.test.js index cb51014f..6f89a2a9 100644 --- a/apps/dashboard-api/src/__tests__/analytics.controller.test.js +++ b/apps/dashboard-api/src/__tests__/analytics.controller.test.js @@ -1,26 +1,26 @@ -'use strict'; +"use strict"; -const mongoose = require('mongoose'); +const mongoose = require("mongoose"); class mockAppError extends Error { - constructor(statusCode, message) { - super(message); - this.statusCode = statusCode; - } + constructor(statusCode, message) { + super(message); + this.statusCode = statusCode; + } } class mockApiResponse { - constructor(data, message = '') { - this.data = data; - this.message = message; - } - send(res) { - return res.json({ - success: true, - data: this.data, - message: this.message - }); - } + constructor(data, message = "") { + this.data = data; + this.message = message; + } + send(res) { + return res.json({ + success: true, + data: this.data, + message: this.message, + }); + } } const mockProjectAggregate = jest.fn(); @@ -36,334 +36,372 @@ const mockPlatformEventFindOne = jest.fn(); const mockDeveloperActivityFindOne = jest.fn(); const mockDeveloperActivityAggregate = jest.fn(); -jest.mock('@urbackend/common', () => ({ - Project: { - aggregate: mockProjectAggregate, - find: mockProjectFind, - }, - Developer: { - findById: mockDeveloperFindById, - }, - Log: { - countDocuments: mockLogCountDocuments, - find: mockLogFind, - distinct: mockLogDistinct, - }, - Webhook: { - countDocuments: mockWebhookCountDocuments, - }, - PlatformEvent: { - find: mockPlatformEventFind, - findOne: mockPlatformEventFindOne, - }, - DeveloperActivity: { - findOne: mockDeveloperActivityFindOne, - aggregate: mockDeveloperActivityAggregate, - }, - getConnection: mockGetConnection, - resolveEffectivePlan: jest.fn(dev => dev?.plan || 'free'), - getPlanLimits: jest.fn(() => ({ maxProjects: 5, maxCollections: 10 })), - getProjectAccessQuery: jest.fn(userId => ({ owner: userId })), - AppError: mockAppError, - ApiResponse: mockApiResponse, +jest.mock("@urbackend/common", () => ({ + Project: { + aggregate: mockProjectAggregate, + find: mockProjectFind, + }, + Developer: { + findById: mockDeveloperFindById, + }, + Log: { + countDocuments: mockLogCountDocuments, + find: mockLogFind, + distinct: mockLogDistinct, + }, + Webhook: { + countDocuments: mockWebhookCountDocuments, + }, + PlatformEvent: { + find: mockPlatformEventFind, + findOne: mockPlatformEventFindOne, + }, + DeveloperActivity: { + findOne: mockDeveloperActivityFindOne, + aggregate: mockDeveloperActivityAggregate, + }, + getConnection: mockGetConnection, + resolveEffectivePlan: jest.fn((dev) => dev?.plan || "free"), + getPlanLimits: jest.fn(() => ({ maxProjects: 5, maxCollections: 10 })), + getProjectAccessQuery: jest.fn((userId) => ({ owner: userId })), + AppError: mockAppError, + ApiResponse: mockApiResponse, })); -const controller = require('../controllers/analytics.controller'); - -describe('Analytics Controller', () => { - let req, res, next; - - beforeEach(() => { - jest.clearAllMocks(); - req = { - user: { _id: new mongoose.Types.ObjectId().toString() }, - }; - res = { - json: jest.fn().mockReturnThis(), - }; - next = jest.fn(); - }); - - describe('getGlobalStats', () => { - it('should return aggregated global stats with user counts fetched in parallel', async () => { - const userId = req.user._id; - - mockProjectAggregate.mockResolvedValueOnce([ - { - _id: null, - totalProjects: 2, - totalDatabaseUsed: 100, - totalStorageUsed: 200, - totalCollections: 5, - } - ]); - - mockDeveloperFindById.mockReturnValue({ - select: jest.fn().mockResolvedValue({ - plan: 'pro', - maxProjects: 10, - maxCollections: 20, - planExpiresAt: null, - }), - }); - - mockProjectFind.mockReturnValue({ - select: jest.fn().mockReturnValue({ - lean: jest.fn().mockResolvedValue([ - { _id: new mongoose.Types.ObjectId('60c72b2f9b1d8b22fc5a87ba') }, - { _id: new mongoose.Types.ObjectId('60c72b2f9b1d8b22fc5a87bb') } - ]) - }) - }); - - mockLogCountDocuments.mockResolvedValueOnce(150); - mockWebhookCountDocuments.mockResolvedValueOnce(25); - - const mockDbConn1 = { - collection: jest.fn().mockReturnValue({ - countDocuments: jest.fn().mockResolvedValue(10), - }), - }; - const mockDbConn2 = { - collection: jest.fn().mockReturnValue({ - countDocuments: jest.fn().mockResolvedValue(20), - }), - }; - - mockGetConnection - .mockResolvedValueOnce(mockDbConn1) - .mockResolvedValueOnce(mockDbConn2); - - await controller.getGlobalStats(req, res, next); - - expect(mockGetConnection).toHaveBeenCalledTimes(2); - expect(mockGetConnection).toHaveBeenNthCalledWith(1, '60c72b2f9b1d8b22fc5a87ba'); - expect(mockGetConnection).toHaveBeenNthCalledWith(2, '60c72b2f9b1d8b22fc5a87bb'); - - expect(res.json).toHaveBeenCalledTimes(1); - const responseData = res.json.mock.calls[0][0]; - expect(responseData.success).toBe(true); - expect(responseData.data.usage).toEqual({ - totalProjects: 2, - totalCollections: 5, - totalStorageUsed: 200, - totalDatabaseUsed: 100, - totalRequests: 150, - totalWebhooks: 25, - totalUsers: 30, // 10 + 20 - }); +const controller = require("../controllers/analytics.controller"); + +describe("Analytics Controller", () => { + let req, res, next; + + beforeEach(() => { + jest.clearAllMocks(); + req = { + user: { _id: new mongoose.Types.ObjectId().toString() }, + }; + res = { + json: jest.fn().mockReturnThis(), + }; + next = jest.fn(); + }); + + describe("getGlobalStats", () => { + it("should return aggregated global stats with user counts fetched in parallel", async () => { + const userId = req.user._id; + + mockProjectAggregate.mockResolvedValueOnce([ + { + _id: null, + totalProjects: 2, + totalDatabaseUsed: 100, + totalStorageUsed: 200, + totalCollections: 5, + }, + ]); + + mockDeveloperFindById.mockReturnValue({ + select: jest.fn().mockResolvedValue({ + plan: "pro", + maxProjects: 10, + maxCollections: 20, + planExpiresAt: null, + }), + }); + + mockProjectFind.mockReturnValue({ + select: jest.fn().mockReturnValue({ + lean: jest + .fn() + .mockResolvedValue([ + { _id: new mongoose.Types.ObjectId("60c72b2f9b1d8b22fc5a87ba") }, + { _id: new mongoose.Types.ObjectId("60c72b2f9b1d8b22fc5a87bb") }, + ]), + }), + }); + + mockLogCountDocuments.mockResolvedValueOnce(150); + mockWebhookCountDocuments.mockResolvedValueOnce(25); + + const mockDbConn1 = { + collection: jest.fn().mockReturnValue({ + countDocuments: jest.fn().mockResolvedValue(10), + }), + }; + const mockDbConn2 = { + collection: jest.fn().mockReturnValue({ + countDocuments: jest.fn().mockResolvedValue(20), + }), + }; + + let firstResolve; + let secondConnectionStarted = false; + const firstConnectionPromise = new Promise((resolve) => { + firstResolve = resolve; + }); + + mockGetConnection + .mockImplementationOnce(async (projectId) => { + expect(projectId).toBe("60c72b2f9b1d8b22fc5a87ba"); + return firstConnectionPromise; + }) + .mockImplementationOnce(async (projectId) => { + expect(projectId).toBe("60c72b2f9b1d8b22fc5a87bb"); + secondConnectionStarted = true; + return mockDbConn2; }); - it('should handle db connection errors gracefully without failing the entire request', async () => { - mockProjectAggregate.mockResolvedValueOnce([]); - mockDeveloperFindById.mockReturnValue({ - select: jest.fn().mockResolvedValue(null), - }); - mockProjectFind.mockReturnValue({ - select: jest.fn().mockReturnValue({ - lean: jest.fn().mockResolvedValue([ - { _id: new mongoose.Types.ObjectId('60c72b2f9b1d8b22fc5a87bc') }, - { _id: new mongoose.Types.ObjectId('60c72b2f9b1d8b22fc5a87bd') } - ]) - }) - }); - - mockLogCountDocuments.mockResolvedValueOnce(0); - mockWebhookCountDocuments.mockResolvedValueOnce(0); - - // First project connection fails, second succeeds - const mockDbConn = { - collection: jest.fn().mockReturnValue({ - countDocuments: jest.fn().mockResolvedValue(5), - }), - }; - mockGetConnection - .mockRejectedValueOnce(new Error('Connection timeout')) - .mockResolvedValueOnce(mockDbConn); - - await controller.getGlobalStats(req, res, next); - - expect(res.json).toHaveBeenCalledTimes(1); - const responseData = res.json.mock.calls[0][0]; - expect(responseData.data.usage.totalUsers).toBe(5); // 0 + 5 - }); + const resultPromise = controller.getGlobalStats(req, res, next); + await new Promise((resolve) => setImmediate(resolve)); + expect(secondConnectionStarted).toBe(true); + + firstResolve(mockDbConn1); + await resultPromise; + + expect(mockGetConnection).toHaveBeenCalledTimes(2); + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + expect(responseData.data.usage).toEqual({ + totalProjects: 2, + totalCollections: 5, + totalStorageUsed: 200, + totalDatabaseUsed: 100, + totalRequests: 150, + totalWebhooks: 25, + totalUsers: 30, // 10 + 20 + }); }); - describe('getRecentActivity', () => { - it('should return formatted recent logs', async () => { - mockProjectFind.mockReturnValue({ - distinct: jest.fn().mockResolvedValue(['proj1']) - }); - - mockLogFind.mockReturnValue({ - sort: jest.fn().mockReturnThis(), - limit: jest.fn().mockReturnThis(), - populate: jest.fn().mockReturnThis(), - lean: jest.fn().mockResolvedValue([ - { - _id: 'log1', - projectId: { _id: 'proj1', name: 'My Project' }, - method: 'GET', - path: '/api/users', - status: 200, - timestamp: '2026-06-17T00:00:00Z', - } - ]) - }); - - await controller.getRecentActivity(req, res, next); - - expect(res.json).toHaveBeenCalledTimes(1); - const responseData = res.json.mock.calls[0][0]; - expect(responseData.success).toBe(true); - expect(responseData.data).toEqual([ - { - id: 'log1', - projectName: 'My Project', - projectId: 'proj1', - method: 'GET', - path: '/api/users', - status: 200, - timestamp: '2026-06-17T00:00:00Z', - } - ]); - }); + it("should handle db connection errors gracefully without failing the entire request", async () => { + mockProjectAggregate.mockResolvedValueOnce([]); + mockDeveloperFindById.mockReturnValue({ + select: jest.fn().mockResolvedValue(null), + }); + mockProjectFind.mockReturnValue({ + select: jest.fn().mockReturnValue({ + lean: jest + .fn() + .mockResolvedValue([ + { _id: new mongoose.Types.ObjectId("60c72b2f9b1d8b22fc5a87bc") }, + { _id: new mongoose.Types.ObjectId("60c72b2f9b1d8b22fc5a87bd") }, + ]), + }), + }); + + mockLogCountDocuments.mockResolvedValueOnce(0); + mockWebhookCountDocuments.mockResolvedValueOnce(0); + + // First project connection fails, second succeeds + const mockDbConn = { + collection: jest.fn().mockReturnValue({ + countDocuments: jest.fn().mockResolvedValue(5), + }), + }; + mockGetConnection + .mockRejectedValueOnce(new Error("Connection timeout")) + .mockResolvedValueOnce(mockDbConn); + + await controller.getGlobalStats(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.data.usage.totalUsers).toBe(5); // 0 + 5 }); - - describe('getActivationFunnel', () => { - it('should return status of activation funnel steps', async () => { - mockPlatformEventFind.mockReturnValue({ - sort: jest.fn().mockReturnThis(), - select: jest.fn().mockReturnThis(), - lean: jest.fn().mockResolvedValue([ - { event: 'signup_completed', timestamp: '2026-06-10T00:00:00Z' }, - { event: 'email_verified', timestamp: '2026-06-10T00:05:00Z' } - ]) - }); - - await controller.getActivationFunnel(req, res, next); - - expect(res.json).toHaveBeenCalledTimes(1); - const responseData = res.json.mock.calls[0][0]; - expect(responseData.success).toBe(true); - const steps = responseData.data.steps; - expect(steps[0]).toEqual({ step: 'signup_completed', order: 1, completed: true, completedAt: '2026-06-10T00:00:00Z' }); - expect(steps[1]).toEqual({ step: 'email_verified', order: 2, completed: true, completedAt: '2026-06-10T00:05:00Z' }); - expect(steps[2]).toEqual({ step: 'project_created', order: 3, completed: false, completedAt: null }); - }); + }); + + describe("getRecentActivity", () => { + it("should return formatted recent logs", async () => { + mockProjectFind.mockReturnValue({ + distinct: jest.fn().mockResolvedValue(["proj1"]), + }); + + mockLogFind.mockReturnValue({ + sort: jest.fn().mockReturnThis(), + limit: jest.fn().mockReturnThis(), + populate: jest.fn().mockReturnThis(), + lean: jest.fn().mockResolvedValue([ + { + _id: "log1", + projectId: { _id: "proj1", name: "My Project" }, + method: "GET", + path: "/api/users", + status: 200, + timestamp: "2026-06-17T00:00:00Z", + }, + ]), + }); + + await controller.getRecentActivity(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + expect(responseData.data).toEqual([ + { + id: "log1", + projectName: "My Project", + projectId: "proj1", + method: "GET", + path: "/api/users", + status: 200, + timestamp: "2026-06-17T00:00:00Z", + }, + ]); }); + }); + + describe("getActivationFunnel", () => { + it("should return status of activation funnel steps", async () => { + mockPlatformEventFind.mockReturnValue({ + sort: jest.fn().mockReturnThis(), + select: jest.fn().mockReturnThis(), + lean: jest.fn().mockResolvedValue([ + { event: "signup_completed", timestamp: "2026-06-10T00:00:00Z" }, + { event: "email_verified", timestamp: "2026-06-10T00:05:00Z" }, + ]), + }); + + await controller.getActivationFunnel(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + const steps = responseData.data.steps; + expect(steps[0]).toEqual({ + step: "signup_completed", + order: 1, + completed: true, + completedAt: "2026-06-10T00:00:00Z", + }); + expect(steps[1]).toEqual({ + step: "email_verified", + order: 2, + completed: true, + completedAt: "2026-06-10T00:05:00Z", + }); + expect(steps[2]).toEqual({ + step: "project_created", + order: 3, + completed: false, + completedAt: null, + }); + }); + }); + + describe("getRetention", () => { + it("should return d1/d7/d30 retention flags", async () => { + mockPlatformEventFindOne.mockReturnValue({ + sort: jest.fn().mockReturnThis(), + lean: jest.fn().mockResolvedValue({ + timestamp: "2026-06-10T12:00:00Z", + }), + }); + + mockDeveloperActivityFindOne.mockImplementation(({ date }) => { + return { + lean: jest.fn().mockResolvedValue({ date }), + }; + }); - describe('getRetention', () => { - it('should return d1/d7/d30 retention flags', async () => { - mockPlatformEventFindOne.mockReturnValue({ - sort: jest.fn().mockReturnThis(), - lean: jest.fn().mockResolvedValue({ - timestamp: '2026-06-10T12:00:00Z' - }) - }); - - mockDeveloperActivityFindOne - .mockImplementation(({ date }) => { - return { - lean: jest.fn().mockResolvedValue({ date }) - }; - }); - - await controller.getRetention(req, res, next); - - expect(res.json).toHaveBeenCalledTimes(1); - const responseData = res.json.mock.calls[0][0]; - expect(responseData.success).toBe(true); - expect(responseData.data.d1).toBe(true); - expect(responseData.data.d7).toBe(true); - expect(responseData.data.d30).toBe(true); - }); - - it('should return false flags if no signup event exists', async () => { - mockPlatformEventFindOne.mockReturnValue({ - sort: jest.fn().mockReturnThis(), - lean: jest.fn().mockResolvedValue(null) - }); - - await controller.getRetention(req, res, next); + await controller.getRetention(req, res, next); - expect(res.json).toHaveBeenCalledTimes(1); - const responseData = res.json.mock.calls[0][0]; - expect(responseData.data).toEqual({ d1: false, d7: false, d30: false, signupDate: null }); - }); + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + expect(responseData.data.d1).toBe(true); + expect(responseData.data.d7).toBe(true); + expect(responseData.data.d30).toBe(true); }); - describe('getEngagement', () => { - it('should return aggregated engagement metrics', async () => { - mockDeveloperActivityAggregate.mockResolvedValueOnce([ - { - totalApiCalls: 500, - totalMailSent: 50, - totalStorageUploads: 10, - totalWebhooksFired: 5, - activeDays: 3, - allProjectIds: [['proj1'], ['proj2', 'proj1']] - } - ]); - - await controller.getEngagement(req, res, next); - - expect(res.json).toHaveBeenCalledTimes(1); - const responseData = res.json.mock.calls[0][0]; - expect(responseData.success).toBe(true); - expect(responseData.data).toEqual({ - window: '30d', - totalApiCalls: 500, - totalMailSent: 50, - totalStorageUploads: 10, - totalWebhooksFired: 5, - activeDays: 3, - uniqueActiveProjects: 2, - }); - }); + it("should return false flags if no signup event exists", async () => { + mockPlatformEventFindOne.mockReturnValue({ + sort: jest.fn().mockReturnThis(), + lean: jest.fn().mockResolvedValue(null), + }); + + await controller.getRetention(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.data).toEqual({ + d1: false, + d7: false, + d30: false, + signupDate: null, + }); + }); + }); + + describe("getEngagement", () => { + it("should return aggregated engagement metrics", async () => { + mockDeveloperActivityAggregate.mockResolvedValueOnce([ + { + totalApiCalls: 500, + totalMailSent: 50, + totalStorageUploads: 10, + totalWebhooksFired: 5, + activeDays: 3, + allProjectIds: [["proj1"], ["proj2", "proj1"]], + }, + ]); + + await controller.getEngagement(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + expect(responseData.data).toEqual({ + window: "30d", + totalApiCalls: 500, + totalMailSent: 50, + totalStorageUploads: 10, + totalWebhooksFired: 5, + activeDays: 3, + uniqueActiveProjects: 2, + }); + }); + }); + + describe("getNorthStar", () => { + it("should return north star metrics", async () => { + mockProjectFind.mockReturnValue({ + select: jest.fn().mockReturnValue({ + lean: jest.fn().mockResolvedValue([ + { _id: "proj1", name: "Proj 1" }, + { _id: "proj2", name: "Proj 2" }, + ]), + }), + }); + + mockLogDistinct.mockResolvedValueOnce(["proj1"]); + + await controller.getNorthStar(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.success).toBe(true); + expect(responseData.data).toEqual({ + activeProjects: 1, + totalProjects: 2, + percentage: 50, + }); }); - describe('getNorthStar', () => { - it('should return north star metrics', async () => { - mockProjectFind.mockReturnValue({ - select: jest.fn().mockReturnValue({ - lean: jest.fn().mockResolvedValue([ - { _id: 'proj1', name: 'Proj 1' }, - { _id: 'proj2', name: 'Proj 2' } - ]) - }) - }); - - mockLogDistinct.mockResolvedValueOnce(['proj1']); - - await controller.getNorthStar(req, res, next); - - expect(res.json).toHaveBeenCalledTimes(1); - const responseData = res.json.mock.calls[0][0]; - expect(responseData.success).toBe(true); - expect(responseData.data).toEqual({ - activeProjects: 1, - totalProjects: 2, - percentage: 50, - }); - }); - - it('should return 0 metrics if developer has no projects', async () => { - mockProjectFind.mockReturnValue({ - select: jest.fn().mockReturnValue({ - lean: jest.fn().mockResolvedValue([]) - }) - }); - - await controller.getNorthStar(req, res, next); - - expect(res.json).toHaveBeenCalledTimes(1); - const responseData = res.json.mock.calls[0][0]; - expect(responseData.data).toEqual({ - activeProjects: 0, - totalProjects: 0, - percentage: 0, - }); - }); + it("should return 0 metrics if developer has no projects", async () => { + mockProjectFind.mockReturnValue({ + select: jest.fn().mockReturnValue({ + lean: jest.fn().mockResolvedValue([]), + }), + }); + + await controller.getNorthStar(req, res, next); + + expect(res.json).toHaveBeenCalledTimes(1); + const responseData = res.json.mock.calls[0][0]; + expect(responseData.data).toEqual({ + activeProjects: 0, + totalProjects: 0, + percentage: 0, + }); }); + }); }); diff --git a/apps/dashboard-api/src/controllers/analytics.controller.js b/apps/dashboard-api/src/controllers/analytics.controller.js index e7b5c7bc..e42ce902 100644 --- a/apps/dashboard-api/src/controllers/analytics.controller.js +++ b/apps/dashboard-api/src/controllers/analytics.controller.js @@ -1,4 +1,17 @@ -const { Project, Log, Developer, Webhook, getConnection, resolveEffectivePlan, getPlanLimits, PlatformEvent, DeveloperActivity, AppError, ApiResponse, getProjectAccessQuery } = require("@urbackend/common"); +const { + Project, + Log, + Developer, + Webhook, + getConnection, + resolveEffectivePlan, + getPlanLimits, + PlatformEvent, + DeveloperActivity, + AppError, + ApiResponse, + getProjectAccessQuery, +} = require("@urbackend/common"); const mongoose = require("mongoose"); /** @@ -11,7 +24,7 @@ module.exports.getGlobalStats = async (req, res, next) => { const [stats, dev] = await Promise.all([ Project.aggregate([ - { + { $match: { owner: userId }, }, { @@ -20,45 +33,65 @@ module.exports.getGlobalStats = async (req, res, next) => { totalProjects: { $sum: 1 }, totalDatabaseUsed: { $sum: { $ifNull: ["$databaseUsed", 0] } }, totalStorageUsed: { $sum: { $ifNull: ["$storageUsed", 0] } }, - totalCollections: { $sum: { $size: { $ifNull: ["$collections", []] } } } - } - } + totalCollections: { + $sum: { $size: { $ifNull: ["$collections", []] } }, + }, + }, + }, ]), - Developer.findById(user_id).select("maxProjects maxCollections plan planExpiresAt") + Developer.findById(user_id).select( + "maxProjects maxCollections plan planExpiresAt", + ), ]); const globalStats = stats[0] || { totalProjects: 0, totalDatabaseUsed: 0, totalStorageUsed: 0, - totalCollections: 0 + totalCollections: 0, }; - const projects = await Project.find({ owner: user_id }).select("_id").lean(); - const projectIds = projects.map(p => p._id); - - const totalRequests = await Log.countDocuments({ projectId: { $in: projectIds } }); - const totalWebhooks = await Webhook.countDocuments({ projectId: { $in: projectIds } }); - - const userCountPromises = projects.map(async (project) => { - try { - const conn = await getConnection(project._id.toString()); - return await conn.collection('users').countDocuments(); - } catch (err) { - console.error(`Failed to count users for project ${project._id}:`, err.message); - return 0; - } + const projects = await Project.find({ owner: user_id }) + .select("_id") + .lean(); + const projectIds = projects.map((p) => p._id); + + const totalRequests = await Log.countDocuments({ + projectId: { $in: projectIds }, + }); + const totalWebhooks = await Webhook.countDocuments({ + projectId: { $in: projectIds }, }); - const userCounts = await Promise.all(userCountPromises); - const totalUsers = userCounts.reduce((sum, count) => sum + count, 0); + + const USER_COUNT_CONCURRENCY = 5; + let totalUsers = 0; + + for (let i = 0; i < projects.length; i += USER_COUNT_CONCURRENCY) { + const batch = projects.slice(i, i + USER_COUNT_CONCURRENCY); + const batchCounts = await Promise.all( + batch.map(async (project) => { + try { + const conn = await getConnection(project._id.toString()); + return await conn.collection("users").countDocuments(); + } catch (err) { + console.error( + `Failed to count users for project ${project._id}:`, + err.message, + ); + return 0; + } + }), + ); + totalUsers += batchCounts.reduce((sum, count) => sum + count, 0); + } const effectivePlan = resolveEffectivePlan(dev); const limits = getPlanLimits({ plan: effectivePlan, legacyLimits: { maxProjects: dev?.maxProjects ?? null, - maxCollections: dev?.maxCollections ?? null - } + maxCollections: dev?.maxCollections ?? null, + }, }); return new ApiResponse({ @@ -72,8 +105,8 @@ module.exports.getGlobalStats = async (req, res, next) => { totalDatabaseUsed: globalStats.totalDatabaseUsed, totalRequests, totalWebhooks, - totalUsers - } + totalUsers, + }, }).send(res); } catch (err) { next(err); @@ -86,22 +119,24 @@ module.exports.getGlobalStats = async (req, res, next) => { module.exports.getRecentActivity = async (req, res, next) => { try { const userId = req.user._id; - const projectIds = await Project.find(getProjectAccessQuery(userId)).distinct("_id"); + const projectIds = await Project.find( + getProjectAccessQuery(userId), + ).distinct("_id"); const logs = await Log.find({ projectId: { $in: projectIds } }) .sort({ timestamp: -1 }) .limit(20) - .populate('projectId', 'name') + .populate("projectId", "name") .lean(); - const formattedLogs = logs.map(log => ({ + const formattedLogs = logs.map((log) => ({ id: log._id, - projectName: log.projectId?.name || 'Unknown Project', + projectName: log.projectId?.name || "Unknown Project", projectId: log.projectId?._id || log.projectId, method: log.method, path: log.path, status: log.status, - timestamp: log.timestamp + timestamp: log.timestamp, })); return new ApiResponse(formattedLogs).send(res); @@ -119,11 +154,11 @@ module.exports.getActivationFunnel = async (req, res, next) => { const developerId = req.user._id; const FUNNEL_STEPS = [ - 'signup_completed', - 'email_verified', - 'project_created', - 'collection_created', - 'first_api_success', + "signup_completed", + "email_verified", + "project_created", + "collection_created", + "first_api_success", ]; // Fetch one event per step (we only need existence, not count) @@ -132,7 +167,7 @@ module.exports.getActivationFunnel = async (req, res, next) => { event: { $in: FUNNEL_STEPS }, }) .sort({ timestamp: 1 }) - .select('event timestamp') + .select("event timestamp") .lean(); const completed = {}; @@ -164,11 +199,18 @@ module.exports.getRetention = async (req, res, next) => { // Find signup event to anchor the cohort start date const signupEvent = await PlatformEvent.findOne({ developerId, - event: 'signup_completed', - }).sort({ timestamp: 1 }).lean(); + event: "signup_completed", + }) + .sort({ timestamp: 1 }) + .lean(); if (!signupEvent) { - return new ApiResponse({ d1: false, d7: false, d30: false, signupDate: null }).send(res); + return new ApiResponse({ + d1: false, + d7: false, + d30: false, + signupDate: null, + }).send(res); } const signupDate = new Date(signupEvent.timestamp); @@ -220,12 +262,12 @@ module.exports.getEngagement = async (req, res, next) => { { $group: { _id: null, - totalApiCalls: { $sum: '$apiCallCount' }, - totalMailSent: { $sum: '$mailSentCount' }, - totalStorageUploads: { $sum: '$storageUploadsCount' }, - totalWebhooksFired: { $sum: '$webhookTriggeredCount' }, + totalApiCalls: { $sum: "$apiCallCount" }, + totalMailSent: { $sum: "$mailSentCount" }, + totalStorageUploads: { $sum: "$storageUploadsCount" }, + totalWebhooksFired: { $sum: "$webhookTriggeredCount" }, activeDays: { $sum: 1 }, - allProjectIds: { $push: '$activeProjectIds' }, + allProjectIds: { $push: "$activeProjectIds" }, }, }, ]); @@ -243,7 +285,7 @@ module.exports.getEngagement = async (req, res, next) => { const uniqueActiveProjects = new Set(flatProjectIds.map(String)).size; return new ApiResponse({ - window: '30d', + window: "30d", totalApiCalls: result.totalApiCalls, totalMailSent: result.totalMailSent, totalStorageUploads: result.totalStorageUploads, @@ -268,25 +310,36 @@ module.exports.getNorthStar = async (req, res, next) => { sevenDaysAgo.setUTCDate(sevenDaysAgo.getUTCDate() - 7); // Projects owned by this developer (North Star should be owner-only) - const allProjects = await Project.find({ owner: developerId }).select('_id name').lean(); + const allProjects = await Project.find({ owner: developerId }) + .select("_id name") + .lean(); const projectIds = allProjects.map((p) => p._id); const totalProjects = projectIds.length; if (totalProjects === 0) { - return new ApiResponse({ activeProjects: 0, totalProjects: 0, percentage: 0 }).send(res); + return new ApiResponse({ + activeProjects: 0, + totalProjects: 0, + percentage: 0, + }).send(res); } // Projects with at least one 2xx log in the last 7 days - const activeProjectIds = await Log.distinct('projectId', { + const activeProjectIds = await Log.distinct("projectId", { projectId: { $in: projectIds }, status: { $gte: 200, $lt: 300 }, timestamp: { $gte: sevenDaysAgo }, }); const activeProjects = activeProjectIds.length; - const percentage = totalProjects > 0 ? Math.round((activeProjects / totalProjects) * 100) : 0; - - return new ApiResponse({ activeProjects, totalProjects, percentage }).send(res); + const percentage = + totalProjects > 0 + ? Math.round((activeProjects / totalProjects) * 100) + : 0; + + return new ApiResponse({ activeProjects, totalProjects, percentage }).send( + res, + ); } catch (err) { next(err); }