diff --git a/.depot/workflows/ci-backend.yml b/.depot/workflows/ci-backend.yml index caf1d31c0931..792b389eb42a 100644 --- a/.depot/workflows/ci-backend.yml +++ b/.depot/workflows/ci-backend.yml @@ -3085,10 +3085,6 @@ jobs: # The default ref, which is the PR merge commit. turbo-discover selects the test files # from this tree, so a checkout of any other ref can lack a file that the selection lists. - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - fetch-depth: 1 - lfs: true - clean: false - name: Install signal-fanout shell: bash run: sudo install -m 0755 .github/scripts/signal-fanout /usr/local/bin/signal-fanout diff --git a/Dockerfile b/Dockerfile index d824cc8b24fb..d0b2a17d63a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,7 @@ # The stages are used to: # # - node-base: shared Node.js base with pnpm already provisioned +# - product-manifests: collect the package.json of each products/* workspace package # - frontend-build: build the frontend (static assets) # - sourcemap-upload: upload sourcemaps to PostHog (isolated, no artifacts) # - node-scripts-build: build plugin transpiler and other Node.js build artifacts @@ -39,6 +40,15 @@ RUN corepack enable && corepack install ENV COREPACK_ENABLE_NETWORK=0 +# +# --------------------------------------------------------- +# +# Only the package.json files, so a product source change does not rerun the frontend pnpm install. +FROM node:24.22.1-bookworm-slim@sha256:55fb5f5a527ea2c5c4a95f963e48ef90aa38c7b346054fdc8ee9957a2b55d205 AS product-manifests +COPY products/ /tmp/products/ +RUN cd /tmp/products && for manifest in */package.json; do install -D -m 644 "$manifest" "/manifests/products/$manifest"; done + + # # --------------------------------------------------------- # @@ -56,11 +66,12 @@ COPY common/tailwind/ common/tailwind/ COPY packages/quill/ packages/quill/ COPY packages/llm-normalizer/ packages/llm-normalizer/ COPY packages/agent/packages/agent-contracts/src/ packages/agent/packages/agent-contracts/src/ -COPY products/ products/ +COPY --from=product-manifests /manifests/products/ products/ COPY docs/onboarding/ docs/onboarding/ RUN --mount=type=cache,id=pnpm,target=/tmp/pnpm-store-v24 \ CI=1 pnpm --filter=@posthog/frontend... install --frozen-lockfile --store-dir /tmp/pnpm-store-v24 +COPY products/ products/ COPY frontend/ frontend/ RUN bin/turbo --filter=@posthog/frontend build diff --git a/docs/internal/ai-offline-evaluation-reporting.md b/docs/internal/ai-offline-evaluation-reporting.md index 9afc6c3ed976..31521cacf264 100644 --- a/docs/internal/ai-offline-evaluation-reporting.md +++ b/docs/internal/ai-offline-evaluation-reporting.md @@ -248,6 +248,9 @@ Numeric scores can use the domain of their configured scorer version. The API va `OfflineEvalSuite.experiment_options` accepts the SDK's `ExperimentOptions` for external dataset identifiers, `prompt_version`, model/application versions, and `run_source`. The harness owns the experiment ID, start time, suite key, and expected counts. +Each case's `output["model"]` becomes the item's `model_version`, so one-shot suites can report the model they actually used and compare different models within one experiment. +Cases without a model, or with `model=None`, inherit the experiment's model version. +Invalid model identifiers are omitted and recorded in the item's `omitted_fields`; the case's scores still upload. For external datasets, case metadata can supply `dataset_item_identifier` and `dataset_item_version_identifier`. The harness publisher does not support hosted datasets. A non-null `dataset_revision_id` stops preparation with a local error before any upload request. diff --git a/docs/internal/conversations-reliability.md b/docs/internal/conversations-reliability.md index 8bb2da092d6d..9f6e92260441 100644 --- a/docs/internal/conversations-reliability.md +++ b/docs/internal/conversations-reliability.md @@ -115,6 +115,7 @@ That is provider acceptance, not customer delivery. Timeouts after a possible accept retry with the same `client_msg_id`. Honor `Retry-After` on 429s, cap at one hour, and treat 5xx plus Slack timeout codes as transient. Permanent Slack application errors (revoked token, missing channel, invalid blocks) fail the part. +Automated PR replies check the approved channel list and current channel type before posting. A failed channel lookup retries; a known external channel fails the part. Keep `post_reply_to_slack` registered for in-flight Celery messages. New work uses `process_slack_delivery_part`. diff --git a/docs/internal/workflows-email-sending-tiers.md b/docs/internal/workflows-email-sending-tiers.md index 6914badeaa51..73e358d91950 100644 --- a/docs/internal/workflows-email-sending-tiers.md +++ b/docs/internal/workflows-email-sending-tiers.md @@ -12,6 +12,7 @@ Only workflows that send email are subject to the tiers; SMS, push, and webhook - Batch audience cap: `get_hogflow_batch_trigger_limit` in `products/workflows/backend/utils/batch_trigger_limit.py`, applied at batch dispatch and shown in the blast radius preview. - Send-time caps: two Valkey token buckets per team in the CDP email worker (`claimTeamSendingBudget` in `nodejs/src/cdp/services/messaging/email.service.ts`). - Staff controls: the team's Django admin page (view state, set/pin a tier, recompute now). +- Audit trail: every tier or pin change writes an `email_sending_tier_changed` row to the team's activity log, with the before and after values and the decision reason (`staff_set` for a tier staff set by hand). Staff rows name the acting user. Sweep and backfill rows carry a trigger with the job type and a per-run id. The rows are staff-only in the activity log (`products/workflows/backend/services/email_sending_tier_audit.py`). - Customer surface: the Reputation tab's sending allowance card, shown only while the team is enforced. ## Configuration diff --git a/frontend/snapshots.yml b/frontend/snapshots.yml index e5bdaafd7701..fe15a410a10a 100644 --- a/frontend/snapshots.yml +++ b/frontend/snapshots.yml @@ -3968,6 +3968,10 @@ snapshots: hash: v1.k794b7964.77db5fd39abf5b41db372ff16af2044f3c03b303aabe0d592f6c25de1898339a.3VJBqA15CtN76iDMdubH40FmnriyLRFCW_mGRXdYrsM insights-funnellinechart--with-legend--light: hash: v1.k794b7964.d3d505f78dc4785055c98a602b0edd9c39549001dab1a3f1c3bf753f1cff60be.Lu6z04Zq2foZ5axwQK8Hg4ys33BYX2OIMjQkpYBycKM + insights-funnelstepsbarchart--alert-thresholds--dark: + hash: v1.k794b7964.f07a01625198eb67f2bf3709a2b562abf1e5c9fb6e0ad50d32bc3c0455810e4c.kcHPaKTNWm1MsjfK7ZXC-YOSHARImNlxhU63hmrI9zk + insights-funnelstepsbarchart--alert-thresholds--light: + hash: v1.k794b7964.1615420c253f38e627b2d08757e17d1bd748369b520353077fb7cb69b60ce73d.fckv-4nelywj6ArxmnSQa8Xt5h6ZySRMvXTpCuoTF-A insights-funnelstepsbarchart--bar-labels--dark: hash: v1.k794b7964.5de2ee4fdd7ba021e1af1363b1e7977b10a245ce3ae6486008b83552cc41296d.tMGnyCUxbncHrGz42VrPrOAdXqsv6oqpZP2N6hjMW48 insights-funnelstepsbarchart--bar-labels--light: @@ -9029,57 +9033,57 @@ snapshots: scenes-app-ai-observability-evaluations-onboarding--no-online-evaluations--light: hash: v1.k794b7964.1dbea4bbbd7b0731b841cfdf0dc28826e9bba9caf9735cc7257e0fc4e0a67026.ezM9M-EOjWsTZXJk602I44gvDXvOzobiHJq09E0OEac scenes-app-ai-observability-evaluations-overview--directory-open--dark: - hash: v1.k794b7964.aa11f061ae0ad10fe341206b1223e0a608df0286d2c5d283b54696d4e21a3f97.jl6VVZGQDVDqUkvv3N-uFqifk4b-MykizHrmyhgG_zc + hash: v1.k794b7964.9e8467472787599992a607e83fc725f1808762fc1c33cd0ec4f097e457e1411c.VdW-6vKl9WLQcs3_PcFk76quSN32MHTKKi65Mr2AtIw scenes-app-ai-observability-evaluations-overview--directory-open--light: - hash: v1.k794b7964.76820f10ccf488b27e0905c7bad9dd2891737b4a4e5fa7cd88121d163cd67080.WVgilucVDuLKEEUvDcyzOPGkFFvzXe40TmaEyVoJJng + hash: v1.k794b7964.cf730b6a5ea171f8188571f7517979529e8d5ec0921f8dd540f7566f1c82c668.7lLao0Lap4qcDipaQoM5qHydVIUqSOF10kmighTbX7I scenes-app-ai-observability-evaluations-overview--drawer-categories--dark: - hash: v1.k794b7964.833c9ec088f5ced0b6349de96401cea531c80532b1ccab3e0ede6280075fd177.g5BobVDmlWFirkzQXWggOgdaGlZTyzgXsFSsl3wXnNU + hash: v1.k794b7964.1396006b1c513ffac8152d3328c2c4dae1d2f6d089568d83368ed601cbd4c4fc.TO_VIpBW3waQiAPRFmArwyxs1ypiEdjQzEdBU1Y9Dys scenes-app-ai-observability-evaluations-overview--drawer-categories--light: - hash: v1.k794b7964.bc7f0d4f7a4eaef7ecf8f8b62cfb879294d2844e856de264ba499a995d638bcb._YBpi1OPAd5HQqHxeTkTxBuCr3090LVXixCm3cMTVGQ + hash: v1.k794b7964.ac765a38024efc26131f1e94b0de8ae7de76a272b3f8e3ed876ef65114b0d654.vEvC2Oe0TrrHptfcYHPLuicyHRutHV85C0mGtFyRQxE scenes-app-ai-observability-evaluations-overview--drawer-empty-disabled--dark: - hash: v1.k794b7964.7de3916414d975768f14847157679565558e99c7ac9b75d849ffdf754be78401.VUL4gYDHiOk8Ii6VYTrKxtlsyLBwwPzkaNjLNFKWcdA + hash: v1.k794b7964.7469928bacc2c1ea5c0ee877309f2474bec0daadce99c0c642339a13c36bc198.2_TdbaZqvXx1BhEqm8Z4VTEW0EDJruEcgzqCVQ_a3wg scenes-app-ai-observability-evaluations-overview--drawer-empty-disabled--light: - hash: v1.k794b7964.cbf3fc3daeed93e20270ba78e5c2928f8a71db99e19e38be16ca760c07fcd43f.Rbe8TyAOiBinF4WY1zffPcSFXPv5PaSnXMyTTxsRgTA + hash: v1.k794b7964.c9ef8678f104bd65444d83faa8e770403fb3b19a481ef3ee7c9feeee352e39d5.B4Y9VJuw8kjkQzJqOMW4CH5pYdUdCFyk-AAfj6oL10E scenes-app-ai-observability-evaluations-overview--drawer-empty-enabled--dark: - hash: v1.k794b7964.aa7184c72e72454e93eb3913486bdc8c295a5a2a8aabda3cf566af7d5917d08f.R_OIyIubYv5EgQU__8ZV-O0wNqEp3qQBSvBg30AUtwc + hash: v1.k794b7964.5e3466575fd4688b526dceddb7025fbd3ab51c595c42a2971438e0f8fced3d1b.zNvjVBwMJH4sfnBIUwqCJeCRLBWR2tQXGf7yT3JHb5E scenes-app-ai-observability-evaluations-overview--drawer-empty-enabled--light: - hash: v1.k794b7964.c5d3c557c2451f148ecc57ec3cec957f1b7f9f886712ebb174466fd9e4c4bb40.HOpxeq9U5LXpb1QbF3LUC9AYyk5netaRIatrGIUkk2w + hash: v1.k794b7964.d3e6bbc0d6083da0b972859131952b9459157957dbc9280dde510212ca2d4638.in6HA2hHmpo_O8hdFffEjCZSVYywO1RJfXEe7BZ8JWQ scenes-app-ai-observability-evaluations-overview--drawer-empty-errored--dark: - hash: v1.k794b7964.3af08d5f476b419f3e8063e8e49e0a7c9ec7e659c0b7a5680e4a37e862d280a4.irbSSkw1Sl-MHvdClZkXAzFms2Hx7vvNASxfQIcoNJk + hash: v1.k794b7964.d80c0068d1d69550abcc359f47100102e5b5280dc33c5debc791cba160faddf3.gx2US6d2uRQVNqkfidFZ-aUhEs0Ya52y7gELEC2mWj4 scenes-app-ai-observability-evaluations-overview--drawer-empty-errored--light: - hash: v1.k794b7964.a0a3b37ec30366fad6ddc1ee8b2341aaef91dd14860e3015805ce50e16c2d687.vsCMotdO-txIgRF6ZOPV1WCd09ltBeMXSyHyxjSaIz8 + hash: v1.k794b7964.183ed459b8ac0cc01560f9982ba22b237517dc84d32ba13c8f499ce5a544d2b5.jpWsfKXNRY_3PhIN8t_ibgpnbR-Fsj8De0KO0hg9b1s scenes-app-ai-observability-evaluations-overview--drawer-pass-rate--dark: - hash: v1.k794b7964.338a65fdf80c494f4c0bb14cbb394493accab1ead6d57f4334419ff70e312166.LsHX79e2LcxsjZepYXH0dROrSlc28Y214VvdTum_zkI + hash: v1.k794b7964.1a36eabcc772ca25d6a0a7f0bf88e7d09aca70af6451ad6bc7630874442fd56b.RRG8R-7Ppi9PX2mdyLSdCLwghhimaoXXVs0KX6CfX1s scenes-app-ai-observability-evaluations-overview--drawer-pass-rate--light: - hash: v1.k794b7964.8629ca843e0589c5dd4b1457baea907511013f39a2ed42fa97a678d493710d9f.1e6I8pl3FgbYIPgEqzYpmZkW96r7y6N5ii0uKo0sf7g + hash: v1.k794b7964.5754eeaf8f25e6df5632b09532a9420fad33f75de11aa8f432e68925d2533c2e.5kHPXR7lvVsAdhuPVUCVKemutoMKdigqWb4JJ29DwP8 scenes-app-ai-observability-evaluations-overview--drawer-score--dark: - hash: v1.k794b7964.053969fd2ca12903073c6e72757051c7ba95a1f7f7975f837e8d485339fc47fb.HBvwpYIW0D7hayu-EIhuw_x1Zwgksif80XFrt7MNExA + hash: v1.k794b7964.344bfa1890ae9973250672a9f29f937c1c6f1c0ce597ba1d23e440c5275d6cfa.D64Cbn94PGRWxYS-0Tub1jMZJDGPtF-udeJHbD8nRgo scenes-app-ai-observability-evaluations-overview--drawer-score--light: - hash: v1.k794b7964.1054c1259b718c56fc952efb7c05566d8e6a6b8276fdccb55039ea2f10c55183.X7d7tz6c6mo07nN9hKgFnB4HvCEmQ8-2v2PpZ1McI4E + hash: v1.k794b7964.b51db15a42f22bf705ea9afe54098493d704643921237e9d143aecef70e61f32.8iE8eoI8IPCQCYzqrOKwuCaQMp6w8M-z_Ol5JZVOJno scenes-app-ai-observability-evaluations-overview--drawer-sentiment--dark: - hash: v1.k794b7964.e5172bf8b3bc693e71512452904fcf3cd26431ac11312a63c42b85181bebe4b1.wjR_IxYD_1S6XDEz3mgEajeCYSje3TnIPS7hCh0D_ts + hash: v1.k794b7964.d8341fd34ae2cbe558d59567c78318fe2d4aaf329b037bcd2ac6f3b76c18afb4.LcaNUJ-_d6LXhVS-Lv7_eb_1J8beFP6b2x8jSbuQeN0 scenes-app-ai-observability-evaluations-overview--drawer-sentiment--light: - hash: v1.k794b7964.1dcd8c708d2896cf51abdefb8e953869986df3c3c329e204d860d3cf4296c0e7.lELtcnQIt5ZqwXeqMXNm7Rd-iexljTfywGkRz0jT96o + hash: v1.k794b7964.22b7faf544f624a6b94e72fbc6153c6d0329e5c1b14b2eb1f9e243e20c24d9df.uf1oNJwlqjesk2OB_EhiiAPypny8rcq3_y-BnNAsBP0 scenes-app-ai-observability-evaluations-overview--many-evaluations--dark: - hash: v1.k794b7964.217c7fb285b2538e867684455d6a6f47168acc21beb2526dbb7d6ce3676e5801.97AUffXw8nnPAz3QLYb4WWMqVJdCf48Vsd-o2zEp6Xw + hash: v1.k794b7964.b47ee72ea5ce1258149bd327c2b4d2dfc0173f255bfcdb868acbb0829d45c06b.oXeU3j9LVWCGRv7SUKvxOAPFJoXrwSQYO0gfezg8dn8 scenes-app-ai-observability-evaluations-overview--many-evaluations--light: - hash: v1.k794b7964.c11bc3b421ac8269e555f971075f4ca1733baf5ab884b1ea4837d44556c17412.-lOl68J121_CwMPgqZLa8fO2gS_Q4apOSlbehBeG25Q + hash: v1.k794b7964.31d902373f25edf7afc844ca5ac040ea1e643b8ff8b3230e52bc1251b06f2cc0.Gyqkp0iS5fBKUNpJGjUCJLiNZa7OB6Hjolgo8vPt-Ck scenes-app-ai-observability-evaluations-overview--many-evaluations-with-directories--dark: - hash: v1.k794b7964.4664f59130c034f4385171bbb5d0ce20cbdbd3f71ecc7cc86065a80a2a146215.MUFt92N2_BcrkvLJUoTFd9AgppmyQWdgqAUgi51PA1s + hash: v1.k794b7964.2103291df89ea5ac4e2b2c2c0b602a9237578227e53d3ccee0df8ebdf4462469.Fo4CWjXvtaKRzKkIBcOnkSGqPxT3BoMpSgoyXjRJk7w scenes-app-ai-observability-evaluations-overview--many-evaluations-with-directories--light: - hash: v1.k794b7964.f910e9853b427b4df365b2d628bec4c7a46c0eca175dc47bf289bc3ae3f70f07.GH2v1WUUh8zqbr8ZXQ05FDa_6R5poKG5WyGAAzf0raI + hash: v1.k794b7964.f2ac9295978b7f8172971aec9342f79c3570aae2e8dff864bcf45b1fb9ab7aef.g0R6KkYGyzzkbCalShqPtUyrYCaSgp3rtaVtMBNXagI scenes-app-ai-observability-evaluations-overview--nothing-graded--dark: - hash: v1.k794b7964.cb84e0d16d170ab75fac059bfdaedc4b777ae1af0ca83ebf24e2513b0fa1377e.-JZEVRiCP5z3Wem9gj8rwvs9DohtKICs51RT5d7PcEQ + hash: v1.k794b7964.4d1290a38f30ecb1bc13e47492a6684327af210ca28bf2d5de893e2d79545701.d-szMo-h5AvMziJCww7cbMBHk2j_Evr-jdnFP236oNk scenes-app-ai-observability-evaluations-overview--nothing-graded--light: - hash: v1.k794b7964.52f8eacef3006e94ff4447758d2aa81fea1ca24cf995552e04e6fd2c6f18194b.S3E4zPzJfMuCWjiZq67Im7dAQxr2g5qOdKQ8I3YBoKE + hash: v1.k794b7964.6bd929184a75dc6c7a052cd6fc3a81b94bda1df59003335bb31d35b1c658f48c.92qVT3rL6DGlJdCm0dD2qXudg_DuLh87pZSVsJYrWK8 scenes-app-ai-observability-evaluations-overview--rate-limited--dark: - hash: v1.k794b7964.e8aa37a66dc03651a6633b7e997672b8abf80f42a8efe8df7f9a685af6321f4a.iraGLjZfIqSCcP-vxVIYB22Fe31a7f4ScQi6rcGnB3g + hash: v1.k794b7964.0a716fd857838f8c6d836fb271ba27bc8b03807f9d14392528f487fcdb85f2b3.9Y-HU-8rkjlq8f0YS7rS69uE3-ckqDr27mH0TukD6mo scenes-app-ai-observability-evaluations-overview--rate-limited--light: - hash: v1.k794b7964.db4495726f9c49b37dbe0fc0d388c4de8cc2a807f265554d2d40b22a361a59b5.kcY7HlDaxjApOhHIujs93_5fTT2Gt0VFQYl3PxORgYs + hash: v1.k794b7964.48bfc1d4c5823c8a5494cf5bf79ef910d1c06d77315f5b28810570b3b26a2109.NnZNfIuP_J5KMBwtFMEqT1boYiKsI4StZzd62NQzys0 scenes-app-ai-observability-evaluations-overview--two-evaluations--dark: - hash: v1.k794b7964.0610effc7cf8f2c225b1299e5ae680fa6c74f64500eda628afdea1f43107f3f2.DQCrsNJFXpbSKtX_VaYxP5TeYZfRr9TRSsXzCPR9WYg + hash: v1.k794b7964.24177f3a0ab02c226de812af2a7236ca39efa20130a5e3f90b7eb8b9ef8b9fda.4fFcQQuS2W_Ge3tRrUxO5y8iC3h0XcVIp3VQw02oAjE scenes-app-ai-observability-evaluations-overview--two-evaluations--light: - hash: v1.k794b7964.a76865401cf83c825694b9470b4b992e89fc5ef03d2c2b0b3eceef8537b3b6bc.fyh9_jc67w9X9PzZjtH4_SEgKE4THhWmmRtfjjWDgnE + hash: v1.k794b7964.84206a664057f52d1b347eb3b25f0753aa6bc6511e115a84f6d8e81f2db20c88.KHYA6WgaFepaL76lvAalXU2Nzg4xhH60_2gIsul7LAM scenes-app-ai-observability-image-preview--failed-image--dark: hash: v1.k794b7964.e4f4b0993222ae8f8a3f0e59e4c0528312f13a1e358df37e53d4331b4dacbae5.oDWA2N3BGPuGJvd9voAvMpM2rCR7tfrLjEwDqJT6MYk scenes-app-ai-observability-image-preview--failed-image--light: @@ -11252,6 +11256,14 @@ snapshots: hash: v1.k794b7964.4b4f9908a2552fc97dd30fe179ca38caeb074f454f46ade87b3ca90ef0cd7f70.0GRyu-AtfFknvRJUEg03KJaNlwPBv9-SzOo4hXWpgoY scenes-app-experiments--experiment-results-row-recording-links-unselectable-metric--light: hash: v1.k794b7964.08f0ff958ed2d355bc6a0c9f5048f35e7f75b2cd34bacb867dd56f4ae5e4bf93.BNe46IwANTcBPvWEWBrpf78YOLf5cgWoIF1K-cni1J0 + scenes-app-experiments--experiment-running-time-on-track--dark: + hash: v1.k794b7964.afa5d2bc13065d7380ed28208c4e2eabdb1f4c45121ed0208c403ea955b674e6.MO0qXYuaERQK6SXloMRlNHcmGocr0DyiesdR3ZayFtk + scenes-app-experiments--experiment-running-time-on-track--light: + hash: v1.k794b7964.36487dd075f2235217d9b4fad812b91e9d51f2b22c9500120c81bb0bcc3548c7.0IhzfZMJNdt1pAKXr5f3bvcK9hFC2IvE_EfSlNUcwgQ + scenes-app-experiments--experiment-running-time-out-of-reach--dark: + hash: v1.k794b7964.76ee3239ae3823c6749b82d2d0f39d9e345f86aec33c8553847b59867ca1993d.RHLG1HcCDNxMpGjZ18XoRArDCNhQC2xDXjeGKXC5aNI + scenes-app-experiments--experiment-running-time-out-of-reach--light: + hash: v1.k794b7964.9dc2d830c6fc93df43da8d7ca01861af94d85aacaa7ab6ba202ce9d89ff46236.XVTEPecb7FA5Q0n3544d00ftbnOvv5tGWMxl0oR9VeE scenes-app-experiments--experiment-stopped-with-conclusion--dark: hash: v1.k794b7964.578ffd24470c381648d034ab3f9454d9f448430ae15d09e34b4e61b6441c3258.aOoEZotWjnQ2NmVYfdoTIz3AF8y1Y8DTCrF9YeavSQk scenes-app-experiments--experiment-stopped-with-conclusion--light: @@ -12049,13 +12061,13 @@ snapshots: scenes-app-inbox-first-report--no-data-narrow--light: hash: v1.k794b7964.8a760fde0be568a914e4453aac2c128a6ec065c0a6c01ef91eb016cbf11cf5ae.f8H0eCk6dHfGuY7f9eWdK5mjitwkzbGGrjg2ANeiNMM scenes-app-inbox-first-report--running--dark: - hash: v1.k794b7964.2bb4016dfbab0d8c8f1378706ab67bb51e84955b08685645596828789bb856d9.TZHjE1omnp-7XaQgd50zWuWvXUPdKlr-5ycTO7gJGJE + hash: v1.k794b7964.8873c63918500c67c125bf1810cfcbe0ebca8cb6326c7474a6d32760ce3108fc.cdTBqbm_zunougoPEEbKrq5YoNTbEhvWOoEZpP4w9cw scenes-app-inbox-first-report--running--light: - hash: v1.k794b7964.4d6369a69c7adb8a3edeadc4acc9cacc45bb2cbe753a68b1e8d336fbf9404053.JH83D4JcoC0XR8h7FgHmPxWiV5CAP6ZUqPzVkBYM5qU + hash: v1.k794b7964.6cad8925cb5f4e92a40d869f7bd5fec9080d4b0d810c6911f7a7935ddb912ebb.p7cBdOTLZxGPaazscXGseUZdlDMYNEOT3B-NFZCbxAQ scenes-app-inbox-first-report--running-narrow--dark: - hash: v1.k794b7964.1489893faeb3232bcd9ae74915614d155359ce14793fdcc57962a757d646e078.p5QygOekc_jO7LrBCAySkt7BCZpUi8TMtoCeVntqh90 + hash: v1.k794b7964.83e568a89db9806e44fa63bb21de51c0611974fd86f5e78818c83d13cd981962.uHejFGwT0d13MiZd9dYf3wh0zj-GRRsKNKBUbj4YkRE scenes-app-inbox-first-report--running-narrow--light: - hash: v1.k794b7964.d994a9d5e1c127959958aa3e468fce4945cea47afe00e61fd1d7df37ec27c4b9.Isj5PvJ7ELE3a--ZB3q2pv1l6D3-vjIV0-J7X8pcR94 + hash: v1.k794b7964.6247637182b919d676f1f745366b1f66417a72e61e3d69326931e3d71e763284.5vftvvyKgzlcrWlVtUR4EgW4S4aI_7ShVqifNBqYJH8 scenes-app-inbox-focused-setup--built-in-scouts--dark: hash: v1.k794b7964.9c362d9b9c26bb883910cfbdd1836a4527c166f4c96a2f5360e69023c79d5f0e.DuYijIxQuSoZCwSuCLHfE5vpQ9S7-wEi7vbYSPvUoQY scenes-app-inbox-focused-setup--built-in-scouts--light: @@ -13605,9 +13617,9 @@ snapshots: scenes-app-marketing-analytics-search-performance--placement-unavailable--light: hash: v1.k794b7964.9132ab7a37c8519e4fb52a37d6623c8eea2969d69532342ce877bd4c31fcc184.glpephRinlnPE2nB831WYYOKL8P3gKw_4_yDgVAA24E scenes-app-marketing-analytics-search-performance--position-metrics--dark: - hash: v1.k794b7964.adf8c4f8927a8a0e131e3548b4d1e75798e478b8cdc7232b388f9381780902cc.2uhi2pewKTyDwAHs2W5Iuv7eZLYCQGCThyAJ9CBLE7U + hash: v1.k794b7964.95bd2000c35ea1689e482196cf423e67a26aa274afb3007b2bd43a595f0baec2.sOzOlU1b1PPJZkEfOZ_sj8Tf_Fi4r-CWrIFPPPnNmsA scenes-app-marketing-analytics-search-performance--position-metrics--light: - hash: v1.k794b7964.d5099dea2f3faf5bbb5bd4dc76bbb0224762efbfed459dc9a99c3bf19023bd67.Iz50mTZ2ak6wLLJAw0_h2QWLczqy8xwEhqjf0SrR0ng + hash: v1.k794b7964.9a55e5c29016c60fc359238d47f15def8b79e5f69b9102eca41a934ab4e014d2.iIdIdmdc2rFnn02foj7OY36r_qsFVNt7f8OaX74zjA8 scenes-app-marketing-analytics-search-performance--post-hog-conversions--dark: hash: v1.k794b7964.9d3e260b6c10d6ef412dd784d10973a8649a9c67995ff07c1dc4ce42b29bd886.8SXJebwFKiup-BKVjtiRUkx5D8kJiWjYjhtaaIoy5_g scenes-app-marketing-analytics-search-performance--post-hog-conversions--light: @@ -15781,9 +15793,9 @@ snapshots: scenes-app-tasks--pi-task-selected--light: hash: v1.k794b7964.dc94140b8460e76c6e8b81adb9c46997c89a5634f20fac20ddada1f1ffd12223.4KOsU0tDj1zB26GFLV3daFR9ypOtXtETf08ZsGEMZt4 scenes-app-tasks--task-detail-loading--dark: - hash: v1.k794b7964.cf68a26a8d95ea04f3a13aa312f1ef15587ae7c533366fe679299a19f0e37dcc.6TZiaw-L4PN6eRjj342Xu9Zg9hSM2oqOFcM4l7GhyTs + hash: v1.k794b7964.a34fe05f1131d0368d2d803f27320c93f7a9bd5bcebc31d2459be0ed16e059c5.uSNsO42bTdJ1tYa-Bmz4uTocYRX1SIbo2vaa9IKq1AM scenes-app-tasks--task-detail-loading--light: - hash: v1.k794b7964.c8fb0a7ee9be9b7e43c86e1ef0ab08afa36fc1ee73e37c7db3f295cacd7f428e.LTBA5SJdNLyUSgSShns_HEJxKkHVYFN0T0lGWEKMTPs + hash: v1.k794b7964.f896d7c2b7cff888483338a1fededcca9f993a0b1dc1ec1bfcca6820b654c641.BHwh8AdR4NFJJrLw-CM4YYGlfVSb9XksqPOw2DeyKvg scenes-app-tasks--task-load-error--dark: hash: v1.k794b7964.8c160753039593f051efc147d69f447a861f14a8281d97f8f612b1b2279bb241.wB8WL7fVnS3sNEHJtbST0gUGVoyv2V2TwcnemxoSeBw scenes-app-tasks--task-load-error--light: diff --git a/frontend/src/generated/core/api.schemas.ts b/frontend/src/generated/core/api.schemas.ts index f721cac84c30..cc2cd50ecf3e 100644 --- a/frontend/src/generated/core/api.schemas.ts +++ b/frontend/src/generated/core/api.schemas.ts @@ -7051,6 +7051,19 @@ export type FileSystemDestroyParams = { recursive?: boolean } +export type FileSystemFrequentListParams = { + /** + * Maximum number of frequently viewed items to return. + * @minimum 1 + * @maximum 20 + */ + limit?: number + /** + * A search term. + */ + search?: string +} + export type FileSystemSearchRetrieveParams = { /** * Maximum number of results to return, best first. diff --git a/frontend/src/generated/core/api.ts b/frontend/src/generated/core/api.ts index f10262f8a804..8ce22cc15922 100644 --- a/frontend/src/generated/core/api.ts +++ b/frontend/src/generated/core/api.ts @@ -36,6 +36,7 @@ import type { ExportsListParams, FileSystemApi, FileSystemDestroyParams, + FileSystemFrequentListParams, FileSystemHomeFolderApi, FileSystemListParams, FileSystemRankedSearchResponseApi, @@ -1946,6 +1947,36 @@ export const fileSystemCountByPathCreate = async ( }) } +export const getFileSystemFrequentListUrl = (projectId: string, params?: FileSystemFrequentListParams) => { + const normalizedParams = new URLSearchParams() + + Object.entries(params || {}).forEach(([key, value]) => { + if (value !== undefined) { + normalizedParams.append(key, value === null ? 'null' : String(value)) + } + }) + + const stringifiedParams = normalizedParams.toString() + + return stringifiedParams.length > 0 + ? `/api/projects/${projectId}/file_system/frequent/?${stringifiedParams}` + : `/api/projects/${projectId}/file_system/frequent/` +} + +/** + * @summary List frequently viewed files + */ +export const fileSystemFrequentList = async ( + projectId: string, + params?: FileSystemFrequentListParams, + options?: RequestInit +): Promise => { + return apiMutator(getFileSystemFrequentListUrl(projectId, params), { + ...options, + method: 'GET', + }) +} + export const getFileSystemHomeFolderCreateUrl = (projectId: string) => { return `/api/projects/${projectId}/file_system/home_folder/` } diff --git a/frontend/src/lib/components/ActivityLog/humanizeActivity.test.tsx b/frontend/src/lib/components/ActivityLog/humanizeActivity.test.tsx index 25a7a16d2c25..bffb44378702 100644 --- a/frontend/src/lib/components/ActivityLog/humanizeActivity.test.tsx +++ b/frontend/src/lib/components/ActivityLog/humanizeActivity.test.tsx @@ -168,6 +168,48 @@ describe('humanizeActivity', () => { } }) + it.each<{ reason: string; changes: Partial[]; action: string; description: string }>([ + { + reason: 'staff_set', + changes: [ + { field: 'email_sending_tier', before: 2, after: 3 }, + { field: 'email_sending_tier_pinned', before: false, after: true }, + ], + action: 'Set the workflow email sending tier from 2 to 3 and pinned it there', + description: 'A user set the workflow email sending tier on Example from 2 to 3 and pinned it there', + }, + { + reason: 'clean_and_used', + changes: [{ field: 'email_sending_tier', before: 1, after: 2 }], + action: 'Moved the workflow email sending tier from 1 to 2', + description: 'A user moved the workflow email sending tier on Example from 1 to 2 (reason: clean_and_used)', + }, + { + reason: 'staff_set', + changes: [{ field: 'email_sending_tier_pinned', before: true, after: false }], + action: 'Unpinned the workflow email sending tier', + description: 'A user unpinned the workflow email sending tier on Example', + }, + ])('describes an email sending tier change: $action', ({ reason, changes, action, description }) => { + const logItem = makeLogItem({ + scope: ActivityScope.TEAM, + activity: 'email_sending_tier_changed', + detail: { + merge: null, + trigger: null, + name: 'Example', + context: { reason }, + changes: changes.map((change) => ({ type: ActivityScope.TEAM, action: 'changed', ...change })), + }, + }) + const result = teamActivityDescriber(logItem) + const text = (value: Description | undefined): string => + (render(<>{value}).container.textContent || '').replace(/\s+/g, ' ').trim() + + expect(text(result.description)).toBe(description) + expect(text(result.summary?.action)).toBe(action) + }) + it.each([ ['minted', 'A user created Developer mode token for a@posthog.com'], ['revoked', 'A user revoked Developer mode token for a@posthog.com'], diff --git a/frontend/src/lib/components/Cards/InsightCard/InsightMeta.tsx b/frontend/src/lib/components/Cards/InsightCard/InsightMeta.tsx index 6327a771dbc0..1c421dcaf2d4 100644 --- a/frontend/src/lib/components/Cards/InsightCard/InsightMeta.tsx +++ b/frontend/src/lib/components/Cards/InsightCard/InsightMeta.tsx @@ -31,6 +31,7 @@ import { accessLevelSatisfied } from 'lib/utils/accessControlUtils' import { copyToClipboard } from 'lib/utils/copyToClipboard' import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { capitalizeFirstLetter } from 'lib/utils/strings' +import { reportDashboardInsightMetaUpdated } from 'scenes/dashboard/dashboardEvents' import { getEffectiveDateOverride } from 'scenes/dashboard/dashboardUtils' import { dataRetentionBannerLogic } from 'scenes/insights/dataRetention/dataRetentionBannerLogic' import { exceedsRetention } from 'scenes/insights/dataRetention/exceedsRetention' @@ -216,7 +217,7 @@ export function InsightMeta({ const { copyImage } = useActions(captureImageLogic) const { isCapturing: isCapturingImage } = useValues(captureImageLogic) const { updateInsightDirect } = useActions(insightsModel) - const { reportDashboardInsightMetaUpdated, reportInsightResultsCopiedToClipboard } = useActions(eventUsageLogic) + const { reportInsightResultsCopiedToClipboard } = useActions(eventUsageLogic) const { featureFlags } = useValues(featureFlagLogic) const showCompactTile = diff --git a/frontend/src/lib/components/MarkdownNotebook/EditableTableBlock.tsx b/frontend/src/lib/components/MarkdownNotebook/EditableTableBlock.tsx index ec7ed81152b1..9860211b4c08 100644 --- a/frontend/src/lib/components/MarkdownNotebook/EditableTableBlock.tsx +++ b/frontend/src/lib/components/MarkdownNotebook/EditableTableBlock.tsx @@ -12,7 +12,7 @@ import { useState, } from 'react' -import { IconMinus, IconPlus } from '@posthog/icons' +import { IconMinus, IconPlus, IconTrash } from '@posthog/icons' import { LemonButton } from '@posthog/lemon-ui' import { shouldUseMarkdownPaste } from './documentModel' @@ -111,6 +111,7 @@ export function EditableTableBlock({ setBlockRef, setTableCellRef, updateNode, + deleteNode, handleSelectionChange, startTextSelectionPointer, restoreSelectionRef, @@ -120,6 +121,7 @@ export function EditableTableBlock({ setBlockRef: (element: HTMLElement | null) => void setTableCellRef: (position: TableCellPosition, element: HTMLElement | null) => void updateNode: (nodeId: string, updater: (node: NotebookBlockNode) => NotebookBlockNode | null) => void + deleteNode: () => void handleSelectionChange: () => void startTextSelectionPointer: (event: TextSelectionPointerStartEvent) => void restoreSelectionRef: MutableRefObject @@ -481,6 +483,7 @@ export function EditableTableBlock({ label={control.label} tooltip={control.tooltip} icon={} + dataAttr="markdown-notebook-table-add-row" onClick={() => addTableRowAfter(control.rowIndex, 0)} /> @@ -499,6 +502,7 @@ export function EditableTableBlock({ label={control.label} tooltip={control.tooltip} icon={} + dataAttr="markdown-notebook-table-add-column" onClick={() => addTableColumnAfter(control.columnIndex)} /> @@ -519,6 +523,7 @@ export function EditableTableBlock({ label={`Remove row ${rowIndex + 1}`} tooltip="Remove row" icon={} + dataAttr="markdown-notebook-table-remove-row" onClick={() => removeTableRow(rowIndex)} /> @@ -540,6 +545,7 @@ export function EditableTableBlock({ label={`Remove column ${columnIndex + 1}`} tooltip="Remove column" icon={} + dataAttr="markdown-notebook-table-remove-column" disabledReason={ columnCount <= 1 ? 'Tables need at least one column' : undefined } @@ -548,6 +554,21 @@ export function EditableTableBlock({ ) })} +
+ } + dataAttr="markdown-notebook-table-delete" + onClick={deleteNode} + /> +
) : null} @@ -560,12 +581,14 @@ export function TableStructureControlButton({ label, tooltip, icon, + dataAttr, disabledReason, onClick, }: { label: string tooltip: string icon: JSX.Element + dataAttr: string disabledReason?: string onClick: () => void }): JSX.Element { @@ -573,6 +596,7 @@ export function TableStructureControlButton({ { + const onChange = jest.fn() + const { container } = render( + createElement(MarkdownNotebook, { + value: withNotebookTitle(`Intro paragraph\n\n${tableMarkdown}`), + onChange, + }) + ) + const lastCell = Array.from( + container.querySelectorAll('.MarkdownNotebook__table-cell-content') + ).at(-1) as HTMLElement + + placeCaretInElement(lastCell) + fireEvent.keyDown(lastCell, { key: 'Backspace' }) + + expect(container.querySelector('.MarkdownNotebook__table-block') === null).toEqual(!!expectedMarkdown) + expect(onChange.mock.calls.at(-1)?.[0]).toEqual(expectedMarkdown) + }) + + it('deletes a table with content from its delete control', () => { + const onChange = jest.fn() + const { container } = render( + createElement(MarkdownNotebook, { + value: withNotebookTitle('Intro paragraph\n\n| A | B |\n| --- | --- |\n| 1 | 2 |'), + onChange, + }) + ) + + fireEvent.click(container.querySelector('button[aria-label="Delete table"]') as HTMLButtonElement) + + expect(container.querySelector('.MarkdownNotebook__table-block')).toBeNull() + expect(onChange).toHaveBeenLastCalledWith(`${TEST_NOTEBOOK_TITLE_MARKDOWN}\n\nIntro paragraph`) + }) + it('keeps newly inserted components active for keyboard row actions', () => { const onChange = jest.fn() const { container } = render(createElement(MarkdownNotebook, { value: withNotebookTitle(' '), onChange })) diff --git a/frontend/src/lib/components/MarkdownNotebook/MarkdownNotebook.tsx b/frontend/src/lib/components/MarkdownNotebook/MarkdownNotebook.tsx index 78fef0d2897f..972c93cab307 100644 --- a/frontend/src/lib/components/MarkdownNotebook/MarkdownNotebook.tsx +++ b/frontend/src/lib/components/MarkdownNotebook/MarkdownNotebook.tsx @@ -234,6 +234,7 @@ import { getTableCellRefKey, getTableColumnCount, getTableEdgeCellPosition, + isTableEmpty, makeEmptyTableRow, normalizeTableRow, tableCellPositionsEqual, @@ -4736,6 +4737,27 @@ function MarkdownNotebookEditor({ return true }, [commitDocument, requestFocusAfterRemovingNode]) + const deleteEmptyTableAtCurrentSelection = useCallback((): boolean => { + const element = getSelectedInlineEditableElementOfType( + notebookRef.current, + 'MarkdownNotebook__table-cell-content' + ) + const nodeId = element?.dataset.markdownNotebookNodeId + if (!element || !nodeId) { + return false + } + + const nodes = documentRef.current.nodes.length ? documentRef.current.nodes : [emptyNodeRef.current] + const node = nodes.find((currentNode) => currentNode.id === nodeId) + if (!node || node.type !== 'table' || !isTableEmpty(node)) { + return false + } + + requestFocusAfterRemovingNode(nodeId) + deleteNodeWithRefCleanup(nodeId) + return true + }, [deleteNodeWithRefCleanup, requestFocusAfterRemovingNode]) + const insertParagraphBelowTrailingCodeBlockAtCurrentSelection = useCallback((): boolean => { const element = getSelectedInlineEditableElementOfType(notebookRef.current, 'MarkdownNotebook__code-block') const nodeId = element?.dataset.markdownNotebookNodeId @@ -6774,7 +6796,8 @@ function MarkdownNotebookEditor({ !event.ctrlKey && (deleteListItemRangeAtCurrentSelection() || deleteListItemAtCurrentSelection(event.key === 'Backspace' ? 'backward' : 'forward') || - deleteEmptyCodeBlockAtCurrentSelection()) + deleteEmptyCodeBlockAtCurrentSelection() || + deleteEmptyTableAtCurrentSelection()) ) { event.preventDefault() event.stopPropagation() diff --git a/frontend/src/lib/components/MarkdownNotebook/renderNode.tsx b/frontend/src/lib/components/MarkdownNotebook/renderNode.tsx index 346a783b0ed5..55f9c2c6e063 100644 --- a/frontend/src/lib/components/MarkdownNotebook/renderNode.tsx +++ b/frontend/src/lib/components/MarkdownNotebook/renderNode.tsx @@ -235,6 +235,7 @@ export function renderNode({ setBlockRef={setBlockRef} setTableCellRef={setTableCellRef} updateNode={updateNode} + deleteNode={deleteNodeAndFocusAdjacent} handleSelectionChange={handleSelectionChange} startTextSelectionPointer={startTextSelectionPointer} restoreSelectionRef={restoreSelectionRef} diff --git a/frontend/src/lib/components/MarkdownNotebook/tableModel.ts b/frontend/src/lib/components/MarkdownNotebook/tableModel.ts index 0546d539e819..754b6efdd653 100644 --- a/frontend/src/lib/components/MarkdownNotebook/tableModel.ts +++ b/frontend/src/lib/components/MarkdownNotebook/tableModel.ts @@ -1,5 +1,6 @@ import { InsertMenuSelectionDirection, TableCellPosition } from './editorTypes' import { NotebookTableBlockNode, NotebookTableCell } from './types' +import { getInlineText } from './utils' export function getTableCellRefKey(nodeId: string, position: TableCellPosition): string { return `${nodeId}:${position.section}:${String(position.rowIndex)}:${String(position.columnIndex)}` @@ -17,6 +18,10 @@ export function makeEmptyTableRow(columnCount: number): NotebookTableCell[] { return Array.from({ length: columnCount }, () => ({ children: [] })) } +export function isTableEmpty(node: NotebookTableBlockNode): boolean { + return [node.headers, ...node.rows].every((row) => row.every((cell) => !getInlineText(cell.children).length)) +} + export function getTableCellPositions(node: NotebookTableBlockNode): TableCellPosition[] { const columnCount = getTableColumnCount(node) return [ diff --git a/frontend/src/lib/constants.tsx b/frontend/src/lib/constants.tsx index d7faf9a6af30..2f9f7c3ab33f 100644 --- a/frontend/src/lib/constants.tsx +++ b/frontend/src/lib/constants.tsx @@ -271,7 +271,6 @@ export const FEATURE_FLAGS = { BILLING_ALERTS: 'billing-alerts', // owner: #team-billing, gates the Billing > Alerts tab BILLING_REAL_TIME_USAGE: 'billing-real-time-usage', // owner: #team-billing, gates the Real-time usage scene BROADCASTS_AI_FIRST_NEW: 'broadcasts-ai-first-new', // owner: @dmarchuk #team-workflows - BUSINESS_KNOWLEDGE_GITHUB_REPOS: 'business-knowledge-github-repos', // owner: @veryayskiy #team-conversations CANVAS_COMMENTS: 'posthog-desktop-canvas-comments', // owner: @k11kirky, shows canvas comments and selection comments on web canvases, shared with PostHog Desktop CDP_DWH_TABLE_SOURCE: 'cdp-dwh-table-source', // owner: #team-workflows-cdp CDP_DWH_VIEW_SOURCE: 'cdp-dwh-view-source', // owner: #team-workflows-cdp @@ -545,6 +544,7 @@ export const FEATURE_FLAGS = { TAXONOMIC_FILTER_SEARCH_INTENT: 'taxonomic-filter-search-intent', // owner: @pauldambra multivariate=control,banner,promote, classifies a picker search with the decision model. control changes nothing, banner suggests the right tab, promote moves the predicted group to the top of the All tab TAXONOMIC_FILTER_VALUE_SEARCH_GATING: 'taxonomic-filter-value-search-gating', // owner: @pauldambra TEXT_CARD_WORD_ART: 'text-card-word-art', // owner: @jonmcwest, gates the word art insert button in dashboard text cards + TODAY_JUMP_BACK_RANKING: 'today-jump-back-ranking', TODAY_PULSE_RANKING: 'today-pulse-ranking', TODAY_RAIL_NAV: 'today-rail-nav', // owner: @k11kirky, replaces the left navigation with a rail (Home, Spaces, Library, Tools) and makes the Today page the homepage TODAY_REPORT_JEV: 'today-report-jev', // owner: @puemos, asks the decision model to label the problem, cause and fix in a Today report, to mark the numbers a source states, and to pick the code excerpt a finding describes diff --git a/frontend/src/lib/utils/eventUsageLogic.test.ts b/frontend/src/lib/utils/eventUsageLogic.test.ts index ce6a599603d0..833309477af0 100644 --- a/frontend/src/lib/utils/eventUsageLogic.test.ts +++ b/frontend/src/lib/utils/eventUsageLogic.test.ts @@ -14,21 +14,13 @@ import { BaseMathType, BehavioralEventType, ChartDisplayType, - type DashboardTile, - type DashboardType, FilterLogicalOperator, FunnelVizType, PropertyFilterType, StepOrderValue, } from '~/types' -import { - dashboardViewedProperties, - eventUsageLogic, - getEventPropertiesForMetric, - sanitizeDashboard, - sanitizeQuery, -} from './eventUsageLogic' +import { eventUsageLogic, getEventPropertiesForMetric, sanitizeQuery } from './eventUsageLogic' describe('eventUsageLogic', () => { describe('experiments list viewed', () => { @@ -377,108 +369,6 @@ describe('eventUsageLogic', () => { }) }) - it('keeps metric variable definitions out of a sanitized dashboard', () => { - const metricVariables = [{ name: 'env', source: 'sql', query: 'SELECT private_value' }] - const tile = { id: 1, insight: { id: 2, filter_override_context: { dashboard: { metricVariables } } } } - const dashboard = { id: 7, tiles: [tile], filters: { metricVariables }, persisted_filters: { metricVariables } } - - const sanitized = sanitizeDashboard(dashboard as unknown as DashboardType) - - expect(JSON.stringify(sanitized)).not.toContain('private_value') - }) - - describe('dashboardViewedProperties', () => { - const dashboard = (tiles: Partial[]): DashboardType => - ({ - id: 7, - created_at: '2026-01-01T00:00:00Z', - is_shared: false, - pinned: true, - creation_mode: 'default', - created_by: { uuid: 'creator' }, - tiles, - }) as unknown as DashboardType - - const insightTile = (query: unknown, is_sample = false): Partial => - ({ insight: { query, is_sample } }) as unknown as Partial - - const cases: [string, Partial[], Record][] = [ - ['no tiles', [], { item_count: 0, sample_items_count: 0 }], - [ - 'an insight tile with a query', - [insightTile({ kind: NodeKind.InsightVizNode, source: { kind: NodeKind.TrendsQuery, series: [] } })], - { item_count: 1, text_count: 1, uses_data_warehouse_source: false, data_warehouse_tiles_count: 0 }, - ], - ['an insight tile without a query', [insightTile(null)], { item_count: 1, empty_count: 1 }], - [ - 'a sample insight tile that reads the warehouse', - [insightTile({ kind: NodeKind.TrendsQuery, series: [{ kind: NodeKind.DataWarehouseNode }] }, true)], - { - item_count: 1, - text_count: 1, - sample_items_count: 1, - uses_data_warehouse_source: true, - data_warehouse_tiles_count: 1, - }, - ], - [ - 'text and widget tiles', - [{ text: { body: 'hi' } }, { text: { body: 'there' } }, { widget: {} }] as Partial[], - { item_count: 3, text_tiles_count: 2, widget_tiles_count: 1 }, - ], - [ - 'groups once however many tiles share a key', - [ - { group_key: 'a' }, - { group_key: 'a' }, - { group_key: 'b' }, - { group_key: null }, - ] as Partial[], - { item_count: 4, text_tiles_count: 4, group_count: 2 }, - ], - ] - - it.each(cases)('counts %s', (_, tiles, expected) => { - expect(dashboardViewedProperties(dashboard(tiles), null, 'viewer')).toEqual({ - created_at: '2026-01-01T00:00:00Z', - is_shared: false, - pinned: true, - creation_mode: 'default', - viewer_is_creator: false, - created_by_system: false, - dashboard_id: 7, - group_display: 'sections', - group_count: 0, - lastRefreshed: undefined, - refreshAge: undefined, - sample_items_count: 0, - uses_data_warehouse_source: false, - data_warehouse_tiles_count: 0, - ...expected, - }) - }) - - it('reports the dashboard group display', () => { - const withTabs = { ...dashboard([]), customization: { group_display: 'tabs' } } as unknown as DashboardType - expect(dashboardViewedProperties(withTabs, null, 'viewer')).toMatchObject({ group_display: 'tabs' }) - }) - - const viewerCases: [string, { uuid: string } | null, string | undefined, boolean | undefined][] = [ - ['the creator views it', { uuid: 'creator' }, 'creator', true], - ['another user views it', { uuid: 'creator' }, 'viewer', false], - ['the viewer is unknown', { uuid: 'creator' }, undefined, undefined], - ['the system created it', null, 'viewer', undefined], - ] - - it.each(viewerCases)('reports viewer_is_creator when %s', (_, created_by, viewerUuid, expected) => { - const withCreator = { ...dashboard([]), created_by } as unknown as DashboardType - expect(dashboardViewedProperties(withCreator, null, viewerUuid)).toMatchObject({ - viewer_is_creator: expected, - created_by_system: created_by === null, - }) - }) - }) - it('reports only the kind for a legacy metric', () => { const metric = { kind: NodeKind.ExperimentFunnelsQuery, diff --git a/frontend/src/lib/utils/eventUsageLogic.ts b/frontend/src/lib/utils/eventUsageLogic.ts index 819c36c661de..787a9367d7be 100644 --- a/frontend/src/lib/utils/eventUsageLogic.ts +++ b/frontend/src/lib/utils/eventUsageLogic.ts @@ -2,8 +2,6 @@ import { MakeLogicType, actions, connect, kea, listeners, path } from 'kea' import posthog from 'posthog-js' import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' -import type { Dayjs } from 'lib/dayjs' -import { now } from 'lib/dayjs' import type { IntegrationConnectSurface, IntegrationLinkExistingCounts } from 'lib/integrations/utils' import { TimeToSeeDataPayload } from 'lib/internalMetrics' import { preflightLogic } from 'lib/logic/preflightLogic' @@ -56,10 +54,6 @@ import { import { ChartDisplayType, CohortType, - DashboardTileSpacing, - DashboardMode, - DashboardTile, - DashboardType, Experiment, ExperimentHoldoutType, ExperimentIdType, @@ -68,7 +62,6 @@ import { PropertyFilterType, InsightModel, type SDK, - TileGroupDisplay, } from '~/types' import { captureBIWorksheetAction } from 'products/business_intelligence/frontend/biEditorAnalytics' @@ -98,16 +91,6 @@ export enum DashboardEventSource { DashboardVariableOverride = 'dashboard_variable_override', } -export type DashboardFilterChangeType = - | 'date' - | 'properties' - | 'breakdown' - | 'variable' - | 'interval' - | 'test_accounts' - | 'compare' - | 'metric_variables' - export enum InsightEventSource { LongPress = 'long_press', MoreDropdown = 'more_dropdown', @@ -608,34 +591,6 @@ export function sanitizeInsight(insight: Partial | null): object | return sanitizedInsight } -function sanitizeTile(tile: DashboardTile | null): object | undefined { - if (!tile) { - return undefined - } - - return { - ...tile, - insight: tile.insight ? sanitizeInsight(tile.insight) : undefined, - } -} - -export function sanitizeDashboard(dashboard: DashboardType | null): object | null { - if (!dashboard) { - return null - } - - // Remove breakdown color entries, as they carry raw breakdown values (user data) — report only their count - const { breakdown_colors, ...sanitizedDashboard } = dashboard - - return { - ...sanitizedDashboard, - breakdown_colors_count: breakdown_colors?.length ?? 0, - filters: filtersForAnalytics(dashboard.filters ?? {}), - persisted_filters: dashboard.persisted_filters ? filtersForAnalytics(dashboard.persisted_filters) : null, - tiles: dashboard.tiles?.map((tile) => sanitizeTile(tile)) || [], - } -} - function countBehavioralFilters(value: unknown): number { if (Array.isArray(value)) { return value.reduce((count, item) => count + countBehavioralFilters(item), 0) @@ -712,69 +667,6 @@ function funnelProperties(querySource: InsightQueryNode): SanitizedQuery { } } -function increment(counts: Record, key: string): void { - counts[key] = (counts[key] || 0) + 1 -} - -function insightTileCountKey(tile: DashboardTile): string { - const query = isNodeWithSource(tile.insight?.query) ? tile.insight.query.source : tile.insight?.query - return `${query?.kind || !!tile.text ? 'text' : 'empty'}_count` -} - -function countDashboardTiles(tiles: DashboardTile[], properties: Record): void { - for (const tile of tiles) { - if (tile.insight) { - increment(properties, insightTileCountKey(tile)) - properties.sample_items_count += tile.insight.is_sample ? 1 : 0 - if (queryUsesDataWarehouse(tile.insight.query)) { - properties.uses_data_warehouse_source = true - properties.data_warehouse_tiles_count += 1 - } - } else if (tile.widget) { - increment(properties, 'widget_tiles_count') - } else { - increment(properties, 'text_tiles_count') - } - } -} - -export function tileGroupProperties( - groupDisplay: TileGroupDisplay | undefined, - tiles: DashboardTile[] -): { group_display: TileGroupDisplay; group_count: number } { - return { - group_display: groupDisplay ?? 'sections', - group_count: new Set(tiles.map((tile) => tile.group_key).filter(Boolean)).size, - } -} - -export function dashboardViewedProperties( - dashboard: DashboardType, - lastRefreshed: Dayjs | null, - viewerUuid: string | undefined -): Record { - const { created_at, is_shared, pinned, creation_mode, id } = dashboard - const properties: Record = { - created_at, - is_shared, - pinned, - creation_mode, - viewer_is_creator: - dashboard.created_by?.uuid && viewerUuid ? dashboard.created_by.uuid === viewerUuid : undefined, - sample_items_count: 0, - item_count: dashboard.tiles?.length || 0, - created_by_system: !dashboard.created_by, - dashboard_id: id, - ...tileGroupProperties(dashboard.customization?.group_display, dashboard.tiles || []), - lastRefreshed: lastRefreshed?.toISOString(), - refreshAge: lastRefreshed ? now().diff(lastRefreshed, 'seconds') : undefined, - uses_data_warehouse_source: false, - data_warehouse_tiles_count: 0, - } - countDashboardTiles(dashboard.tiles || [], properties) - return properties -} - /** Takes a query and returns an object with "useful" properties that don't contain sensitive data. */ export function sanitizeQuery(query: Node | null): SanitizedQuery { const payload: SanitizedQuery = { @@ -938,95 +830,6 @@ export interface eventUsageLogicActions { journeyName: string stepCount: number } - reportDashboardAddMenuOpened: ( - source: 'header', - dashboardId: number - ) => { - dashboardId: number - source: 'header' - } - reportDashboardEditModeDiscardPrompt: ( - dashboard: DashboardType | null, - action: 'discarded' | 'kept_editing' | 'shown' - ) => { - action: 'discarded' | 'kept_editing' | 'shown' - dashboard: DashboardType | null - } - reportDashboardEmptyAddChartClicked: (dashboardId: number | undefined) => { - dashboardId: number | undefined - } - reportDashboardEmptyAiPromptSubmitted: ( - dashboardId: number | undefined, - promptType: 'custom_prompt' | 'starter_question' - ) => { - dashboardId: number | undefined - promptType: 'custom_prompt' | 'starter_question' - } - reportDashboardEmptyWebAnalyticsClicked: (dashboardId: number | undefined) => { - dashboardId: number | undefined - } - reportDashboardFiltersChanged: ( - dashboard: DashboardType | null, - changeType: DashboardFilterChangeType, - properties: Record - ) => { - changeType: DashboardFilterChangeType - dashboard: DashboardType | null - properties: Record - } - reportDashboardInsightDeleteAfterRemovalClicked: (otherDashboardCount: number) => { - otherDashboardCount: number - } - reportDashboardInsightMetaUpdated: ( - dashboardId: number | undefined, - insightId: number, - attribute: 'description' | 'name' - ) => { - attribute: 'description' | 'name' - dashboardId: number | undefined - insightId: number - } - reportDashboardLayoutEditModeEntered: ( - dashboard: DashboardType | null, - source: DashboardEventSource, - layoutZoom: number | null - ) => { - dashboard: DashboardType | null - layoutZoom: number | null - source: DashboardEventSource - } - reportDashboardModeToggled: ( - dashboard: DashboardType | null, - mode: DashboardMode | null, - source: DashboardEventSource | null, - layoutZoom: number | null, - layoutEditMode?: boolean - ) => { - dashboard: DashboardType | null - layoutEditMode: boolean | undefined - layoutZoom: number | null - mode: DashboardMode | null - source: DashboardEventSource | null - } - reportDashboardMoveInitiated: ( - method: 'bulk' | 'single', - count: number - ) => { - count: number - method: 'bulk' | 'single' - } - reportDashboardTileDensityConfigured: (tileDensity: DashboardTileSpacing) => { - tileDensity: DashboardTileSpacing - } - reportDashboardTileRepositioned: ( - dashboardId: number, - action: 'moved' | 'resized', - layoutZoom: number - ) => { - action: 'moved' | 'resized' - dashboardId: number - layoutZoom: number - } reportDataManagementDefinitionClickEdit: (type: TaxonomicFilterGroupType) => { type: TaxonomicFilterGroupType } @@ -1729,47 +1532,6 @@ export const eventUsageLogic = kea([ reportProjectCreationSubmitted: (projectCount: number, nameLength: number) => ({ projectCount, nameLength }), reportProjectNoticeDismissed: (key: string) => ({ key }), reportProjectNoticeShown: (variant: string) => ({ variant }), - reportDashboardModeToggled: ( - dashboard: DashboardType | null, - mode: DashboardMode | null, - source: DashboardEventSource | null, - layoutZoom: number | null, - layoutEditMode?: boolean - ) => ({ dashboard, mode, source, layoutZoom, layoutEditMode }), - reportDashboardLayoutEditModeEntered: ( - dashboard: DashboardType | null, - source: DashboardEventSource, - layoutZoom: number | null - ) => ({ dashboard, source, layoutZoom }), - reportDashboardFiltersChanged: ( - dashboard: DashboardType | null, - changeType: DashboardFilterChangeType, - properties: Record - ) => ({ dashboard, changeType, properties }), - reportDashboardTileDensityConfigured: (tileDensity: DashboardTileSpacing) => ({ tileDensity }), - reportDashboardEditModeDiscardPrompt: ( - dashboard: DashboardType | null, - action: 'shown' | 'discarded' | 'kept_editing' - ) => ({ dashboard, action }), - reportDashboardMoveInitiated: (method: 'single' | 'bulk', count: number) => ({ method, count }), - reportDashboardInsightDeleteAfterRemovalClicked: (otherDashboardCount: number) => ({ otherDashboardCount }), - reportDashboardTileRepositioned: (dashboardId: number, action: 'moved' | 'resized', layoutZoom: number) => ({ - dashboardId, - action, - layoutZoom, - }), - reportDashboardInsightMetaUpdated: ( - dashboardId: number | undefined, - insightId: number, - attribute: 'name' | 'description' - ) => ({ dashboardId, insightId, attribute }), - /** Empty-state AI prompt chips (ai-first empty dashboard only). */ - reportDashboardEmptyAiPromptSubmitted: ( - dashboardId: number | undefined, - promptType: 'custom_prompt' | 'starter_question' - ) => ({ dashboardId, promptType }), - reportDashboardEmptyAddChartClicked: (dashboardId: number | undefined) => ({ dashboardId }), - reportDashboardEmptyWebAnalyticsClicked: (dashboardId: number | undefined) => ({ dashboardId }), reportTimezoneComponentViewed: ( component: 'label' | 'indicator', project_timezone?: string, @@ -1782,7 +1544,6 @@ export const eventUsageLogic = kea([ reportSessionTableVersionUpdated: (version: string) => ({ version }), reportCustomChannelTypeRulesUpdated: (numRules: number) => ({ numRules }), reportCreatedDashboardFromModal: true, - reportDashboardAddMenuOpened: (source: 'header', dashboardId: number) => ({ source, dashboardId }), /** Dashboard created via PostHog web app from a template (new dashboard modal / template chooser). */ reportSavedInsightToDashboard: (insight: Partial | null, dashboardId: number | null) => ({ insight, @@ -2220,82 +1981,6 @@ export const eventUsageLogic = kea([ reportDataTableColumnsUpdated: async ({ context_type }) => { posthog.capture('data table columns updated', { context_type }) }, - reportDashboardModeToggled: async ({ dashboard, mode, source, layoutZoom, layoutEditMode }) => { - posthog.capture('dashboard mode toggled', { - dashboard_id: dashboard?.id, - dashboard: sanitizeDashboard(dashboard), - mode, - source, - layout_zoom: layoutZoom ?? undefined, - layout_edit_mode: layoutEditMode ?? undefined, - }) - }, - reportDashboardLayoutEditModeEntered: async ({ dashboard, source, layoutZoom }) => { - posthog.capture('dashboard layout edit mode entered', { - dashboard_id: dashboard?.id, - dashboard: sanitizeDashboard(dashboard), - source, - layout_zoom: layoutZoom ?? undefined, - }) - }, - reportDashboardFiltersChanged: async ({ dashboard, changeType, properties }) => { - posthog.capture('dashboard filters changed', { - dashboard_id: dashboard?.id, - change_type: changeType, - ...properties, - }) - }, - reportDashboardTileDensityConfigured: async ({ tileDensity }) => { - posthog.capture('dashboard tile density configured', { tile_density: tileDensity }) - }, - reportDashboardEditModeDiscardPrompt: async ({ dashboard, action }) => { - posthog.capture('dashboard edit mode discard prompt', { - dashboard_id: dashboard?.id, - dashboard: sanitizeDashboard(dashboard), - action, - }) - }, - reportDashboardMoveInitiated: async ({ method, count }) => { - posthog.capture('dashboard move initiated', { method, count }) - }, - reportDashboardTileRepositioned: async ({ dashboardId, action, layoutZoom }) => { - posthog.capture('dashboard tile repositioned', { - dashboard_id: dashboardId, - action, - layout_zoom: layoutZoom, - }) - }, - reportDashboardInsightMetaUpdated: async ({ dashboardId, insightId, attribute }) => { - posthog.capture('dashboard insight meta updated', { - dashboard_id: dashboardId, - insight_id: insightId, - attribute, - }) - }, - reportDashboardInsightDeleteAfterRemovalClicked: async ({ otherDashboardCount }) => { - posthog.capture('dashboard insight delete after removal clicked', { - other_dashboard_count: otherDashboardCount, - }) - }, - reportDashboardEmptyAiPromptSubmitted: async ({ dashboardId, promptType }) => { - posthog.capture('dashboard empty ai prompt submitted', { - dashboard_id: dashboardId, - prompt_type: promptType, - source: 'web', - }) - }, - reportDashboardEmptyAddChartClicked: async ({ dashboardId }) => { - posthog.capture('dashboard empty add chart clicked', { - dashboard_id: dashboardId, - source: 'web', - }) - }, - reportDashboardEmptyWebAnalyticsClicked: async ({ dashboardId }) => { - posthog.capture('dashboard empty web analytics clicked', { - dashboard_id: dashboardId, - source: 'web', - }) - }, reportTimezoneComponentViewed: async (payload) => { posthog.capture('timezone component viewed', payload) }, @@ -2329,9 +2014,6 @@ export const eventUsageLogic = kea([ reportCreatedDashboardFromModal: async () => { posthog.capture('created new dashboard from modal') }, - reportDashboardAddMenuOpened: async ({ source, dashboardId }) => { - posthog.capture('dashboard add menu opened', { source, dashboard_id: dashboardId }) - }, reportSavedInsightToDashboard: async ({ insight, dashboardId }) => { if (isBIVisualizationNode(insight?.query)) { captureBIWorksheetAction('added_to_dashboard', insight.query.config, { insight_id: insight.id }) diff --git a/frontend/src/scenes/dashboard/DashboardAiPromptComposer.tsx b/frontend/src/scenes/dashboard/DashboardAiPromptComposer.tsx index 6b6518bda821..f93ab53b64ea 100644 --- a/frontend/src/scenes/dashboard/DashboardAiPromptComposer.tsx +++ b/frontend/src/scenes/dashboard/DashboardAiPromptComposer.tsx @@ -5,9 +5,9 @@ import type { FormEvent } from 'react' import { LemonButton } from 'lib/lemon-ui/LemonButton' import { LemonTextArea } from 'lib/lemon-ui/LemonTextArea/LemonTextArea' import { Link } from 'lib/lemon-ui/Link' -import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { dashboardAiPromptComposerLogic } from './dashboardAiPromptComposerLogic' +import { reportDashboardEmptyAiPromptSubmitted } from './dashboardEvents' function reportDashboardEmptyAiPromptClicked( promptLabel: string, @@ -127,7 +127,6 @@ export function DashboardAiPromptComposer({ }: DashboardAiPromptComposerProps): JSX.Element { const { prompt, promptSource } = useValues(dashboardAiPromptComposerLogic) const { setPrompt } = useActions(dashboardAiPromptComposerLogic) - const { reportDashboardEmptyAiPromptSubmitted } = useActions(eventUsageLogic) const submitPrompt = (event: FormEvent): void => { event.preventDefault() diff --git a/frontend/src/scenes/dashboard/DashboardHeaderActions.tsx b/frontend/src/scenes/dashboard/DashboardHeaderActions.tsx index 82deb57f72b6..b21e889df087 100644 --- a/frontend/src/scenes/dashboard/DashboardHeaderActions.tsx +++ b/frontend/src/scenes/dashboard/DashboardHeaderActions.tsx @@ -10,7 +10,7 @@ import { LemonBadge } from 'lib/lemon-ui/LemonBadge' import { LemonButton } from 'lib/lemon-ui/LemonButton' import { LemonMenu, LemonMenuItem, LemonMenuItems } from 'lib/lemon-ui/LemonMenu' import { getAccessControlDisabledReason } from 'lib/utils/accessControlUtils' -import { DashboardEventSource, eventUsageLogic } from 'lib/utils/eventUsageLogic' +import { DashboardEventSource } from 'lib/utils/eventUsageLogic' import { MaxTool } from 'scenes/max/MaxTool' import { Scene } from 'scenes/sceneTypes' import { urls } from 'scenes/urls' @@ -22,6 +22,7 @@ import { DashboardCustomizeMenu } from 'products/dashboards/frontend/components/ import { DashboardAddTileHandlers, DashboardAddTileKey, getDashboardAddTileOptions } from './dashboardAddTileOptions' import { DashboardCustomizeButton } from './DashboardCustomizeButton' +import { reportDashboardAddMenuOpened } from './dashboardEvents' import { DashboardLoadAction, dashboardLogic } from './dashboardLogic' import { DashboardSubscribeButton } from './DashboardSubscribeButton' @@ -61,7 +62,6 @@ export function DashboardAddTileButton(): JSX.Element | null { openImageTileModal, openButtonTileModal, } = useActions(dashboardLogic) - const { reportDashboardAddMenuOpened } = useActions(eventUsageLogic) if (!dashboard || tiles.length === 0) { return null diff --git a/frontend/src/scenes/dashboard/DashboardItems.test.tsx b/frontend/src/scenes/dashboard/DashboardItems.test.tsx index 1382f994dac8..f95ac1ed668e 100644 --- a/frontend/src/scenes/dashboard/DashboardItems.test.tsx +++ b/frontend/src/scenes/dashboard/DashboardItems.test.tsx @@ -5,7 +5,6 @@ import { useActions, useAsyncActions, useValues } from 'kea' import { router } from 'kea-router' import type { PositionStrategy } from 'react-grid-layout/core' -import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { dashboardLogic } from 'scenes/dashboard/dashboardLogic' import { dashboardsModel } from '~/models/dashboardsModel' @@ -39,7 +38,6 @@ jest.mock('~/models/insightsModel', () => ({ })) jest.mock('lib/utils/eventUsageLogic', () => ({ - eventUsageLogic: { __mock: 'eventUsageLogic' }, DashboardEventSource: { CardEdgeHover: 'CardEdgeHover', CardDragHandle: 'CardDragHandle', @@ -303,12 +301,6 @@ describe('DashboardItems', () => { } } - if (logic === eventUsageLogic) { - return { - reportDashboardTileRepositioned: jest.fn(), - } - } - if (logic === router) { return { push: jest.fn(), diff --git a/frontend/src/scenes/dashboard/DashboardItems.tsx b/frontend/src/scenes/dashboard/DashboardItems.tsx index 5f5ae9f0f805..87178912ee08 100644 --- a/frontend/src/scenes/dashboard/DashboardItems.tsx +++ b/frontend/src/scenes/dashboard/DashboardItems.tsx @@ -15,8 +15,9 @@ import { ApiError } from 'lib/api' import { EditModeEdge, useResizeHandleScrollbarPassThrough } from 'lib/components/Cards/InsightCard/EditModeEdgeOverlay' import { TileDecorations } from 'lib/components/TileDecorations/TileDecorations' import { LemonBanner } from 'lib/lemon-ui/LemonBanner' -import { DashboardEventSource, eventUsageLogic } from 'lib/utils/eventUsageLogic' +import { DashboardEventSource } from 'lib/utils/eventUsageLogic' import { objectsEqual } from 'lib/utils/objects' +import { reportDashboardTileRepositioned } from 'scenes/dashboard/dashboardEvents' import { DashboardLoadAction, dashboardLogic } from 'scenes/dashboard/dashboardLogic' import { BREAKPOINTS, @@ -146,7 +147,6 @@ export function DashboardItems({ showCreateAnomalyAlertButton }: DashboardItemsP moveTileToSection, } = useActions(dashboardLogic) const { updateWidgetTile } = useAsyncActions(dashboardLogic) - const { reportDashboardTileRepositioned } = useActions(eventUsageLogic) const { push } = useActions(router) const { data: surveyLinkedInsights, loading: surveyLinkedInsightsLoading } = useSurveyLinkedInsights({ // Dashboard pages hide the survey suggestion in compact headers; notebook embeds also use this component and can show it. diff --git a/frontend/src/scenes/dashboard/DashboardZoomControl.tsx b/frontend/src/scenes/dashboard/DashboardZoomControl.tsx index 5edd66d5fb80..32793310373f 100644 --- a/frontend/src/scenes/dashboard/DashboardZoomControl.tsx +++ b/frontend/src/scenes/dashboard/DashboardZoomControl.tsx @@ -4,9 +4,9 @@ import posthog from 'posthog-js' import { LemonButton } from '@posthog/lemon-ui' import { Shortcut } from 'lib/components/Shortcuts/Shortcut' -import { sanitizeDashboard } from 'lib/utils/eventUsageLogic' import { Scene } from 'scenes/sceneTypes' +import { sanitizeDashboard } from './dashboardEvents' import { dashboardLogic } from './dashboardLogic' interface DashboardZoomControlProps { diff --git a/frontend/src/scenes/dashboard/EmptyDashboardComponent.tsx b/frontend/src/scenes/dashboard/EmptyDashboardComponent.tsx index 97100cec8b65..560acf4e7c7e 100644 --- a/frontend/src/scenes/dashboard/EmptyDashboardComponent.tsx +++ b/frontend/src/scenes/dashboard/EmptyDashboardComponent.tsx @@ -9,7 +9,6 @@ import { AccessControlAction } from 'lib/components/AccessControlAction' import { ProductIntroduction } from 'lib/components/ProductIntroduction/ProductIntroduction' import { LemonButton } from 'lib/lemon-ui/LemonButton' import { LemonMenu } from 'lib/lemon-ui/LemonMenu' -import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { maxGlobalLogic } from 'scenes/max/maxGlobalLogic' import { urls } from 'scenes/urls' @@ -18,6 +17,7 @@ import { AccessControlLevel, AccessControlResourceType, DashboardType, SidePanel import { addInsightToDashboardLogic } from './addInsightToDashboardModalLogic' import { DashboardAiPromptComposer } from './DashboardAiPromptComposer' +import { reportDashboardEmptyAddChartClicked, reportDashboardEmptyWebAnalyticsClicked } from './dashboardEvents' import { DASHBOARD_CANNOT_EDIT_MESSAGE } from './DashboardHeader' import { getAddTileMenuItems } from './DashboardHeaderActions' import { dashboardLogic } from './dashboardLogic' @@ -47,7 +47,6 @@ function DashboardEmptyActions({ onAddWidget: () => void onOpenAiWithPrompt: (prompt: string) => void }): JSX.Element { - const { reportDashboardEmptyAddChartClicked, reportDashboardEmptyWebAnalyticsClicked } = useActions(eventUsageLogic) const chipDisabledReason = !canEdit ? DASHBOARD_CANNOT_EDIT_MESSAGE : aiDisabledReason || undefined const handleAddInsight = (): void => { reportDashboardEmptyAddChartClicked(dashboard?.id) diff --git a/frontend/src/scenes/dashboard/QuillDashboardAddTileButton.tsx b/frontend/src/scenes/dashboard/QuillDashboardAddTileButton.tsx index 41741cb50086..283034ca718f 100644 --- a/frontend/src/scenes/dashboard/QuillDashboardAddTileButton.tsx +++ b/frontend/src/scenes/dashboard/QuillDashboardAddTileButton.tsx @@ -13,13 +13,13 @@ import { } from '@posthog/quill' import { getAccessControlDisabledReason } from 'lib/utils/accessControlUtils' -import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { MaxTool } from 'scenes/max/MaxTool' import { iconForType } from '~/layout/panel-layout/ProjectTree/defaultTree' import { AccessControlLevel, AccessControlResourceType } from '~/types' import { DashboardAddTileKey, getDashboardAddTileOptions } from './dashboardAddTileOptions' +import { reportDashboardAddMenuOpened } from './dashboardEvents' import { DashboardLoadAction, dashboardLogic } from './dashboardLogic' // pinned: autocapture data-attrs for the header's Add menu items @@ -41,7 +41,6 @@ export function QuillDashboardAddTileButton(): JSX.Element | null { openImageTileModal, openButtonTileModal, } = useActions(dashboardLogic) - const { reportDashboardAddMenuOpened } = useActions(eventUsageLogic) if (!dashboard || tiles.length === 0) { return null diff --git a/frontend/src/scenes/dashboard/dashboardEvents.test.ts b/frontend/src/scenes/dashboard/dashboardEvents.test.ts new file mode 100644 index 000000000000..cf67e2dcb8be --- /dev/null +++ b/frontend/src/scenes/dashboard/dashboardEvents.test.ts @@ -0,0 +1,108 @@ +import { NodeKind } from '~/queries/schema/schema-general' +import type { DashboardTile, DashboardType } from '~/types' + +import { dashboardViewedProperties, sanitizeDashboard } from './dashboardEvents' + +describe('dashboardEvents', () => { + it('keeps metric variable definitions out of a sanitized dashboard', () => { + const metricVariables = [{ name: 'env', source: 'sql', query: 'SELECT private_value' }] + const tile = { id: 1, insight: { id: 2, filter_override_context: { dashboard: { metricVariables } } } } + const dashboard = { id: 7, tiles: [tile], filters: { metricVariables }, persisted_filters: { metricVariables } } + + const sanitized = sanitizeDashboard(dashboard as unknown as DashboardType) + + expect(JSON.stringify(sanitized)).not.toContain('private_value') + }) + + describe('dashboardViewedProperties', () => { + const dashboard = (tiles: Partial[]): DashboardType => + ({ + id: 7, + created_at: '2026-01-01T00:00:00Z', + is_shared: false, + pinned: true, + creation_mode: 'default', + created_by: { uuid: 'creator' }, + tiles, + }) as unknown as DashboardType + + const insightTile = (query: unknown, is_sample = false): Partial => + ({ insight: { query, is_sample } }) as unknown as Partial + + const cases: [string, Partial[], Record][] = [ + ['no tiles', [], { item_count: 0, sample_items_count: 0 }], + [ + 'an insight tile with a query', + [insightTile({ kind: NodeKind.InsightVizNode, source: { kind: NodeKind.TrendsQuery, series: [] } })], + { item_count: 1, text_count: 1, uses_data_warehouse_source: false, data_warehouse_tiles_count: 0 }, + ], + ['an insight tile without a query', [insightTile(null)], { item_count: 1, empty_count: 1 }], + [ + 'a sample insight tile that reads the warehouse', + [insightTile({ kind: NodeKind.TrendsQuery, series: [{ kind: NodeKind.DataWarehouseNode }] }, true)], + { + item_count: 1, + text_count: 1, + sample_items_count: 1, + uses_data_warehouse_source: true, + data_warehouse_tiles_count: 1, + }, + ], + [ + 'text and widget tiles', + [{ text: { body: 'hi' } }, { text: { body: 'there' } }, { widget: {} }] as Partial[], + { item_count: 3, text_tiles_count: 2, widget_tiles_count: 1 }, + ], + [ + 'groups once however many tiles share a key', + [ + { group_key: 'a' }, + { group_key: 'a' }, + { group_key: 'b' }, + { group_key: null }, + ] as Partial[], + { item_count: 4, text_tiles_count: 4, group_count: 2 }, + ], + ] + + it.each(cases)('counts %s', (_, tiles, expected) => { + expect(dashboardViewedProperties(dashboard(tiles), null, 'viewer')).toEqual({ + created_at: '2026-01-01T00:00:00Z', + is_shared: false, + pinned: true, + creation_mode: 'default', + viewer_is_creator: false, + created_by_system: false, + dashboard_id: 7, + group_display: 'sections', + group_count: 0, + lastRefreshed: undefined, + refreshAge: undefined, + sample_items_count: 0, + uses_data_warehouse_source: false, + data_warehouse_tiles_count: 0, + ...expected, + }) + }) + + it('reports the dashboard group display', () => { + const withTabs = { ...dashboard([]), customization: { group_display: 'tabs' } } as unknown as DashboardType + expect(dashboardViewedProperties(withTabs, null, 'viewer')).toMatchObject({ group_display: 'tabs' }) + }) + + const viewerCases: [string, { uuid: string } | null, string | undefined, boolean | undefined][] = [ + ['the creator views it', { uuid: 'creator' }, 'creator', true], + ['another user views it', { uuid: 'creator' }, 'viewer', false], + ['the viewer is unknown', { uuid: 'creator' }, undefined, undefined], + ['the system created it', null, 'viewer', undefined], + ] + + it.each(viewerCases)('reports viewer_is_creator when %s', (_, created_by, viewerUuid, expected) => { + const withCreator = { ...dashboard([]), created_by } as unknown as DashboardType + expect(dashboardViewedProperties(withCreator, null, viewerUuid)).toMatchObject({ + viewer_is_creator: expected, + created_by_system: created_by === null, + }) + }) + }) +}) diff --git a/frontend/src/scenes/dashboard/dashboardEvents.ts b/frontend/src/scenes/dashboard/dashboardEvents.ts new file mode 100644 index 000000000000..2c2bf51e438b --- /dev/null +++ b/frontend/src/scenes/dashboard/dashboardEvents.ts @@ -0,0 +1,230 @@ +import posthog from 'posthog-js' + +import type { Dayjs } from 'lib/dayjs' +import { now } from 'lib/dayjs' +import { DashboardEventSource, filtersForAnalytics, sanitizeInsight } from 'lib/utils/eventUsageLogic' + +import { isNodeWithSource, queryUsesDataWarehouse } from '~/queries/utils' +import { DashboardMode, DashboardTile, DashboardTileSpacing, DashboardType, TileGroupDisplay } from '~/types' + +export type DashboardFilterChangeType = + | 'date' + | 'properties' + | 'breakdown' + | 'variable' + | 'interval' + | 'test_accounts' + | 'compare' + | 'metric_variables' + +function sanitizeTile(tile: DashboardTile | null): object | undefined { + if (!tile) { + return undefined + } + + return { + ...tile, + insight: tile.insight ? sanitizeInsight(tile.insight) : undefined, + } +} + +export function sanitizeDashboard(dashboard: DashboardType | null): object | null { + if (!dashboard) { + return null + } + + // Remove breakdown color entries, as they carry raw breakdown values (user data) — report only their count + const { breakdown_colors, ...sanitizedDashboard } = dashboard + + return { + ...sanitizedDashboard, + breakdown_colors_count: breakdown_colors?.length ?? 0, + filters: filtersForAnalytics(dashboard.filters ?? {}), + persisted_filters: dashboard.persisted_filters ? filtersForAnalytics(dashboard.persisted_filters) : null, + tiles: dashboard.tiles?.map((tile) => sanitizeTile(tile)) || [], + } +} + +function increment(counts: Record, key: string): void { + counts[key] = (counts[key] || 0) + 1 +} + +function insightTileCountKey(tile: DashboardTile): string { + const query = isNodeWithSource(tile.insight?.query) ? tile.insight.query.source : tile.insight?.query + return `${query?.kind || !!tile.text ? 'text' : 'empty'}_count` +} + +function countDashboardTiles(tiles: DashboardTile[], properties: Record): void { + for (const tile of tiles) { + if (tile.insight) { + increment(properties, insightTileCountKey(tile)) + properties.sample_items_count += tile.insight.is_sample ? 1 : 0 + if (queryUsesDataWarehouse(tile.insight.query)) { + properties.uses_data_warehouse_source = true + properties.data_warehouse_tiles_count += 1 + } + } else if (tile.widget) { + increment(properties, 'widget_tiles_count') + } else { + increment(properties, 'text_tiles_count') + } + } +} + +export function tileGroupProperties( + groupDisplay: TileGroupDisplay | undefined, + tiles: DashboardTile[] +): { group_display: TileGroupDisplay; group_count: number } { + return { + group_display: groupDisplay ?? 'sections', + group_count: new Set(tiles.map((tile) => tile.group_key).filter(Boolean)).size, + } +} + +export function dashboardViewedProperties( + dashboard: DashboardType, + lastRefreshed: Dayjs | null, + viewerUuid: string | undefined +): Record { + const { created_at, is_shared, pinned, creation_mode, id } = dashboard + const properties: Record = { + created_at, + is_shared, + pinned, + creation_mode, + viewer_is_creator: + dashboard.created_by?.uuid && viewerUuid ? dashboard.created_by.uuid === viewerUuid : undefined, + sample_items_count: 0, + item_count: dashboard.tiles?.length || 0, + created_by_system: !dashboard.created_by, + dashboard_id: id, + ...tileGroupProperties(dashboard.customization?.group_display, dashboard.tiles || []), + lastRefreshed: lastRefreshed?.toISOString(), + refreshAge: lastRefreshed ? now().diff(lastRefreshed, 'seconds') : undefined, + uses_data_warehouse_source: false, + data_warehouse_tiles_count: 0, + } + countDashboardTiles(dashboard.tiles || [], properties) + return properties +} + +export function reportDashboardModeToggled( + dashboard: DashboardType | null, + mode: DashboardMode | null, + source: DashboardEventSource | null, + layoutZoom: number | null, + layoutEditMode?: boolean +): void { + posthog.capture('dashboard mode toggled', { + dashboard_id: dashboard?.id, + dashboard: sanitizeDashboard(dashboard), + mode, + source, + layout_zoom: layoutZoom ?? undefined, + layout_edit_mode: layoutEditMode ?? undefined, + }) +} + +export function reportDashboardLayoutEditModeEntered( + dashboard: DashboardType | null, + source: DashboardEventSource, + layoutZoom: number | null +): void { + posthog.capture('dashboard layout edit mode entered', { + dashboard_id: dashboard?.id, + dashboard: sanitizeDashboard(dashboard), + source, + layout_zoom: layoutZoom ?? undefined, + }) +} + +export function reportDashboardFiltersChanged( + dashboard: DashboardType | null, + changeType: DashboardFilterChangeType, + properties: Record +): void { + posthog.capture('dashboard filters changed', { + dashboard_id: dashboard?.id, + change_type: changeType, + ...properties, + }) +} + +export function reportDashboardTileDensityConfigured(tileDensity: DashboardTileSpacing): void { + posthog.capture('dashboard tile density configured', { tile_density: tileDensity }) +} + +export function reportDashboardEditModeDiscardPrompt( + dashboard: DashboardType | null, + action: 'shown' | 'discarded' | 'kept_editing' +): void { + posthog.capture('dashboard edit mode discard prompt', { + dashboard_id: dashboard?.id, + dashboard: sanitizeDashboard(dashboard), + action, + }) +} + +export function reportDashboardMoveInitiated(method: 'single' | 'bulk', count: number): void { + posthog.capture('dashboard move initiated', { method, count }) +} + +export function reportDashboardInsightDeleteAfterRemovalClicked(otherDashboardCount: number): void { + posthog.capture('dashboard insight delete after removal clicked', { + other_dashboard_count: otherDashboardCount, + }) +} + +export function reportDashboardTileRepositioned( + dashboardId: number, + action: 'moved' | 'resized', + layoutZoom: number +): void { + posthog.capture('dashboard tile repositioned', { + dashboard_id: dashboardId, + action, + layout_zoom: layoutZoom, + }) +} + +export function reportDashboardInsightMetaUpdated( + dashboardId: number | undefined, + insightId: number, + attribute: 'name' | 'description' +): void { + posthog.capture('dashboard insight meta updated', { + dashboard_id: dashboardId, + insight_id: insightId, + attribute, + }) +} + +/** Empty-state AI prompt chips (ai-first empty dashboard only). */ +export function reportDashboardEmptyAiPromptSubmitted( + dashboardId: number | undefined, + promptType: 'custom_prompt' | 'starter_question' +): void { + posthog.capture('dashboard empty ai prompt submitted', { + dashboard_id: dashboardId, + prompt_type: promptType, + source: 'web', + }) +} + +export function reportDashboardEmptyAddChartClicked(dashboardId: number | undefined): void { + posthog.capture('dashboard empty add chart clicked', { + dashboard_id: dashboardId, + source: 'web', + }) +} + +export function reportDashboardEmptyWebAnalyticsClicked(dashboardId: number | undefined): void { + posthog.capture('dashboard empty web analytics clicked', { + dashboard_id: dashboardId, + source: 'web', + }) +} + +export function reportDashboardAddMenuOpened(source: 'header', dashboardId: number): void { + posthog.capture('dashboard add menu opened', { source, dashboard_id: dashboardId }) +} diff --git a/frontend/src/scenes/dashboard/dashboardLogic.test.ts b/frontend/src/scenes/dashboard/dashboardLogic.test.ts index 90b8e7eb59c3..9742f3b7c474 100644 --- a/frontend/src/scenes/dashboard/dashboardLogic.test.ts +++ b/frontend/src/scenes/dashboard/dashboardLogic.test.ts @@ -1604,10 +1604,7 @@ describe('dashboardLogic', () => { it('previews tile spacing immediately and persists only the final choice', async () => { await expectLogic(logic).toFinishAllListeners() ;(api.update as jest.Mock).mockClear() - const reportTileDensityConfigured = jest.spyOn( - eventUsageLogic.actions, - 'reportDashboardTileDensityConfigured' - ) + const capture = jest.spyOn(posthog, 'capture').mockClear() jest.useFakeTimers() try { @@ -1627,7 +1624,7 @@ describe('dashboardLogic', () => { layout_compaction: DashboardGridCompaction.Vertical, grid_spacing: 'relaxed', }) - expect(reportTileDensityConfigured).toHaveBeenCalledWith('relaxed') + expect(capture).toHaveBeenCalledWith('dashboard tile density configured', { tile_density: 'relaxed' }) } finally { jest.useRealTimers() } @@ -1635,10 +1632,7 @@ describe('dashboardLogic', () => { it('reports the tile density when reset to standard', async () => { await expectLogic(logic).toFinishAllListeners() - const reportTileDensityConfigured = jest.spyOn( - eventUsageLogic.actions, - 'reportDashboardTileDensityConfigured' - ) + const capture = jest.spyOn(posthog, 'capture').mockClear() jest.useFakeTimers() try { @@ -1647,7 +1641,7 @@ describe('dashboardLogic', () => { await jest.advanceTimersByTimeAsync(750) await expectLogic(logic).toFinishAllListeners() - expect(reportTileDensityConfigured).toHaveBeenCalledWith('standard') + expect(capture).toHaveBeenCalledWith('dashboard tile density configured', { tile_density: 'standard' }) } finally { jest.useRealTimers() } @@ -2992,12 +2986,8 @@ describe('dashboardLogic', () => { }) it('reports filter changes separately from layout edit mode entry', async () => { - const reportFiltersChanged = jest.spyOn(eventUsageLogic.actions, 'reportDashboardFiltersChanged') - const reportLayoutEditEntered = jest.spyOn( - eventUsageLogic.actions, - 'reportDashboardLayoutEditModeEntered' - ) - const reportModeToggled = jest.spyOn(eventUsageLogic.actions, 'reportDashboardModeToggled') + const capture = jest.spyOn(posthog, 'capture').mockClear() + const capturedEvents = (): string[] => capture.mock.calls.map(([event]) => event) await expectLogic(logic).toFinishAllListeners() @@ -3008,22 +2998,19 @@ describe('dashboardLogic', () => { ) }).toFinishAllListeners() - expect(reportLayoutEditEntered).not.toHaveBeenCalled() - expect(reportModeToggled).not.toHaveBeenCalled() + expect(capturedEvents()).not.toContain('dashboard layout edit mode entered') + expect(capturedEvents()).not.toContain('dashboard mode toggled') await expectLogic(logic, () => { logic.actions.setDates('-7d', null) }).toFinishAllListeners() - expect(reportFiltersChanged).toHaveBeenCalledWith( - expect.objectContaining({ id: 5 }), - 'date', - expect.objectContaining({ date_from: '-7d', date_to: null }) + expect(capture).toHaveBeenCalledWith( + 'dashboard filters changed', + expect.objectContaining({ dashboard_id: 5, change_type: 'date', date_from: '-7d', date_to: null }) ) - reportFiltersChanged.mockClear() - reportLayoutEditEntered.mockClear() - reportModeToggled.mockClear() + capture.mockClear() await expectLogic(logic, () => { logic.actions.setDashboardEditing( @@ -3032,17 +3019,18 @@ describe('dashboardLogic', () => { ) }).toFinishAllListeners() - expect(reportLayoutEditEntered).toHaveBeenCalledWith( - expect.objectContaining({ id: 5 }), - DashboardEventSource.SceneCommonButtons, - 1 + expect(capture).toHaveBeenCalledWith( + 'dashboard layout edit mode entered', + expect.objectContaining({ + dashboard_id: 5, + source: DashboardEventSource.SceneCommonButtons, + layout_zoom: 1, + }) ) - expect(reportModeToggled).not.toHaveBeenCalled() - expect(reportFiltersChanged).not.toHaveBeenCalled() + expect(capturedEvents()).not.toContain('dashboard mode toggled') + expect(capturedEvents()).not.toContain('dashboard filters changed') - reportFiltersChanged.mockRestore() - reportLayoutEditEntered.mockRestore() - reportModeToggled.mockRestore() + capture.mockRestore() }) it('filter edits preserve layout editing', async () => { @@ -5936,10 +5924,7 @@ describe('dashboardLogic', () => { it('offers to delete a removed insight and explains its other dashboard usage', async () => { const { fireEvent, render } = await import('@testing-library/react') - const reportDeleteClicked = jest.spyOn( - eventUsageLogic.actions, - 'reportDashboardInsightDeleteAfterRemovalClicked' - ) + const capture = jest.spyOn(posthog, 'capture').mockClear() const insightTile = logic.values.insightTiles[0] const insight = insightTile.insight! @@ -5959,7 +5944,9 @@ describe('dashboardLogic', () => { const toastContent = lemonToastInfoSpy.mock.calls.at(-1)?.[0] const { getByText } = render(toastContent) fireEvent.click(getByText('Delete insight everywhere')) - expect(reportDeleteClicked).toHaveBeenCalledWith(1) + expect(capture).toHaveBeenCalledWith('dashboard insight delete after removal clicked', { + other_dashboard_count: 1, + }) expect(lemonToastDismissSpy).not.toHaveBeenCalled() const dialogProps = lemonDialogOpenSpy.mock.calls.at(-1)?.[0] @@ -5977,7 +5964,7 @@ describe('dashboardLogic', () => { getDialogText('This deletes the insight and removes it from every dashboard. You can undo this action.') ).not.toBeNull() - reportDeleteClicked.mockRestore() + capture.mockRestore() }) }) diff --git a/frontend/src/scenes/dashboard/dashboardLogic.tsx b/frontend/src/scenes/dashboard/dashboardLogic.tsx index 2c652990b09e..1818e541e9af 100644 --- a/frontend/src/scenes/dashboard/dashboardLogic.tsx +++ b/frontend/src/scenes/dashboard/dashboardLogic.tsx @@ -46,20 +46,23 @@ import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { accessLevelSatisfied } from 'lib/utils/accessControlUtils' import { deleteInsightWithUndo } from 'lib/utils/deleteWithUndo' import { clearDOMTextSelection, getJSHeapMemory, uuid } from 'lib/utils/dom' +import { DashboardEventSource, eventUsageLogic, filtersForAnalytics, sanitizeQuery } from 'lib/utils/eventUsageLogic' +import { objectsEqual } from 'lib/utils/objects' +import { shouldCancelQuery } from 'lib/utils/requests' +import { toParams } from 'lib/utils/url' +import { addInsightToDashboardLogic } from 'scenes/dashboard/addInsightToDashboardModalLogic' import { - DashboardEventSource, DashboardFilterChangeType, dashboardViewedProperties, - eventUsageLogic, + reportDashboardEditModeDiscardPrompt, + reportDashboardFiltersChanged, + reportDashboardInsightDeleteAfterRemovalClicked, + reportDashboardLayoutEditModeEntered, + reportDashboardModeToggled, + reportDashboardTileDensityConfigured, sanitizeDashboard, - filtersForAnalytics, - sanitizeQuery, tileGroupProperties, -} from 'lib/utils/eventUsageLogic' -import { objectsEqual } from 'lib/utils/objects' -import { shouldCancelQuery } from 'lib/utils/requests' -import { toParams } from 'lib/utils/url' -import { addInsightToDashboardLogic } from 'scenes/dashboard/addInsightToDashboardModalLogic' +} from 'scenes/dashboard/dashboardEvents' import { BREAKPOINTS, dashboardToSaveableTemplate, getDashboardTileDisplayName } from 'scenes/dashboard/dashboardUtils' import { liveDashboardIds } from 'scenes/dashboard/liveDashboardIds' import { calculateDuplicateLayout, calculateLayouts } from 'scenes/dashboard/tileLayouts' @@ -4985,9 +4988,7 @@ export const dashboardLogic = kea([ data-attr="delete-removed-insight" className="!m-0" onClick={() => { - eventUsageLogic.actions.reportDashboardInsightDeleteAfterRemovalClicked( - otherDashboardIds.size - ) + reportDashboardInsightDeleteAfterRemovalClicked(otherDashboardIds.size) deleteInsight() }} > @@ -5126,7 +5127,7 @@ export const dashboardLogic = kea([ dashboardsModel.actions.updateDashboardSuccess(dashboard, { layoutsPersisted: true, }) - eventUsageLogic.actions.reportDashboardTileDensityConfigured(tileSpacing) + reportDashboardTileDensityConfigured(tileSpacing) } catch { if (!cache.pendingDashboardTileSpacing) { actions.setDashboardTileSpacing(persistedTileSpacing) @@ -5923,7 +5924,7 @@ export const dashboardLogic = kea([ discard() return } - eventUsageLogic.actions.reportDashboardEditModeDiscardPrompt(values.dashboard, 'shown') + reportDashboardEditModeDiscardPrompt(values.dashboard, 'shown') LemonDialog.open({ title: 'Discard unsaved changes?', description: @@ -5932,7 +5933,7 @@ export const dashboardLogic = kea([ children: 'Discard changes', status: 'danger', onClick: () => { - eventUsageLogic.actions.reportDashboardEditModeDiscardPrompt(values.dashboard, 'discarded') + reportDashboardEditModeDiscardPrompt(values.dashboard, 'discarded') discard() }, 'data-attr': 'dashboard-edit-mode-discard-confirm', @@ -5940,7 +5941,7 @@ export const dashboardLogic = kea([ secondaryButton: { children: 'Keep editing', onClick: () => { - eventUsageLogic.actions.reportDashboardEditModeDiscardPrompt(values.dashboard, 'kept_editing') + reportDashboardEditModeDiscardPrompt(values.dashboard, 'kept_editing') }, }, }) @@ -6017,7 +6018,7 @@ export const dashboardLogic = kea([ } if (editing?.layout) { - eventUsageLogic.actions.reportDashboardLayoutEditModeEntered( + reportDashboardLayoutEditModeEntered( values.dashboard, source, values.layoutEditMode ? values.layoutZoom : null @@ -6026,7 +6027,7 @@ export const dashboardLogic = kea([ }, setDashboardMode: async ({ mode, source }) => { if (mode || source) { - eventUsageLogic.actions.reportDashboardModeToggled( + reportDashboardModeToggled( values.dashboard, mode, source, @@ -6183,7 +6184,7 @@ export const dashboardLogic = kea([ remainingInsightIds: new Set(values.insightTiles.map((tile) => tile.insight!.short_id)), } cache.pendingFilterChange = filterChange - eventUsageLogic.actions.reportDashboardFiltersChanged(values.dashboard, changeType, { + reportDashboardFiltersChanged(values.dashboard, changeType, { ...properties, filter_change_id: filterChange.id, supersedes_filter_change_id: previousChange?.id, diff --git a/frontend/src/scenes/dashboard/dashboards/dashboardsLogic.ts b/frontend/src/scenes/dashboard/dashboards/dashboardsLogic.ts index bf9925ed692a..72d0b53253e8 100644 --- a/frontend/src/scenes/dashboard/dashboards/dashboardsLogic.ts +++ b/frontend/src/scenes/dashboard/dashboards/dashboardsLogic.ts @@ -8,8 +8,8 @@ import { Sorting } from 'lib/lemon-ui/LemonTable/sorting' import { featureFlagLogic } from 'lib/logic/featureFlagLogic' import { buildUserScopedPersistenceConfig } from 'lib/logic/persistence' import { trackedActionToUrl } from 'lib/logic/scenes/trackedActionToUrl' -import { eventUsageLogic } from 'lib/utils/eventUsageLogic' import { objectClean, objectsEqual } from 'lib/utils/objects' +import { reportDashboardMoveInitiated } from 'scenes/dashboard/dashboardEvents' import { teamLogic } from 'scenes/teamLogic' import { urls } from 'scenes/urls' import { userLogic } from 'scenes/userLogic' @@ -160,13 +160,6 @@ export interface dashboardsLogicActions { onStillSelected: (ids: number[]) => void } } // dashboardsModel - reportDashboardMoveInitiated: ( - method: 'bulk' | 'single', - count: number - ) => { - count: number - method: 'bulk' | 'single' - } // eventUsageLogic openMoveToModal: (items: FileSystemEntry[]) => { items: FileSystemEntry[] } // moveToLogic @@ -316,8 +309,6 @@ export const dashboardsLogic = kea([ ['bulkDeleteDashboards', 'bulkDeleteDashboardsSuccess', 'reloadDashboardsSuccess'], moveToLogic, ['openMoveToModal'], - eventUsageLogic, - ['reportDashboardMoveInitiated'], projectTreeDataLogic, ['movedItem'], ], @@ -775,7 +766,7 @@ export const dashboardsLogic = kea([ if (moving.length === 0) { return } - actions.reportDashboardMoveInitiated(method, moving.length) + reportDashboardMoveInitiated(method, moving.length) actions.openMoveToModal(moving) cache.awaitingMove = { ids: new Set(ids), onStillSelected } }, diff --git a/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.test.ts b/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.test.ts index d3a87620eeec..2af8eef2b0cf 100644 --- a/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.test.ts +++ b/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.test.ts @@ -1,6 +1,17 @@ import { CachedNewExperimentQueryResponse } from '~/queries/schema/schema-general' -import { calculateCurrentExposures, calculateDaysElapsed, calculateExposureRate } from './calculations' +import type { RunningTimeOutlookApi } from 'products/experiments/frontend/generated/api.schemas' + +import { + buildRunningTimeOutlook, + calculateCurrentExposures, + calculateDaysElapsed, + DetectableEffectCheckpoint, + describeCheckpointDay, + describeUseButton, + formatDetectableEffect, + RunningTimeOutlook, +} from './calculations' // The sample-size / variance math is verified in the backend // (products/experiments/backend/test/test_running_time_calculator.py). These tests @@ -20,20 +31,6 @@ describe('running time calculations', () => { }) }) - describe('calculateExposureRate', () => { - it('returns exposures per day', () => { - expect(calculateExposureRate(1000, 10)).toBe(100) - }) - - it('returns null below the minimum elapsed window', () => { - expect(calculateExposureRate(1000, 0.05)).toBeNull() - }) - - it('returns null without exposures', () => { - expect(calculateExposureRate(null, 10)).toBeNull() - }) - }) - describe('calculateDaysElapsed', () => { it('returns null without a start date', () => { expect(calculateDaysElapsed(null)).toBeNull() @@ -45,4 +42,174 @@ describe('running time calculations', () => { expect(elapsed!).toBeGreaterThan(0) }) }) + + describe('buildRunningTimeOutlook', () => { + // Two weeks in at about 143 exposures a day, as the backend projects it. + const PROJECTION: RunningTimeOutlookApi = { + days_since_launch: 14, + exposures_per_day: 2000 / 14, + detectable_effect_now: 126.5, + upcoming_checkpoints: [ + { weeks_since_launch: 4, detectable_effect: 89.4 }, + { weeks_since_launch: 6, detectable_effect: 73 }, + { weeks_since_launch: 8, detectable_effect: 63.2 }, + ], + reachable_effect: 73, + } + const outlook = ( + overrides: Partial[0]> + ): RunningTimeOutlook | null => + buildRunningTimeOutlook({ + projection: PROJECTION, + currentSampleSize: 2000, + savedEffect: 30, + savedSampleSize: 35556, + defaultEffect: 30, + enteredEffect: null, + selectedKey: null, + minimumDetectableEffect: 30, + ...overrides, + }) + const rowsOf = (result: RunningTimeOutlook): string[] => + result.checkpoints.map(({ kind, isTarget }) => (isTarget ? `${kind} (target)` : kind)) + + it('lays out today, the week marks and the custom row at the saved effect', () => { + const result = outlook({})! + + expect( + result.checkpoints.map(({ kind, days, exposures, detectableEffect, isTarget }) => [ + kind, + Math.round(days), + exposures, + detectableEffect, + isTarget, + ]) + ).toEqual([ + ['now', 14, 2000, 126.5, false], + ['week', 28, 4000, 89.4, false], + ['week', 42, 6000, 73, false], + ['week', 56, 8000, 63.2, false], + ['custom', 249, 35556, 30, true], + ]) + expect(result.daysToTarget).toBeCloseTo(248.9, 1) + }) + + it.each([ + [ + '"Use" chooses a week mark', + { selectedKey: 'week-6' }, + ['now', 'week', 'week (target)', 'week', 'custom'], + 30, + 42, + ], + ['"Use" chooses today', { selectedKey: 'now' }, ['now (target)', 'week', 'week', 'week', 'custom'], 30, 14], + [ + 'the slider moves the custom row', + { enteredEffect: 50 }, + ['now', 'week', 'week', 'week', 'custom (target)'], + 50, + 89.6, + ], + ])('keeps the same rows when %s', (_name, overrides, rows, customEffect, daysToTarget) => { + const result = outlook(overrides)! + + expect(rowsOf(result)).toEqual(rows) + expect(result.checkpoints.find(({ kind }) => kind === 'custom')?.detectableEffect).toEqual(customEffect) + expect(result.daysToTarget).toBeCloseTo(daysToTarget, 1) + }) + + it('reopens on the row whose effect was saved, with the custom row at the default effect', () => { + const result = outlook({ savedEffect: 73, savedSampleSize: 5990, minimumDetectableEffect: 73 })! + + expect(rowsOf(result)).toEqual(['now', 'week', 'week (target)', 'week', 'custom']) + expect(result.checkpoints.find(({ kind }) => kind === 'custom')?.detectableEffect).toEqual(30) + }) + + it.each([ + [80, false], + [73, false], + [30, true], + ])('puts a %s%% target out of reach: %s', (minimumDetectableEffect, isOutOfReach) => { + expect(outlook({ minimumDetectableEffect })!.isOutOfReach).toBe(isOutOfReach) + }) + + it.each([ + ['no projection', { projection: null }], + ['no saved target', { savedSampleSize: null }], + ['no exposures so far', { currentSampleSize: null }], + ])('gives no outlook without %s', (_name, overrides) => { + expect(outlook(overrides)).toBeNull() + }) + }) + + describe('describeCheckpointDay', () => { + const checkpoint = (overrides: Partial): DetectableEffectCheckpoint => ({ + key: 'custom', + kind: 'custom', + days: 18, + exposures: 3341, + detectableEffect: 30, + isTarget: false, + ...overrides, + }) + + beforeEach(() => { + jest.useFakeTimers().setSystemTime(new Date('2026-10-27T12:00:00Z')) + }) + + afterEach(() => { + jest.useRealTimers() + }) + + it.each([ + ['a week mark', { kind: 'week' as const, days: 14 }, 'week 2'], + ['today', { kind: 'now' as const, days: 11.17, exposures: 2000 }, 'day 12'], + ['the custom row before it is reached', {}, 'in 7 days'], + ['the custom row reached earlier today', { days: 11, exposures: 1970 }, 'day 12'], + ])('describes %s', (_name, overrides, description) => { + expect( + describeCheckpointDay(checkpoint(overrides), { + startDate: '2026-10-16T08:00:00Z', + currentSampleSize: 2000, + }) + ).toEqual(description) + }) + }) + + describe('describeUseButton', () => { + it.each([ + [ + 'the effect of a week mark as the table shows it', + { kind: 'week' as const, detectableEffect: 32.74 }, + { effect: 32.8, tooltip: 'Set 32.8% as the minimum detectable effect' }, + ], + [ + 'the custom effect as it is', + { kind: 'custom' as const, detectableEffect: 12.345 }, + { effect: 12.345, tooltip: 'Set 12.345% as the minimum detectable effect' }, + ], + [ + 'only why an effect over 100% cannot be set', + { kind: 'now' as const, detectableEffect: 126.5 }, + { effect: 127, disabledReason: 'The minimum detectable effect can be at most 100%' }, + ], + ])('offers %s', (_name, overrides, button) => { + expect(describeUseButton({ key: 'row', days: 14, exposures: 2000, isTarget: false, ...overrides })).toEqual( + button + ) + }) + }) + + describe('formatDetectableEffect', () => { + it.each([ + [63.21, '63.3%'], + [30.000000001, '30%'], + [8.714, '8.72%'], + [100, '100%'], + [100.01, '>100%'], + [126.49, '>100%'], + ])('formats the detectable change %s as %s', (detectableEffect, formatted) => { + expect(formatDetectableEffect(detectableEffect)).toEqual(formatted) + }) + }) }) diff --git a/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.ts b/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.ts index 829a5d07bed4..7d4804fff640 100644 --- a/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.ts +++ b/frontend/src/scenes/experiments/RunningTimeCalculator/calculations.ts @@ -2,9 +2,12 @@ import { dayjs } from 'lib/dayjs' import type { CachedNewExperimentQueryResponse } from '~/queries/schema/schema-general' -// The target sample lives in the backend (products/experiments/backend/running_time_target.py), reached via -// GET /experiments/:id/running_time_target, and the manual preview's math via POST -// /experiments/calculate_running_time. This module keeps only the client-side helpers that read live experiment state. +import type { RunningTimeOutlookApi } from 'products/experiments/frontend/generated/api.schemas' + +// The target sample and its outlook live in the backend (products/experiments/backend/running_time_target.py), reached +// via GET /experiments/:id/running_time_target, and the manual preview's math via POST +// /experiments/calculate_running_time. This module keeps the helpers that read live experiment state, rescale the target +// for an effect being edited, and place that target on the backend's outlook. // Manual calculator only supports these types (ratio/retention require full baseline data). export type ManualCalculatorMetricType = 'funnel' | 'mean_count' | 'mean_sum_or_avg' @@ -28,13 +31,9 @@ export function calculateDaysElapsed(startDate: string | null): number | null { return dayjs().diff(dayjs(startDate), 'days', true) // fractional days } -export function calculateExposureRate(currentExposures: number | null, daysElapsed: number | null): number | null { - if (!currentExposures || !daysElapsed || daysElapsed < 0.1) { - return null - } - - return currentExposures / daysElapsed -} +// The minimum detectable effects, as a percentage, that the running time settings accept. +export const MINIMUM_DETECTABLE_EFFECT_MIN = 1 +export const MINIMUM_DETECTABLE_EFFECT_MAX = 100 /** * The target sample for another minimum detectable effect, by the rule the target endpoint documents: the variances come @@ -48,3 +47,182 @@ export function rescaleSampleSize(sampleSize: number, fromEffect: number, toEffe // Round before the ceiling, as the backend does, so that float noise doesn't turn an exact 3,200 into 3,201. return Math.ceil(Number((sampleSize * (fromEffect / toEffect) ** 2).toFixed(6))) } + +export interface DetectableEffectCheckpoint { + key: string + kind: 'now' | 'week' | 'custom' + // Since launch. + days: number + // So far for today, projected for the week marks, and the target sample for the custom row. + exposures: number + // The smallest relative change, as a percentage, that the exposures by then can detect. + detectableEffect: number + isTarget: boolean +} + +export interface RunningTimeOutlook { + // Since launch, to the row the user chose as the target. + daysToTarget: number + isOutOfReach: boolean + // Today, the backend's next week marks and the custom row, by date. + checkpoints: DetectableEffectCheckpoint[] + // The smallest change detectable within four weeks from today, which the header shows for a target out of reach. + reachableEffect: number +} + +/** + * Rounded up to three significant figures, because the exposures can't detect a smaller change than the exact value. + * Set as the minimum detectable effect, the rounded value needs at most 2% fewer exposures than the row it came from. + */ +export function roundUpDetectableEffect(detectableEffect: number): number { + if (!(detectableEffect > 0)) { + return detectableEffect + } + const precision = 10 ** (2 - Math.floor(Math.log10(detectableEffect))) + // Round before the ceiling, so that float noise doesn't turn an exact 30 into 30.1. + return Math.ceil(Number((detectableEffect * precision).toFixed(6))) / precision +} + +/** + * The backend's projection of what the experiment can detect over time, as a fixed set of rows: today, the next week + * marks, and a custom row for the effect that the slider sets. The slider moves the custom row, and "Use" chooses a row + * as the target, so no interaction adds or removes a row. Every row sits where the projection reaches its exposures. + */ +export function buildRunningTimeOutlook({ + projection, + currentSampleSize, + savedEffect, + savedSampleSize, + defaultEffect, + enteredEffect, + selectedKey, + minimumDetectableEffect, +}: { + projection: RunningTimeOutlookApi | null + currentSampleSize: number | null + // The effect the backend sized its target for, and that target. + savedEffect: number | null + savedSampleSize: number | null + defaultEffect: number + // The last effect from the slider or the input, null until the user changes it. + enteredEffect: number | null + // The row the user chose with "Use", null when the custom row is the target. + selectedKey: string | null + minimumDetectableEffect: number +}): RunningTimeOutlook | null { + if (!projection || currentSampleSize === null || !savedEffect || !savedSampleSize) { + return null + } + const rate = projection.exposures_per_day + const now: DetectableEffectCheckpoint = { + key: 'now', + kind: 'now', + days: projection.days_since_launch, + exposures: currentSampleSize, + detectableEffect: projection.detectable_effect_now, + isTarget: false, + } + const weeks = projection.upcoming_checkpoints.map( + (checkpoint): DetectableEffectCheckpoint => ({ + key: `week-${checkpoint.weeks_since_launch}`, + kind: 'week', + days: checkpoint.weeks_since_launch * 7, + exposures: Math.round(rate * checkpoint.weeks_since_launch * 7), + detectableEffect: checkpoint.detectable_effect, + isTarget: false, + }) + ) + // A saved effect that today or a week mark shows came from that row's "Use", so that row is the target. The custom + // row then starts at the default effect rather than repeat the row. + const savedRow = [now, ...weeks].find( + (checkpoint) => roundUpDetectableEffect(checkpoint.detectableEffect) === roundUpDetectableEffect(savedEffect) + ) + const customEffect = enteredEffect ?? (savedRow ? defaultEffect : savedEffect) + const customSampleSize = rescaleSampleSize(savedSampleSize, savedEffect, customEffect) + if (!customSampleSize) { + return null + } + const custom: DetectableEffectCheckpoint = { + key: 'custom', + kind: 'custom', + days: customSampleSize / rate, + exposures: customSampleSize, + detectableEffect: customEffect, + isTarget: false, + } + const rows = [now, ...weeks, custom] + const targetKey = selectedKey ?? (enteredEffect === null && savedRow ? savedRow.key : custom.key) + const target = rows.find((checkpoint) => checkpoint.key === targetKey) ?? custom + + return { + daysToTarget: target.days, + isOutOfReach: minimumDetectableEffect < projection.reachable_effect, + checkpoints: rows + .map((checkpoint) => (checkpoint === target ? { ...checkpoint, isTarget: true } : checkpoint)) + .sort((a, b) => a.days - b.days), + reachableEffect: projection.reachable_effect, + } +} + +/** + * Calendar days from today to a day since launch, at least one, so that the header and the table count alike. + */ +export function daysFromToday(startDate: string, daysSinceLaunch: number): number { + const date = dayjs(startDate).add(daysSinceLaunch * 24, 'hour') + return Math.max(1, date.startOf('day').diff(dayjs().startOf('day'), 'day')) +} + +export function describeCheckpointDay( + checkpoint: DetectableEffectCheckpoint, + { startDate, currentSampleSize }: { startDate: string; currentSampleSize: number } +): string { + if (checkpoint.kind === 'week') { + return `week ${checkpoint.days / 7}` + } + if (checkpoint.kind === 'custom' && checkpoint.exposures > currentSampleSize) { + const days = daysFromToday(startDate, checkpoint.days) + return `in ${days.toLocaleString()} ${days === 1 ? 'day' : 'days'}` + } + // Calendar days, like the dates beside them, so that rows on one date show one day and the launch date is day 1. + const launch = dayjs(startDate) + const day = + launch + .add(checkpoint.days * 24, 'hour') + .startOf('day') + .diff(launch.startOf('day'), 'day') + 1 + return `day ${day.toLocaleString()}` +} + +export function describeUseButton(checkpoint: DetectableEffectCheckpoint): { + effect: number + disabledReason?: string + tooltip?: string +} { + const effect = + checkpoint.kind === 'custom' + ? checkpoint.detectableEffect + : roundUpDetectableEffect(checkpoint.detectableEffect) + if (effect > MINIMUM_DETECTABLE_EFFECT_MAX) { + return { + effect, + disabledReason: `The minimum detectable effect can be at most ${MINIMUM_DETECTABLE_EFFECT_MAX}%`, + } + } + if (effect < MINIMUM_DETECTABLE_EFFECT_MIN) { + return { + effect, + disabledReason: `The minimum detectable effect must be at least ${MINIMUM_DETECTABLE_EFFECT_MIN}%`, + } + } + return { effect, tooltip: `Set ${effect.toLocaleString()}% as the minimum detectable effect` } +} + +/** + * Past the largest effect the settings accept, the exact value is too large to act on, so it shows as ">100%". + */ +export function formatDetectableEffect(detectableEffect: number): string { + const effect = roundUpDetectableEffect(detectableEffect) + return effect > MINIMUM_DETECTABLE_EFFECT_MAX + ? `>${MINIMUM_DETECTABLE_EFFECT_MAX}%` + : `${effect.toLocaleString(undefined, { maximumSignificantDigits: 3 })}%` +} diff --git a/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.test.ts b/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.test.ts index bddda9c4b730..d6a099830ca5 100644 --- a/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.test.ts +++ b/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.test.ts @@ -1,6 +1,7 @@ import { api } from 'lib/api.mock' import { expectLogic } from 'kea-test-utils' +import posthog from 'posthog-js' import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' @@ -11,6 +12,7 @@ import { Experiment } from '~/types' import { experimentLogic } from '../experimentLogic' import { experimentMetricsLogic } from '../experimentMetricsLogic' import { modalsLogic } from '../modalsLogic' +import { roundUpDetectableEffect } from './calculations' import { runningTimeLogic } from './runningTimeLogic' jest.mock('lib/lemon-ui/LemonToast/LemonToast', () => ({ @@ -61,6 +63,15 @@ const metricResultWithBaseline = { variant_results: [{ key: 'test', number_of_samples: 600, sum: 130, sum_squares: 52 }], } +// The backend's projection for the result above: 1,200 exposures over ten days. +const PROJECTION = { + days_since_launch: 10, + exposures_per_day: 120, + detectable_effect_now: 38.7, + upcoming_checkpoints: [{ weeks_since_launch: 2, detectable_effect: 32.7 }], + reachable_effect: 25.4, +} + // The latest completed run covers the one primary metric, so the metrics logic applies the baseline on mount. const latestRecalculation = { id: 'recalc-1', @@ -123,6 +134,7 @@ describe('runningTimeLogic', () => { runningTimeTargetMock.mockResolvedValue({ recommended_sample_size: 2000, minimum_detectable_effect: 30, + outlook: PROJECTION, }) api.update.mockResolvedValue({ ...experiment, @@ -172,6 +184,7 @@ describe('runningTimeLogic', () => { runningTimeTargetMock.mockResolvedValue({ recommended_sample_size: 2000, minimum_detectable_effect: 30, + outlook: PROJECTION, }) api.update.mockRejectedValue({ status: 409, @@ -202,7 +215,7 @@ describe('runningTimeLogic', () => { logic = runningTimeLogic({ experiment }) logic.mount() - await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationSuccess']).toFinishAllListeners() + await expectLogic(logic).toDispatchActions(['loadAutomaticTargetSuccess']).toFinishAllListeners() runningTimeTargetMock.mockClear() logic.actions.setConfig({ mde: 15 }) @@ -223,18 +236,75 @@ describe('runningTimeLogic', () => { logic = runningTimeLogic({ experiment }) logic.mount() - await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationSuccess']).toFinishAllListeners() + await expectLogic(logic).toDispatchActions(['loadAutomaticTargetSuccess']).toFinishAllListeners() expect(logic.values.currentExposures).toEqual(1100) }) + it('counts the days left on the projection, also before 100 exposures and with results from before today', async () => { + // 90 exposures in results that end on day 9 make 10 a day, so the 180 of the target arrive on day 18. + runningTimeTargetMock.mockResolvedValue({ + recommended_sample_size: 180, + minimum_detectable_effect: 30, + current_sample_size: 90, + outlook: { ...PROJECTION, days_since_launch: 10, exposures_per_day: 10 }, + }) + api.update.mockResolvedValue({ ...experiment, version: 4 }) + + logic = runningTimeLogic({ experiment }) + logic.mount() + await expectLogic(logic).toDispatchActions(['loadAutomaticTargetSuccess']).toFinishAllListeners() + + expect(logic.values.remainingDays).toEqual(8) + }) + + it('keeps the same rows while the user takes a row with "Use" and moves the slider', async () => { + // At 120 exposures a day, the 1,996 that 30% needs arrive after week 2, which can detect 32.7%. + runningTimeTargetMock.mockResolvedValue({ + recommended_sample_size: 1996, + minimum_detectable_effect: 30, + current_sample_size: 1200, + outlook: PROJECTION, + }) + api.update.mockResolvedValue({ ...experiment, version: 4 }) + logic = runningTimeLogic({ experiment }) + logic.mount() + await expectLogic(logic).toDispatchActions(['loadAutomaticTargetSuccess']).toFinishAllListeners() + const rows = (): [number, boolean][] | undefined => + logic.values.outlook?.checkpoints.map(({ detectableEffect, isTarget }) => [detectableEffect, isTarget]) + + expect(rows()).toEqual([ + [38.7, false], + [32.7, false], + [30, true], + ]) + + logic.actions.selectCheckpoint('week-2', roundUpDetectableEffect(32.7)) + + expect(rows()).toEqual([ + [38.7, false], + [32.7, true], + [30, false], + ]) + expect(logic.values.remainingDays).toEqual(4) + expect(logic.values.config.mde).toEqual(32.7) + + logic.actions.setConfig({ mde: 36 }) + + expect(rows()).toEqual([ + [38.7, false], + [36, true], + [32.7, false], + ]) + }) + it('asks again when the metric sums change without new exposures', async () => { runningTimeTargetMock.mockResolvedValue({ recommended_sample_size: 2000, minimum_detectable_effect: 30 }) api.update.mockResolvedValue({ ...experiment, version: 4 }) logic = runningTimeLogic({ experiment }) logic.mount() - await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationSuccess']).toFinishAllListeners() + await expectLogic(logic).toDispatchActions(['loadAutomaticTargetSuccess']).toFinishAllListeners() metricsLogicInstance.actions.setPrimaryMetricsResults([ { @@ -242,7 +312,7 @@ describe('runningTimeLogic', () => { baseline: { ...metricResultWithBaseline.baseline, sum: 150, sum_squares: 90 }, } as any, ]) - await expectLogic(logic).toDispatchActions(['loadAutomaticCalculation']).toFinishAllListeners() + await expectLogic(logic).toDispatchActions(['loadAutomaticTarget']).toFinishAllListeners() expect(runningTimeTargetMock).toHaveBeenCalledTimes(2) }) @@ -253,6 +323,7 @@ describe('runningTimeLogic', () => { runningTimeTargetMock.mockResolvedValueOnce({ recommended_sample_size: 2000, minimum_detectable_effect: 30, + outlook: PROJECTION, }) api.update.mockImplementation(async (_url: string, payload: unknown) => ({ ...experiment, @@ -260,19 +331,22 @@ describe('runningTimeLogic', () => { version: 4, })) + const captureSpy = jest.spyOn(posthog, 'capture') logic = runningTimeLogic({ experiment }) logic.mount() - await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationSuccess']).toFinishAllListeners() + await expectLogic(logic).toDispatchActions(['loadAutomaticTargetSuccess']).toFinishAllListeners() + const savedMde = logic.values.config.mde modalsLogic.actions.openRunningTimeConfigModal() logic.actions.setConfig({ mde: 15 }) runningTimeTargetMock.mockResolvedValueOnce({ recommended_sample_size: 8000, minimum_detectable_effect: 15, + outlook: PROJECTION, }) api.update.mockClear() await expectLogic(logic, () => logic.actions.save()) - .toDispatchActions(['loadAutomaticCalculationSuccess']) + .toDispatchActions(['loadAutomaticTargetSuccess']) .toFinishAllListeners() expect(api.update).toHaveBeenLastCalledWith( @@ -284,6 +358,11 @@ describe('runningTimeLogic', () => { }), }) ) + expect(captureSpy).toHaveBeenCalledWith( + 'experiment running time settings saved', + expect.objectContaining({ minimum_detectable_effect: 15, previous_minimum_detectable_effect: savedMde }) + ) + captureSpy.mockRestore() }) it('keeps the modal open with the edited config when the experiment save fails', async () => { @@ -308,7 +387,14 @@ describe('runningTimeLogic', () => { }) describe('isCalculating', () => { - it('stays true while the automatic estimate is in flight, then settles on success', async () => { + it.each([ + ['an automatic experiment', {}, null], + [ + 'a manual experiment switched to automatic', + { exposure_estimate_config: { conversionRateInputType: 'manual' } }, + { mode: 'automatic' as const }, + ], + ])('stays true while the estimate of %s is in flight, then settles', async (_name, saved, edit) => { let resolveCalc: (value: unknown) => void = () => {} runningTimeTargetMock.mockReturnValue( new Promise((resolve) => { @@ -316,15 +402,22 @@ describe('runningTimeLogic', () => { }) ) api.update.mockResolvedValue({ ...experiment, version: 4 }) + experimentLogicInstance.actions.setExperiment({ + ...experiment, + running_time_calculation: saved as Experiment['running_time_calculation'], + }) logic = runningTimeLogic({ experiment }) logic.mount() + if (edit) { + logic.actions.setConfig(edit) + } - await expectLogic(logic).toDispatchActions(['loadAutomaticCalculation']) + await expectLogic(logic).toDispatchActions(['loadAutomaticTarget']) expect(logic.values.isCalculating).toBe(true) resolveCalc({ recommended_sample_size: 2000, minimum_detectable_effect: 30 }) - await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationSuccess']).toFinishAllListeners() + await expectLogic(logic).toDispatchActions(['loadAutomaticTargetSuccess']).toFinishAllListeners() expect(logic.values.isCalculating).toBe(false) }) @@ -336,7 +429,7 @@ describe('runningTimeLogic', () => { logic = runningTimeLogic({ experiment }) logic.mount() - await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationFailure']).toFinishAllListeners() + await expectLogic(logic).toDispatchActions(['loadAutomaticTargetFailure']).toFinishAllListeners() expect(logic.values.isCalculating).toBe(false) }) @@ -344,12 +437,13 @@ describe('runningTimeLogic', () => { runningTimeTargetMock.mockResolvedValue({ recommended_sample_size: 2000, minimum_detectable_effect: 30, + outlook: PROJECTION, }) logic = runningTimeLogic({ experiment }) logic.mount() // The baseline fires the automatic estimate on mount; let it settle so only the recalculation state moves. - await expectLogic(logic).toDispatchActions(['loadAutomaticCalculationSuccess']).toFinishAllListeners() + await expectLogic(logic).toDispatchActions(['loadAutomaticTargetSuccess']).toFinishAllListeners() expect(logic.values.isCalculating).toBe(false) metricsLogicInstance.actions.setRecalculationLoading(true) diff --git a/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.ts b/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.ts index 1c032901e282..7389cbb6bc7f 100644 --- a/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.ts +++ b/frontend/src/scenes/experiments/RunningTimeCalculator/runningTimeLogic.ts @@ -1,6 +1,7 @@ import { MakeLogicType, actions, connect, kea, key, listeners, path, props, reducers, selectors } from 'kea' import { loaders } from 'kea-loaders' import { subscriptions } from 'kea-subscriptions' +import posthog from 'posthog-js' import api from 'lib/api' import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' @@ -9,7 +10,7 @@ import { objectsEqual } from 'lib/utils/objects' import { experimentsConfigLogic } from '~/scenes/settings/environment/experimentsConfigLogic' import { ConversionRateInputType, Experiment } from '~/types' -import { isLaunched } from 'products/experiments/frontend/experimentStatus' +import { isExperimentExposureFrozen, isLaunched } from 'products/experiments/frontend/experimentStatus' import { experimentsCalculateRunningTimeCreate, experimentsRunningTimeTargetRetrieve, @@ -22,15 +23,18 @@ import type { import type { CachedNewExperimentQueryResponse } from '../../../queries/schema/schema-general' import { experimentLogic, saveExperimentUpdate } from '../experimentLogic' import { experimentMetricsLogic } from '../experimentMetricsLogic' +import { getDefaultMetricTitle } from '../MetricsView/shared/utils' import { modalsLogic } from '../modalsLogic' import { getFlagVariants, metricResults, isExperimentConflictError, toConcurrencyPayload } from '../utils' import type { MetricWithResult } from '../utils' import { ManualCalculatorMetricType, calculateCurrentExposures, + buildRunningTimeOutlook, calculateDaysElapsed, - calculateExposureRate, + daysFromToday, rescaleSampleSize, + RunningTimeOutlook, } from './calculations' export interface RunningTimeLogicProps { @@ -50,6 +54,14 @@ export interface ManualPreview { runningTime: number | null } +// The metric the automatic estimate is sized from. +export interface EstimateMetric { + name: string + // The experiment has more than one primary metric, and the estimate reads the first. + isFirstOfSeveral: boolean + hasOtherMetrics: boolean +} + // The backend sizes the target for the saved minimum detectable effect. The settings rescale it for an edited one, so an // edit sends no request. export interface AutomaticTargetRequest { @@ -68,17 +80,22 @@ export interface runningTimeLogicValues { primaryMetricsResultsErrors: (unknown | null)[] // experimentMetricsLogic defaultMinimumDetectableEffect: number // experimentsConfigLogic isRunningTimeConfigModalOpen: boolean // modalsLogic - automaticCalculation: RunningTimeTargetApi | null - automaticCalculationInput: AutomaticTargetRequest | null - automaticCalculationLoading: boolean + automaticTarget: RunningTimeTargetApi | null + automaticTargetFailed: boolean + automaticTargetLoading: boolean + automaticTargetRequest: AutomaticTargetRequest | null + canIncludeMoreUsers: boolean config: RunningTimeConfig configOverrides: Partial | null currentExposures: number | null dailyExposureRate: number | null editedMde: number | null + enteredMde: number | null + estimateMetric: EstimateMetric | null initialConfig: RunningTimeConfig isCalculating: boolean isComplete: boolean + isManualEstimate: boolean isManualMode: boolean isSaving: boolean manualFormPreview: ManualPreview @@ -89,7 +106,9 @@ export interface runningTimeLogicValues { metricsWithResults: MetricWithResult[] mode: RunningTimeConfig['mode'] numberOfVariants: number + outlook: RunningTimeOutlook | null remainingDays: number | null + selectedCheckpointKey: string | null targetEffect: number targetSampleSize: number | null } @@ -105,22 +124,25 @@ export interface runningTimeLogicActions { closeRunningTimeConfigModal: () => { value: true } // modalsLogic + openRunningTimeConfigModal: () => { + value: true + } // modalsLogic cancel: () => { value: true } - loadAutomaticCalculation: (_: AutomaticTargetRequest) => AutomaticTargetRequest - loadAutomaticCalculationFailure: ( + loadAutomaticTarget: (_: AutomaticTargetRequest) => AutomaticTargetRequest + loadAutomaticTargetFailure: ( error: string, errorObject?: any ) => { error: string errorObject?: any } - loadAutomaticCalculationSuccess: ( - automaticCalculation: RunningTimeTargetApi | null, + loadAutomaticTargetSuccess: ( + automaticTarget: RunningTimeTargetApi | null, payload?: AutomaticTargetRequest ) => { - automaticCalculation: RunningTimeTargetApi | null + automaticTarget: RunningTimeTargetApi | null payload?: AutomaticTargetRequest } loadManualPreview: (input: RunningTimeCalculationInputApi) => RunningTimeCalculationInputApi @@ -153,6 +175,13 @@ export interface runningTimeLogicActions { save: () => { value: true } + selectCheckpoint: ( + key: string, + effect: number + ) => { + effect: number + key: string + } setConfig: (config: Partial) => { config: Partial } @@ -178,23 +207,20 @@ export interface runningTimeLogicMeta { ) => RunningTimeConfig mde: (config: RunningTimeConfig) => number editedMde: (configOverrides: Partial | null) => number | null - targetEffect: ( - editedMde: number | null, - automaticCalculation: RunningTimeTargetApi | null, - mde: number - ) => number + targetEffect: (editedMde: number | null, automaticTarget: RunningTimeTargetApi | null, mde: number) => number mode: (config: RunningTimeConfig) => RunningTimeConfig['mode'] metricsWithResults: ( experiment: Experiment, primaryMetricsResults: CachedNewExperimentQueryResponse[], primaryMetricsResultsErrors: unknown[] ) => MetricWithResult[] + isManualEstimate: (isManualMode: boolean, mode: 'automatic' | 'manual') => boolean currentExposures: ( - isManualMode: boolean, + isManualEstimate: boolean, metricsWithResults: MetricWithResult[], - automaticCalculation: RunningTimeTargetApi | null + automaticTarget: RunningTimeTargetApi | null ) => number | null - automaticCalculationInput: ( + automaticTargetRequest: ( mode: 'automatic' | 'manual', metricsWithResults: MetricWithResult[] ) => AutomaticTargetRequest | null @@ -203,29 +229,37 @@ export interface runningTimeLogicMeta { numberOfVariants: number ) => RunningTimeCalculationInputApi | null targetSampleSize: ( - isManualMode: boolean, + isManualEstimate: boolean, targetEffect: number, experiment: Experiment, - automaticCalculationInput: AutomaticTargetRequest | null, - automaticCalculation: RunningTimeTargetApi | null + automaticTargetRequest: AutomaticTargetRequest | null, + automaticTarget: RunningTimeTargetApi | null ) => number | null dailyExposureRate: ( - isManualMode: boolean, + isManualEstimate: boolean, experiment: Experiment, - currentExposures: number | null + automaticTarget: RunningTimeTargetApi | null ) => number | null remainingDays: ( targetSampleSize: number | null, currentExposures: number | null, dailyExposureRate: number | null, - experiment: Experiment + experiment: Experiment, + isManualEstimate: boolean, + outlook: RunningTimeOutlook | null ) => number | null + outlook: ( + isManualEstimate: boolean, + automaticTarget: RunningTimeTargetApi | null, + targetEffect: number, + defaultMinimumDetectableEffect: number, + enteredMde: number | null, + selectedCheckpointKey: string | null + ) => RunningTimeOutlook | null + canIncludeMoreUsers: (experiment: Experiment) => boolean + estimateMetric: (experiment: Experiment, metricsWithResults: MetricWithResult[]) => EstimateMetric | null isComplete: (currentExposures: number | null, targetSampleSize: number | null) => boolean - isCalculating: ( - isManualMode: boolean, - isRecalculating: boolean, - automaticCalculationLoading: boolean - ) => boolean + isCalculating: (isManualEstimate: boolean, isRecalculating: boolean, automaticTargetLoading: boolean) => boolean manualFormPreview: ( manualPreviewInput: RunningTimeCalculationInputApi | null, manualPreview: ManualPreview @@ -262,13 +296,14 @@ export const runningTimeLogic = kea([ experimentLogic({ experimentId }), ['setExperiment', 'setUnmodifiedExperiment'], modalsLogic, - ['closeRunningTimeConfigModal'], + ['openRunningTimeConfigModal', 'closeRunningTimeConfigModal'], ], } }), actions({ setConfig: (config: Partial) => ({ config }), + selectCheckpoint: (key: string, effect: number) => ({ key, effect }), resetConfig: true, save: true, cancel: true, @@ -281,6 +316,23 @@ export const runningTimeLogic = kea([ null as Partial | null, { setConfig: (state, { config }) => ({ ...state, ...config }), + selectCheckpoint: (state, { effect }) => ({ ...state, mde: effect }), + resetConfig: () => null, + }, + ], + enteredMde: [ + null as number | null, + { + setConfig: (state, { config }) => (config.mde === undefined ? state : config.mde), + resetConfig: () => null, + }, + ], + // The row chosen with "Use". The slider and the input make the custom row the target again. + selectedCheckpointKey: [ + null as string | null, + { + selectCheckpoint: (_, { key }) => key, + setConfig: (state, { config }) => (config.mde === undefined ? state : null), resetConfig: () => null, }, ], @@ -290,14 +342,22 @@ export const runningTimeLogic = kea([ setSaving: (_, { saving }) => saving, }, ], + automaticTargetFailed: [ + false, + { + loadAutomaticTarget: () => false, + loadAutomaticTargetSuccess: () => false, + loadAutomaticTargetFailure: () => true, + }, + ], }), loaders(({ values, props }) => ({ // Automatic mode: the target the backend sizes from the primary metric's stored result and the flag's split. - automaticCalculation: [ + automaticTarget: [ null as RunningTimeTargetApi | null, { - loadAutomaticCalculation: async (_: AutomaticTargetRequest, breakpoint) => { + loadAutomaticTarget: async (_: AutomaticTargetRequest, breakpoint) => { const experimentId = props.experiment.id if (typeof experimentId !== 'number') { return null @@ -380,12 +440,12 @@ export const runningTimeLogic = kea([ // The effect the automatic target is sized for: an edited one, else the one the backend sized for. The backend's // can differ from `mde` before the team default loads, or when the saved effect isn't a positive number. targetEffect: [ - (s) => [s.editedMde, s.automaticCalculation, s.mde], - (editedMde: number | null, automaticCalculation: RunningTimeTargetApi | null, mde: number): number => - editedMde ?? automaticCalculation?.minimum_detectable_effect ?? mde, + (s) => [s.editedMde, s.automaticTarget, s.mde], + (editedMde: number | null, automaticTarget: RunningTimeTargetApi | null, mde: number): number => + editedMde ?? automaticTarget?.minimum_detectable_effect ?? mde, ], - // A primitive, so that automaticCalculationInput stays stable when the experiment object is replaced (e.g. - // after persisting an estimate) but the mode is unchanged. + // A primitive, so that automaticTargetRequest stays stable when the experiment object is replaced (e.g. after + // persisting an estimate) but the mode is unchanged. mode: [(s) => [s.config], (config: RunningTimeConfig): RunningTimeConfig['mode'] => config.mode], // Primary metrics zipped with their results, so automatic mode always has a baseline to read. @@ -398,10 +458,17 @@ export const runningTimeLogic = kea([ ) => (experiment ? metricResults(experiment)(results, errors, 'primary') : []), ], + // Switching a manual experiment to automatic in the settings previews the automatic estimate, so the manual one + // applies only while both the saved and the edited mode are manual. + isManualEstimate: [ + (s) => [s.isManualMode, s.mode], + (isManualMode: boolean, mode: RunningTimeConfig['mode']): boolean => isManualMode && mode === 'manual', + ], + currentExposures: [ - (s) => [s.isManualMode, s.metricsWithResults, s.automaticCalculation], + (s) => [s.isManualEstimate, s.metricsWithResults, s.automaticTarget], ( - isManualMode: boolean, + isManualEstimate: boolean, results: { displayIndex: number error: any @@ -409,18 +476,18 @@ export const runningTimeLogic = kea([ metricIndex: number result: any }[], - automaticCalculation: RunningTimeTargetApi | null + automaticTarget: RunningTimeTargetApi | null ): number | null => { // The backend counts the exposures the automatic target sizes, so a variant at 0% does not count. - if (!isManualMode && automaticCalculation?.current_sample_size != null) { - return automaticCalculation.current_sample_size + if (!isManualEstimate && automaticTarget?.current_sample_size != null) { + return automaticTarget.current_sample_size } return calculateCurrentExposures(results?.[0]?.result ?? null) }, ], // Request for the automatic-mode target, or null when not applicable (manual mode / no results yet). - automaticCalculationInput: [ + automaticTargetRequest: [ (s) => [s.mode, s.metricsWithResults], ( mode: RunningTimeConfig['mode'], @@ -472,59 +539,82 @@ export const runningTimeLogic = kea([ ], targetSampleSize: [ - (s) => [s.isManualMode, s.targetEffect, s.experiment, s.automaticCalculationInput, s.automaticCalculation], + (s) => [s.isManualEstimate, s.targetEffect, s.experiment, s.automaticTargetRequest, s.automaticTarget], ( - isManualMode: boolean, + isManualEstimate: boolean, targetEffect: number, experiment: Experiment, automaticInput: AutomaticTargetRequest | null, - automaticCalculation: RunningTimeTargetApi | null + automaticTarget: RunningTimeTargetApi | null ): number | null => { - if (isManualMode) { + if (isManualEstimate) { // Persisted by the save listener from the same backend calculation. return experiment?.running_time_calculation?.recommended_sample_size ?? null } - if (!automaticInput || !automaticCalculation?.recommended_sample_size) { + if (!automaticInput || !automaticTarget?.recommended_sample_size) { return null } return rescaleSampleSize( - automaticCalculation.recommended_sample_size, - automaticCalculation.minimum_detectable_effect, + automaticTarget.recommended_sample_size, + automaticTarget.minimum_detectable_effect, targetEffect ) }, ], dailyExposureRate: [ - (s) => [s.isManualMode, s.experiment, s.currentExposures], - (isManualMode: boolean, experiment: Experiment, currentExposures: number | null): number | null => { - if (isManualMode) { + (s) => [s.isManualEstimate, s.experiment, s.automaticTarget], + ( + isManualEstimate: boolean, + experiment: Experiment, + automaticTarget: RunningTimeTargetApi | null + ): number | null => { + if (isManualEstimate) { return experiment?.running_time_calculation?.exposure_estimate_config?.manualExposureRate ?? null } - - // Automatic mode: calculate from actual rate - const daysElapsed = calculateDaysElapsed(experiment?.start_date ?? null) - return calculateExposureRate(currentExposures, daysElapsed) + return automaticTarget?.outlook?.exposures_per_day ?? null }, ], // Days until we reach target sample size remainingDays: [ - (s) => [s.targetSampleSize, s.currentExposures, s.dailyExposureRate, s.experiment], + (s) => [ + s.targetSampleSize, + s.currentExposures, + s.dailyExposureRate, + s.experiment, + s.isManualEstimate, + s.outlook, + ], ( target: number | null, current: number | null, rate: number | null, - experiment: Experiment + experiment: Experiment, + isManualEstimate: boolean, + outlook: RunningTimeOutlook | null ): number | null => { if (!target || target <= 0 || !rate || rate <= 0) { return null } + // The automatic rate comes from the backend's outlook, which projects from the first 0.1 days on. + if (!isManualEstimate) { + if (current !== null && current >= target) { + return 0 + } + if (!experiment?.start_date) { + return Math.ceil(target / rate) + } + // The projection reaches the target `target / rate` days after launch, or at the checkpoint whose + // effect the target took. Calendar days to that date keep the header, the table's date and its + // "in N days" in step, also when the results end before today. + return daysFromToday(experiment.start_date, outlook?.daysToTarget ?? target / rate) + } - // Need minimum data for automatic mode + // The manual estimate counts down from the exposures so far once there are a day and 100 of them. const daysElapsed = calculateDaysElapsed(experiment?.start_date ?? null) if (!daysElapsed || daysElapsed < 1 || !current || current < 100) { - // Not enough data yet - show total estimated time if we have rate + // Before that, the whole planned run. if (rate > 0) { return Math.ceil(target / rate) } @@ -539,6 +629,65 @@ export const runningTimeLogic = kea([ return Math.ceil(remaining / rate) }, ], + // What the experiment can detect over time, for the automatic estimate. + outlook: [ + (s) => [ + s.isManualEstimate, + s.automaticTarget, + s.targetEffect, + s.defaultMinimumDetectableEffect, + s.enteredMde, + s.selectedCheckpointKey, + ], + ( + isManualEstimate: boolean, + automaticTarget: RunningTimeTargetApi | null, + targetEffect: number, + defaultMinimumDetectableEffect: number, + enteredMde: number | null, + selectedCheckpointKey: string | null + ): RunningTimeOutlook | null => + isManualEstimate + ? null + : buildRunningTimeOutlook({ + projection: automaticTarget?.outlook ?? null, + currentSampleSize: automaticTarget?.current_sample_size ?? null, + savedEffect: automaticTarget?.minimum_detectable_effect ?? null, + savedSampleSize: automaticTarget?.recommended_sample_size ?? null, + defaultEffect: defaultMinimumDetectableEffect, + enteredEffect: enteredMde, + selectedKey: selectedCheckpointKey, + minimumDetectableEffect: targetEffect, + }), + ], + canIncludeMoreUsers: [ + (s) => [s.experiment], + (experiment: Experiment): boolean => { + // A group without conditions at 100% already includes every user. + const includesEveryone = (experiment?.feature_flag?.filters?.groups ?? []).some( + (group) => !group.properties?.length && (group.rollout_percentage ?? 100) >= 100 + ) + return !includesEveryone && !isExperimentExposureFrozen(experiment) + }, + ], + // The first primary metric in display order, which is the one the backend sizes the target from. + estimateMetric: [ + (s) => [s.experiment, s.metricsWithResults], + (experiment: Experiment, metricsWithResults: MetricWithResult[]): EstimateMetric | null => { + const metric = metricsWithResults?.[0]?.metric + if (!metric) { + return null + } + const secondaryCount = + (experiment?.metrics_secondary?.length ?? 0) + + (experiment?.saved_metrics ?? []).filter((saved) => saved.metadata?.type === 'secondary').length + return { + name: metric.name || getDefaultMetricTitle(metric), + isFirstOfSeveral: metricsWithResults.length > 1, + hasOtherMetrics: metricsWithResults.length > 1 || secondaryCount > 0, + } + }, + ], isComplete: [ (s) => [s.currentExposures, s.targetSampleSize], (current: number | null, target: number | null): boolean => @@ -549,12 +698,12 @@ export const runningTimeLogic = kea([ // instead of flashing a pending state during the gap between the two. Manual mode has no async // estimate to wait on, so it never counts as calculating. isCalculating: [ - (s) => [s.isManualMode, s.isRecalculating, s.automaticCalculationLoading], - (isManualMode: boolean, isRecalculating: boolean, automaticCalculationLoading: boolean): boolean => { - if (isManualMode) { + (s) => [s.isManualEstimate, s.isRecalculating, s.automaticTargetLoading], + (isManualEstimate: boolean, isRecalculating: boolean, automaticTargetLoading: boolean): boolean => { + if (isManualEstimate) { return false } - return isRecalculating || automaticCalculationLoading + return isRecalculating || automaticTargetLoading }, ], manualFormPreview: [ @@ -567,10 +716,10 @@ export const runningTimeLogic = kea([ subscriptions(({ actions }) => ({ // kea-subscriptions fires on reference change. The recalc poll replaces the metric-results array every // tick, so these inputs get a fresh object reference even when their values are identical, which would - // re-fire the calculation POST on every poll. Skip the dispatch unless the value actually changed. - automaticCalculationInput: (input: AutomaticTargetRequest | null, oldInput: AutomaticTargetRequest | null) => { + // re-fire the request on every poll. Skip the dispatch unless the value actually changed. + automaticTargetRequest: (input: AutomaticTargetRequest | null, oldInput: AutomaticTargetRequest | null) => { if (input && !objectsEqual(input, oldInput)) { - actions.loadAutomaticCalculation(input) + actions.loadAutomaticTarget(input) } }, manualPreviewInput: ( @@ -585,7 +734,7 @@ export const runningTimeLogic = kea([ listeners(({ actions, values, props }) => ({ // Persist the automatic estimate once the backend calculation lands and the selectors are fresh. - loadAutomaticCalculationSuccess: () => { + loadAutomaticTargetSuccess: () => { actions.persistRunningTimeEstimate() }, @@ -659,6 +808,8 @@ export const runningTimeLogic = kea([ save: async () => { const { config, numberOfVariants, experiment, currentProjectId } = values + // Read before the save, which replaces the experiment and with it the initial config. + const previousMde = values.initialConfig.mde actions.setSaving(true) let saved = false @@ -729,15 +880,33 @@ export const runningTimeLogic = kea([ if (!saved) { return } + // pinned: analytics event name and properties. + posthog.capture('experiment running time settings saved', { + experiment_id: props.experiment.id, + mode: config.mode, + minimum_detectable_effect: config.mde, + previous_minimum_detectable_effect: previousMde, + }) actions.closeRunningTimeConfigModal() actions.resetConfig() // The target on hand is sized for the previous effect. The backend sizes it for the saved one, and its // success persists the estimate. - if (config.mode === 'automatic' && values.automaticCalculationInput) { - actions.loadAutomaticCalculation(values.automaticCalculationInput) + if (config.mode === 'automatic' && values.automaticTargetRequest) { + actions.loadAutomaticTarget(values.automaticTargetRequest) } }, + openRunningTimeConfigModal: () => { + const { config, outlook } = values + // pinned: analytics event name and properties. + posthog.capture('experiment running time settings opened', { + experiment_id: props.experiment.id, + mode: config.mode, + out_of_reach: outlook?.isOutOfReach ?? null, + days_to_target: outlook ? Math.round(outlook.daysToTarget) : null, + }) + }, + cancel: () => { actions.closeRunningTimeConfigModal() actions.resetConfig() diff --git a/frontend/src/scenes/experiments/stories/ExperimentRunningTimeOutlook.stories.tsx b/frontend/src/scenes/experiments/stories/ExperimentRunningTimeOutlook.stories.tsx new file mode 100644 index 000000000000..43d93bb430e5 --- /dev/null +++ b/frontend/src/scenes/experiments/stories/ExperimentRunningTimeOutlook.stories.tsx @@ -0,0 +1,126 @@ +import { Meta, StoryObj } from '@storybook/react' +import { waitFor } from '@testing-library/dom' +import userEvent from '@testing-library/user-event' + +import { App } from 'scenes/App' +import { urls } from 'scenes/urls' + +import { mswDecorator } from '~/mocks/browser' +import EXPERIMENT_WITH_MEAN_METRIC from '~/mocks/fixtures/api/experiments/experiment_with_mean_metric.json' +import EXPOSURE_QUERY_RESULT from '~/mocks/fixtures/api/experiments/exposure_query_result.json' +import MEAN_METRIC_RESULT from '~/mocks/fixtures/api/experiments/mean_metric_result.json' +import { NodeKind } from '~/queries/schema/schema-general' + +import type { RunningTimeTargetApi } from 'products/experiments/frontend/generated/api.schemas' + +import { recalculationMocks, resultsByMetricType } from './recalculationMocks' + +const EXPERIMENT_PATH = `/api/projects/:team_id/experiments/${EXPERIMENT_WITH_MEAN_METRIC.id}/` +const RECALCULATION = recalculationMocks( + EXPERIMENT_WITH_MEAN_METRIC, + resultsByMetricType(EXPERIMENT_WITH_MEAN_METRIC, { mean: MEAN_METRIC_RESULT }) +) + +// The fixture launched on 2025-05-20, so `mockDate` puts the run two weeks in, as each target response's outlook says. +const targetMocks = (target: RunningTimeTargetApi): Parameters[0] => ({ + get: { + ...RECALCULATION.get, + [EXPERIMENT_PATH]: EXPERIMENT_WITH_MEAN_METRIC, + [`${EXPERIMENT_PATH}running_time_target/`]: target, + '/api/projects/:team_id/experiment_holdouts': [], + '/api/projects/:team_id/experiment_saved_metrics/': [], + [`/api/projects/:team_id/feature_flags/${EXPERIMENT_WITH_MEAN_METRIC.feature_flag.id}/`]: + EXPERIMENT_WITH_MEAN_METRIC.feature_flag, + [`/api/projects/:team_id/feature_flags/${EXPERIMENT_WITH_MEAN_METRIC.feature_flag.id}/status/`]: {}, + '/api/environments/:team_id/default_release_conditions/': [], + '/api/projects/:team_id/experiments_config/': {}, + [`${EXPERIMENT_PATH}health/`]: { findings: [], checks: [] }, + '/api/projects/:team_id/property_definitions/seen_together': {}, + }, + post: { + ...RECALCULATION.post, + '/api/environments/:team_id/query/:kind': async ({ request }) => { + const body = (await request.json()) as Record + return body.query.kind === NodeKind.ExperimentExposureQuery + ? [200, EXPOSURE_QUERY_RESULT] + : [200, { results: [] }] + }, + }, + patch: { [EXPERIMENT_PATH]: EXPERIMENT_WITH_MEAN_METRIC }, +}) + +const openRunningTimeSettings = async (): Promise => { + const settings = await waitFor(() => { + const button = document.querySelector('[data-attr="experiment-running-time-config"]') + if (!button) { + throw new Error('The running time settings button has not rendered yet') + } + return button + }) + await userEvent.click(settings) +} + +const meta: Meta = { + component: App, + title: 'Scenes-App/Experiments', + parameters: { + layout: 'fullscreen', + viewMode: 'story', + mockDate: '2025-06-03', + pageUrl: urls.experiment(EXPERIMENT_WITH_MEAN_METRIC.id), + }, +} +export default meta + +type Story = StoryObj<{}> + +// A target days away: the settings place it between the upcoming weeks. +export const ExperimentRunningTimeOnTrack: Story = { + decorators: [ + mswDecorator( + targetMocks({ + minimum_detectable_effect: 30, + recommended_sample_size: 3000, + current_sample_size: 2000, + outlook: { + days_since_launch: 14, + exposures_per_day: 2000 / 14, + detectable_effect_now: 36.7, + upcoming_checkpoints: [ + { weeks_since_launch: 4, detectable_effect: 26 }, + { weeks_since_launch: 6, detectable_effect: 21.2 }, + { weeks_since_launch: 8, detectable_effect: 18.4 }, + ], + reachable_effect: 21.2, + }, + }) + ), + ], + play: openRunningTimeSettings, +} + +// A target months away: the header says what the experiment can detect by week 4, and the settings list the ways to +// finish sooner. +export const ExperimentRunningTimeOutOfReach: Story = { + decorators: [ + mswDecorator( + targetMocks({ + minimum_detectable_effect: 30, + recommended_sample_size: 35556, + current_sample_size: 2000, + outlook: { + days_since_launch: 14, + exposures_per_day: 2000 / 14, + detectable_effect_now: 126.5, + upcoming_checkpoints: [ + { weeks_since_launch: 4, detectable_effect: 89.4 }, + { weeks_since_launch: 6, detectable_effect: 73 }, + { weeks_since_launch: 8, detectable_effect: 63.2 }, + ], + reachable_effect: 73, + }, + }) + ), + ], + play: openRunningTimeSettings, +} diff --git a/frontend/src/scenes/project-homepage/today/Today.stories.tsx b/frontend/src/scenes/project-homepage/today/Today.stories.tsx index 9ccd93135209..980e0d8771a7 100644 --- a/frontend/src/scenes/project-homepage/today/Today.stories.tsx +++ b/frontend/src/scenes/project-homepage/today/Today.stories.tsx @@ -912,6 +912,7 @@ const meta: Meta = { const results = type ? LIBRARY.filter((entry) => entry.type === type) : LIBRARY return [200, { results, count: results.length }] }, + '/api/projects/:team_id/file_system/frequent/': [], '/api/environments/:team_id/file_system/unfiled/': { results: [], count: 0 }, '/api/projects/:team_id/canvases/canvas-building/': VIEW_CANVASES[1], '/api/projects/:team_id/tasks/task-canvas-building/': { @@ -972,6 +973,52 @@ type Story = StoryObj<{}> export const Home: Story = { decorators: [keepNumbers([])] } +export const HomeWithFrequentItems: Story = { + parameters: { + featureFlags: { + [FEATURE_FLAGS.TODAY_RAIL_NAV]: true, + [FEATURE_FLAGS.POSTHOG_CODE_TASK_ANALYSIS]: true, + [FEATURE_FLAGS.TODAY_JUMP_BACK_RANKING]: 'test', + }, + }, + decorators: [ + keepNumbers([]), + mswDecorator({ + get: { + '/api/projects/:team_id/file_system/frequent/': [ + { + id: 'fs-frequent-1', + path: 'Dashboards/Daily operations', + type: 'dashboard', + ref: '81', + href: '/dashboard/81', + last_viewed_at: '2026-09-27T09:00:00Z', + }, + { + id: 'fs-frequent-2', + path: 'Notebooks/Weekly review', + type: 'notebook', + ref: 'weekly-review', + href: '/notebooks/weekly-review', + last_viewed_at: '2026-09-21T09:00:00Z', + }, + ], + }, + }), + ], +} + +export const HomeWithRecentRanking: Story = { + ...HomeWithFrequentItems, + parameters: { + featureFlags: { + [FEATURE_FLAGS.TODAY_RAIL_NAV]: true, + [FEATURE_FLAGS.POSTHOG_CODE_TASK_ANALYSIS]: true, + [FEATURE_FLAGS.TODAY_JUMP_BACK_RANKING]: 'control', + }, + }, +} + export const HomeWithKeptNumbers: Story = { decorators: [ keepNumbers([ diff --git a/frontend/src/scenes/project-homepage/today/TodayRecents.tsx b/frontend/src/scenes/project-homepage/today/TodayRecents.tsx index c69438d05048..88832d47ab82 100644 --- a/frontend/src/scenes/project-homepage/today/TodayRecents.tsx +++ b/frontend/src/scenes/project-homepage/today/TodayRecents.tsx @@ -4,8 +4,8 @@ import { useId, useMemo } from 'react' import { IconChevronDown } from '@posthog/icons' +import { TZLabel } from 'lib/components/TZLabel' import { LinkPrimitive } from 'lib/lemon-ui/Link' -import { capitalizeFirstLetter } from 'lib/utils/strings' import { todayObjectGlanceLogic } from '~/layout/today/todayObjectGlanceLogic' import { TodayObjectIcon } from '~/layout/today/TodayObjectIcon' @@ -18,8 +18,17 @@ import { FileSystemEntry } from '~/queries/schema/schema-general' import { TodayHomeHeading } from './TodayHomeHeading' import { RECENT_OBJECT_COUNT, todayLogic } from './todayLogic' -function RecentObjectRow({ entry }: { entry: FileSystemEntry }): JSX.Element { +function RecentObjectRow({ + entry, + position, + frequent, +}: { + entry: FileSystemEntry + position: number + frequent: boolean +}): JSX.Element { const preview = useMemo(() => objectPreview(entry), [entry]) + const { jumpBackOpened } = useActions(todayLogic) return (
  • @@ -28,12 +37,20 @@ function RecentObjectRow({ entry }: { entry: FileSystemEntry }): JSX.Element { to={preview.href ?? undefined} className="TodayJumpRow__link" data-attr="today-home-recent-object" + onClick={() => jumpBackOpened(entry, position, frequent)} > {preview.name} - {preview.typeLabel && ( - {capitalizeFirstLetter(preview.typeLabel)} - )} + + {entry.last_viewed_at ? ( + + ) : ( + Time unknown + )} +
    • ) @@ -41,7 +58,8 @@ function RecentObjectRow({ entry }: { entry: FileSystemEntry }): JSX.Element { export function TodayRecents(): JSX.Element { useMountedLogic(todayObjectGlanceLogic) - const { recentObjects, recentsHasLoaded, listExpanded } = useValues(todayLogic) + const { recentObjects, frequentObjectsLoading, recentsHasLoaded, listExpanded, useFrequentJumpBack } = + useValues(todayLogic) const { setListExpanded } = useActions(todayLogic) const { pickPane } = useActions(todayShellLogic) const titleId = useId() @@ -63,8 +81,8 @@ export function TodayRecents(): JSX.Element { )} - {!recentsHasLoaded ? ( -

    Finding what you looked at last…

    + {!recentsHasLoaded || (useFrequentJumpBack && frequentObjectsLoading) ? ( +

    Finding your items…

    ) : recentObjects.length === 0 ? (

    @@ -84,15 +102,25 @@ export function TodayRecents(): JSX.Element { ) : (

      - {shown.map((entry) => ( - + {shown.map((entry, index) => ( + ))}
    {more.length > 0 && (
      - {more.map((entry) => ( - + {more.map((entry, index) => ( + ))}
    diff --git a/frontend/src/scenes/project-homepage/today/todayLogic.test.ts b/frontend/src/scenes/project-homepage/today/todayLogic.test.ts index 746c0947f658..0708699fb908 100644 --- a/frontend/src/scenes/project-homepage/today/todayLogic.test.ts +++ b/frontend/src/scenes/project-homepage/today/todayLogic.test.ts @@ -2,7 +2,11 @@ import { router } from 'kea-router' import { expectLogic } from 'kea-test-utils' import posthog from 'posthog-js' +import { FEATURE_FLAGS } from 'lib/constants' +import { featureFlagLogic } from 'lib/logic/featureFlagLogic' + import { useMocks } from '~/mocks/jest' +import { recentItemsModel } from '~/models/recentItemsModel' import { initKeaTests } from '~/test/init' import { AccessControlLevel, AccessControlResourceType } from '~/types' @@ -18,6 +22,7 @@ import { makeHighlightItem, makeHighlights } from './todayTestFixtures' import { todayVerdictLogic } from './todayVerdictLogic' describe('todayLogic', () => { + let frequentResponse: [number, unknown] let listResponse: [number, any] let listParams: URLSearchParams | null let reportResponse: [number, any] | null @@ -25,6 +30,7 @@ describe('todayLogic', () => { let highlightCalls: number beforeEach(() => { + frequentResponse = [200, []] highlightResponses = [[200, makeHighlights()]] listResponse = [200, { results: [], count: 0 }] highlightCalls = 0 @@ -32,6 +38,7 @@ describe('todayLogic', () => { reportResponse = null useMocks({ get: { + '/api/projects/:team_id/file_system/frequent/': () => frequentResponse, '/api/projects/:team_id/signals/reports/': ({ request }) => { listParams = new URL(request.url).searchParams return listResponse @@ -49,10 +56,65 @@ describe('todayLogic', () => { initKeaTests() }) + it.each([ + { status: 200, variant: 'test' }, + { status: 500, variant: 'test' }, + { status: 200, variant: 'control' }, + { status: 200, variant: false }, + ])('selects $variant ranking when the request returns $status', async ({ status, variant }) => { + const entry = (id: string): { id: string; path: string; type: string; ref: string } => ({ + id, + path: `Dashboards/${id}`, + type: 'dashboard', + ref: id, + }) + frequentResponse = [status, status === 200 ? [entry('daily'), entry('weekly')] : {}] + featureFlagLogic.actions.setFeatureFlags([], { [FEATURE_FLAGS.TODAY_JUMP_BACK_RANKING]: variant }) + const logic = todayLogic() + logic.mount() + await expectLogic(logic).toFinishAllListeners() + recentItemsModel.actions.loadRecentsSuccess(['a', 'b', 'c', 'd'].map(entry)) + const mixed = status === 200 && variant === 'test' + expect(logic.values.recentObjects.map((item) => item.id)).toEqual( + mixed ? ['a', 'b', 'daily', 'weekly', 'c', 'd'] : ['a', 'b', 'c', 'd'] + ) + logic.actions.jumpBackOpened(logic.values.recentObjects[2], 3, mixed) + expect(posthog.capture).toHaveBeenCalledWith( + 'today jump back opened', + expect.objectContaining({ + position: 3, + reason: mixed ? 'frequent' : 'recent', + ranking_variant: variant === 'test' ? 'test' : 'control', + }) + ) + }) + afterEach(() => { jest.useRealTimers() }) + it.each(['toggle', 'url'])('reloads frequent items when sample mode is disabled by %s', async (source) => { + const entry = { id: 'daily', path: 'Daily dashboard', type: 'dashboard', ref: 'daily' } + frequentResponse = [200, [entry]] + const logic = todayLogic() + logic.mount() + await expectLogic(logic).toFinishAllListeners() + + await expectLogic(logic, () => logic.actions.setUseSampleData(true)) + .toFinishAllListeners() + .toMatchValues({ frequentObjects: [] }) + + await expectLogic(logic, () => { + if (source === 'url') { + router.actions.push('/home', { sample: '0' }) + } else { + logic.actions.setUseSampleData(false) + } + }) + .toFinishAllListeners() + .toMatchValues({ useSampleData: false, frequentObjects: [entry] }) + }) + it.each([ { shown: 'the report list', diff --git a/frontend/src/scenes/project-homepage/today/todayLogic.ts b/frontend/src/scenes/project-homepage/today/todayLogic.ts index 3394fdfcb8ea..b2cce3b58e2b 100644 --- a/frontend/src/scenes/project-homepage/today/todayLogic.ts +++ b/frontend/src/scenes/project-homepage/today/todayLogic.ts @@ -5,14 +5,19 @@ import type { LocationChangedPayload } from 'kea-router/lib/types' import posthog from 'posthog-js' import { ApiError } from 'lib/api' +import { FEATURE_FLAGS } from 'lib/constants' import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast' +import { FeatureFlagsSet, featureFlagLogic } from 'lib/logic/featureFlagLogic' import { getProductAccessDisabledReason } from 'lib/utils/accessControlUtils' +import { withTimeout } from 'lib/utils/async' +import { isObject } from 'lib/utils/guards' import { removeProjectIdIfPresent } from 'lib/utils/kea-router' import { objectsEqual } from 'lib/utils/objects' import { teamLogic } from 'scenes/teamLogic' import { urls } from 'scenes/urls' import { userLogic } from 'scenes/userLogic' +import { fileSystemFrequentList } from '~/generated/core/api' import { recentItemsModel } from '~/models/recentItemsModel' import { FileSystemEntry } from '~/queries/schema/schema-general' import { TeamPublicType, TeamType, UserType } from '~/types' @@ -40,7 +45,7 @@ import { type TodayForYouRows, } from './todayForYouRows' import { todayHighlightsLogic } from './todayHighlightsLogic' -import { recentObjectsForHome } from './todayRecentObjects' +import { JumpBackObject, jumpBackObjectsForHome, recentObjectsForHome } from './todayRecentObjects' import { type TodayReportPreview, reportPreview } from './todayReportPreviews' import type { TodayReportState } from './todayReportState' import { isSampleReportId, parseSampleParam, sampleTopReports } from './todaySampleReports' @@ -148,6 +153,7 @@ interface TodayRankedReport { // Generated by kea-typegen. Update if you're an agent, ignore if you're human. export interface todayLogicValues { + featureFlags: FeatureFlagsSet // featureFlagLogic recents: FileSystemEntry[] // recentItemsModel recentsHasLoaded: boolean // recentItemsModel currentProjectId: number | string // teamLogic @@ -173,6 +179,8 @@ export interface todayLogicValues { forYouReady: boolean forYouRows: TodayForYouRows forYouVisibleRows: TodayForYouRow[] + frequentObjects: FileSystemEntry[] + frequentObjectsLoading: boolean greeting: string homeScrollTop: number | null hour: number @@ -184,7 +192,7 @@ export interface todayLogicValues { moreReportsInInbox: number moreReportsLoading: boolean now: number - recentObjects: FileSystemEntry[] + recentObjects: JumpBackObject[] reportId: string | null reportOpenContext: TodayReportOpenContext | null reportPreviews: Record @@ -195,6 +203,7 @@ export interface todayLogicValues { shownReportIds: string[] topReports: TodayReports | null topReportsLoading: boolean + useFrequentJumpBack: boolean useSampleData: boolean } @@ -221,6 +230,13 @@ export interface todayLogicActions { searchParams: Record url: string } // router + loadCurrentTeamSuccess: ( + currentTeam: TeamPublicType | null, + payload?: any + ) => { + currentTeam: TeamPublicType | null + payload?: any + } // teamLogic loadHighlights: () => any // todayHighlightsLogic askAi: ( prompt: string, @@ -241,6 +257,30 @@ export interface todayLogicActions { position: number product: string } + jumpBackOpened: ( + entry: FileSystemEntry, + position: number, + frequent: boolean + ) => { + entry: FileSystemEntry + frequent: boolean + position: number + } + loadFrequentObjects: () => any + loadFrequentObjectsFailure: ( + error: string, + errorObject?: any + ) => { + error: string + errorObject?: any + } + loadFrequentObjectsSuccess: ( + frequentObjects: FileSystemEntry[], + payload?: any + ) => { + frequentObjects: FileSystemEntry[] + payload?: any + } loadMoreReports: () => any loadMoreReportsFailure: ( error: string, @@ -323,7 +363,12 @@ export interface todayLogicMeta { moreReportCount: (topReports: TodayReports | null) => number hour: (now: number) => number greeting: (hour: number, user: UserType | null) => string - recentObjects: (recents: FileSystemEntry[]) => FileSystemEntry[] + useFrequentJumpBack: (featureFlags: FeatureFlagsSet) => boolean + recentObjects: ( + recents: FileSystemEntry[], + frequentObjects: FileSystemEntry[], + useFrequentJumpBack: boolean + ) => JumpBackObject[] shownReportIds: (reports: SignalReport[]) => string[] currentMoreReports: ( moreReports: TodayMoreReports | null, @@ -376,6 +421,8 @@ export const todayLogic = kea([ path(['scenes', 'project-homepage', 'today', 'todayLogic']), connect(() => ({ values: [ + featureFlagLogic, + ['featureFlags'], userLogic, ['user'], teamLogic, @@ -395,7 +442,14 @@ export const todayLogic = kea([ todayVerdictLogic, ['reportStateOverrides'], ], - actions: [router, ['locationChanged'], todayHighlightsLogic, ['loadHighlights']], + actions: [ + router, + ['locationChanged'], + todayHighlightsLogic, + ['loadHighlights'], + teamLogic, + ['loadCurrentTeamSuccess'], + ], })), actions({ // `report` is the report the person reads. Without it, the context is the report list. @@ -408,12 +462,52 @@ export const todayLogic = kea([ setReturnFocusReportId: (reportId: string | null) => ({ reportId }), setReportOpenContext: (context: TodayReportOpenContext | null) => ({ context }), setListExpanded: (list: TodayList, expanded: boolean) => ({ list, expanded }), + jumpBackOpened: (entry: FileSystemEntry, position: number, frequent: boolean) => ({ + entry, + position, + frequent, + }), showMoreForYou: true, retryFailedLoads: true, setUseSampleData: (useSampleData: boolean) => ({ useSampleData }), tick: true, }), loaders(({ values }) => ({ + frequentObjects: [ + [] as FileSystemEntry[], + { + loadFrequentObjects: async (): Promise => { + if (values.useSampleData || values.currentProjectId === null) { + return [] + } + try { + const projectId = values.currentProjectId + const entries = await withTimeout( + (signal) => fileSystemFrequentList(String(projectId), { limit: 8 }, { signal }), + 10_000, + 'loadFrequentObjects timed out' + ) + if (values.currentProjectId !== projectId) { + return [] + } + return entries.map((entry) => ({ + id: entry.id, + path: entry.path, + type: entry.type, + ref: entry.ref ?? undefined, + href: entry.href ?? undefined, + meta: isObject(entry.meta) ? entry.meta : undefined, + last_viewed_at: entry.last_viewed_at, + })) + } catch (error) { + if (!(error instanceof ApiError && error.status === 403)) { + posthog.captureException(error) + } + return [] + } + }, + }, + ], topReports: [ null as TodayReports | null, { @@ -543,9 +637,23 @@ export const todayLogic = kea([ (s) => [s.hour, s.user], (hour: number, user: UserType | null): string => greetingForHour(hour, user?.first_name), ], + useFrequentJumpBack: [ + (s) => [s.featureFlags], + (featureFlags: FeatureFlagsSet): boolean => featureFlags[FEATURE_FLAGS.TODAY_JUMP_BACK_RANKING] === 'test', + ], recentObjects: [ - (s) => [s.recents], - (recents: FileSystemEntry[]): FileSystemEntry[] => recentObjectsForHome(recents, RECENT_OBJECT_MORE_COUNT), + (s) => [s.recents, s.frequentObjects, s.useFrequentJumpBack], + ( + recents: FileSystemEntry[], + frequentObjects: FileSystemEntry[], + useFrequentJumpBack: boolean + ): JumpBackObject[] => + useFrequentJumpBack + ? jumpBackObjectsForHome(recents, frequentObjects, RECENT_OBJECT_MORE_COUNT) + : recentObjectsForHome(recents, RECENT_OBJECT_MORE_COUNT).map((entry) => ({ + ...entry, + jumpBackReason: 'recent', + })), ], shownReportIds: [(s) => [s.reports], (reports: SignalReport[]): string[] => reports.map((report) => report.id)], currentMoreReports: [ @@ -716,6 +824,18 @@ export const todayLogic = kea([ }) } return { + loadCurrentTeamSuccess: () => actions.loadFrequentObjects(), + jumpBackOpened: ({ entry, position, frequent }) => { + if (!values.useSampleData) { + posthog.capture('today jump back opened', { + object_type: entry.type, + object_ref: entry.ref ?? entry.id, + position, + reason: frequent ? 'frequent' : 'recent', + ranking_variant: values.useFrequentJumpBack ? 'test' : 'control', + }) + } + }, askAi: async ({ prompt, source, report }, breakpoint) => { let context: TodayAskContext // Sample reports have ids that do not exist, so PostHog AI gets no context to look up. A real @@ -745,6 +865,7 @@ export const todayLogic = kea([ }, setUseSampleData: () => { actions.loadTopReports() + actions.loadFrequentObjects() }, exploreOpened: ({ product, position }) => { posthog.capture('today explore opened', { product, position }) @@ -809,6 +930,7 @@ export const todayLogic = kea([ posthog.capture('today home loaded', { report_count: topReports.results.length, more_report_count: values.moreReportCount, + jump_back_ranking: values.useFrequentJumpBack ? 'recent-and-frequent-v1' : 'recent-v1', }) }, loadMoreReports: () => { @@ -858,6 +980,7 @@ export const todayLogic = kea([ } else { actions.loadTopReports() } + actions.loadFrequentObjects() cache.disposables.add(() => { const clock = window.setInterval(() => actions.tick(), CLOCK_MS) return () => clearInterval(clock) diff --git a/frontend/src/scenes/project-homepage/today/todayRecentObjects.test.ts b/frontend/src/scenes/project-homepage/today/todayRecentObjects.test.ts index 2dc178dbb432..1bee2d46897b 100644 --- a/frontend/src/scenes/project-homepage/today/todayRecentObjects.test.ts +++ b/frontend/src/scenes/project-homepage/today/todayRecentObjects.test.ts @@ -1,15 +1,66 @@ -import { recentObjectsForHome } from './todayRecentObjects' +import { jumpBackObjectsForHome, recentObjectsForHome } from './todayRecentObjects' describe('recentObjectsForHome', () => { it('keeps agent sessions and entries with no page out of the recent objects', () => { const recents = [ { id: 'task', path: 'Tasks/Fix the flaky test', type: 'task', href: '/tasks/1' }, { id: 'dashboard', path: 'Dashboards/Growth', type: 'dashboard', href: '/dashboard/1' }, + { id: 'dashboard-copy', path: 'Pinned/Growth', type: 'dashboard', ref: '1' }, { id: 'unlinked', path: 'Insights/Draft', type: 'insight' }, { id: 'by-ref', path: 'Flags/checkout', type: 'feature_flag', ref: '7' }, { id: 'insight', path: 'Insights/Funnel', type: 'insight/funnels', href: '/insights/a' }, + { id: 'insight-copy', path: 'Pinned/Funnel', type: 'insight', ref: 'a' }, + { id: 'query-one', path: 'Queries/One', href: '/sql?open_saved_query=one' }, + { id: 'query-two', path: 'Queries/Two', href: '/sql?open_saved_query=two' }, ] - expect(recentObjectsForHome(recents, 3).map((entry) => entry.id)).toEqual(['dashboard', 'by-ref', 'insight']) + expect(recentObjectsForHome(recents, 5).map((entry) => entry.id)).toEqual([ + 'dashboard', + 'by-ref', + 'insight', + 'query-one', + 'query-two', + ]) + }) + + it.each([ + [ + ['a', 'b', 'c', 'd', 'e'], + ['d', 'e'], + ['a', 'b', 'd', 'e', 'c'], + ], + [ + ['a', 'b', 'c', 'd', 'e'], + ['a', 'b', 'd', 'e'], + ['a', 'b', 'd', 'e', 'c'], + ], + [['a', 'b', 'c', 'd', 'e'], ['d'], ['a', 'b', 'd', 'c', 'e']], + [['a', 'b', 'c', 'd', 'e'], [], ['a', 'b', 'c', 'd', 'e']], + [ + ['a', 'b', 'c', 'd', 'e'], + ['d', 'd', 'e'], + ['a', 'b', 'd', 'e', 'c'], + ], + [ + ['a', 'a', 'b', 'c', 'd', 'e'], + ['c', 'd'], + ['a', 'b', 'c', 'd', 'e'], + ], + [ + ['a', 'b', 'c', 'c', 'd', 'e'], + ['d', 'e'], + ['a', 'b', 'd', 'e', 'c'], + ], + [['a', 'a', 'b', 'c', 'd', 'e'], [], ['a', 'b', 'c', 'd', 'e']], + ])('mixes recent %j and frequent %j without duplicates', (recentIds, frequentIds, expected) => { + const entry = (id: string): { id: string; path: string; type: string; ref: string } => ({ + id, + path: `Dashboards/${id}`, + type: 'dashboard', + ref: id, + }) + expect(jumpBackObjectsForHome(recentIds.map(entry), frequentIds.map(entry), 8).map((item) => item.id)).toEqual( + expected + ) }) }) diff --git a/frontend/src/scenes/project-homepage/today/todayRecentObjects.ts b/frontend/src/scenes/project-homepage/today/todayRecentObjects.ts index 0ebe4312d5f8..894d00363457 100644 --- a/frontend/src/scenes/project-homepage/today/todayRecentObjects.ts +++ b/frontend/src/scenes/project-homepage/today/todayRecentObjects.ts @@ -1,10 +1,63 @@ +import { removeProjectIdIfPresent } from 'lib/utils/kea-router' import { baseObjectType, libraryObjectHref } from 'scenes/library/libraryUtils' import { FileSystemEntry } from '~/queries/schema/schema-general' +function objectKey(entry: FileSystemEntry): string { + const href = libraryObjectHref({ type: baseObjectType(entry.type), ref: entry.ref }) ?? libraryObjectHref(entry) + if (!href) { + return `${baseObjectType(entry.type)}:${entry.ref ?? entry.id}` + } + const destination = removeProjectIdIfPresent(href).split('#')[0] + const queryStart = destination.indexOf('?') + const pathname = queryStart === -1 ? destination : destination.slice(0, queryStart) + const search = queryStart === -1 ? '' : destination.slice(queryStart) + return `${pathname.replace(/\/+$/, '')}${search}` +} + /** Recently viewed objects for the home, without tasks, which are agent sessions rather than things people look at. */ export function recentObjectsForHome(recents: FileSystemEntry[], limit: number): FileSystemEntry[] { + const seen = new Set() return recents - .filter((entry) => libraryObjectHref(entry) !== null && baseObjectType(entry.type) !== 'task') + .filter((entry) => { + if (libraryObjectHref(entry) === null || baseObjectType(entry.type) === 'task') { + return false + } + const key = objectKey(entry) + if (seen.has(key)) { + return false + } + seen.add(key) + return true + }) .slice(0, limit) } + +export type JumpBackObject = FileSystemEntry & { jumpBackReason: 'recent' | 'frequent' } + +export function jumpBackObjectsForHome( + recents: FileSystemEntry[], + frequent: FileSystemEntry[], + limit: number +): JumpBackObject[] { + const recentObjects = recentObjectsForHome(recents, limit) + const first = recentObjects.slice(0, 2) + const seen = new Set(first.map(objectKey)) + const result: JumpBackObject[] = first.map((entry) => ({ ...entry, jumpBackReason: 'recent' })) + for (const entry of recentObjectsForHome(frequent, frequent.length)) { + if (!seen.has(objectKey(entry))) { + seen.add(objectKey(entry)) + result.push({ ...entry, jumpBackReason: 'frequent' }) + if (result.length === first.length + 2) { + break + } + } + } + for (const entry of recentObjects) { + if (!seen.has(objectKey(entry))) { + seen.add(objectKey(entry)) + result.push({ ...entry, jumpBackReason: 'recent' }) + } + } + return result.slice(0, limit) +} diff --git a/frontend/src/scenes/settings/SettingsMap.tsx b/frontend/src/scenes/settings/SettingsMap.tsx index 8653cd4f7e4f..acb7b275edc7 100644 --- a/frontend/src/scenes/settings/SettingsMap.tsx +++ b/frontend/src/scenes/settings/SettingsMap.tsx @@ -1679,7 +1679,6 @@ export const SETTINGS_MAP: SettingSection[] = [ description: 'Let business knowledge read these repositories when answering a question. It searches file names and the README, then reads a file. It does not index the code.', component: , - flag: 'BUSINESS_KNOWLEDGE_GITHUB_REPOS', keywords: ['business', 'knowledge', 'github', 'repository', 'code'], }, ], diff --git a/frontend/src/scenes/team-activity/teamActivityDescriber.tsx b/frontend/src/scenes/team-activity/teamActivityDescriber.tsx index f491a1405671..b03e9b40c018 100644 --- a/frontend/src/scenes/team-activity/teamActivityDescriber.tsx +++ b/frontend/src/scenes/team-activity/teamActivityDescriber.tsx @@ -16,7 +16,7 @@ import PropertyFiltersDisplay from 'lib/components/PropertyFilters/components/Pr import { Link } from 'lib/lemon-ui/Link' import { CURRENCY_SYMBOL_TO_EMOJI_MAP, CURRENCY_SYMBOL_TO_NAME_MAP } from 'lib/utils/currency' import { isObject } from 'lib/utils/guards' -import { pluralize } from 'lib/utils/strings' +import { capitalizeFirstLetter, pluralize } from 'lib/utils/strings' import { ExperimentsConfig } from 'scenes/settings/environment/experimentsConfigLogic' import { urls } from 'scenes/urls' @@ -868,6 +868,33 @@ function describeWorkflowEmailSuspension(logItem: ActivityLogItem): HumanizedCha } } +function describeWorkflowEmailTierChange(logItem: ActivityLogItem): HumanizedChange { + const changes = logItem.detail?.changes || [] + const tierChange = changes.find((change) => change.field === 'email_sending_tier') + const pinChange = changes.find((change) => change.field === 'email_sending_tier_pinned') + const contextReason = logItem.detail?.context?.reason as string | undefined + // A staff-set tier has no decision behind it, so the row says "set" and shows no reason. + const setByStaff = contextReason === 'staff_set' + const reason = setByStaff ? undefined : contextReason + const target = shortIdActivityLink(logItem, urls.notebook) + const pinVerb = pinChange ? (pinChange.after ? 'pinned' : 'unpinned') : null + const verb = tierChange ? (setByStaff ? 'set' : 'moved') : (pinVerb ?? 'changed') + const tierMove = tierChange ? ` from ${String(tierChange.before)} to ${String(tierChange.after)}` : '' + const pinSuffix = tierChange && pinVerb ? ` and ${pinVerb === 'pinned' ? 'pinned it there' : 'unpinned it'}` : '' + const action = `${verb} the workflow email sending tier` + return { + summary: activityLogSummary(logItem, `${capitalizeFirstLetter(action)}${tierMove}${pinSuffix}`, target, reason), + description: ( + <> + {action} on {target} + {tierMove} + {pinSuffix} + {reason ? <> (reason: {reason}) : null} + + ), + } +} + export function teamActivityDescriber(logItem: ActivityLogItem, asNotification?: boolean): HumanizedChange { if (logItem.scope !== ActivityScope.TEAM) { console.error('team describer received a non-Team activity') @@ -878,6 +905,10 @@ export function teamActivityDescriber(logItem: ActivityLogItem, asNotification?: return describeWorkflowEmailSuspension(logItem) } + if (logItem.activity === 'email_sending_tier_changed') { + return describeWorkflowEmailTierChange(logItem) + } + if (logItem.activity == 'changed' || logItem.activity == 'updated') { const changes = describeMappedChanges( logItem, diff --git a/nodejs/src/cdp/cdp-e2e.serial.test.ts b/nodejs/src/cdp/cdp-e2e.serial.test.ts index ad0d206a0511..b29004a886ef 100644 --- a/nodejs/src/cdp/cdp-e2e.serial.test.ts +++ b/nodejs/src/cdp/cdp-e2e.serial.test.ts @@ -197,6 +197,7 @@ describe('CDP Consumer loop', () => { "headers": { "Authorization": "Bearer super-secret-token", }, + "maxResponseBytes": 5242880, "method": "POST", }, ] diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.ts index 0cd0a382dfad..3a9e9b48d86d 100644 --- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.ts +++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-plugins.test.ts @@ -155,6 +155,7 @@ describe('CdpCyclotronWorkerPlugins', () => { "Authorization": "Bearer 1234567890", "Content-Type": "application/json", }, + "maxResponseBytes": 5242880, "method": "POST", }, ] @@ -169,6 +170,7 @@ describe('CdpCyclotronWorkerPlugins', () => { "Authorization": "Bearer 1234567890", "Content-Type": "application/json", }, + "maxResponseBytes": 5242880, "method": "POST", }, ] diff --git a/nodejs/src/cdp/services/hog-executor-async.service.ts b/nodejs/src/cdp/services/hog-executor-async.service.ts index 2c8878bc4bc2..c914db4cba34 100644 --- a/nodejs/src/cdp/services/hog-executor-async.service.ts +++ b/nodejs/src/cdp/services/hog-executor-async.service.ts @@ -18,7 +18,12 @@ import type { } from '../types' import { createAddLogFunction, destinationE2eLagMsSummary } from '../utils' import { resolveAwsSigV4Credentials, signAwsRequest } from '../utils/aws-sigv4' -import { cdpTrackedFetch, fetchErrorDetail, isFetchResponseRetriable } from '../utils/cdp-fetch' +import { + MAX_FETCH_RESPONSE_BYTES, + cdpTrackedFetch, + fetchErrorDetail, + isFetchResponseRetriable, +} from '../utils/cdp-fetch' import { createInvocationResult } from '../utils/invocation-utils' import { isNonFailureStatus } from '../utils/non-failure-status-codes' import { ScopedServiceJwt } from '../utils/scoped-service-jwt' @@ -579,7 +584,12 @@ export class HogExecutorAsyncService { } } } catch (e) { - addLog('error', `Failed to parse response body: ${e.message}`) + addLog( + 'error', + e.name === 'ResponseTooLargeError' + ? `The response body is larger than ${MAX_FETCH_RESPONSE_BYTES / 1024 / 1024} MB, so it was not read.` + : `Failed to parse response body: ${e.message}` + ) body = undefined } diff --git a/nodejs/src/cdp/services/legacy-plugin-executor.service.test.ts b/nodejs/src/cdp/services/legacy-plugin-executor.service.test.ts index 55f24f637f07..ac6fe565a54a 100644 --- a/nodejs/src/cdp/services/legacy-plugin-executor.service.test.ts +++ b/nodejs/src/cdp/services/legacy-plugin-executor.service.test.ts @@ -255,6 +255,7 @@ describe('LegacyPluginExecutorService', () => { "Authorization": "Basic MTIzNDU2Nzg5MDpjaW8tdG9rZW4=", "User-Agent": "PostHog Customer.io App", }, + "maxResponseBytes": 5242880, "method": "GET", }, ], @@ -267,6 +268,7 @@ describe('LegacyPluginExecutorService', () => { "Content-Type": "application/json", "User-Agent": "PostHog Customer.io App", }, + "maxResponseBytes": 5242880, "method": "PUT", }, ], @@ -279,6 +281,7 @@ describe('LegacyPluginExecutorService', () => { "Content-Type": "application/json", "User-Agent": "PostHog Customer.io App", }, + "maxResponseBytes": 5242880, "method": "POST", }, ], diff --git a/nodejs/src/cdp/services/native-destination-executor.service.test.ts b/nodejs/src/cdp/services/native-destination-executor.service.test.ts index 64a21c4a7480..114aa5b05c0b 100644 --- a/nodejs/src/cdp/services/native-destination-executor.service.test.ts +++ b/nodejs/src/cdp/services/native-destination-executor.service.test.ts @@ -105,6 +105,7 @@ describe('NativeDestinationExecutorService', () => { "Content-Type": "application/json", "User-Agent": "PostHog.com/1.0", }, + "maxResponseBytes": 5242880, "method": "POST", }, ] @@ -170,6 +171,7 @@ describe('NativeDestinationExecutorService', () => { "Content-Type": "application/json", "User-Agent": "PostHog.com/1.0", }, + "maxResponseBytes": 5242880, "method": "POST", }, ] @@ -192,6 +194,31 @@ describe('NativeDestinationExecutorService', () => { }) }) + it('retries a retryable status whose body is over the size cap', async () => { + const invocation = createExampleNativeInvocation( + createHogFunction({ name: 'Plugin test', template_id: 'native-webhook' }), + inputs + ) + const tooLarge = Object.assign(new Error('Response body is larger than the 5242880 byte limit'), { + name: 'ResponseTooLargeError', + }) + mockFetch.mockResolvedValue({ + status: 503, + json: () => Promise.reject(tooLarge), + text: () => Promise.reject(tooLarge), + headers: {}, + dump: () => Promise.resolve(), + }) + + const result = await service.execute(invocation) + + expect(result.finished).toBe(false) + expect(result.invocation.queueMetadata).toEqual({ tries: 1 }) + expect(result.logs.map((x) => x.message)).toContainEqual( + expect.stringContaining('HTTP request failed with status 503 (Response body is larger than') + ) + }) + it('should retry retryable fetch errors', async () => { const fn = createHogFunction({ name: 'Plugin test', @@ -231,6 +258,7 @@ describe('NativeDestinationExecutorService', () => { "Content-Type": "application/json", "User-Agent": "PostHog.com/1.0", }, + "maxResponseBytes": 5242880, "method": "POST", }, ] diff --git a/nodejs/src/cdp/services/native-destination-executor.service.ts b/nodejs/src/cdp/services/native-destination-executor.service.ts index 3fe920fa1bc0..6b35027f431f 100644 --- a/nodejs/src/cdp/services/native-destination-executor.service.ts +++ b/nodejs/src/cdp/services/native-destination-executor.service.ts @@ -15,7 +15,13 @@ import { redactError, redactSensitiveValues, } from '../utils' -import { CdpFetchConfig, cdpTrackedFetch, getNextRetryTime, isFetchResponseRetriable } from '../utils/cdp-fetch' +import { + CdpFetchConfig, + cdpTrackedFetch, + getNextRetryTime, + isFetchResponseRetriable, + readFetchResponseText, +} from '../utils/cdp-fetch' import { createInvocationResult } from '../utils/invocation-utils' const nativeDestinationExecutionDuration = new Histogram({ @@ -175,7 +181,7 @@ export class NativeDestinationExecutorService { hogFunctionId: invocation.hogFunction.id, }) - const fetchResponseText = (await fetchResponse?.text()) ?? 'unknown' + const fetchResponseText = await readFetchResponseText(fetchResponse) if (fetchError || !fetchResponse || fetchResponse.status >= 400) { if ( diff --git a/nodejs/src/cdp/services/segment-destination-executor.service.test.ts b/nodejs/src/cdp/services/segment-destination-executor.service.test.ts index 439e068e5131..8659b951e8ef 100644 --- a/nodejs/src/cdp/services/segment-destination-executor.service.test.ts +++ b/nodejs/src/cdp/services/segment-destination-executor.service.test.ts @@ -107,6 +107,7 @@ describe('SegmentDestinationExecutorService', () => { "headers": { "Content-Type": "application/json", }, + "maxResponseBytes": 5242880, "method": "POST", }, ] @@ -198,6 +199,7 @@ describe('SegmentDestinationExecutorService', () => { "headers": { "Content-Type": "application/json", }, + "maxResponseBytes": 5242880, "method": "POST", }, ] @@ -262,6 +264,7 @@ describe('SegmentDestinationExecutorService', () => { "headers": { "Content-Type": "application/json", }, + "maxResponseBytes": 5242880, "method": "POST", }, ] @@ -390,6 +393,7 @@ describe('SegmentDestinationExecutorService', () => { "Content-Type": "application/json", "x-gb-agent": "GB/Segment", }, + "maxResponseBytes": 5242880, "method": "POST", }, ] @@ -466,6 +470,7 @@ describe('SegmentDestinationExecutorService', () => { { "body": undefined, "headers": {}, + "maxResponseBytes": 5242880, "method": "GET", }, ] @@ -478,6 +483,7 @@ describe('SegmentDestinationExecutorService', () => { "headers": { "Content-Type": "application/json", }, + "maxResponseBytes": 5242880, "method": "POST", }, ] diff --git a/nodejs/src/cdp/services/segment-destination-executor.service.ts b/nodejs/src/cdp/services/segment-destination-executor.service.ts index d20d9f648a76..f7d95f674db9 100644 --- a/nodejs/src/cdp/services/segment-destination-executor.service.ts +++ b/nodejs/src/cdp/services/segment-destination-executor.service.ts @@ -16,7 +16,13 @@ import { redactError, redactSensitiveValues, } from '../utils' -import { CdpFetchConfig, cdpTrackedFetch, getNextRetryTime, isFetchResponseRetriable } from '../utils/cdp-fetch' +import { + CdpFetchConfig, + cdpTrackedFetch, + getNextRetryTime, + isFetchResponseRetriable, + readFetchResponseText, +} from '../utils/cdp-fetch' import { createInvocationResult } from '../utils/invocation-utils' const pluginExecutionDuration = new Histogram({ @@ -310,7 +316,7 @@ export class SegmentDestinationExecutorService { hogFunctionId: invocation.hogFunction.id, }) - const fetchResponseText = (await fetchResponse?.text()) ?? 'unknown' + const fetchResponseText = await readFetchResponseText(fetchResponse) if (fetchError || !fetchResponse || fetchResponse.status >= 400) { if ( diff --git a/nodejs/src/cdp/utils/cdp-fetch.ts b/nodejs/src/cdp/utils/cdp-fetch.ts index 2464864cc8be..5f3fd0df2937 100644 --- a/nodejs/src/cdp/utils/cdp-fetch.ts +++ b/nodejs/src/cdp/utils/cdp-fetch.ts @@ -9,6 +9,23 @@ import { tryCatch } from '~/common/utils/try-catch' import { PluginsServerConfig } from '../../types' export const MAX_FETCH_TIMEOUT_MS = 10000 +// The response lands in the invocation state, which can be queued through Kafka with a 6 MB message limit. +export const MAX_FETCH_RESPONSE_BYTES = 5 * 1024 * 1024 + +// An error body is only logged, so an oversized one must not hide the status that decides the retry. +export async function readFetchResponseText(response: FetchResponse | null | undefined): Promise { + if (!response) { + return 'unknown' + } + try { + return await response.text() + } catch (e) { + if (e.name === 'ResponseTooLargeError' && response.status >= 400) { + return e.message + } + throw e + } +} /** Narrowed config type for CDP fetch retry settings, used by native/segment destination executors */ export type CdpFetchConfig = Pick< @@ -134,7 +151,7 @@ export async function cdpTrackedFetch({ // which run inside undici's connect flow where no caller context is otherwise available. const doFetch = () => fetchAttribution.run({ teamId, hogFunctionId, templateId }, () => - tryCatch(async () => await fetch(url, fetchParams)) + tryCatch(async () => await fetch(url, { maxResponseBytes: MAX_FETCH_RESPONSE_BYTES, ...fetchParams })) ) let [fetchError, fetchResponse] = await doFetch() diff --git a/nodejs/src/common/utils/request.test.ts b/nodejs/src/common/utils/request.test.ts index f184d2ff8ab7..7c4037ac8416 100644 --- a/nodejs/src/common/utils/request.test.ts +++ b/nodejs/src/common/utils/request.test.ts @@ -10,6 +10,7 @@ import { parseJSON } from './json-parse' import { logger } from './logger' import { FetchOptions, + ResponseTooLargeError, SecureRequestError, fetch, internalFetch, @@ -38,6 +39,9 @@ beforeAll(async () => { } else if (url.pathname === '/status/404') { res.writeHead(404) res.end() + } else if (url.pathname === '/bom') { + res.writeHead(200, { 'content-type': 'application/json' }) + res.end('\ufeff{"ok":true}') } else if (url.pathname === '/stream/50') { res.writeHead(200, { 'content-type': 'application/json' }) for (let i = 0; i < 50; i++) { @@ -728,6 +732,29 @@ describe('_fetch response body handling', () => { expect(response.headers['content-type']).toBeDefined() }) + const streamedBodyBytes = range(50).reduce((total, i) => total + JSON.stringify({ id: i }).length + 1, 0) + + it.each([ + ['under the limit', streamedBodyBytes + 1, false], + ['exactly at the limit', streamedBodyBytes, false], + ['over the limit', streamedBodyBytes - 1, true], + ])('caps a streamed body %s', async (_name, maxResponseBytes, rejects) => { + const response = await internalFetch(`${baseUrl}/stream/50`, { maxResponseBytes }) + if (rejects) { + await expect(response.text()).rejects.toBeInstanceOf(ResponseTooLargeError) + } else { + expect(await response.text()).toHaveLength(streamedBodyBytes) + } + }) + + it.each([ + ['uncapped', undefined], + ['capped', 1024], + ])('strips a leading BOM from a %s body', async (_name, maxResponseBytes) => { + const response = await internalFetch(`${baseUrl}/bom`, { maxResponseBytes }) + expect(await response.json()).toEqual({ ok: true }) + }) + it('should fully read streamed/chunked response bodies', async () => { const response = await internalFetch(`${baseUrl}/stream/50`) const text = await response.text() diff --git a/nodejs/src/common/utils/request.ts b/nodejs/src/common/utils/request.ts index b8ca00ba7b71..d418a74378f2 100644 --- a/nodejs/src/common/utils/request.ts +++ b/nodejs/src/common/utils/request.ts @@ -135,6 +135,7 @@ export type FetchOptions = { // Requests with different groups never share an HTTP/2 session. APNs can reject a provider token from one developer // team on a session that already carried another team's token. Requires `allowH2`. http2ConnectionGroup?: string + maxResponseBytes?: number } export type FetchResponse = { @@ -161,6 +162,13 @@ export class InvalidRequestError extends Error { } } +export class ResponseTooLargeError extends Error { + constructor(maxBytes: number) { + super(`Response body is larger than the ${maxBytes} byte limit`) + this.name = 'ResponseTooLargeError' + } +} + export class ResolutionError extends Error { constructor(message: string) { super(message) @@ -750,7 +758,18 @@ export async function _fetch( const readBody = (): Promise => { if (!bodyPromise) { - bodyPromise = readAndDestroyBody(result.body).finally(() => lifecycle?.onBodyDone()) + const maxBytes = options.maxResponseBytes + const read = + maxBytes === undefined + ? readAndDestroyBody(result.body) + : readCappedBody(result.body, maxBytes, false).then(({ bytes, overLimit }) => { + if (overLimit) { + throw new ResponseTooLargeError(maxBytes) + } + const text = bytes.toString('utf8') + return text.charCodeAt(0) === 0xfeff ? text.slice(1) : text + }) + bodyPromise = read.finally(() => lifecycle?.onBodyDone()) } return bodyPromise } diff --git a/packages/agent/packages/agent-contracts/src/domain-types.ts b/packages/agent/packages/agent-contracts/src/domain-types.ts index 0271a504a55e..2acc07d14306 100644 --- a/packages/agent/packages/agent-contracts/src/domain-types.ts +++ b/packages/agent/packages/agent-contracts/src/domain-types.ts @@ -786,6 +786,10 @@ export interface SignalReport { is_suggested_reviewer?: boolean; /** Distinct source products contributing signals to this report. */ source_products?: string[]; + /** Whether the report proposes an improvement to a scout. */ + is_self_improvement_report?: boolean; + /** Selected code repository, when the report has one. */ + repo_slug?: string | null; /** PR URL from the latest implementation task run, if available. */ implementation_pr_url?: string | null; work_state?: "unclaimed" | "working" | "in_review" | "done"; diff --git a/packages/agent/packages/agent-contracts/src/model-catalog.generated.ts b/packages/agent/packages/agent-contracts/src/model-catalog.generated.ts index aefcb8047ec9..f4733754f04e 100644 --- a/packages/agent/packages/agent-contracts/src/model-catalog.generated.ts +++ b/packages/agent/packages/agent-contracts/src/model-catalog.generated.ts @@ -478,6 +478,25 @@ export const MODELS: readonly CatalogModel[] = [ costSummary: "Per 1M tokens: $2 input/$10 output to 272K; $4 input/$15 output above", }, + { + id: "gpt-6.1-terra", + runtimeAdapter: "codex", + reasoningEfforts: ["low", "medium", "high", "xhigh", "max"], + label: "GPT-6.1 Terra", + cost: { + inputPerMtok: 1, + outputPerMtok: 5, + longContext: { + aboveInputTokens: 272000, + inputPerMtok: 2, + outputPerMtok: 7.5, + }, + }, + costMultiplier: "0.5× base", + costSummary: + "Per 1M tokens: $1 input/$5 output to 272K; $2 input/$7.50 output above", + retired: true, + }, { id: "gpt-6-luna", runtimeAdapter: "codex", diff --git a/packages/quill/packages/charts/src/charts/BarList/StackedBarList.tsx b/packages/quill/packages/charts/src/charts/BarList/StackedBarList.tsx index 769f58f5d362..ef0a67e5bb51 100644 --- a/packages/quill/packages/charts/src/charts/BarList/StackedBarList.tsx +++ b/packages/quill/packages/charts/src/charts/BarList/StackedBarList.tsx @@ -41,6 +41,7 @@ export interface StackedBarListProps { onPointClick?: (data: PointClickData) => void dataAttr?: string onError?: (error: Error, info: React.ErrorInfo) => void + children?: React.ReactNode } const DEFAULT_ROW_HEIGHT = { start: 28, top: 40 } @@ -71,6 +72,7 @@ function StackedBarListInner({ onPointClick, dataAttr, onError, + children, }: StackedBarListProps): React.ReactElement { const labelPosition = config?.labelPosition ?? 'start' const { @@ -135,7 +137,9 @@ function StackedBarListInner({ onPointClick={onPointClick} dataAttr={dataAttr} onError={onError} - /> + > + {children} + ) : null if (labelPosition === 'top') { diff --git a/packages/quill/packages/charts/src/docs/overlays.md b/packages/quill/packages/charts/src/docs/overlays.md index 54f7ce294cd1..0e80a7adcbe5 100644 --- a/packages/quill/packages/charts/src/docs/overlays.md +++ b/packages/quill/packages/charts/src/docs/overlays.md @@ -10,6 +10,8 @@ They compose as children of a chart: ``` +`StackedBarList` also accepts overlay children. It places them in the chart area, which excludes the label and value cells. + Trend lines, moving averages, and confidence bands are not overlays. They are derived `Series` entries (`overlay: true` for the first two), so the legend, the tooltip, and the axis already know about them. diff --git a/posthog/admin/admins/team_admin.py b/posthog/admin/admins/team_admin.py index a1835e0f0bc7..402bc27a4a8a 100644 --- a/posthog/admin/admins/team_admin.py +++ b/posthog/admin/admins/team_admin.py @@ -7,7 +7,7 @@ import dataclasses from datetime import UTC, datetime, timedelta from decimal import Decimal, InvalidOperation -from typing import Any +from typing import Any, cast from urllib import parse from django.conf import settings @@ -36,7 +36,7 @@ from posthog.admin.inlines.team_marketing_analytics_config_inline import TeamMarketingAnalyticsConfigInline from posthog.helpers.impersonation import is_impersonated from posthog.llm.gateway_internal_client import AIGatewayInternalError, AIGatewayNotConfigured, add_credit, get_wallet -from posthog.models import Team +from posthog.models import Team, User from posthog.models.activity_logging.activity_log import ActivityContextBase, ActivityLog, Detail, log_activity from posthog.models.events_retention_config import ( OrganizationEventsRetentionConfig, @@ -839,7 +839,9 @@ def suspend_email_sending_view(self, request, object_id): # The facade row-locks the config while checking + flipping, so concurrent submits can't # both dispatch the customer email + notification. Side effects stay outside that lock. - change = suspend_email_sending(team.pk, reason) + change = suspend_email_sending( + team.pk, reason, user_id=cast(User, request.user).pk, was_impersonated=is_impersonated(request) + ) already_suspended_at = change.previously_suspended_at suspended_at = change.changed_at @@ -1036,7 +1038,13 @@ def set_email_sending_tier_view(self, request: HttpRequest, object_id: str) -> H return redirect(team_url) pinned = request.POST.get("pinned") == "on" - previous_tier = set_email_sending_tier(team.pk, tier=tier, pinned=pinned) + previous_tier = set_email_sending_tier( + team.pk, + tier=tier, + pinned=pinned, + user_id=cast(User, request.user).pk, + was_impersonated=is_impersonated(request), + ) logger.info( "admin_set_email_sending_tier", @@ -1070,7 +1078,9 @@ def recompute_email_sending_tier_view(self, request: HttpRequest, object_id: str # suspend and set-tier actions. ensure_workflows_config(team.pk) try: - decision = recompute_email_sending_tier(team.id) + decision = recompute_email_sending_tier( + team.id, user_id=cast(User, request.user).pk, was_impersonated=is_impersonated(request) + ) except Exception: logger.exception("admin_recompute_email_sending_tier_failed", team_id=team.id) self.message_user(request, "Could not recompute the tier. Check the logs.", level=messages.ERROR) diff --git a/posthog/admin/test_team_admin.py b/posthog/admin/test_team_admin.py index b45e0d71bbff..2723fed59760 100644 --- a/posthog/admin/test_team_admin.py +++ b/posthog/admin/test_team_admin.py @@ -25,7 +25,6 @@ ) from posthog.models import Organization from posthog.models.activity_logging.activity_log import ActivityLog -from posthog.models.team.extensions import get_or_create_team_extension from posthog.models.team.team import Team from posthog.personhog_client.fake_client import FakePersonHogClient from posthog.personhog_client.proto import GetGroupTypeMappingsByProjectIdRequest @@ -680,8 +679,8 @@ def test_suspend_sets_fields_logs_activity_and_notifies(self) -> None: assert self.mock_notification.call_count == 1 def test_suspend_drops_the_tier_to_zero_even_when_pinned(self) -> None: - config = get_or_create_team_extension( - self.team, TeamWorkflowsConfig, defaults={"email_sending_tier": 3, "email_sending_tier_pinned": True} + config, _ = TeamWorkflowsConfig.objects.update_or_create( + team=self.team, defaults={"email_sending_tier": 3, "email_sending_tier_pinned": True} ) response = self.admin.suspend_email_sending_view(self._post({"reason": "spam complaints"}), str(self.team.pk)) @@ -692,6 +691,39 @@ def test_suspend_drops_the_tier_to_zero_even_when_pinned(self) -> None: assert config.email_sending_tier == 0 assert config.email_sending_tier_updated_at is not None + entry = ActivityLog.objects.get(scope="Team", team_id=self.team.id, activity="email_sending_tier_changed") + assert entry.user == self.user + assert (entry.detail or {})["context"] == {"reason": "staff_suspension"} + + @parameterized.expand( + [ + ("tier_change", {"tier": "2"}, [("email_sending_tier", 1, 2)]), + ("pin_only", {"tier": "1", "pinned": "on"}, [("email_sending_tier_pinned", False, True)]), + ("no_change", {"tier": "1"}, None), + ] + ) + def test_set_tier_audits_what_changed( + self, _name: str, form: dict[str, str], expected_changes: list[tuple[str, object, object]] | None + ) -> None: + TeamWorkflowsConfig.objects.update_or_create( + team=self.team, defaults={"email_sending_tier": 1, "email_sending_tier_pinned": False} + ) + + response = self.admin.set_email_sending_tier_view(self._post(form), str(self.team.pk)) + assert response.status_code == 302 + + entries = list( + ActivityLog.objects.filter(scope="Team", team_id=self.team.id, activity="email_sending_tier_changed") + ) + if expected_changes is None: + assert entries == [] + return + assert len(entries) == 1 + assert entries[0].user == self.user + detail = entries[0].detail or {} + assert detail["context"] == {"reason": "staff_set"} + assert [(c["field"], c["before"], c["after"]) for c in detail["changes"]] == expected_changes + def test_tier_actions_render_without_a_nested_form(self) -> None: # The field renders inside the admin's team change form. A nested
    would break the page, # so the actions must submit the surrounding form through formaction instead. @@ -732,7 +764,7 @@ def test_recompute_creates_a_missing_workflows_config(self) -> None: with patch("posthog.admin.admins.team_admin.recompute_email_sending_tier", return_value=None) as mock_recompute: response = self.admin.recompute_email_sending_tier_view(self._post(), str(self.team.pk)) assert response.status_code == 302 - mock_recompute.assert_called_once_with(self.team.id) + mock_recompute.assert_called_once_with(self.team.id, user_id=self.user.pk, was_impersonated=False) assert self._config() is not None def test_suspend_is_idempotent(self) -> None: diff --git a/posthog/api/file_system/file_system.py b/posthog/api/file_system/file_system.py index a5aa5043fdad..e50372d49598 100644 --- a/posthog/api/file_system/file_system.py +++ b/posthog/api/file_system/file_system.py @@ -6,6 +6,7 @@ from functools import cached_property from typing import Any, Optional, cast from uuid import UUID +from zoneinfo import ZoneInfo from django.apps import apps from django.db import connection, transaction @@ -146,6 +147,12 @@ class FileSystemDeleteQuerySerializer(serializers.Serializer): ) +class FileSystemFrequentQuerySerializer(serializers.Serializer): + limit = serializers.IntegerField( + default=8, min_value=1, max_value=20, help_text="Maximum number of frequently viewed items to return." + ) + + class FileSystemSerializer(FileSystemAccessLevelSerializerMixin, serializers.ModelSerializer): last_viewed_at = serializers.DateTimeField(read_only=True, allow_null=True) created_by = UserBasicSerializer(read_only=True, allow_null=True) @@ -367,6 +374,7 @@ class FileSystemViewSet(TeamAndOrgViewSetMixin, viewsets.ModelViewSet): "list", "retrieve", "search", + "frequent", "unfiled", "count", "count_by_path", @@ -883,6 +891,26 @@ def list(self, request: Request, *args: Any, **kwargs: Any) -> Response: ) return response + @validated_request( + query_serializer=FileSystemFrequentQuerySerializer, + responses={200: FileSystemSerializer(many=True)}, + summary="List frequently viewed files", + ) + @action(methods=["GET"], detail=False, pagination_class=None) + def frequent(self, request: Request, **kwargs: object) -> Response: + queryset = FileSystem.objects.filter(surface_q(self.file_system_surface), team_id=self.team.id) + queryset = self._filter_by_access_control(queryset) + items = get_recent_file_system_items( + team_id=self.team.id, + user_id=cast(User, request.user).id, + surface=self.file_system_surface, + limit=request.validated_query_data["limit"], + exclude_types=[*RETIRED_FILE_SYSTEM_TYPES, "folder", "task"], + file_system_queryset=queryset, + frequent_on=timezone.now().astimezone(ZoneInfo(self.team.timezone or "UTC")).date(), + ) + return Response(self.get_serializer(items, many=True).data) + def _created_by_users(self, results: builtins.list[dict[str, Any]]) -> builtins.list[dict[str, Any]]: # Collect user IDs from the "created_by" meta field so the client can render avatars # without a second round-trip. Rows written before `meta` was validated can hold a diff --git a/posthog/api/file_system/test/test_file_system.py b/posthog/api/file_system/test/test_file_system.py index d2677057d07d..d77ba3f9736f 100644 --- a/posthog/api/file_system/test/test_file_system.py +++ b/posthog/api/file_system/test/test_file_system.py @@ -1,5 +1,5 @@ from collections.abc import Callable, Iterable, Mapping -from datetime import UTC +from datetime import UTC, timedelta from typing import Any, TypedDict, cast import pytest @@ -12,6 +12,7 @@ from django.db import connection from django.test import SimpleTestCase from django.test.utils import CaptureQueriesContext +from django.utils import timezone from parameterized import parameterized from rest_framework import status @@ -63,6 +64,37 @@ def setUp(self): self.user.is_staff = True self.user.save() + @time_machine.travel("2026-01-31T12:00:00Z", tick=False) + def test_frequent_files_rank_separate_visit_days_with_recency_decay(self) -> None: + now = timezone.now() + cases = [ + ("one-off", [0], "web"), + ("regular", [1, 2, 3, 4], "web"), + ("older", [10, 20], "web"), + ("expired", [40, 41], "web"), + ("future", [-1, -2], "web"), + ("reloaded", [0, 0, 0], "web"), + ("desktop", [0, 1, 2], "desktop"), + ] + for ref, days, surface in cases: + FileSystem.objects.create(team=self.team, path=f"Docs/{ref}", type="doc", ref=ref, surface=surface) + FileSystemViewLog.objects.create( + team=self.team, + user=self.user, + type="doc", + ref=ref, + surface=surface, + viewed_at=now - timedelta(days=min(days)), + view_days=[(now - timedelta(days=day)).date() for day in days], + ) + url = f"/api/projects/{self.team.id}/file_system/frequent/" + response = self.client.get(url) + self.assertEqual(response.status_code, 200, response.content) + self.assertEqual([item["ref"] for item in response.json()], ["regular", "older"]) + self.assertEqual([item["ref"] for item in self.client.get(url, {"limit": 1}).json()], ["regular"]) + self.client.force_login(self._create_user("different@example.com")) + self.assertEqual(self.client.get(url).json(), []) + def test_list_files_initially_empty(self): """ When no FileSystem objects exist in the DB for the team, the list should be empty. @@ -1499,6 +1531,29 @@ def _create_access_control( def _create_sibling_team(self) -> Team: return Team.objects.create(project=self.project, organization=self.organization, name="Env-2") + @patch("posthoganalytics.feature_enabled", return_value=True) + def test_frequent_files_filter_access_before_selecting_top_items(self, mock_flag: MagicMock) -> None: + assert self.file_b.ref is not None + Dashboard.objects.filter(id=self.file_b.ref).update(created_by=self.other_user) + FileSystem.objects.filter(team=self.team, type="dashboard", ref=self.file_b.ref).update( + created_by=self.other_user + ) + self._create_access_control(resource="dashboard", resource_id=self.file_b.ref, access_level="none") + now = timezone.now() + for entry, days in ((self.file_a, [1, 2]), (self.file_b, [0, 1, 2, 3])): + assert entry.ref is not None + FileSystemViewLog.objects.create( + team=self.team, + user=self.user, + type=entry.type, + ref=entry.ref, + viewed_at=now - timedelta(days=min(days)), + view_days=[(now - timedelta(days=day)).date() for day in days], + ) + response = self.client.get(f"/api/projects/{self.team.id}/file_system/frequent/", {"limit": 1}) + self.assertEqual(response.status_code, 200, response.content) + self.assertEqual([item["ref"] for item in response.json()], [self.file_a.ref]) + @patch("posthoganalytics.feature_enabled", return_value=True) def test_list_excludes_items_with_none_access(self, mock_flag): self._create_access_control(resource="dashboard", resource_id=self.file_b.ref, access_level="none") diff --git a/posthog/api/project.py b/posthog/api/project.py index e52ce6ae7240..19014ebe0f3f 100644 --- a/posthog/api/project.py +++ b/posthog/api/project.py @@ -2079,6 +2079,7 @@ def activity(self, request: request.Request, **kwargs): item_ids=[str(project.pk)], limit=page_params.limit, page=page_params.page, + user=request.user, ) return activity_page_response(activity_page, page_params.limit, page_params.page, request) diff --git a/posthog/api/sharing.py b/posthog/api/sharing.py index f57f9a239cf4..aad37d93db16 100644 --- a/posthog/api/sharing.py +++ b/posthog/api/sharing.py @@ -74,7 +74,7 @@ from products.dashboards.backend.api.dashboard import DashboardSerializer from products.dashboards.backend.models.dashboard import Dashboard from products.exports.backend.api.exports import ExportedAssetCreateSerializer -from products.exports.backend.facade.api import export_limit_context +from products.exports.backend.facade.api import export_limit_context, query_with_reference_lines from products.exports.backend.models.exported_asset import ( EXPORTED_ASSET_PURPOSE_RENDER, EXPORTED_ASSET_PURPOSE_SUBSCRIPTION_DELIVERY, @@ -1168,6 +1168,15 @@ def retrieve(self, request: Request, *args: Any, **kwargs: Any) -> Any: # Add hideExtraDetails to context so that PII related information is not returned to the client insight_context = {**context, "hide_extra_details": state.get("hideExtraDetails", False)} insight_data = InsightSerializer(resource.insight, many=False, context=insight_context).data + if ( + is_export_render + and isinstance(resource, ExportedAsset) + and resource.export_context + and resource.export_context.get("reference_lines") + ): + insight_data["query"] = query_with_reference_lines( + insight_data.get("query") or {}, resource.export_context["reference_lines"] + ) exported_data.update({"insight": insight_data}) exported_data.update({"themes": get_themes_for_team(resource.team)}) exported_data.update({"cohorts": _collect_cohorts_for_sharing([resource.insight], resource.team)}) diff --git a/posthog/api/team/conversations_settings.py b/posthog/api/team/conversations_settings.py index 7be00087f255..ff0cf6019f8b 100644 --- a/posthog/api/team/conversations_settings.py +++ b/posthog/api/team/conversations_settings.py @@ -13,12 +13,23 @@ from posthog.event_usage import report_user_action from posthog.models import Team, User +from products.conversations.backend.models.constants import AUTOMATED_UPDATE_SLACK_CHANNEL_IDS + @extend_schema_field(OpenApiTypes.OBJECT) class ConversationsSettingsField(serializers.JSONField): pass +def validate_automated_update_slack_channel_ids(channel_ids: Any) -> list[str]: + field = serializers.ListField(child=serializers.RegexField(regex=r"\A[CG][A-Z0-9]{6,}\Z")) + validated = field.run_validation([] if channel_ids is None else channel_ids) + unique_channel_ids = list(dict.fromkeys(validated)) + if len(unique_channel_ids) > 20: + raise serializers.ValidationError("Use 20 channels or fewer.") + return unique_channel_ids + + MANAGED_CONVERSATIONS_SETTINGS = ( # Strip widget_public_token from user input - it's auto-generated only "widget_public_token", @@ -216,6 +227,13 @@ def validate_conversations_settings( # null so a clearing PATCH still overwrites the stored list (see # merge_conversations_settings, which only replaces keys present here). value.pop("slack_channel_ids", None) + if AUTOMATED_UPDATE_SLACK_CHANNEL_IDS in value: + try: + value[AUTOMATED_UPDATE_SLACK_CHANNEL_IDS] = validate_automated_update_slack_channel_ids( + value[AUTOMATED_UPDATE_SLACK_CHANNEL_IDS] + ) + except serializers.ValidationError as error: + raise serializers.ValidationError({AUTOMATED_UPDATE_SLACK_CHANNEL_IDS: error.detail}) from error icon_url = value.get("slack_bot_icon_url") if icon_url is not None: if not isinstance(icon_url, str): diff --git a/posthog/api/team/viewsets.py b/posthog/api/team/viewsets.py index 112b861d762b..7b47d43bfdf3 100644 --- a/posthog/api/team/viewsets.py +++ b/posthog/api/team/viewsets.py @@ -433,6 +433,7 @@ def activity(self, request: request.Request, **kwargs): item_ids=[str(team.pk)], limit=page_params.limit, page=page_params.page, + user=request.user, ) return activity_page_response(activity_page, page_params.limit, page_params.page, request) diff --git a/posthog/api/test/test_sharing.py b/posthog/api/test/test_sharing.py index 0c32f2beab0b..b9976f88a795 100644 --- a/posthog/api/test/test_sharing.py +++ b/posthog/api/test/test_sharing.py @@ -1614,6 +1614,24 @@ def _calculated_result(mock_calculate: MagicMock) -> None: timezone="UTC", ) + @parameterized.expand([("render", True), ("shared", False)]) + @patch("posthog.caching.calculate_results.calculate_for_query_based_insight") + @mock_exporter_template + def test_threshold_snapshot_is_only_used_by_the_export_renderer( + self, _name: str, is_renderer: bool, mock_calculate: MagicMock + ) -> None: + self.insight.query = {"kind": "BIVisualizationNode"} + self.insight.save(update_fields=["query"]) + self.exported_asset.export_context = {"reference_lines": [{"label": "Threshold snapshot", "value": 250}]} + self.exported_asset.save(update_fields=["export_context"]) + self._calculated_result(mock_calculate) + url = self._render_url("") if is_renderer else f"/shared/{self.sharing_config.access_token}" + response = self.client.get(url) + assert response.status_code == 200 + assert (b"Threshold snapshot" in response.content) is is_renderer + self.insight.refresh_from_db() + assert "chartSettings" not in self.insight.query + @patch("posthog.caching.calculate_results.calculate_for_query_based_insight") @patch("products.product_analytics.backend.presentation.insight.QueryCache") @mock_exporter_template diff --git a/posthog/api/test/test_team.py b/posthog/api/test/test_team.py index fc93166638d3..ac7f301ac20b 100644 --- a/posthog/api/test/test_team.py +++ b/posthog/api/test/test_team.py @@ -1191,6 +1191,27 @@ def test_is_generating_demo_data(self): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertEqual(response.json(), {"is_generating_demo_data": False}) + @parameterized.expand([("member", False, []), ("staff", True, ["email_sending_tier_changed"])]) + def test_activity_hides_staff_only_rows_from_members( + self, _name: str, is_staff: bool, expected_activities: list[str] + ) -> None: + ActivityLog.objects.create( + team_id=self.team.pk, + organization_id=self.organization.id, + user=User.objects.create(email="staff@posthog.com", is_staff=True), + scope="Team", + item_id=str(self.team.pk), + activity="email_sending_tier_changed", + detail={"name": self.team.name}, + ) + self.user.is_staff = is_staff + self.user.save() + + response = self.client.get(f"/api/environments/{self.team.pk}/activity") + + assert response.status_code == status.HTTP_200_OK, response.json() + assert [item["activity"] for item in response.json()["results"]] == expected_activities + @time_machine.travel("2022-02-08", tick=False) def test_team_float_config_can_be_serialized_to_activity_log(self): # regression test since this isn't true by default @@ -2322,6 +2343,31 @@ def test_conversations_settings_merges_with_existing(self): assert settings["widget_greeting_text"] == "Hello!" assert settings["widget_color"] == "#ff0000" + @parameterized.expand( + [ + ("deduplicates_ids", ["C0123ABCD", "C0123ABCD"], status.HTTP_200_OK, ["C0123ABCD"]), + ("clears_with_null", None, status.HTTP_200_OK, []), + ("rejects_names", ["#support"], status.HTTP_400_BAD_REQUEST, None), + ("rejects_direct_messages", ["D0123ABCD"], status.HTTP_400_BAD_REQUEST, None), + ("rejects_non_list", "C0123ABCD", status.HTTP_400_BAD_REQUEST, None), + ("rejects_non_string", [123], status.HTTP_400_BAD_REQUEST, None), + ("caps_channels", [f"C{i:08d}" for i in range(21)], status.HTTP_400_BAD_REQUEST, None), + ] + ) + def test_automated_update_channel_allowlist_validation( + self, _name, channel_ids, expected_status, expected_channels + ): + response = self.client.patch( + "/api/environments/@current/", + {"conversations_settings": {"automated_update_slack_channel_ids": channel_ids}}, + ) + + assert response.status_code == expected_status + if expected_channels is not None: + assert ( + response.json()["conversations_settings"]["automated_update_slack_channel_ids"] == expected_channels + ) + @parameterized.expand([(["legacy", "list"],), ("legacy string",), (1,)]) def test_conversations_settings_merges_with_legacy_non_object_existing(self, legacy_value): # Rows written before validation required an object/null can hold a stray diff --git a/posthog/email.py b/posthog/email.py index 4a1f0df47af7..b93a8b5c4e5b 100644 --- a/posthog/email.py +++ b/posthog/email.py @@ -15,7 +15,7 @@ from django.core import exceptions, mail from django.core.cache import cache from django.core.mail.backends.smtp import EmailBackend -from django.db import transaction +from django.db import InterfaceError, OperationalError, transaction from django.template.loader import get_template from django.utils import timezone from django.utils.module_loading import import_string @@ -25,7 +25,7 @@ import structlog import css_inline import posthoganalytics -from celery import shared_task +from celery import current_task, shared_task from lxml import html as lxml_html from prometheus_client import Counter @@ -108,7 +108,9 @@ def is_email_available(with_absolute_urls: bool = False) -> bool: "ignore_result": True, "autoretry_for": (Exception,), "max_retries": 3, - "retry_backoff": True, + # Retries wait 5s, 10s and 20s. Without jitter the last attempt cannot come within a second of the first. + "retry_backoff": 5, + "retry_jitter": False, } # Failure rate as an alertable time series. Labelled only by outcome+transport to bound @@ -281,6 +283,12 @@ def __init__(self, message: str, *, code: str, status_code: int | None = None) - self.status_code = status_code +# Errors that a later attempt can fix. A database error can come before the send, when the record is read, or after +# it, when the record is marked sent. A Workflows send is pinned before that write, so its retry replays under the +# same idempotency key. A Customer.io send can go out a second time. +RETRYABLE_HTTP_SEND_ERRORS = (WorkflowsEmailError, OperationalError, InterfaceError) + + def _workflows_error_allows_fallback(err: WorkflowsEmailError) -> bool: """Whether Customer.io can send the email without the risk of a duplicate.""" if err.status_code is None: @@ -506,7 +514,10 @@ def _send_one( workflows_template_id = get_workflows_template_id(template_name, dest, pinned_template_id) if workflows_template_id: try: - return "workflows", _send_one_via_workflows(dest, campaign_key, workflows_template_id, properties) + response = _send_one_via_workflows(dest, campaign_key, workflows_template_id, properties) + # Marking the record sent can still fail, and the retry must then replay this send, not move providers. + _pin_to_workflows(idempotency_key, workflows_template_id) + return "workflows", response except WorkflowsEmailError as err: if pinned_template_id or not _workflows_error_allows_fallback(err): _pin_to_workflows(idempotency_key, workflows_template_id) @@ -529,6 +540,7 @@ def _send_via_http( campaign_key: str, template_name: str, properties: dict, + will_retry: bool = True, ) -> None: """Sends emails using the Customer.io API, or the Workflows transactional email API where the flag matches""" if not is_http_email_service_available(): @@ -536,11 +548,16 @@ def _send_via_http( # A dedicated client, because the global one may never flush before a Celery worker exits. with ph_scoped_capture(event_region=get_instance_region()) as capture: - _send_via_http_batch(capture, to, campaign_key, template_name, properties) + _send_via_http_batch(capture, to, campaign_key, template_name, properties, will_retry) def _send_via_http_batch( - capture: ScopedCapture, to: list[dict[str, str]], campaign_key: str, template_name: str, properties: dict + capture: ScopedCapture, + to: list[dict[str, str]], + campaign_key: str, + template_name: str, + properties: dict, + will_retry: bool = True, ) -> None: sent_count = 0 already_sent_count = 0 # delivered in a prior run — not a failure if the batch later aborts @@ -583,7 +600,8 @@ def _send_via_http_batch( except Exception as err: capture_exception(err) # already logs the traceback via logger.exception # The exception carries no template name, so this event is what lets a dashboard count failures per template. - retrying = isinstance(err, WorkflowsEmailError) + retryable = isinstance(err, RETRYABLE_HTTP_SEND_ERRORS) + retrying = retryable and will_retry if dest: _capture_email_event( capture, @@ -600,9 +618,9 @@ def _send_via_http_batch( # Count every recipient that did not get through (the failing one + any not yet attempted), # so `failed` shares the per-recipient unit with `sent` instead of registering once per batch. EMAIL_SEND_COUNTER.labels(outcome="failed", transport="http").inc(len(to) - already_sent_count - sent_count) - if retrying: - # Workflows may have queued this email. The Celery retry sends the same idempotency key, so Workflows - # returns the queued email instead of sending a second one, and recipients already sent are skipped. + if retryable: + # The retry sends the same idempotency key, so Workflows returns an email it already queued instead of + # sending a second one, and recipients already marked sent are skipped. raise @@ -826,6 +844,16 @@ def sanitize_value(value: Any, *, key: Any = None) -> Any: return {k: sanitize_value(v, key=k) for k, v in properties.items()} +def _send_will_retry(retry_in_worker: bool) -> bool: + """Whether a retryable failure of this send gets another attempt.""" + if retry_in_worker: + return True + task = current_task + if not task or task.name != "posthog.email._send_email" or task.request.is_eager: + return False + return task.request.retries < task.max_retries + + def _send_email_now( campaign_key: str, to: list[dict[str, str]], @@ -846,8 +874,9 @@ def _send_email_now( campaign_key=campaign_key, template_name=template_name, properties=properties or {}, + will_retry=_send_will_retry(retry_in_worker), ) - except WorkflowsEmailError: + except RETRYABLE_HTTP_SEND_ERRORS: if not retry_in_worker: raise # The retry sends the same idempotency key, so Workflows does not send the email twice. diff --git a/posthog/hogql/database/test/__snapshots__/test_database.ambr b/posthog/hogql/database/test/__snapshots__/test_database.ambr index 3077b9a516f5..0908cc96bd17 100644 --- a/posthog/hogql/database/test/__snapshots__/test_database.ambr +++ b/posthog/hogql/database/test/__snapshots__/test_database.ambr @@ -2889,6 +2889,26 @@ "table": null, "type": "json" }, + "brand_sentiment": { + "chain": null, + "fields": null, + "hogql_value": "brand_sentiment", + "id": null, + "name": "brand_sentiment", + "schema_valid": true, + "table": null, + "type": "string" + }, + "brand_sentiment_score": { + "chain": null, + "fields": null, + "hogql_value": "brand_sentiment_score", + "id": null, + "name": "brand_sentiment_score", + "schema_valid": true, + "table": null, + "type": "float" + }, "cost_usd": { "chain": null, "fields": null, @@ -17537,6 +17557,26 @@ "table": null, "type": "json" }, + "brand_sentiment": { + "chain": null, + "fields": null, + "hogql_value": "brand_sentiment", + "id": null, + "name": "brand_sentiment", + "schema_valid": true, + "table": null, + "type": "string" + }, + "brand_sentiment_score": { + "chain": null, + "fields": null, + "hogql_value": "brand_sentiment_score", + "id": null, + "name": "brand_sentiment_score", + "schema_valid": true, + "table": null, + "type": "float" + }, "cost_usd": { "chain": null, "fields": null, diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py index 0b5c6eebd33d..5328e5c6cd84 100644 --- a/posthog/management/commands/start_temporal_worker.py +++ b/posthog/management/commands/start_temporal_worker.py @@ -156,6 +156,10 @@ WORKFLOWS as WEEKLY_DIGEST_WORKFLOWS, ) +from products.aeo.backend.facade.temporal import ( + ACTIVITIES as AEO_BRAND_SENTIMENT_ACTIVITIES, + WORKFLOWS as AEO_BRAND_SENTIMENT_WORKFLOWS, +) from products.alerts.backend.facade.temporal import ( PLATFORM_AI_QUEUE_ACTIVITIES as INSIGHT_PLATFORM_AI_QUEUE_ACTIVITIES, PLATFORM_EVALUATION_ACTIVITIES as INSIGHT_PLATFORM_EVALUATION_ACTIVITIES, @@ -580,9 +584,10 @@ WEEKLY_DIGEST_ACTIVITIES + WA_DIGEST_ACTIVITIES + DATA_CATALOG_DIGEST_ACTIVITIES, ), ( + # AEO brand sentiment rides this queue because its image is the one that carries the ONNX sentiment model. settings.LLMA_EVALS_TASK_QUEUE, - LLM_ANALYTICS_EVAL_WORKFLOWS + LLM_ANALYTICS_TAGGER_WORKFLOWS, - LLM_ANALYTICS_EVAL_ACTIVITIES + LLM_ANALYTICS_TAGGER_ACTIVITIES, + LLM_ANALYTICS_EVAL_WORKFLOWS + LLM_ANALYTICS_TAGGER_WORKFLOWS + AEO_BRAND_SENTIMENT_WORKFLOWS, + LLM_ANALYTICS_EVAL_ACTIVITIES + LLM_ANALYTICS_TAGGER_ACTIVITIES + AEO_BRAND_SENTIMENT_ACTIVITIES, ), ( settings.LLMA_TASK_QUEUE, diff --git a/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt b/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt index 6400428f91da..b2de685e7426 100644 --- a/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt +++ b/posthog/management/migration_analysis/atomic_false_acknowledged_migrations.txt @@ -57,3 +57,4 @@ ee.0065_migrate_dashboard_collaborators conversations.0076_tag_legacy_conversation_attachments posthog.1398_remove_userproductlist_state data_modeling.0047_remove_datawarehousemodelpath_state +posthog.1414_remove_filesystem_rows_for_deleted_insights diff --git a/posthog/migrations/1414_remove_filesystem_rows_for_deleted_insights.py b/posthog/migrations/1414_remove_filesystem_rows_for_deleted_insights.py new file mode 100644 index 000000000000..330f1146bfc1 --- /dev/null +++ b/posthog/migrations/1414_remove_filesystem_rows_for_deleted_insights.py @@ -0,0 +1,87 @@ +import logging + +from django.db import OperationalError, migrations, transaction + +logger = logging.getLogger(__name__) + +ID_CHUNK = 500_000 +MIN_SPLIT = 1_000 +TEAM_TIMEOUTS = ("15s", "60s", "300s") + + +def _run(connection, timeout, fn): + with transaction.atomic(using=connection.alias), connection.cursor() as cur: + cur.execute(f"SET LOCAL statement_timeout = '{timeout}'") + cur.execute("SET LOCAL lock_timeout = '2s'") + return fn(cur) + + +def _teams_in_range(connection, lo, hi, found): + def query(cur): + cur.execute( + "SELECT DISTINCT team_id FROM posthog_dashboarditem WHERE deleted = true AND id > %s AND id <= %s", + [lo, hi], + ) + return [row[0] for row in cur.fetchall()] + + try: + found.update(_run(connection, "60s", query)) + except OperationalError: + if hi - lo <= MIN_SPLIT: + logger.warning("1414: could not scan insight ids %s to %s", lo, hi) + return + mid = (lo + hi) // 2 + _teams_in_range(connection, lo, mid, found) + _teams_in_range(connection, mid, hi, found) + + +def _clean_team(connection, team_id, timeout): + def work(cur): + # The subquery runs inside the DELETE, so an insight restored after team discovery keeps its entry. + for table in ("posthog_filesystem", "posthog_filesystemshortcut"): + cur.execute( + f""" + DELETE FROM {table} + WHERE team_id = %s AND type = 'insight' + AND ref IN (SELECT short_id FROM posthog_dashboarditem WHERE team_id = %s AND deleted = true) + """, + [team_id, team_id], + ) + + _run(connection, timeout, work) + + +def remove_rows(apps, schema_editor): + connection = schema_editor.connection + + with connection.cursor() as cur: + cur.execute("SELECT max(id) FROM posthog_dashboarditem") + max_id = cur.fetchone()[0] or 0 + + team_ids: set[int] = set() + for lo in range(0, max_id, ID_CHUNK): + _teams_in_range(connection, lo, min(lo + ID_CHUNK, max_id), team_ids) + + # A team that times out is retried with a longer timeout, never failed: the cleanup is optional. + pending = sorted(team_ids) + for timeout in TEAM_TIMEOUTS: + still_pending = [] + for team_id in pending: + try: + _clean_team(connection, team_id, timeout) + except OperationalError: + still_pending.append(team_id) + pending = still_pending + if not pending: + break + + for team_id in pending: + logger.warning("1414: skipped team %s", team_id) + + +class Migration(migrations.Migration): + atomic = False + + dependencies = [("posthog", "1413_event_filter_pipeline_override")] + + operations = [migrations.RunPython(remove_rows, migrations.RunPython.noop)] diff --git a/posthog/migrations/max_migration.txt b/posthog/migrations/max_migration.txt index 295058da0cb4..352f862057bc 100644 --- a/posthog/migrations/max_migration.txt +++ b/posthog/migrations/max_migration.txt @@ -1 +1 @@ -1413_event_filter_pipeline_override +1414_remove_filesystem_rows_for_deleted_insights diff --git a/posthog/models/activity_logging/activity_log.py b/posthog/models/activity_logging/activity_log.py index f3f0f635455c..bfe63e3595e2 100644 --- a/posthog/models/activity_logging/activity_log.py +++ b/posthog/models/activity_logging/activity_log.py @@ -681,10 +681,11 @@ def mask_change_values(scope: str, field: str, before: Any, after: Any) -> Maske "allow_staff": True, }, { - # Staff-only email sending suspension flips: the acting staff user must not leak into the - # org activity log. The customer is told via email and in-app notification instead. + # Staff-only email sending controls: the acting staff user must not leak into the org + # activity log. Suspensions and automatic tier moves reach the customer through email and + # in-app notification, and the Reputation tab shows the current tier. "scope": "Team", - "activities": ["email_sending_suspended", "email_sending_unsuspended"], + "activities": ["email_sending_suspended", "email_sending_unsuspended", "email_sending_tier_changed"], "exclude_when": {}, "allow_staff": True, }, @@ -1730,6 +1731,7 @@ def load_activity( item_ids: Optional[list[str]] = None, limit: int = 10, page: int = 1, + user: Union["User", AnonymousUser, None] = None, ) -> ActivityPage: # TODO in follow-up to posthog #8931 selecting specific fields into a return type from this query @@ -1740,6 +1742,9 @@ def load_activity( if item_ids is not None: activity_query = activity_query.filter(item_id__in=item_ids) + if user is not None: + activity_query = apply_activity_visibility_restrictions(activity_query, user) + return get_activity_page(activity_query, limit, page) diff --git a/posthog/models/file_system/file_system.py b/posthog/models/file_system/file_system.py index 7f501ce7071f..bbe8ef1008f4 100644 --- a/posthog/models/file_system/file_system.py +++ b/posthog/models/file_system/file_system.py @@ -1,3 +1,4 @@ +from collections.abc import Sequence from datetime import datetime from typing import Optional @@ -124,6 +125,12 @@ def delete_file(*, team: Team, file_type: str, ref: str, surface: str = DEFAULT_ FileSystemShortcut.objects.filter(surface_q(surface), team=team, type=file_type, ref=ref).delete() +def delete_files(*, team: Team, file_type: str, refs: Sequence[str], surface: str = DEFAULT_SURFACE) -> None: + count, _ = FileSystem.objects.filter(surface_q(surface), team=team, type=file_type, ref__in=refs).delete() + if count > 0: + FileSystemShortcut.objects.filter(surface_q(surface), team=team, type=file_type, ref__in=refs).delete() + + def split_path(path: str) -> list[str]: segments = [] current = "" diff --git a/posthog/models/file_system/file_system_mixin.py b/posthog/models/file_system/file_system_mixin.py index 095cd45f33cb..366482b6405e 100644 --- a/posthog/models/file_system/file_system_mixin.py +++ b/posthog/models/file_system/file_system_mixin.py @@ -35,29 +35,7 @@ def __init_subclass__(cls, **kwargs): @receiver(post_save, sender=cls, weak=False) def _file_system_post_save(sender, instance: "FileSystemSyncMixin", created, **kwargs): - from posthog.models.file_system.file_system import create_or_update_file, delete_file - - fs_data = instance.get_file_system_representation() - try: - team = instance.team # type: ignore - if fs_data.should_delete: - delete_file(team=team, file_type=fs_data.type, ref=fs_data.ref, surface=fs_data.surface) - else: - create_or_update_file( - team=team, - base_folder=fs_data.base_folder, - name=fs_data.name, - file_type=fs_data.type, - ref=fs_data.ref, - href=fs_data.href, - meta=fs_data.meta, - created_at=fs_data.meta.get("created_at") or getattr(instance, "created_at", None), - created_by_id=fs_data.meta.get("created_by") or getattr(instance, "created_by_id", None), - surface=fs_data.surface, - ) - except Exception as e: - # Don't raise exceptions in signals - capture_exception(e, additional_properties=dataclasses.asdict(fs_data)) + instance.sync_file_system() @receiver(post_delete, sender=cls, weak=False) def _file_system_post_delete(sender, instance: "FileSystemSyncMixin", **kwargs): @@ -75,6 +53,31 @@ def _file_system_post_delete(sender, instance: "FileSystemSyncMixin", **kwargs): # Don't raise exceptions in signals capture_exception(e, additional_properties=dataclasses.asdict(fs_data)) + def sync_file_system(self) -> None: + from posthog.models.file_system.file_system import create_or_update_file, delete_file + + fs_data = self.get_file_system_representation() + try: + team = self.team # type: ignore + if fs_data.should_delete: + delete_file(team=team, file_type=fs_data.type, ref=fs_data.ref, surface=fs_data.surface) + else: + create_or_update_file( + team=team, + base_folder=fs_data.base_folder, + name=fs_data.name, + file_type=fs_data.type, + ref=fs_data.ref, + href=fs_data.href, + meta=fs_data.meta, + created_at=fs_data.meta.get("created_at") or getattr(self, "created_at", None), + created_by_id=fs_data.meta.get("created_by") or getattr(self, "created_by_id", None), + surface=fs_data.surface, + ) + except Exception as e: + # Don't raise exceptions in signals + capture_exception(e, additional_properties=dataclasses.asdict(fs_data)) + @classmethod def get_file_system_unfiled(cls, team: "Team", surface: str = DEFAULT_SURFACE) -> QuerySet[Any]: """ diff --git a/posthog/models/file_system/file_system_view_log.py b/posthog/models/file_system/file_system_view_log.py index bf6871d59126..4aa160dc0fb4 100644 --- a/posthog/models/file_system/file_system_view_log.py +++ b/posthog/models/file_system/file_system_view_log.py @@ -1,5 +1,6 @@ from __future__ import annotations +import heapq from collections.abc import Collection, Sequence from dataclasses import is_dataclass from datetime import date, datetime, timedelta @@ -23,6 +24,15 @@ from posthog.models.user import User VIEW_DAYS_KEPT = 35 +FRECENCY_HALF_LIFE_DAYS = 14 + + +def view_days_score(view_days: Sequence[date], today: date) -> float: + return sum( + 0.5 ** ((today - day).days / FRECENCY_HALF_LIFE_DAYS) + for day in set(view_days) + if 0 <= (today - day).days < VIEW_DAYS_KEPT + ) class FileSystemViewLog(UUIDModel): @@ -158,6 +168,7 @@ def get_recent_file_system_items( exclude_types: Optional[Sequence[str]] = ("folder",), file_system_queryset: Optional[QuerySet[FileSystem]] = None, descending: bool = True, + frequent_on: date | None = None, ) -> list[FileSystem]: """Recently-viewed FileSystem rows for a user, newest first by default. @@ -170,9 +181,15 @@ def get_recent_file_system_items( before the ``(type, ref)`` keys are matched. ``descending`` is threaded down to the view-log query so the ``limit`` slice picks the globally oldest/newest views, not a re-sorted page. """ - entries = FileSystem.objects.filter( - surface_q(surface), team_id=team_id, type=OuterRef("type"), ref=OuterRef("ref") - ).exclude(shortcut=True) + entries = ( + ( + file_system_queryset + if frequent_on is not None and file_system_queryset is not None + else FileSystem.objects.all() + ) + .filter(surface_q(surface), team_id=team_id, type=OuterRef("type"), ref=OuterRef("ref")) + .exclude(shortcut=True) + ) log_queryset = recent_view_logs( team_id=team_id, user_id=user_id, @@ -180,9 +197,22 @@ def get_recent_file_system_items( exclude_types=exclude_types, descending=descending, ).filter(Exists(entries)) - if limit is not None: - log_queryset = log_queryset[:limit] - log_rows = list(log_queryset.values_list("type", "ref", "viewed_at")) + if frequent_on is not None: + candidates = ( + (row_type, row_ref, viewed_at, view_days_score(view_days or [], frequent_on)) + for row_type, row_ref, viewed_at, view_days in log_queryset.filter( + viewed_at__gte=timezone.now() - timedelta(days=VIEW_DAYS_KEPT) + ) + .values_list("type", "ref", "viewed_at", "view_days") + .iterator() + if len({day for day in view_days or [] if 0 <= (frequent_on - day).days < VIEW_DAYS_KEPT}) >= 2 + ) + ranked = heapq.nlargest(limit or 20, candidates, key=lambda row: (row[3], row[2], row[0], row[1])) + log_rows = [(row_type, row_ref, viewed_at) for row_type, row_ref, viewed_at, _ in ranked] + else: + if limit is not None: + log_queryset = log_queryset[:limit] + log_rows = list(log_queryset.values_list("type", "ref", "viewed_at")) if not log_rows: return [] diff --git a/posthog/settings/web.py b/posthog/settings/web.py index ff605a97f505..4f3f5e1f03d3 100644 --- a/posthog/settings/web.py +++ b/posthog/settings/web.py @@ -1467,8 +1467,13 @@ def static_varies_origin(headers, path, url): # teams in this allowlist AND with the `aeo-citation-tracking` flag enabled. AEO_CITATION_TEAM_IDS = get_list(get_from_env("AEO_CITATION_TEAM_IDS", "")) AEO_TARGET_DOMAINS = get_list(get_from_env("AEO_TARGET_DOMAINS", "posthog.com")) +# Scores brand sentiment on the evals Temporal queue after each run. Off until that worker carries the model. +AEO_BRAND_SENTIMENT_ENABLED = get_from_env("AEO_BRAND_SENTIMENT_ENABLED", False, type_cast=str_to_bool) AEO_ANTHROPIC_MODEL = get_from_env("AEO_ANTHROPIC_MODEL", "claude-sonnet-5") AEO_OPENAI_MODEL = get_from_env("AEO_OPENAI_MODEL", "gpt-5") +# A full daily run covers the whole prompt set, so the cap only guards against a runaway import. +AEO_MAX_PROMPTS_PER_RUN = get_from_env("AEO_MAX_PROMPTS_PER_RUN", 2000, type_cast=int) +AEO_RUN_CONCURRENCY = get_from_env("AEO_RUN_CONCURRENCY", 16, type_cast=int) EXA_API_KEY = get_from_env("EXA_API_KEY", "") CONTENT_AUTOPILOT_MODEL = get_from_env("CONTENT_AUTOPILOT_MODEL", "claude-sonnet-5") CONTENT_AUTOPILOT_SAFETY_MODEL = get_from_env("CONTENT_AUTOPILOT_SAFETY_MODEL", "claude-haiku-4-5") diff --git a/posthog/tasks/alerts/utils.py b/posthog/tasks/alerts/utils.py index f9ee2576966c..799577da00a6 100644 --- a/posthog/tasks/alerts/utils.py +++ b/posthog/tasks/alerts/utils.py @@ -1,6 +1,7 @@ from collections.abc import Collection from contextlib import ExitStack from datetime import UTC, datetime, timedelta +from typing import Any from zoneinfo import ZoneInfo from django.db.models import Q @@ -448,10 +449,32 @@ def render_alert_insight_chart( ) if not any(destination.destination_type == "slack" for destination in destinations): return None + alert_threshold_lines: list[dict[str, Any]] = [] + for insight_alert in ( + AlertConfiguration.objects.filter(team_id=alert.team_id, insight_id=alert.insight_id, enabled=True) + .select_related("threshold") + .order_by("created_at", "id") + ): + if insight_alert.threshold is None or insight_alert.condition.get("type") != "absolute_value": + continue + threshold_config = insight_alert.threshold.configuration + if threshold_config.get("type") != "absolute": + continue + for side, value in (threshold_config.get("bounds") or {}).items(): + if side in {"upper", "lower"} and value is not None: + alert_threshold_lines.append( + { + "label": f"{insight_alert.name} {side} threshold", + "value": value, + "borderColor": "#d14343", + "displayLabel": False, + } + ) asset, content = exports.render_png_export( team=alert.team, created_by=render_user, insight_id=alert.insight_id, + reference_lines=alert_threshold_lines, # System render: keep it out of the user's export listings and quota. is_system=True, expires_after=datetime.now(UTC) + _INSIGHT_CHART_ASSET_TTL, diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt index a9b80ad025aa..ee44caedfa94 100644 --- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt +++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt @@ -821,8 +821,6 @@ 1 products/warehouse_sources/backend/temporal/data_imports/sources/openweather/settings.py 1 products/warehouse_sources/backend/temporal/data_imports/sources/opinion_stage/opinion_stage.py 1 products/warehouse_sources/backend/temporal/data_imports/sources/opinion_stage/settings.py -1 products/warehouse_sources/backend/temporal/data_imports/sources/opsgenie/opsgenie.py -1 products/warehouse_sources/backend/temporal/data_imports/sources/opsgenie/settings.py 1 products/warehouse_sources/backend/temporal/data_imports/sources/opuswatch/opuswatch.py 1 products/warehouse_sources/backend/temporal/data_imports/sources/orb/orb.py 1 products/warehouse_sources/backend/temporal/data_imports/sources/orb/settings.py diff --git a/posthog/test/test_email.py b/posthog/test/test_email.py index 72b060a57c99..bdd324af7634 100644 --- a/posthog/test/test_email.py +++ b/posthog/test/test_email.py @@ -1,6 +1,7 @@ import hashlib import smtplib import datetime +import contextlib import dataclasses from decimal import Decimal from typing import Any @@ -15,6 +16,7 @@ from django.core import mail from django.core.cache import cache from django.core.exceptions import ImproperlyConfigured +from django.db import OperationalError from django.test import override_settings from django.utils import timezone @@ -309,6 +311,38 @@ def test_send_via_http_failed_metric_counts_undelivered_recipients(self, mock_po after = REGISTRY.get_sample_value("posthog_email_send_total", failed_labels) or 0.0 self.assertEqual(after - before, 2.0) # b failed + c never attempted; a succeeded + @parameterized.expand([("attempts_left", True), ("last_attempt", False)]) + @patch("posthog.email.ph_scoped_capture") + @patch("posthog.email.requests.post") + def test_send_via_http_reraises_a_lost_database_connection_so_celery_retries( + self, _name: str, will_retry: bool, mock_post: MagicMock, mock_scoped_capture: MagicMock + ) -> None: + mock_capture = mock_scoped_capture.return_value.__enter__.return_value + lost_connection = OperationalError("could not translate host name to address") + + with ( + override_instance_config("EMAIL_HOST", "localhost"), + self.settings(CUSTOMER_IO_API_KEY="test-key"), + patch("posthog.email.MessagingRecord.objects.get_or_create", side_effect=lost_connection), + self.assertRaises(OperationalError), + ): + _send_via_http( + to=[{"raw_email": "a@posthog.com"}], + campaign_key="db_blip", + template_name="2fa_enabled", + properties={}, + will_retry=will_retry, + ) + + mock_post.assert_not_called() + failed = [ + c.kwargs for c in mock_capture.call_args_list if c.kwargs.get("event") == "transactional email failed" + ] + self.assertEqual( + [(f["properties"]["error_type"], f["properties"]["retrying"]) for f in failed], + [("OperationalError", will_retry)], + ) + @patch("posthog.email.ph_scoped_capture") @patch("posthog.email.requests.post") def test_send_via_http_success(self, mock_post, mock_scoped_capture) -> None: @@ -683,18 +717,76 @@ def test_retry_after_an_uncertain_workflows_send_stays_on_workflows_while_it_is_ ] self.assertEqual(providers, expected_providers) + @patch("posthog.email.ph_scoped_capture") + @patch("posthog.email.posthoganalytics.feature_enabled") + @patch("posthog.email.requests.post") + def test_retry_after_failing_to_mark_a_workflows_send_replays_it_on_workflows( + self, mock_post: MagicMock, mock_feature_enabled: MagicMock, _mock_capture: MagicMock + ) -> None: + mock_feature_enabled.side_effect = [True, False] + accepted = MagicMock(status_code=202, json=MagicMock(return_value={"message_id": "m1"})) + mock_post.side_effect = [accepted, accepted] + original_save = MessagingRecord.save + failed: list[MessagingRecord] = [] + + def lose_connection_on_first_sent_mark(record: MessagingRecord, *args: Any, **kwargs: Any) -> None: + if record.sent_at and not failed: + failed.append(record) + raise OperationalError("could not translate host name to address") + original_save(record, *args, **kwargs) + + with ( + override_instance_config("EMAIL_HOST", "localhost"), + self.settings( + CUSTOMER_IO_API_KEY="cio-key", + POSTHOG_TRANSACTIONAL_EMAIL_API_KEY="phs_test", + POSTHOG_HOST="https://us.posthog.com", + ), + patch.dict("posthog.email.WORKFLOWS_TEMPLATE_ID_MAP", {"password_reset": "tmpl-1"}), + patch.object(MessagingRecord, "save", autospec=True, side_effect=lose_connection_on_first_sent_mark), + ): + message = EmailMessage( + campaign_key="mark_failed_campaign", + subject="Test subject", + template_name="password_reset", + use_http=True, + ) + message.add_recipient("test@posthog.com") + with self.assertRaises(OperationalError): + message.send(send_async=False, retry=False) + message.send(send_async=False, retry=False) + + # Workflows already queued the first email, so Customer.io sending the retry would deliver it twice. + providers = [ + "workflows" if call.args[0].startswith("https://us.posthog.com") else "customer_io" + for call in mock_post.call_args_list + ] + self.assertEqual(providers, ["workflows", "workflows"]) + + @parameterized.expand( + [ + ("workflows_timeout", None, 1), + ("lost_database_connection", OperationalError("could not translate host name to address"), 0), + ] + ) @patch.object(_send_email, "apply_async") @patch("posthog.email.ph_scoped_capture") @patch("posthog.email.posthoganalytics.feature_enabled", return_value=True) @patch("posthog.email.requests.post", side_effect=requests.ReadTimeout("no response")) - def test_inline_send_hands_an_uncertain_workflows_send_to_a_worker( + def test_inline_send_hands_a_retryable_failure_to_a_worker( self, + _name: str, + record_error: Exception | None, + expected_posts: int, mock_post: MagicMock, _mock_feature_enabled: MagicMock, _mock_capture: MagicMock, mock_apply_async: MagicMock, ) -> None: with ( + patch("posthog.email.MessagingRecord.objects.get_or_create", side_effect=record_error) + if record_error + else contextlib.nullcontext(), override_instance_config("EMAIL_HOST", "localhost"), self.settings( CUSTOMER_IO_API_KEY="cio-key", @@ -709,8 +801,8 @@ def test_inline_send_hands_an_uncertain_workflows_send_to_a_worker( message.add_recipient("test@posthog.com") message.send(send_async=False) - # Eager retries would post once per retry inside the request, each up to the 10s timeout. - self.assertEqual(mock_post.call_count, 1) + # Eager retries would run at once inside the request, with no backoff for the failure to clear. + self.assertEqual(mock_post.call_count, expected_posts) mock_apply_async.assert_called_once() retry_kwargs = mock_apply_async.call_args.kwargs["kwargs"] self.assertEqual(retry_kwargs["campaign_key"], "inline_campaign") diff --git a/products/aeo/README.md b/products/aeo/README.md index 371976408e08..f8d8e72b9a70 100644 --- a/products/aeo/README.md +++ b/products/aeo/README.md @@ -67,7 +67,9 @@ A prompt keeps its id, so its checks, stats and detail view stay in one place. I ### Which prompts a run takes -A run takes at most 50 prompts. It takes one prompt per topic in turn, so every topic is sampled on each run. Within a topic, prompts never checked come first, then the least recently checked, so a set larger than one run rotates. +A run takes the whole prompt set, up to `AEO_MAX_PROMPTS_PER_RUN` (default 2,000), and checks `AEO_RUN_CONCURRENCY` prompts at a time (default 16). +It takes one prompt per topic in turn, so a capped run still samples every topic. +Within a topic, prompts never checked come first, then the least recently checked, so a set larger than one run rotates, and a run cut short by a worker restart resumes on the next dispatch. ## The table @@ -115,7 +117,9 @@ Web analytics' Content autopilot (`/web/content-autopilot`, flag `web-analytics- ## Cost -Roughly \$2–4/day at 50 prompts × 3 engines × 1 run/day: provider web-search fees + tokens (attributed as `$ai_web_search_cost_usd` and `$ai_total_cost_usd` on the `$ai_generation` event) plus Exa at \$5 per 1,000 requests (`cost_usd` on the check event). +Measured in October 2026: about \$0.06 per OpenAI check and \$0.14 per Claude check, provider web-search fees and tokens included (`$ai_web_search_cost_usd` and `$ai_total_cost_usd` on the `$ai_generation` event). +A daily run of 1,000 prompts on both engines is about \$200. +Exa adds \$5 per 1,000 requests (`cost_usd` on the check event). ## Out of scope diff --git a/products/aeo/backend/brand.py b/products/aeo/backend/brand.py new file mode 100644 index 000000000000..87f339740e39 --- /dev/null +++ b/products/aeo/backend/brand.py @@ -0,0 +1,47 @@ +"""How an answer is judged to name the brand. + +The brand names are the registrable names of the target domains (`example` for +`docs.example.co.uk`), so the same list drives the SQL mention filter, the brand +regex the sentiment scorer uses, and the names the frontend builds its HogQL +mention filter from. +""" + +from __future__ import annotations + +import re + +from django.db.models import Q + +import tldextract + +_DOMAIN_PARTS = tldextract.TLDExtract(suffix_list_urls=()) + + +def brand_names(target_domains: list[str]) -> tuple[str, ...]: + """Lower-cased registrable names of the target domains, sorted, without duplicates.""" + return tuple(sorted({(_DOMAIN_PARTS(domain).domain or domain).lower() for domain in target_domains if domain})) + + +def brand_regex(target_domains: list[str]) -> str | None: + """Postgres regex (word-bounded, for `iregex`) that matches any brand name. None without a brand.""" + names = brand_names(target_domains) + if not names: + return None + return r"\m(" + "|".join(re.escape(name) for name in names) + r")\M" + + +def brand_pattern(target_domains: list[str]) -> re.Pattern[str] | None: + """Python equivalent of `brand_regex`, for text already loaded from the database.""" + names = brand_names(target_domains) + if not names: + return None + return re.compile(r"\b(" + "|".join(re.escape(name) for name in names) + r")\b", re.IGNORECASE) + + +def mention_filter(target_domains: list[str]) -> Q: + """Checks whose answer names the brand. A cited answer counts as a mention even when the + text only carries the URL, so the mention rate is never below the citation rate.""" + mention = Q(cited=True) + if regex := brand_regex(target_domains): + mention |= Q(answer_text__iregex=regex) + return mention diff --git a/products/aeo/backend/engines.py b/products/aeo/backend/engines.py index e8706d4eeadb..30392e30886a 100644 --- a/products/aeo/backend/engines.py +++ b/products/aeo/backend/engines.py @@ -13,6 +13,8 @@ from __future__ import annotations +import time +import threading from dataclasses import field from typing import Any, Protocol from urllib.parse import urlparse @@ -97,12 +99,68 @@ class CitationEngine(Protocol): def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck: ... -def _session() -> requests.Session: +class _ThreadSessions: + """One requests.Session per thread. A run calls engines from a thread pool, and + requests does not document Session as safe to share across threads.""" + + def __init__(self, *, trust_env: bool) -> None: + self._local = threading.local() + self._trust_env = trust_env + + def get(self) -> requests.Session: + session: requests.Session | None = getattr(self._local, "session", None) + if session is None: + session = requests.Session() + session.trust_env = self._trust_env + self._local.session = session + return session + + +def _gateway_sessions() -> _ThreadSessions: # trust_env=False keeps in-cluster gateway calls off any egress proxy, and - # the session reuses connections across a run's many sequential calls. - session = requests.Session() - session.trust_env = False - return session + # each thread's session reuses connections across its calls. + return _ThreadSessions(trust_env=False) + + +# Throttling and transient upstream errors. A full run sends many calls in parallel, so +# these get a short bounded retry instead of becoming a failed check straight away. +RETRYABLE_STATUS_CODES = frozenset({429, 500, 502, 503, 504}) +GATEWAY_MAX_ATTEMPTS = 3 +GATEWAY_RETRY_BASE_SECONDS = 5 +GATEWAY_RETRY_MAX_SECONDS = 60 + + +def _retry_delay_seconds(response: requests.Response, attempt: int) -> float: + retry_after = response.headers.get("Retry-After", "") + if retry_after.isdigit(): + return min(float(retry_after), GATEWAY_RETRY_MAX_SECONDS) + return min(GATEWAY_RETRY_BASE_SECONDS * 2**attempt, GATEWAY_RETRY_MAX_SECONDS) + + +class _SharedBackoff: + """A pause every worker thread honours before its next gateway request. One throttled + response pauses the whole run, so the other workers stop adding load while it backs off.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._resume_at = 0.0 + + def pause(self, seconds: float) -> None: + with self._lock: + # A shorter pause from another worker must not cut a longer one short. + self._resume_at = max(self._resume_at, time.monotonic() + seconds) + + def wait(self) -> None: + # Another worker can extend the pause during the sleep, so read the deadline again after each one. + while True: + with self._lock: + remaining = self._resume_at - time.monotonic() + if remaining <= 0: + return + time.sleep(remaining) + + +GATEWAY_BACKOFF = _SharedBackoff() def gateway_post_json( @@ -112,10 +170,17 @@ def gateway_post_json( payload: dict[str, Any], timeout: int = GATEWAY_TIMEOUT_SECONDS, ) -> dict[str, Any]: - """POST and decode JSON; raises requests.RequestException on any failure.""" - response = session.post(url, headers=headers, json=payload, timeout=timeout) - response.raise_for_status() - return response.json() + """POST and decode JSON, retrying throttled and transient errors a bounded number of + times; raises requests.RequestException once the attempts run out or on any other failure.""" + for attempt in range(GATEWAY_MAX_ATTEMPTS): + GATEWAY_BACKOFF.wait() + response = session.post(url, headers=headers, json=payload, timeout=timeout) + if response.status_code in RETRYABLE_STATUS_CODES and attempt < GATEWAY_MAX_ATTEMPTS - 1: + GATEWAY_BACKOFF.pause(_retry_delay_seconds(response, attempt)) + continue + response.raise_for_status() + return response.json() + raise AssertionError("unreachable: the last attempt returns or raises") def _dedupe_urls(urls: list[str]) -> list[str]: @@ -245,13 +310,19 @@ def parse_exa_citations(body: dict[str, Any]) -> ParsedAnswer: ) +def is_target_host(host: str, target_domains: list[str]) -> bool: + """True when the host is one of the target domains or a subdomain of it.""" + host = host.lower() + return any(host == domain or host.endswith(f".{domain}") for domain in (d.lower() for d in target_domains)) + + def is_target_url(url: str, target_domains: list[str]) -> bool: """True when the URL's host is one of the target domains or a subdomain of it.""" try: - host = (urlparse(url).hostname or "").lower() + host = urlparse(url).hostname or "" except ValueError: return False - return any(host == domain or host.endswith(f".{domain}") for domain in (d.lower() for d in target_domains)) + return is_target_host(host, target_domains) def target_position(cited_urls: list[str], target_domains: list[str]) -> int | None: @@ -343,7 +414,7 @@ def __init__(self, model: str | None = None) -> None: if gateway is None: raise ValueError("AI gateway is not configured (AI_GATEWAY_URL / AI_GATEWAY_API_KEY)") self._gateway = gateway - self._session = _session() + self._sessions = _gateway_sessions() def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck: payload = { @@ -369,7 +440,9 @@ def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) **(ai_gateway_headers(trace_id=trace_id, properties=custom_properties) or {}), } try: - body = gateway_post_json(self._session, self._gateway.url.rstrip("/") + "/messages", headers, payload) + body = gateway_post_json( + self._sessions.get(), self._gateway.url.rstrip("/") + "/messages", headers, payload + ) except requests.RequestException as e: return CitationCheck(engine=self.name, model=self.model, trace_id=trace_id, error=_request_error(e)) parsed = parse_anthropic_citations(body) @@ -397,7 +470,7 @@ def __init__(self, model: str | None = None) -> None: if gateway is None: raise ValueError("AI gateway is not configured (AI_GATEWAY_URL / AI_GATEWAY_API_KEY)") self._gateway = gateway - self._session = _session() + self._sessions = _gateway_sessions() def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck: payload = { @@ -411,7 +484,9 @@ def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) **(ai_gateway_headers(trace_id=trace_id, properties=custom_properties) or {}), } try: - body = gateway_post_json(self._session, self._gateway.url.rstrip("/") + "/responses", headers, payload) + body = gateway_post_json( + self._sessions.get(), self._gateway.url.rstrip("/") + "/responses", headers, payload + ) except requests.RequestException as e: return CitationCheck(engine=self.name, model=self.model, trace_id=trace_id, error=_request_error(e)) if (error := openai_response_error(body)) is not None: @@ -440,13 +515,13 @@ class ExaAnswerEngine: model = "exa-answer" def __init__(self) -> None: - self._session = requests.Session() + self._sessions = _ThreadSessions(trust_env=True) def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck: headers = {"x-api-key": settings.EXA_API_KEY} try: body = gateway_post_json( - self._session, + self._sessions.get(), EXA_ANSWER_URL, headers, {"query": prompt, "text": False}, diff --git a/products/aeo/backend/facade/api.py b/products/aeo/backend/facade/api.py index af9ffa63ff4f..3b759994f85d 100644 --- a/products/aeo/backend/facade/api.py +++ b/products/aeo/backend/facade/api.py @@ -6,22 +6,25 @@ from __future__ import annotations +import re import uuid import datetime as dt -from collections import defaultdict +from collections import Counter, defaultdict from typing import Any from zoneinfo import ZoneInfo +from django.conf import settings from django.core.exceptions import ValidationError from django.db import IntegrityError, transaction -from django.db.models import Count, Max, Min, Q +from django.db.models import Count, Max, Min, Q, QuerySet from django.db.models.functions import TruncDate from django.utils import timezone from posthog.models.team import Team from products.aeo.backend import gaps -from products.aeo.backend.engines import MAX_PROMPT_LENGTH +from products.aeo.backend.brand import brand_names, mention_filter +from products.aeo.backend.engines import MAX_PROMPT_LENGTH, top_domains from products.aeo.backend.facade.contracts import ( PROMPT_TOPIC_MAX_LENGTH, CitationGap, @@ -31,10 +34,13 @@ PromptCheck, PromptDailyEngineStats, PromptEngineCheck, + PromptEngineDomainShare, + PromptEngineSentiment, PromptEngineStats, PromptImport, PromptImportResult, PromptNotFoundError, + PromptSearchWord, PromptStats, PromptTextConflictError, PromptTopic, @@ -182,12 +188,17 @@ def list_prompt_checks( def get_prompt_stats(team_id: int, prompt_id: str, *, timezone_name: str, days: int) -> PromptStats: - """Totals per engine over the prompt's whole history, plus per-day counts for the last `days` days + """Totals per engine over the prompt's whole history, plus per-day counts, engine consensus on + the cited domains, brand sentiment and the words of the web searches over the last `days` days in the team's timezone.""" prompt = _get_prompt_row(team_id, prompt_id) + target_domains: list[str] = settings.AEO_TARGET_DOMAINS checks = AEOCitationCheck.objects.for_team(team_id).filter(prompt_id=prompt.id) + # `mentioned` filters on the `cited` column, and an annotation named `cited` would shadow that + # column for every later annotation in the same call, so it must come first. counts = { "checks": Count("id"), + "mentioned": Count("id", filter=Q(check_failed=False) & mention_filter(target_domains)), "cited": Count("id", filter=Q(cited=True)), "failed": Count("id", filter=Q(check_failed=True)), } @@ -197,6 +208,7 @@ def get_prompt_stats(team_id: int, prompt_id: str, *, timezone_name: str, days: engine=row["engine"], checks=row["checks"], cited=row["cited"], + mentioned=row["mentioned"], failed=row["failed"], first_checked_at=row["first_checked_at"], last_checked_at=row["last_checked_at"], @@ -210,26 +222,117 @@ def get_prompt_stats(team_id: int, prompt_id: str, *, timezone_name: str, days: daily_to = timezone.now().astimezone(tz).date() daily_from = daily_to - dt.timedelta(days=days - 1) window_start = dt.datetime.combine(daily_from, dt.time.min, tzinfo=tz) + in_window = checks.filter(created_at__gte=window_start) daily = tuple( PromptDailyEngineStats( - date=row["date"], engine=row["engine"], checks=row["checks"], cited=row["cited"], failed=row["failed"] + date=row["date"], + engine=row["engine"], + checks=row["checks"], + cited=row["cited"], + mentioned=row["mentioned"], + failed=row["failed"], ) - for row in checks.filter(created_at__gte=window_start) - .annotate(date=TruncDate("created_at", tzinfo=tz)) + for row in in_window.annotate(date=TruncDate("created_at", tzinfo=tz)) .values("date", "engine") .annotate(**counts) .order_by("date", "engine") ) + answered = in_window.filter(check_failed=False) + sentiment = tuple( + PromptEngineSentiment( + engine=row["engine"], + positive=row["positive"], + neutral=row["neutral"], + negative=row["negative"], + unscored=row["unscored"], + ) + for row in answered.values("engine") + .annotate( + positive=Count("id", filter=Q(brand_sentiment="positive")), + neutral=Count("id", filter=Q(brand_sentiment="neutral")), + negative=Count("id", filter=Q(brand_sentiment="negative")), + unscored=Count("id", filter=Q(brand_sentiment__isnull=True) & mention_filter(target_domains)), + ) + .order_by("engine") + ) + report = _PromptWindowReport.from_checks(answered) return PromptStats( total_checks=sum(engine.checks for engine in engines), + brand_names=brand_names(target_domains), engines=engines, daily_from=daily_from, daily_to=daily_to, daily=daily, + consensus=report.consensus(), + sentiment=sentiment, + top_words=report.top_words(), ) +MAX_CONSENSUS_DOMAINS = 10 +MAX_STATS_WORDS = 20 +MIN_SEARCH_WORD_LENGTH = 3 +_WORD = re.compile(r"[^\W_]+", re.UNICODE) +SEARCH_STOP_WORDS = frozenset( + """a an and are as at be best by can do does for from how i in is it its my of on or that the this to top vs + what which who with without your you""".split() +) + + +class _PromptWindowReport: + """Cited domains and web searches across one prompt's answered checks in the window. + + Counts are per check: a domain cited twice in one answer counts once, so every number reads as + "how many answers", the same unit as the citation rate.""" + + def __init__(self) -> None: + self._domain_checks: Counter[str] = Counter() + self._engine_checks: Counter[str] = Counter() + self._engine_domain_checks: Counter[tuple[str, str]] = Counter() + self._queries: Counter[str] = Counter() + + @classmethod + def from_checks(cls, answered: QuerySet[AEOCitationCheck]) -> _PromptWindowReport: + report = cls() + for row in answered.values("engine", "cited_urls", "search_queries"): + report.add(row["engine"], _strings(row["cited_urls"]), _strings(row["search_queries"])) + return report + + def add(self, engine: str, cited_urls: tuple[str, ...], search_queries: tuple[str, ...]) -> None: + self._engine_checks[engine] += 1 + domains = top_domains(list(cited_urls)) + self._domain_checks.update(domains) + self._engine_domain_checks.update((engine, domain) for domain in domains) + self._queries.update(key for key in (" ".join(query.casefold().split()) for query in search_queries) if key) + + def consensus(self) -> tuple[PromptEngineDomainShare, ...]: + domains = [domain for domain, _ in self._ranked(self._domain_checks, MAX_CONSENSUS_DOMAINS)] + return tuple( + PromptEngineDomainShare( + engine=engine, + domain=domain, + checks_citing=self._engine_domain_checks[(engine, domain)], + checks=checks, + ) + for engine, checks in sorted(self._engine_checks.items()) + for domain in domains + ) + + def top_words(self) -> tuple[PromptSearchWord, ...]: + words: Counter[str] = Counter() + for key, count in self._queries.items(): + for word in set(_WORD.findall(key)): + if len(word) >= MIN_SEARCH_WORD_LENGTH and word not in SEARCH_STOP_WORDS: + words[word] += count + return tuple(PromptSearchWord(word=word, count=count) for word, count in self._ranked(words, MAX_STATS_WORDS)) + + @staticmethod + def _ranked(counter: Counter[str], limit: int) -> list[tuple[str, int]]: + # Ties break on the value, so the order is stable between two requests. + return sorted(counter.items(), key=lambda item: (-item[1], item[0]))[:limit] + + def import_prompts(team_id: int, prompts: list[PromptImport]) -> PromptImportResult: """Add prompts to the team's set, or update the topic and evidence of prompts that already exist. Existing prompts keep their active state.""" @@ -373,6 +476,8 @@ def _text_conflict(*, archived: bool) -> PromptTextConflictError: "search_queries", "answer_text", "prompt_version", + "brand_sentiment", + "brand_sentiment_score", ) @@ -397,6 +502,8 @@ def _to_prompt_check(row: dict[str, Any]) -> PromptCheck: search_queries=_strings(row["search_queries"]), answer_text=row["answer_text"], prompt_version=row["prompt_version"], + brand_sentiment=row["brand_sentiment"], + brand_sentiment_score=row["brand_sentiment_score"], ) diff --git a/products/aeo/backend/facade/contracts.py b/products/aeo/backend/facade/contracts.py index 21dff934d5de..7bf4131837ef 100644 --- a/products/aeo/backend/facade/contracts.py +++ b/products/aeo/backend/facade/contracts.py @@ -114,6 +114,9 @@ class PromptCheck: search_queries: tuple[str, ...] answer_text: str | None prompt_version: int | None + # Sentiment of the sentences that name the brand. None until scored, and when the brand is not named. + brand_sentiment: str | None = None + brand_sentiment_score: float | None = None @frozen @@ -121,6 +124,8 @@ class PromptEngineStats: engine: str checks: int cited: int + # Answers that name the brand, cited ones included: the same definition the citation gaps use. + mentioned: int failed: int first_checked_at: dt.datetime last_checked_at: dt.datetime @@ -130,6 +135,11 @@ def citation_rate(self) -> float | None: answered = self.checks - self.failed return self.cited / answered if answered else None + @property + def mention_rate(self) -> float | None: + answered = self.checks - self.failed + return self.mentioned / answered if answered else None + @frozen class PromptDailyEngineStats: @@ -137,16 +147,50 @@ class PromptDailyEngineStats: engine: str checks: int cited: int + mentioned: int failed: int +@frozen +class PromptEngineDomainShare: + """How many of one engine's answered checks in the window cited a domain.""" + + engine: str + domain: str + checks_citing: int + checks: int + + +@frozen +class PromptEngineSentiment: + """Brand sentiment of one engine's answered checks in the window. `unscored` answers name the + brand but have no sentiment yet.""" + + engine: str + positive: int + neutral: int + negative: int + unscored: int + + +@frozen +class PromptSearchWord: + word: str + count: int + + @frozen class PromptStats: total_checks: int + # The names an answer must contain to count as a mention, so a client can filter answers the same way. + brand_names: tuple[str, ...] engines: tuple[PromptEngineStats, ...] daily_from: dt.date daily_to: dt.date daily: tuple[PromptDailyEngineStats, ...] + consensus: tuple[PromptEngineDomainShare, ...] + sentiment: tuple[PromptEngineSentiment, ...] + top_words: tuple[PromptSearchWord, ...] @frozen diff --git a/products/aeo/backend/facade/hogql.py b/products/aeo/backend/facade/hogql.py index d5e1ad64fa75..6d2c1a705eb1 100644 --- a/products/aeo/backend/facade/hogql.py +++ b/products/aeo/backend/facade/hogql.py @@ -88,6 +88,16 @@ "top_cited_domains": StringJSONDatabaseField( name="top_cited_domains", description="JSON array of distinct hosts across the cited URLs." ), + "brand_sentiment": StringDatabaseField( + name="brand_sentiment", + nullable=True, + description="Sentiment of the sentences that name the brand: 'positive', 'neutral' or 'negative'. Null until scored, and when the answer never names the brand.", + ), + "brand_sentiment_score": FloatDatabaseField( + name="brand_sentiment_score", + nullable=True, + description="Classifier confidence (0-1) in brand_sentiment.", + ), "cost_usd": FloatDatabaseField( name="cost_usd", nullable=True, description="Engine-reported cost, where the engine reports one." ), diff --git a/products/aeo/backend/facade/temporal.py b/products/aeo/backend/facade/temporal.py new file mode 100644 index 000000000000..efff02307d98 --- /dev/null +++ b/products/aeo/backend/facade/temporal.py @@ -0,0 +1,6 @@ +"""Temporal workflows and activities this product registers on the evals queue, which is the +only worker image that carries the sentiment model.""" + +from products.aeo.backend.temporal.brand_sentiment import ACTIVITIES, WORKFLOWS + +__all__ = ["ACTIVITIES", "WORKFLOWS"] diff --git a/products/aeo/backend/gaps.py b/products/aeo/backend/gaps.py index 5487b6260d57..0c51e5c12b35 100644 --- a/products/aeo/backend/gaps.py +++ b/products/aeo/backend/gaps.py @@ -1,6 +1,5 @@ from __future__ import annotations -import re import datetime as dt from collections import Counter, defaultdict from typing import Any @@ -8,8 +7,7 @@ from django.conf import settings from django.db.models import BooleanField, Case, F, Q, Value, When -import tldextract - +from products.aeo.backend.brand import mention_filter from products.aeo.backend.engines import is_target_url, top_domains from products.aeo.backend.facade.contracts import CitationGap, EngineAnswer from products.aeo.backend.models import AEOCitationCheck, AEOPromptVersion @@ -17,20 +15,12 @@ MAX_GAP_URLS = 10 MAX_GAP_QUERIES = 10 -_DOMAIN_PARTS = tldextract.TLDExtract(suffix_list_urls=()) def _ranked(counter: Counter[str], limit: int) -> tuple[str, ...]: return tuple(value for value, _ in counter.most_common(limit)) -def _brand_regex(target_domains: list[str]) -> str | None: - names = {(_DOMAIN_PARTS(domain).domain or domain).lower() for domain in target_domains if domain} - if not names: - return None - return r"\m(" + "|".join(re.escape(name) for name in sorted(names)) + r")\M" - - def _gap_keys(first_hashes: dict[str, str], current_hashes: dict[str, str]) -> dict[str, str]: """Key each prompt's gap by its first version's hash, so a text edit keeps the key. @@ -60,12 +50,13 @@ def list_citation_gaps(team_id: int, *, since: dt.datetime) -> list[CitationGap] .filter(created_at__gte=since, check_failed=False, prompt__active=True) .filter(current_version) ) - mention = Q(cited=True) - if brand := _brand_regex(target_domains): - mention |= Q(answer_text__iregex=brand) rows = ( checks_in_window.annotate( - mentioned=Case(When(mention, then=Value(True)), default=Value(False), output_field=BooleanField()) + mentioned=Case( + When(mention_filter(target_domains), then=Value(True)), + default=Value(False), + output_field=BooleanField(), + ) ) .order_by("-created_at") .values( diff --git a/products/aeo/backend/management/commands/run_aeo_citation_checks.py b/products/aeo/backend/management/commands/run_aeo_citation_checks.py index 39b803d44a15..17b53e44245b 100644 --- a/products/aeo/backend/management/commands/run_aeo_citation_checks.py +++ b/products/aeo/backend/management/commands/run_aeo_citation_checks.py @@ -25,7 +25,7 @@ class Command(BaseCommand): def add_arguments(self, parser: CommandParser) -> None: parser.add_argument("--team-id", type=int, required=True) - parser.add_argument("--limit", type=int, help="Max prompts this run (default 50).") + parser.add_argument("--limit", type=int, help="Max prompts this run (default AEO_MAX_PROMPTS_PER_RUN).") parser.add_argument( "--engines", type=str, diff --git a/products/aeo/backend/migrations/0007_citation_check_brand_sentiment.py b/products/aeo/backend/migrations/0007_citation_check_brand_sentiment.py new file mode 100644 index 000000000000..3df58b8408f8 --- /dev/null +++ b/products/aeo/backend/migrations/0007_citation_check_brand_sentiment.py @@ -0,0 +1,31 @@ +# Generated by Django 5.2.18 on 2026-10-23 23:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("aeo", "0006_backfill_prompt_versions"), + ] + + operations = [ + migrations.AddField( + model_name="aeocitationcheck", + name="brand_sentiment", + field=models.CharField( + blank=True, + help_text="Sentiment of the sentences that name the brand: positive, neutral or negative. Null until scored, and when the answer never names the brand.", + max_length=16, + null=True, + ), + ), + migrations.AddField( + model_name="aeocitationcheck", + name="brand_sentiment_score", + field=models.FloatField( + blank=True, + help_text="Classifier confidence (0-1) in `brand_sentiment`, averaged over the brand sentences.", + null=True, + ), + ), + ] diff --git a/products/aeo/backend/migrations/max_migration.txt b/products/aeo/backend/migrations/max_migration.txt index a1727f32d961..637c4c4a811e 100644 --- a/products/aeo/backend/migrations/max_migration.txt +++ b/products/aeo/backend/migrations/max_migration.txt @@ -1 +1 @@ -0006_backfill_prompt_versions +0007_citation_check_brand_sentiment diff --git a/products/aeo/backend/models.py b/products/aeo/backend/models.py index 2e1c729af35d..ead33a5ce13e 100644 --- a/products/aeo/backend/models.py +++ b/products/aeo/backend/models.py @@ -188,6 +188,18 @@ class AEOCitationCheck(TeamScopedRootMixin, CreatedMetaFields, UUIDTModel): target_urls = models.JSONField(default=list, blank=True, help_text="Cited URLs on a target domain.") top_cited_domains = models.JSONField(default=list, blank=True, help_text="Distinct hosts across the cited URLs.") + brand_sentiment = models.CharField( + max_length=16, + null=True, + blank=True, + help_text="Sentiment of the sentences that name the brand: positive, neutral or negative. Null until scored, and when the answer never names the brand.", + ) + brand_sentiment_score = models.FloatField( + null=True, + blank=True, + help_text="Classifier confidence (0-1) in `brand_sentiment`, averaged over the brand sentences.", + ) + cost_usd = models.FloatField(null=True, blank=True, help_text="Engine-reported cost, where the engine reports one.") gateway_trace_id = models.CharField( max_length=64, diff --git a/products/aeo/backend/presentation/views.py b/products/aeo/backend/presentation/views.py index b11ecf0e2371..5b88e8956031 100644 --- a/products/aeo/backend/presentation/views.py +++ b/products/aeo/backend/presentation/views.py @@ -16,6 +16,8 @@ from posthog.permissions import posthog_feature_flag_enabled from products.aeo.backend.facade.api import ( + MAX_CONSENSUS_DOMAINS, + MAX_STATS_WORDS, archive_prompt, get_prompt, get_prompt_stats, @@ -35,10 +37,13 @@ PromptCheck, PromptDailyEngineStats, PromptEngineCheck, + PromptEngineDomainShare, + PromptEngineSentiment, PromptEngineStats, PromptImport, PromptImportResult, PromptNotFoundError, + PromptSearchWord, PromptStats, PromptTextConflictError, PromptTopic, @@ -171,6 +176,12 @@ class Meta: "prompt_version": { "help_text": "Version of the prompt that ran. Null for checks recorded before prompts had versions." }, + "brand_sentiment": { + "help_text": "Sentiment of the sentences that name the brand: 'positive', 'neutral' or 'negative'. Null until scored, and when the answer never names the brand." + }, + "brand_sentiment_score": { + "help_text": "Classifier confidence (0-1) in 'brand_sentiment'. Null when 'brand_sentiment' is null." + }, } @@ -187,12 +198,23 @@ class AEOPromptChecksQuerySerializer(serializers.Serializer): ) +MENTIONED_HELP = ( + "Checks whose answer names the brand (the registrable name of a target domain, case ignored). " + "A cited check always counts as mentioned." +) + + class AEOPromptEngineStatsSerializer(DataclassSerializer): citation_rate = serializers.FloatField( read_only=True, allow_null=True, help_text="Share (0-1) of answered checks that cited a target domain. Failed checks are left out. Null when no check was answered.", ) + mention_rate = serializers.FloatField( + read_only=True, + allow_null=True, + help_text="Share (0-1) of answered checks whose answer names the brand. Failed checks are left out. Null when no check was answered.", + ) class Meta: dataclass = PromptEngineStats @@ -201,6 +223,7 @@ class Meta: "engine": {"help_text": ENGINE_HELP}, "checks": {"help_text": "Checks of the prompt on this engine, failed ones included."}, "cited": {"help_text": "Checks whose answer cited a target domain."}, + "mentioned": {"help_text": MENTIONED_HELP}, "failed": {"help_text": "Checks where the engine did not answer."}, "first_checked_at": {"help_text": "When the engine first ran the prompt."}, "last_checked_at": {"help_text": "When the engine last ran the prompt."}, @@ -216,10 +239,50 @@ class Meta: "engine": {"help_text": ENGINE_HELP}, "checks": {"help_text": "Checks on this engine that day, failed ones included."}, "cited": {"help_text": "Checks that day whose answer cited a target domain."}, + "mentioned": {"help_text": "Checks that day whose answer names the brand. " + MENTIONED_HELP}, "failed": {"help_text": "Checks that day where the engine did not answer."}, } +class AEOPromptEngineDomainShareSerializer(DataclassSerializer): + class Meta: + dataclass = PromptEngineDomainShare + ref_name = "AEOPromptEngineDomainShare" + extra_kwargs = { + "engine": {"help_text": ENGINE_HELP}, + "domain": {"help_text": "One of the most cited domains across all engines in the window."}, + "checks_citing": { + "help_text": f"This engine's answered checks in the last {STATS_DAYS} days that cited the domain." + }, + "checks": {"help_text": f"This engine's answered checks in the last {STATS_DAYS} days."}, + } + + +class AEOPromptEngineSentimentSerializer(DataclassSerializer): + class Meta: + dataclass = PromptEngineSentiment + ref_name = "AEOPromptEngineSentiment" + extra_kwargs = { + "engine": {"help_text": ENGINE_HELP}, + "positive": {"help_text": f"Answers in the last {STATS_DAYS} days whose brand sentences read positive."}, + "neutral": {"help_text": f"Answers in the last {STATS_DAYS} days whose brand sentences read neutral."}, + "negative": {"help_text": f"Answers in the last {STATS_DAYS} days whose brand sentences read negative."}, + "unscored": { + "help_text": f"Answers in the last {STATS_DAYS} days that name the brand but have no sentiment yet." + }, + } + + +class AEOPromptSearchWordSerializer(DataclassSerializer): + class Meta: + dataclass = PromptSearchWord + ref_name = "AEOPromptSearchWord" + extra_kwargs = { + "word": {"help_text": "A word from the web searches, lower-cased, common English words left out."}, + "count": {"help_text": "How many searches used the word, weighted by how often each search ran."}, + } + + class AEOPromptStatsSerializer(DataclassSerializer): engines = AEOPromptEngineStatsSerializer( many=True, help_text="Totals per answer engine over the prompt's whole history." @@ -228,12 +291,27 @@ class AEOPromptStatsSerializer(DataclassSerializer): many=True, help_text=f"Counts per day and engine from 'daily_from' to 'daily_to' ({STATS_DAYS} days). Days and engines without a check are left out.", ) + consensus = AEOPromptEngineDomainShareSerializer( + many=True, + help_text=f"One row per engine and per domain for the {MAX_CONSENSUS_DOMAINS} most cited domains of the last {STATS_DAYS} days: how many of the engine's answers cited the domain. Shows which sources the engines agree on.", + ) + sentiment = AEOPromptEngineSentimentSerializer( + many=True, + help_text=f"Brand sentiment of the answers in the last {STATS_DAYS} days, one row per engine. Only the sentences that name the brand are scored.", + ) + top_words = AEOPromptSearchWordSerializer( + many=True, + help_text=f"Most used words across the web searches the engines ran in the last {STATS_DAYS} days, at most {MAX_STATS_WORDS}. Only engines that report their searches contribute.", + ) class Meta: dataclass = PromptStats ref_name = "AEOPromptStats" extra_kwargs = { "total_checks": {"help_text": "Checks of the prompt across all engines, failed ones included."}, + "brand_names": { + "help_text": "Lower-cased brand names, derived from the project's target domains. An answer that contains one of them as a word counts as a mention." + }, "daily_from": {"help_text": "First day of the daily series, in the project's timezone."}, "daily_to": {"help_text": "Last day of the daily series (today), in the project's timezone."}, } @@ -512,7 +590,7 @@ def checks(self, request: ValidatedRequest, pk: str, **kwargs: Any) -> Response: @validated_request( operation_id="aeo_prompts_stats", summary="Get an AEO prompt's citation stats", - description=f"Gets citation totals per answer engine for one prompt, plus per-day counts for the last {STATS_DAYS} days in the project's timezone.", + description=f"Gets citation and brand-mention totals per answer engine for one prompt, plus, for the last {STATS_DAYS} days in the project's timezone, per-day counts, which engines cite which domains, brand sentiment per engine, and the words of the web searches the engines ran.", responses={200: OpenApiResponse(response=AEOPromptStatsSerializer)}, tags=["web_analytics"], ) diff --git a/products/aeo/backend/runner.py b/products/aeo/backend/runner.py index 414116d6fc89..c62462bc3d12 100644 --- a/products/aeo/backend/runner.py +++ b/products/aeo/backend/runner.py @@ -15,6 +15,7 @@ from __future__ import annotations import uuid +from concurrent.futures import ThreadPoolExecutor, as_completed from typing import Any, Optional from django.conf import settings @@ -24,17 +25,17 @@ import structlog +from posthog.dataclasses import frozen from posthog.models.scoping import team_scope from posthog.models.team import Team from products.aeo.backend.engines import CitationEngine, available_engines, build_check_fields from products.aeo.backend.facade.contracts import CitationRunSummary from products.aeo.backend.models import AEOCitationCheck, AEOPrompt +from products.aeo.backend.temporal.brand_sentiment import start_brand_sentiment_workflow logger = structlog.get_logger(__name__) -DEFAULT_MAX_PROMPTS_PER_RUN = 50 - def run_citation_checks( team: Team, @@ -54,7 +55,7 @@ def run_citation_checks( logger.warning("aeo_citation_run_no_engines", team_id=team.id) return _empty_summary(team, "no engines configured (AI_GATEWAY_URL/AI_GATEWAY_API_KEY, EXA_API_KEY)"), [] - prompts = select_prompts(team, DEFAULT_MAX_PROMPTS_PER_RUN if limit is None else limit) + prompts = select_prompts(team, settings.AEO_MAX_PROMPTS_PER_RUN if limit is None else limit) if not prompts: logger.warning("aeo_citation_run_no_prompts", team_id=team.id) return _empty_summary(team, "no active prompts — run `seed_aeo_prompts` first"), [] @@ -68,59 +69,39 @@ def run_citation_checks( rows_written = 0 write_failures = 0 - for prompt in prompts: - prompt_rows: list[AEOCitationCheck] = [] - for engine in engines: - trace_id = str(uuid.uuid4()) - check = engine.run( - prompt.prompt, - trace_id=trace_id, - custom_properties={ - "aeo_prompt_id": str(prompt.id), - "aeo_run_id": run_id, - "aeo_prompt_source": prompt.prompt_source, - # No $ai_ prefix, so the gateway keeps this on the $ai_generation - # event, which lets usage attribute spend back to the checked team. - "team_id": str(team.id), - }, - ) - if check.error is not None: - engine_failures += 1 - logger.warning( - "aeo_citation_check_failed", - team_id=team.id, - engine=engine.name, - prompt_id=str(prompt.id), - error=check.error, - ) - fields = { - **build_check_fields( - check=check, - run_id=run_id, - prompt_id=str(prompt.id), - prompt_text=prompt.prompt, - prompt_source=prompt.prompt_source, - prompt_hash=prompt.prompt_hash, - target_domains=target_domains, - ), - "prompt_topic": prompt.topic, - "prompt_version": prompt.current_version, - } - prompt_rows.append(AEOCitationCheck(team_id=canonical_team_id, **fields)) - checks.append(fields) - - if record and prompt_rows: - # Write per prompt so a mid-run crash keeps the completed checks. + # Engine calls are network-bound and run in worker threads. Database writes stay on + # this thread, so the workers never open their own Django connections. + pool = ThreadPoolExecutor(max_workers=max(1, settings.AEO_RUN_CONCURRENCY)) + futures = {pool.submit(_check_prompt, team, run_id, engines, target_domains, prompt): prompt for prompt in prompts} + try: + for done, future in enumerate(as_completed(futures), start=1): try: - # The fail-closed manager refuses writes without a team scope, and none of the callers - # (the Celery task, the management command) run inside one. - with team_scope(canonical_team_id): - AEOCitationCheck.objects.bulk_create(prompt_rows) - rows_written += len(prompt_rows) - AEOPrompt.objects.for_team(team.id).filter(id=prompt.id).update(last_checked_at=timezone.now()) + result = future.result() except Exception as e: - write_failures += len(prompt_rows) - logger.exception("aeo_citation_write_failed", team_id=team.id, run_id=run_id, error=str(e)[:300]) + # One prompt's unexpected engine error must not end a run that takes hours. + engine_failures += len(engines) + logger.exception( + "aeo_citation_prompt_crashed", + team_id=team.id, + prompt_id=str(futures[future].id), + error=str(e)[:300], + ) + continue + engine_failures += result.failures + checks.extend(result.fields_per_engine) + if record and result.fields_per_engine: + if _record_prompt_checks(team, canonical_team_id, run_id, result.prompt, result.fields_per_engine): + rows_written += len(result.fields_per_engine) + else: + write_failures += len(result.fields_per_engine) + if done % 100 == 0: + logger.info("aeo_citation_run_progress", team_id=team.id, run_id=run_id, done=done, total=len(prompts)) + except BaseException: + # A Celery time limit or a worker shutdown lands here. Drop the queued prompts so they + # make no more paid calls whose results nobody records. + pool.shutdown(wait=False, cancel_futures=True) + raise + pool.shutdown(wait=True) summary = CitationRunSummary( team_id=team.id, @@ -144,9 +125,87 @@ def run_citation_checks( rows_written=summary.rows_written, write_failures=summary.write_failures, ) + if record and rows_written and settings.AEO_BRAND_SENTIMENT_ENABLED: + start_brand_sentiment_workflow(canonical_team_id, run_id) return summary, checks +@frozen +class PromptCheckResult: + """One prompt's checks across every engine, before they are written.""" + + prompt: AEOPrompt + fields_per_engine: list[dict[str, Any]] + failures: int + + +def _check_prompt( + team: Team, run_id: str, engines: list[CitationEngine], target_domains: list[str], prompt: AEOPrompt +) -> PromptCheckResult: + """Run one prompt on every engine. Runs in a worker thread, so it must not touch the database.""" + fields_per_engine: list[dict[str, Any]] = [] + failures = 0 + for engine in engines: + trace_id = str(uuid.uuid4()) + check = engine.run( + prompt.prompt, + trace_id=trace_id, + custom_properties={ + "aeo_prompt_id": str(prompt.id), + "aeo_run_id": run_id, + "aeo_prompt_source": prompt.prompt_source, + # No $ai_ prefix, so the gateway keeps this on the $ai_generation + # event, which lets usage attribute spend back to the checked team. + "team_id": str(team.id), + }, + ) + if check.error is not None: + failures += 1 + logger.warning( + "aeo_citation_check_failed", + team_id=team.id, + engine=engine.name, + prompt_id=str(prompt.id), + error=check.error, + ) + fields_per_engine.append( + { + **build_check_fields( + check=check, + run_id=run_id, + prompt_id=str(prompt.id), + prompt_text=prompt.prompt, + prompt_source=prompt.prompt_source, + prompt_hash=prompt.prompt_hash, + target_domains=target_domains, + ), + "prompt_topic": prompt.topic, + "prompt_version": prompt.current_version, + } + ) + return PromptCheckResult(prompt=prompt, fields_per_engine=fields_per_engine, failures=failures) + + +def _record_prompt_checks( + team: Team, canonical_team_id: int, run_id: str, prompt: AEOPrompt, fields_per_engine: list[dict[str, Any]] +) -> bool: + """Write one prompt's checks and mark the prompt checked. Called per prompt so a + mid-run crash keeps the completed checks; False when the write failed.""" + prompt_rows = [AEOCitationCheck(team_id=canonical_team_id, **fields) for fields in fields_per_engine] + try: + # The fail-closed manager refuses writes without a team scope, and none of the callers + # (the Celery task, the management command) run inside one. + with team_scope(canonical_team_id): + AEOCitationCheck.objects.bulk_create(prompt_rows) + # A prompt every engine failed on was not really checked, so leave it first in line for the next run. + if any(not fields["check_failed"] for fields in fields_per_engine): + AEOPrompt.objects.for_team(team.id).filter(id=prompt.id).update(last_checked_at=timezone.now()) + except Exception as e: + logger.exception("aeo_citation_write_failed", team_id=team.id, run_id=run_id, error=str(e)[:300]) + return False + return True + + def select_prompts(team: Team, limit: int) -> list[AEOPrompt]: """Pick up to `limit` active, unarchived prompts, alternating between topics. diff --git a/products/aeo/backend/sentiment.py b/products/aeo/backend/sentiment.py new file mode 100644 index 000000000000..641b25825b39 --- /dev/null +++ b/products/aeo/backend/sentiment.py @@ -0,0 +1,56 @@ +"""How an answer engine's reply speaks about the brand. + +Only the sentences that name the brand are scored, so praise of a competitor in +the same answer does not count as praise of the brand. The classifier itself is +the AI observability ONNX model; it lives on the evals Temporal queue, and this +module only prepares its input and folds its output back into one label per answer. +""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence + +from posthog.dataclasses import frozen +from posthog.hogql_queries.ai.sentiment_labeling import select_sentiment_label + +from products.aeo.backend.brand import brand_pattern + +SENTIMENT_LABELS = ("negative", "neutral", "positive") +# Bounds the classifier input per answer: an answer that names the brand in more sentences than +# this is scored on the first ones, and the model truncates a sentence longer than this anyway. +MAX_BRAND_SENTENCES = 20 +MAX_SENTENCE_CHARS = 1000 +_SENTENCE_BREAK = re.compile(r"(?<=[.!?])\s+|\n+") + + +@frozen +class AnswerSentiment: + label: str + score: float + + +def brand_sentences(answer_text: str | None, target_domains: list[str]) -> list[str]: + """The sentences of an answer that name the brand, in order, trimmed and bounded.""" + pattern = brand_pattern(target_domains) + if not answer_text or pattern is None: + return [] + sentences = (sentence.strip() for sentence in _SENTENCE_BREAK.split(answer_text)) + matching = (sentence[:MAX_SENTENCE_CHARS] for sentence in sentences if sentence and pattern.search(sentence)) + return [sentence for _, sentence in zip(range(MAX_BRAND_SENTENCES), matching)] + + +def aggregate_sentiment(sentence_scores: Sequence[Mapping[str, float]]) -> AnswerSentiment | None: + """One label for the answer from its brand sentences' class scores. + + The class scores are averaged over the sentences before the label is picked, so one strongly + negative sentence among neutral ones pulls the answer toward negative without a majority vote + flipping it, and the same neutral margin as AI observability keeps near-ties neutral.""" + if not sentence_scores: + return None + mean_scores = { + label: round(sum(scores.get(label, 0.0) for scores in sentence_scores) / len(sentence_scores), 4) + for label in SENTIMENT_LABELS + } + label = select_sentiment_label(mean_scores) + return AnswerSentiment(label=label, score=mean_scores[label]) diff --git a/products/aeo/backend/tasks/tasks.py b/products/aeo/backend/tasks/tasks.py index 1f2a69198528..50348a8f5298 100644 --- a/products/aeo/backend/tasks/tasks.py +++ b/products/aeo/backend/tasks/tasks.py @@ -25,10 +25,11 @@ logger = structlog.get_logger(__name__) -# A full run is up to 50 prompts x 3 engines of sequential web-search calls -# (typically 10-60s each); four hours gives slow days headroom without letting -# a hung run hold a worker slot indefinitely. -RUN_SOFT_TIME_LIMIT_SECONDS = 4 * 60 * 60 +# A full run covers the whole prompt set: each prompt makes one web-search call per +# engine (about a minute each), AEO_RUN_CONCURRENCY prompts at a time. Ten hours +# leaves headroom for a slow day and still ends a hung run well before the next +# daily dispatch. +RUN_SOFT_TIME_LIMIT_SECONDS = 10 * 60 * 60 RUN_TIME_LIMIT_SECONDS = RUN_SOFT_TIME_LIMIT_SECONDS + 300 diff --git a/products/aeo/backend/temporal/__init__.py b/products/aeo/backend/temporal/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/products/aeo/backend/temporal/brand_sentiment.py b/products/aeo/backend/temporal/brand_sentiment.py new file mode 100644 index 000000000000..43528c65a239 --- /dev/null +++ b/products/aeo/backend/temporal/brand_sentiment.py @@ -0,0 +1,175 @@ +"""Scores brand sentiment for one citation run. + +The ONNX sentiment model and its dependencies are only in the image that serves +the evals Temporal queue, so the runner cannot classify inline: it starts this +workflow, and the activity reads the run's answers back from Postgres, scores the +sentences that name the brand, and writes the label and score to the check rows. +""" + +from __future__ import annotations + +import asyncio +import datetime as dt +import dataclasses +from collections.abc import Sequence +from typing import Any + +from django.conf import settings +from django.utils import timezone + +import structlog +from temporalio import activity, workflow +from temporalio.common import RetryPolicy, WorkflowIDReusePolicy +from temporalio.exceptions import WorkflowAlreadyStartedError + +from posthog.exceptions_capture import capture_exception +from posthog.sync import database_sync_to_async +from posthog.temporal.common.base import PostHogWorkflow +from posthog.temporal.common.client import sync_connect +from posthog.temporal.common.utils import close_db_connections + +from products.aeo.backend.brand import brand_regex +from products.aeo.backend.models import AEOCitationCheck +from products.aeo.backend.sentiment import aggregate_sentiment, brand_sentences + +logger = structlog.get_logger(__name__) + +WORKFLOW_NAME = "aeo-brand-sentiment" +# One run is at most ~150 answers x 20 sentences on a CPU model, well inside this. +CLASSIFY_TIMEOUT = dt.timedelta(minutes=15) +CLASSIFY_BATCH = 32 +# Answers scored before the setting was turned on, or missed by a failed run, are picked up from +# this far back, which matches the window the prompt report shows. +BACKLOG_WINDOW = dt.timedelta(days=30) +MAX_ROWS_PER_RUN = 2000 + + +@dataclasses.dataclass(frozen=True) +class AEOBrandSentimentInputs: + team_id: int + run_id: str + + +@dataclasses.dataclass(frozen=True) +class _PendingAnswer: + check_id: str + sentences: list[str] + + +def _load_pending(inputs: AEOBrandSentimentInputs) -> list[_PendingAnswer]: + """The run's unscored answers, then the team's other unscored answers from the backlog window, + oldest first, capped so one activity never scans an unbounded history. + + Only answers that name the brand count toward the cap: an answer without a mention stays NULL + forever, so letting it fill the slice would starve newer answers that can be scored. + + Two concurrent runs can both pick up the same backlog row. The writes are idempotent (the same + text scores the same), so that only costs a duplicate classification, and no transaction is + held across the model call that a row lock could ride on.""" + target_domains: list[str] = settings.AEO_TARGET_DOMAINS + regex = brand_regex(target_domains) + if regex is None: + return [] + unscored = AEOCitationCheck.objects.for_team(inputs.team_id).filter( + check_failed=False, brand_sentiment__isnull=True, answer_text__iregex=regex + ) + own_rows = list(unscored.filter(run_id=inputs.run_id).order_by("created_at").values_list("id", "answer_text")) + backlog_rows = list( + unscored.exclude(run_id=inputs.run_id) + .filter(created_at__gte=timezone.now() - BACKLOG_WINDOW) + .order_by("created_at") + .values_list("id", "answer_text")[: max(0, MAX_ROWS_PER_RUN - len(own_rows))] + ) + pending = [ + _PendingAnswer(check_id=str(check_id), sentences=brand_sentences(text, target_domains)) + for check_id, text in [*own_rows, *backlog_rows] + ] + return [answer for answer in pending if answer.sentences] + + +def _write_scores(team_id: int, scored: Sequence[tuple[str, str, float]]) -> None: + rows = [ + AEOCitationCheck(id=check_id, brand_sentiment=label, brand_sentiment_score=score) + for check_id, label, score in scored + ] + AEOCitationCheck.objects.for_team(team_id).bulk_update(rows, ["brand_sentiment", "brand_sentiment_score"]) + + +@activity.defn +@close_db_connections +async def classify_aeo_brand_sentiment(inputs: AEOBrandSentimentInputs) -> dict[str, Any]: + """Score every answer of the run that names the brand and has no sentiment yet. + + A missing model is a deployment problem, not a data problem, so the activity reports it and + leaves the rows unscored instead of retrying: the next run picks nothing up either, and the + rows read as "not scored yet" until the worker image carries the model.""" + pending = await database_sync_to_async(_load_pending, thread_sensitive=False)(inputs) + if not pending: + return {"scored": 0, "answers": 0} + + texts = [sentence for answer in pending for sentence in answer.sentences] + try: + import posthog.temporal.ai_observability.sentiment.model as sentiment_model # noqa: PLC0415 -- ONNX deps + + results = [] + for start in range(0, len(texts), CLASSIFY_BATCH): + results.extend(await asyncio.to_thread(sentiment_model.classify, texts[start : start + CLASSIFY_BATCH])) + except (ImportError, FileNotFoundError) as error: + logger.warning( + "aeo_brand_sentiment_model_unavailable", team_id=inputs.team_id, run_id=inputs.run_id, error=str(error) + ) + return {"scored": 0, "answers": len(pending), "error": "model_unavailable"} + + scored: list[tuple[str, str, float]] = [] + position = 0 + for answer in pending: + sentence_scores = [result.scores for result in results[position : position + len(answer.sentences)]] + position += len(answer.sentences) + sentiment = aggregate_sentiment(sentence_scores) + if sentiment is not None: + scored.append((answer.check_id, sentiment.label, sentiment.score)) + await database_sync_to_async(_write_scores, thread_sensitive=False)(inputs.team_id, scored) + logger.info("aeo_brand_sentiment_scored", team_id=inputs.team_id, run_id=inputs.run_id, scored=len(scored)) + return {"scored": len(scored), "answers": len(pending)} + + +@workflow.defn(name=WORKFLOW_NAME) +class AEOBrandSentimentWorkflow(PostHogWorkflow): + inputs_cls = AEOBrandSentimentInputs + + @workflow.run + async def run(self, inputs: AEOBrandSentimentInputs) -> dict[str, Any]: + return await workflow.execute_activity( + classify_aeo_brand_sentiment, + inputs, + start_to_close_timeout=CLASSIFY_TIMEOUT, + retry_policy=RetryPolicy(maximum_attempts=3, initial_interval=dt.timedelta(seconds=30)), + ) + + +def start_brand_sentiment_workflow(team_id: int, run_id: str) -> None: + """Fire-and-forget: the citation run is already recorded, so a Temporal outage only means + this run's answers stay unscored.""" + inputs = AEOBrandSentimentInputs(team_id=team_id, run_id=run_id) + try: + client = sync_connect() + asyncio.run( + client.start_workflow( + WORKFLOW_NAME, + inputs, + id=f"{WORKFLOW_NAME}-{team_id}-{run_id}", + task_queue=settings.LLMA_EVALS_TASK_QUEUE, + id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE_FAILED_ONLY, + ) + ) + except WorkflowAlreadyStartedError: + logger.info("aeo_brand_sentiment_already_started", team_id=team_id, run_id=run_id) + except Exception as error: + logger.exception("aeo_brand_sentiment_start_failed", team_id=team_id, run_id=run_id, error=str(error)[:300]) + capture_exception(error) + else: + logger.info("aeo_brand_sentiment_started", team_id=team_id, run_id=run_id) + + +WORKFLOWS = [AEOBrandSentimentWorkflow] +ACTIVITIES = [classify_aeo_brand_sentiment] diff --git a/products/aeo/backend/test/test_api.py b/products/aeo/backend/test/test_api.py index 16c56c35ddca..9d4e57581b02 100644 --- a/products/aeo/backend/test/test_api.py +++ b/products/aeo/backend/test/test_api.py @@ -6,6 +6,7 @@ from unittest.mock import patch from django.db import connection +from django.test import override_settings from django.test.utils import CaptureQueriesContext from django.utils import timezone @@ -41,6 +42,8 @@ def _record_check( cited: bool = False, check_failed: bool = False, answer_text: str | None = None, + cited_urls: list[str] | None = None, + search_queries: list[str] | None = None, ) -> str: with team_scope(self.team.pk): prompt = AEOPrompt.objects.get(id=prompt_id) @@ -57,6 +60,8 @@ def _record_check( cited=cited, check_failed=check_failed, answer_text=answer_text, + cited_urls=cited_urls or [], + search_queries=search_queries or [], ) AEOCitationCheck.objects.filter(id=check.id).update( created_at=timezone.now() - dt.timedelta(minutes=minutes_ago) @@ -424,26 +429,146 @@ def test_stats_count_checks_per_engine_and_per_day(self) -> None: prompt_id = self._import_one_prompt() day = 24 * 60 self._record_check(prompt_id, "exa-answer", minutes_ago=10, cited=True) - self._record_check(prompt_id, "exa-answer", minutes_ago=20) + self._record_check(prompt_id, "exa-answer", minutes_ago=20, answer_text="Example Analytics is one option.") self._record_check(prompt_id, "exa-answer", minutes_ago=day, check_failed=True) self._record_check(prompt_id, "claude-web-search", minutes_ago=2 * day, cited=True) self._record_check(prompt_id, "claude-web-search", minutes_ago=40 * day) - response = self.client.get(f"{self.url}{prompt_id}/stats/") + with override_settings(AEO_TARGET_DOMAINS=["example.com"]): + response = self.client.get(f"{self.url}{prompt_id}/stats/") assert response.status_code == status.HTTP_200_OK, response.json() stats = response.json() assert stats["total_checks"] == 5 assert (stats["daily_from"], stats["daily_to"]) == ("2026-09-21", "2026-10-20") engines = { - row["engine"]: (row["checks"], row["cited"], row["failed"], row["citation_rate"]) + row["engine"]: ( + row["checks"], + row["cited"], + row["mentioned"], + row["failed"], + row["citation_rate"], + row["mention_rate"], + ) for row in stats["engines"] } - assert engines == {"claude-web-search": (2, 1, 0, 0.5), "exa-answer": (3, 1, 1, 0.5)} - assert [(row["date"], row["engine"], row["checks"], row["cited"], row["failed"]) for row in stats["daily"]] == [ - ("2026-10-18", "claude-web-search", 1, 1, 0), - ("2026-10-19", "exa-answer", 1, 0, 1), - ("2026-10-20", "exa-answer", 2, 1, 0), + assert engines == {"claude-web-search": (2, 1, 1, 0, 0.5, 0.5), "exa-answer": (3, 1, 2, 1, 0.5, 1.0)} + assert [ + (row["date"], row["engine"], row["checks"], row["cited"], row["mentioned"], row["failed"]) + for row in stats["daily"] + ] == [ + ("2026-10-18", "claude-web-search", 1, 1, 1, 0), + ("2026-10-19", "exa-answer", 1, 0, 0, 1), + ("2026-10-20", "exa-answer", 2, 1, 2, 0), + ] + assert stats["brand_names"] == ["example"] + assert (stats["consensus"], stats["top_words"]) == ([], []) + assert [(row["engine"], row["unscored"]) for row in stats["sentiment"]] == [ + ("claude-web-search", 1), + ("exa-answer", 2), + ] + + @parameterized.expand( + [ + ("brand_named_in_the_answer", {"answer_text": "Try Example Analytics for funnels."}, 1), + ("brand_case_ignored", {"answer_text": "EXAMPLE ships session replay."}, 1), + ("brand_inside_another_word", {"answer_text": "Counterexample: nothing fits."}, 0), + ("cited_without_the_name_in_the_text", {"cited": True, "answer_text": "See [1]."}, 1), + ("failed_check", {"check_failed": True}, 0), + ] + ) + def test_stats_count_a_mention_when_the_answer_names_the_brand( + self, _name: str, check: dict, expected_mentioned: int + ) -> None: + prompt_id = self._import_one_prompt() + self._record_check(prompt_id, "exa-answer", minutes_ago=10, **check) + + with override_settings(AEO_TARGET_DOMAINS=["example.com"]): + stats = self.client.get(f"{self.url}{prompt_id}/stats/").json() + + assert [row["mentioned"] for row in stats["engines"]] == [expected_mentioned] + assert [row["mentioned"] for row in stats["daily"]] == [expected_mentioned] + + def test_stats_count_brand_sentiment_per_engine_and_checks_expose_it(self) -> None: + prompt_id = self._import_one_prompt() + scored = self._record_check(prompt_id, "exa-answer", minutes_ago=10, answer_text="Example is great.") + self._record_check(prompt_id, "exa-answer", minutes_ago=20, answer_text="Example is slow.") + self._record_check(prompt_id, "exa-answer", minutes_ago=30, answer_text="Example exists.") + self._record_check(prompt_id, "claude-web-search", minutes_ago=40, answer_text="Nothing about the brand.") + with team_scope(self.team.pk): + for text, label, score in (("great", "positive", 0.9), ("slow", "negative", 0.7)): + AEOCitationCheck.objects.filter(answer_text__contains=text).update( + brand_sentiment=label, brand_sentiment_score=score + ) + + with override_settings(AEO_TARGET_DOMAINS=["example.com"]): + stats = self.client.get(f"{self.url}{prompt_id}/stats/").json() + checks = self.client.get(f"{self.url}{prompt_id}/checks/?engine=exa-answer").json()["results"] + + assert [ + (row["engine"], row["positive"], row["neutral"], row["negative"], row["unscored"]) + for row in stats["sentiment"] + ] == [("claude-web-search", 0, 0, 0, 0), ("exa-answer", 1, 0, 1, 1)] + assert {check["id"]: (check["brand_sentiment"], check["brand_sentiment_score"]) for check in checks} == { + scored: ("positive", 0.9), + **{check["id"]: (check["brand_sentiment"], check["brand_sentiment_score"]) for check in checks[1:]}, + } + assert [check["brand_sentiment"] for check in checks] == ["positive", "negative", None] + + @time_machine.travel("2026-10-20T12:00:00Z", tick=False) + def test_stats_rank_engine_consensus_and_search_words_over_the_window(self) -> None: + prompt_id = self._import_one_prompt() + day = 24 * 60 + ours = "https://docs.example.com/product-analytics" + rival = "https://rival.example.org/best-tools" + self._record_check( + prompt_id, + "claude-web-search", + minutes_ago=10, + cited=True, + cited_urls=[rival, ours, "https://rival.example.org/pricing", rival], + search_queries=["Best product analytics tools", "product analytics for startups"], + ) + self._record_check( + prompt_id, + "claude-web-search", + minutes_ago=day, + cited_urls=[rival], + search_queries=["best product analytics tools"], + ) + self._record_check( + prompt_id, + "openai-web-search", + minutes_ago=2 * day, + cited=True, + cited_urls=["https://community.example.net/thread", ours], + search_queries=["open source analytics"], + ) + self._record_check(prompt_id, "exa-answer", minutes_ago=3 * day, cited_urls=[]) + self._record_check(prompt_id, "openai-web-search", minutes_ago=4 * day, check_failed=True) + self._record_check(prompt_id, "openai-web-search", minutes_ago=40 * day, cited_urls=[rival, rival, rival]) + + with override_settings(AEO_TARGET_DOMAINS=["example.com"]): + stats = self.client.get(f"{self.url}{prompt_id}/stats/").json() + + assert [(row["engine"], row["domain"], row["checks_citing"], row["checks"]) for row in stats["consensus"]] == [ + ("claude-web-search", "docs.example.com", 1, 2), + ("claude-web-search", "rival.example.org", 2, 2), + ("claude-web-search", "community.example.net", 0, 2), + ("exa-answer", "docs.example.com", 0, 1), + ("exa-answer", "rival.example.org", 0, 1), + ("exa-answer", "community.example.net", 0, 1), + ("openai-web-search", "docs.example.com", 1, 1), + ("openai-web-search", "rival.example.org", 0, 1), + ("openai-web-search", "community.example.net", 1, 1), + ] + assert [(row["word"], row["count"]) for row in stats["top_words"]] == [ + ("analytics", 4), + ("product", 3), + ("tools", 2), + ("open", 1), + ("source", 1), + ("startups", 1), ] @parameterized.expand([("above_range", 140), ("not_a_number", "NaN"), ("infinite", "Infinity")]) diff --git a/products/aeo/backend/test/test_brand_sentiment.py b/products/aeo/backend/test/test_brand_sentiment.py new file mode 100644 index 000000000000..d94352334b40 --- /dev/null +++ b/products/aeo/backend/test/test_brand_sentiment.py @@ -0,0 +1,178 @@ +import uuid +import datetime as dt + +import pytest +from unittest.mock import MagicMock, patch + +from django.utils import timezone + +from asgiref.sync import sync_to_async +from temporalio.testing import ActivityEnvironment + +from posthog.models.organization import Organization +from posthog.models.scoping import team_scope +from posthog.models.team import Team +from posthog.temporal.ai_observability.sentiment.schema import SentimentResult + +from products.aeo.backend.engines import CitationCheck +from products.aeo.backend.models import AEOCitationCheck, AEOPrompt +from products.aeo.backend.runner import run_citation_checks +from products.aeo.backend.seeding import PromptCandidate, upsert_prompts +from products.aeo.backend.temporal.brand_sentiment import AEOBrandSentimentInputs, classify_aeo_brand_sentiment + +_MODULE = "products.aeo.backend.temporal.brand_sentiment" + + +def _team() -> Team: + organization = Organization.objects.create(name="aeo sentiment org") + return Team.objects.create(organization=organization, name="aeo sentiment team") + + +def _check(team: Team, prompt: AEOPrompt, run_id: uuid.UUID, answer_text: str | None, **fields) -> AEOCitationCheck: + with team_scope(team.pk): + return AEOCitationCheck.objects.create( + team=team, + prompt=prompt, + run_id=run_id, + prompt_text=prompt.prompt, + prompt_source=prompt.prompt_source, + prompt_hash=prompt.prompt_hash, + engine="exa-answer", + model="exa-answer", + answer_text=answer_text, + **fields, + ) + + +def _seed_run(team: Team) -> tuple[uuid.UUID, dict[str, AEOCitationCheck]]: + with team_scope(team.pk): + prompt = AEOPrompt.objects.create( + team=team, prompt="best tool?", prompt_hash="h", prompt_source=AEOPrompt.Source.MANUAL + ) + run_id = uuid.uuid4() + return run_id, { + "praise": _check(team, prompt, run_id, "Example is fast. Rival is slow."), + "two_sentences": _check(team, prompt, run_id, "Example is slow. Still, Example is cheap."), + "no_brand": _check(team, prompt, run_id, "Rival is the only option."), + "failed": _check(team, prompt, run_id, None, check_failed=True), + "scored": _check(team, prompt, run_id, "Example again.", brand_sentiment="neutral", brand_sentiment_score=0.5), + "other_run": _check(team, prompt, uuid.uuid4(), "Example elsewhere."), + "no_brand_backlog": _check(team, prompt, uuid.uuid4(), "Rival elsewhere."), + "too_old": _check(team, prompt, uuid.uuid4(), "Example long ago."), + } + + +def _age(check: AEOCitationCheck, days: int) -> None: + with team_scope(check.team_id): + AEOCitationCheck.objects.filter(id=check.id).update(created_at=timezone.now() - dt.timedelta(days=days)) + + +def _result(**scores: float) -> SentimentResult: + label = max(scores, key=scores.__getitem__) + return SentimentResult(label=label, score=scores[label], scores=scores) + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.asyncio +async def test_activity_scores_the_runs_answers_then_the_recent_unscored_backlog(settings) -> None: + settings.AEO_TARGET_DOMAINS = ["example.com"] + team = await sync_to_async(_team)() + run_id, checks = await sync_to_async(_seed_run)(team) + await sync_to_async(_age)(checks["other_run"], 10) + await sync_to_async(_age)(checks["no_brand_backlog"], 20) + await sync_to_async(_age)(checks["too_old"], 31) + classify = MagicMock( + return_value=[ + _result(positive=0.9, neutral=0.08, negative=0.02), + _result(positive=0.05, neutral=0.15, negative=0.8), + _result(positive=0.75, neutral=0.2, negative=0.05), + _result(positive=0.1, neutral=0.85, negative=0.05), + ] + ) + + # Cap of 4 leaves one backlog slot after the run's own rows. The older brand-less backlog row + # must not take it from the newer answer that names the brand. + with ( + patch("posthog.temporal.ai_observability.sentiment.model.classify", classify), + patch(f"{_MODULE}.MAX_ROWS_PER_RUN", 4), + ): + result = await ActivityEnvironment().run( + classify_aeo_brand_sentiment, AEOBrandSentimentInputs(team_id=team.id, run_id=str(run_id)) + ) + + assert result == {"scored": 3, "answers": 3} + classify.assert_called_once_with( + ["Example is fast.", "Example is slow.", "Still, Example is cheap.", "Example elsewhere."] + ) + + def sentiments() -> dict[str, tuple[str | None, float | None]]: + with team_scope(team.pk): + rows = AEOCitationCheck.objects.in_bulk([check.id for check in checks.values()]) + return { + name: (rows[check.id].brand_sentiment, rows[check.id].brand_sentiment_score) + for name, check in checks.items() + } + + assert await sync_to_async(sentiments)() == { + "praise": ("positive", 0.9), + "two_sentences": ("negative", 0.425), + "no_brand": (None, None), + "failed": (None, None), + "scored": ("neutral", 0.5), + "other_run": ("neutral", 0.85), + "no_brand_backlog": (None, None), + "too_old": (None, None), + } + + +@pytest.mark.django_db(transaction=True) +@pytest.mark.asyncio +@pytest.mark.parametrize("error", [ImportError("No module named 'onnxruntime'"), FileNotFoundError("model.onnx")]) +async def test_activity_leaves_rows_unscored_when_the_model_is_unavailable(settings, error: Exception) -> None: + settings.AEO_TARGET_DOMAINS = ["example.com"] + team = await sync_to_async(_team)() + run_id, checks = await sync_to_async(_seed_run)(team) + + with patch("posthog.temporal.ai_observability.sentiment.model.classify", MagicMock(side_effect=error)): + result = await ActivityEnvironment().run( + classify_aeo_brand_sentiment, AEOBrandSentimentInputs(team_id=team.id, run_id=str(run_id)) + ) + + assert result == {"scored": 0, "answers": 4, "error": "model_unavailable"} + unscored = await sync_to_async( + lambda: AEOCitationCheck.objects.for_team(team.id).filter(brand_sentiment__isnull=True).count() + )() + assert unscored == len(checks) - 1 + + +class _StubEngine: + name = "stub-engine" + model = "stub" + + def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck: + return CitationCheck(engine=self.name, model=self.model, answer_text="Example is fine.") + + +@pytest.mark.django_db +@pytest.mark.parametrize( + "enabled, record, expected_starts", + [(True, True, 1), (False, True, 0), (True, False, 0)], + ids=["on", "off", "dry_run"], +) +def test_runner_starts_the_sentiment_workflow_only_when_enabled_and_recording( + settings, enabled: bool, record: bool, expected_starts: int +) -> None: + settings.AEO_BRAND_SENTIMENT_ENABLED = enabled + team = _team() + upsert_prompts(team, [PromptCandidate(text="q?", source="imported")]) + + with patch(f"{_MODULE}.sync_connect") as connect, patch(f"{_MODULE}.asyncio.run") as run: + summary, _ = run_citation_checks(team, engines=[_StubEngine()], record=record) + + assert summary.rows_written == (1 if record else 0) + assert run.call_count == expected_starts + if expected_starts: + start = connect.return_value.start_workflow + start.assert_called_once() + assert start.call_args.kwargs["id"] == f"aeo-brand-sentiment-{team.id}-{summary.run_id}" + assert start.call_args.kwargs["task_queue"] == settings.LLMA_EVALS_TASK_QUEUE diff --git a/products/aeo/backend/test/test_engines.py b/products/aeo/backend/test/test_engines.py index 8db3f963aa25..fc8bfed13e4e 100644 --- a/products/aeo/backend/test/test_engines.py +++ b/products/aeo/backend/test/test_engines.py @@ -1,16 +1,23 @@ +from collections.abc import Callable, Iterator +from contextlib import contextmanager from types import SimpleNamespace -from unittest.mock import patch +import pytest +from unittest.mock import MagicMock, patch +import requests from parameterized import parameterized from products.aeo.backend.engines import ( + GATEWAY_MAX_ATTEMPTS, OPENAI_MAX_OUTPUT_TOKENS, CitationCheck, ClaudeWebSearchEngine, OpenAIWebSearchEngine, + _SharedBackoff, anthropic_truncated_error, build_check_fields, + gateway_post_json, is_target_url, openai_response_error, parse_anthropic_citations, @@ -350,3 +357,78 @@ def test_engine_derived_text_is_sanitized_before_it_reaches_the_event() -> None: # Sanitizing must not move the verdict: it runs on the recorded copy only. assert fields["cited"] is True assert fields["target_best_position"] == 1 + + +def _response(status: int, retry_after: str | None = None) -> requests.Response: + response = requests.Response() + response.status_code = status + response._content = b'{"ok": true}' + if retry_after is not None: + response.headers["Retry-After"] = retry_after + return response + + +@parameterized.expand( + [ + ("throttled_then_ok", [_response(429, retry_after="7"), _response(200)], None, 2, [7.0]), + ("transient_until_attempts_run_out", [_response(503)] * GATEWAY_MAX_ATTEMPTS, requests.HTTPError, 3, [5, 10]), + ("client_error_is_not_retried", [_response(400)], requests.HTTPError, 1, []), + ] +) +def test_gateway_post_json_retries_throttling_and_transient_errors( + _name: str, responses: list, raises: type[Exception] | None, posts: int, sleeps: list[float] +) -> None: + session = MagicMock(spec=requests.Session) + session.post.side_effect = responses + + with ( + patch("products.aeo.backend.engines.GATEWAY_BACKOFF", _SharedBackoff()), + _fake_clock() as sleep, + ): + if raises is None: + assert gateway_post_json(session, "https://gw.test", {}, {}) == {"ok": True} + else: + with pytest.raises(raises): + gateway_post_json(session, "https://gw.test", {}, {}) + + assert session.post.call_count == posts + assert [call.args[0] for call in sleep.call_args_list] == sleeps + + +@parameterized.expand( + [ + ("shorter_pause_does_not_cut_a_longer_one", [30, 5], None, [30.0]), + ("pause_extended_during_the_sleep", [30], 60, [30.0, 30.0]), + ] +) +def test_shared_backoff_waits_for_the_latest_deadline( + _name: str, pauses: list[float], extend_during_sleep: float | None, sleeps: list[float] +) -> None: + backoff = _SharedBackoff() + + def other_worker_pauses() -> None: + if extend_during_sleep is not None: + backoff.pause(extend_during_sleep) + + with _fake_clock(on_first_sleep=other_worker_pauses) as sleep: + for seconds in pauses: + backoff.pause(seconds) + backoff.wait() + + assert [call.args[0] for call in sleep.call_args_list] == sleeps + + +@contextmanager +def _fake_clock(on_first_sleep: Callable[[], None] | None = None) -> Iterator[MagicMock]: + now = [100.0] + + def sleep(seconds: float) -> None: + if on_first_sleep is not None and sleep_mock.call_count == 1: + on_first_sleep() + now[0] += seconds + + with ( + patch("products.aeo.backend.engines.time.monotonic", side_effect=lambda: now[0]), + patch("products.aeo.backend.engines.time.sleep", side_effect=sleep) as sleep_mock, + ): + yield sleep_mock diff --git a/products/aeo/backend/test/test_runner.py b/products/aeo/backend/test/test_runner.py index 171554fb8fcc..0218de612a9f 100644 --- a/products/aeo/backend/test/test_runner.py +++ b/products/aeo/backend/test/test_runner.py @@ -1,4 +1,6 @@ import datetime as dt +import threading +from typing import NoReturn import pytest @@ -7,6 +9,7 @@ from posthog.models.organization import Organization from posthog.models.team import Team +from products.aeo.backend import runner from products.aeo.backend.engines import CitationCheck from products.aeo.backend.models import AEOCitationCheck, AEOPrompt from products.aeo.backend.runner import run_citation_checks, select_prompts @@ -60,8 +63,12 @@ def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) @pytest.mark.parametrize("via_child_environment", [False, True]) -def test_run_records_checks_and_rotates_without_a_team_scope(team: Team, settings, via_child_environment: bool) -> None: +@pytest.mark.parametrize("concurrency", [1, 4]) +def test_run_records_checks_and_rotates_without_a_team_scope( + team: Team, settings, via_child_environment: bool, concurrency: int +) -> None: settings.AEO_TARGET_DOMAINS = ["posthog.com"] + settings.AEO_RUN_CONCURRENCY = concurrency runner_team = ( Team.objects.create(organization=team.organization, name="child env", parent_team=team) if via_child_environment @@ -82,3 +89,79 @@ def test_run_records_checks_and_rotates_without_a_team_scope(team: Team, setting assert {c.prompt_topic for c in checks} == {"t0", "t1"} assert len({c.prompt_id for c in checks}) == 4 assert sorted((c.prompt_text, c.prompt_version) for c in checks) == [("q0?", 3), ("q1?", 1), ("q2?", 1), ("q3?", 1)] + + +class _CrashingEngine(_StubEngine): + def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck: + if prompt == "q1?": + raise RuntimeError("unexpected engine bug") + return super().run(prompt, trace_id=trace_id, custom_properties=custom_properties) + + +def test_one_prompt_crashing_does_not_end_the_run(team: Team, settings) -> None: + settings.AEO_RUN_CONCURRENCY = 2 + settings.AEO_MAX_PROMPTS_PER_RUN = 3 + upsert_prompts(team, [PromptCandidate(text=f"q{i}?", source="imported", topic="t") for i in range(3)]) + + summary, _ = run_citation_checks(team, engines=[_CrashingEngine()]) + + assert (summary.prompts, summary.rows_written, summary.engine_failures) == (3, 2, 1) + assert sorted(AEOCitationCheck.objects.for_team(team.id).values_list("prompt_text", flat=True)) == ["q0?", "q2?"] + + +class _FailingEngine(_StubEngine): + def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck: + if prompt == "q1?": + return CitationCheck(engine=self.name, model=self.model, error="HTTPError: status=429") + return super().run(prompt, trace_id=trace_id, custom_properties=custom_properties) + + +def test_a_prompt_every_engine_failed_on_stays_unchecked(team: Team, settings) -> None: + settings.AEO_RUN_CONCURRENCY = 2 + upsert_prompts(team, [PromptCandidate(text=f"q{i}?", source="imported", topic="t") for i in range(2)]) + + summary, _ = run_citation_checks(team, engines=[_FailingEngine()]) + + assert (summary.rows_written, summary.engine_failures) == (2, 1) + last_checked = dict(AEOPrompt.objects.for_team(team.id).values_list("prompt", "last_checked_at")) + assert last_checked["q0?"] is not None + assert last_checked["q1?"] is None + + +class _BlockingEngine(_StubEngine): + """Holds every check until the test releases it, so the test decides which prompts are in flight.""" + + def __init__(self, workers: int) -> None: + self.prompts: list[str] = [] + self.threads: set[threading.Thread] = set() + self.all_workers_busy = threading.Barrier(workers + 1) + self.release = threading.Event() + + def run(self, prompt: str, *, trace_id: str, custom_properties: dict[str, str]) -> CitationCheck: + self.prompts.append(prompt) + self.threads.add(threading.current_thread()) + self.all_workers_busy.wait(timeout=5) + self.release.wait(timeout=5) + return super().run(prompt, trace_id=trace_id, custom_properties=custom_properties) + + +def test_an_interrupted_run_does_not_start_the_queued_prompts(team: Team, settings, monkeypatch) -> None: + settings.AEO_RUN_CONCURRENCY = 2 + upsert_prompts(team, [PromptCandidate(text=f"q{i}?", source="imported", topic="t") for i in range(5)]) + engine = _BlockingEngine(workers=2) + + def interrupt_once_every_worker_is_busy(futures: object) -> NoReturn: + # Stands in for a Celery soft time limit landing while the run waits on its workers. + engine.all_workers_busy.wait(timeout=5) + raise KeyboardInterrupt + + monkeypatch.setattr(runner, "as_completed", interrupt_once_every_worker_is_busy) + + with pytest.raises(KeyboardInterrupt): + run_citation_checks(team, engines=[engine]) + engine.release.set() + for thread in engine.threads: + thread.join(timeout=5) + + assert len(engine.prompts) == 2 + assert len(set(engine.prompts)) == 2 diff --git a/products/aeo/backend/test/test_sentiment.py b/products/aeo/backend/test/test_sentiment.py new file mode 100644 index 000000000000..3adef29e30b2 --- /dev/null +++ b/products/aeo/backend/test/test_sentiment.py @@ -0,0 +1,74 @@ +import pytest + +from products.aeo.backend.brand import brand_names, brand_regex +from products.aeo.backend.sentiment import ( + MAX_BRAND_SENTENCES, + MAX_SENTENCE_CHARS, + AnswerSentiment, + aggregate_sentiment, + brand_sentences, +) + +TARGET = ["docs.example.co.uk", "Example.com"] + + +@pytest.mark.parametrize( + "domains, names, regex", + [ + (TARGET, ("example",), r"\m(example)\M"), + (["posthog.com", "rival.io"], ("posthog", "rival"), r"\m(posthog|rival)\M"), + ([], (), None), + (["", "example.com"], ("example",), r"\m(example)\M"), + ], +) +def test_brand_names_come_from_the_registrable_domain(domains: list[str], names: tuple[str, ...], regex: str | None): + assert brand_names(domains) == names + assert brand_regex(domains) == regex + + +@pytest.mark.parametrize( + "answer, expected", + [ + ( + "Example is fast. Rival is cheaper! Pick Example if you self-host?", + ["Example is fast.", "Pick Example if you self-host?"], + ), + ("EXAMPLE works\nRival too\nexample's docs help", ["EXAMPLE works", "example's docs help"]), + ("A counterexample: nothing here. Examples abound.", []), + ("", []), + (None, []), + ], +) +def test_brand_sentences_keep_only_sentences_that_name_the_brand(answer: str | None, expected: list[str]): + assert brand_sentences(answer, TARGET) == expected + + +def test_brand_sentences_are_bounded(): + answer = " ".join(f"Example point {index}." for index in range(MAX_BRAND_SENTENCES + 5)) + " Example " + "x" * 2000 + sentences = brand_sentences(answer, TARGET) + assert len(sentences) == MAX_BRAND_SENTENCES + assert all(len(sentence) <= MAX_SENTENCE_CHARS for sentence in sentences) + assert brand_sentences("Example is fine.", []) == [] + + +@pytest.mark.parametrize( + "scores, expected", + [ + ([], None), + ([{"positive": 0.9, "neutral": 0.08, "negative": 0.02}], AnswerSentiment(label="positive", score=0.9)), + # The mean decides: one strongly negative sentence beats two mildly neutral ones. + ( + [ + {"positive": 0.1, "neutral": 0.5, "negative": 0.4}, + {"positive": 0.05, "neutral": 0.45, "negative": 0.5}, + {"positive": 0.0, "neutral": 0.05, "negative": 0.95}, + ], + AnswerSentiment(label="negative", score=0.6167), + ), + # A polar label that only just beats neutral stays neutral, as in AI observability. + ([{"positive": 0.5, "neutral": 0.4, "negative": 0.1}], AnswerSentiment(label="neutral", score=0.4)), + ([{"positive": 1.0}], AnswerSentiment(label="positive", score=1.0)), + ], +) +def test_aggregate_sentiment_averages_sentence_scores_before_labeling(scores: list[dict[str, float]], expected): + assert aggregate_sentiment(scores) == expected diff --git a/products/aeo/mcp/tools.yaml b/products/aeo/mcp/tools.yaml index 14b2e1593dc3..87d29dd32963 100644 --- a/products/aeo/mcp/tools.yaml +++ b/products/aeo/mcp/tools.yaml @@ -91,10 +91,16 @@ tools: title: Get a tracked AI prompt's citation stats description: > Get citation stats for one AI citation tracking prompt. `engines` has one row per answer engine over the - prompt's whole history: `checks`, `cited`, `failed`, `citation_rate` (cited share of answered checks, 0-1, - failed checks left out), and the first and last check times. `daily` has counts per day and engine for the - last 30 days (`daily_from` to `daily_to`, in the project's timezone); days without a check are left out. Use - it to compare engines or to see whether a prompt's citations are trending up or down. + prompt's whole history: `checks`, `cited`, `mentioned` (answers that name the brand), `failed`, + `citation_rate` and `mention_rate` (shares of answered checks, 0-1, failed checks left out), and the first + and last check times. `daily` has counts per day and engine for the last 30 days (`daily_from` to + `daily_to`, in the project's timezone); days without a check are left out. Over the same 30 days, + `consensus` says how many of each engine's answers cited each of the ten most cited domains, `sentiment` + counts positive, neutral and negative answers toward the brand per engine (`unscored` answers name the brand + but are not scored yet), and `top_words` lists the most used words of the web searches the engines ran. + `brand_names` are the words an answer must contain to count as a mention; use them to filter `answer_text` + in a HogQL query over `system.aeo_citation_checks`. Use it to compare engines, to see whether a prompt's + citations are trending up or down, and to find which sources the engines agree on. param_overrides: id: description: The prompt id (a UUID), from `aeo-prompts-list`. diff --git a/products/aeo/turbo.json b/products/aeo/turbo.json index a38de87df10f..2abbe99ac41c 100644 --- a/products/aeo/turbo.json +++ b/products/aeo/turbo.json @@ -5,6 +5,7 @@ "inputs": [ "backend/facade/**", "backend/tasks/**", + "backend/temporal/**", "backend/models.py", "backend/migrations/**", "backend/routes.py", diff --git a/products/ai_observability/frontend/evaluations/overview/EvaluationResultsTable.tsx b/products/ai_observability/frontend/evaluations/overview/EvaluationResultsTable.tsx index 742fd78640c5..c268db1f38cb 100644 --- a/products/ai_observability/frontend/evaluations/overview/EvaluationResultsTable.tsx +++ b/products/ai_observability/frontend/evaluations/overview/EvaluationResultsTable.tsx @@ -63,15 +63,13 @@ export function EvaluationResultsTable(): JSX.Element { { title: 'Created by', key: 'created_by', - className: 'hidden @min-[40rem]:table-cell', - width: 0, - align: 'center', + className: 'hidden @min-[40rem]:table-cell whitespace-nowrap', render: (_, row) => { const user = configById.get(row.id)?.created_by return user ? ( - + ) : row.createdBy ? ( - + ) : ( - ) diff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py index 322772b97da4..9342df013747 100644 --- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py +++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py @@ -282,6 +282,8 @@ class SnowflakeQueryServerTimeoutError(TimeoutError): "TINYINT", "BYTEINT", "FIXED", + # Fixed-point number with a non-zero scale, created as 'NUMBER(precision, scale)' + "NUMBER", # All aliases of the same 64 bit float type "FLOAT", "FLOAT4", @@ -301,17 +303,25 @@ class SnowflakeQueryServerTimeoutError(TimeoutError): class SnowflakeType(typing.NamedTuple): name: SnowflakeTypeName repeated: bool + precision: int | None = None + scale: int | None = None + + +SNOWFLAKE_MAX_NUMBER_PRECISION = 38 class SnowflakeDestinationField(typing.NamedTuple): name: str type: SnowflakeTypeName is_nullable: bool + precision: int | None = None + scale: int | None = None def data_type_to_snowflake_type(data_type: pa.DataType) -> SnowflakeType: """Mapping of `pa.DataType` to corresponding `SnowflakeType`.""" - repeated = False + precision: int | None = None + scale: int | None = None if pa.types.is_string(data_type): snowflake_type_name: SnowflakeTypeName = "STRING" @@ -332,6 +342,10 @@ def data_type_to_snowflake_type(data_type: pa.DataType) -> SnowflakeType: elif pa.types.is_floating(data_type): snowflake_type_name = "FLOAT" + elif pa.types.is_decimal(data_type) and data_type.precision <= SNOWFLAKE_MAX_NUMBER_PRECISION: + snowflake_type_name = "NUMBER" + precision, scale = data_type.precision, data_type.scale + elif pa.types.is_boolean(data_type): snowflake_type_name = "BOOLEAN" @@ -342,13 +356,12 @@ def data_type_to_snowflake_type(data_type: pa.DataType) -> SnowflakeType: snowflake_type_name = "TIMESTAMP" elif pa.types.is_list(data_type): - repeated = True - snowflake_type_name = data_type_to_snowflake_type(data_type.value_type).name + return data_type_to_snowflake_type(data_type.value_type)._replace(repeated=True) else: raise TypeError(f"Unsupported type: '{data_type}'") - return SnowflakeType(snowflake_type_name, repeated=repeated) + return SnowflakeType(snowflake_type_name, repeated=False, precision=precision, scale=scale) def snowflake_type_to_data_type(type: SnowflakeType) -> pa.DataType: @@ -370,6 +383,9 @@ def snowflake_type_to_data_type(type: SnowflakeType) -> pa.DataType: base_type = pa.timestamp("ms", tz="UTC") case "FLOAT" | "FLOAT4" | "FLOAT8" | "DOUBLE PRECISION" | "REAL": base_type = pa.float64() + case "NUMBER": + assert type.precision is not None and type.scale is not None + base_type = pa.decimal128(type.precision, type.scale) case _: raise ValueError(f"Unsupported type: '{type.name}'") @@ -401,6 +417,11 @@ def from_destination_field(cls, field: SnowflakeDestinationField) -> typing.Self # it. For now, only support arrays of string. # TODO: Maybe support more arrays? snowflake_type = SnowflakeType(name="STRING", repeated=True) + elif field.type == "NUMBER" or (field.type == "FIXED" and field.scale): + # Snowflake reports every 'NUMBER' column as 'FIXED'. A non-zero scale means + # the column holds decimals, and reading it as 'INTEGER' would truncate them. + # 'NUMBER' itself only comes from `to_destination_field`. + snowflake_type = SnowflakeType(name="NUMBER", repeated=False, precision=field.precision, scale=field.scale) elif field.type == "FIXED": # This isn't documented as a type. I know it is 'INTEGER', not sure why the # snowflake library invented 'FIXED'. @@ -422,8 +443,22 @@ def snowflake_type_name(self) -> SnowflakeTypeName: else: return self.snowflake_type.name + @property + def snowflake_type_ddl(self) -> str: + if self.snowflake_type_name == "NUMBER": + # A bare 'NUMBER' is 'NUMBER(38, 0)', which would round away all decimals. + assert self.snowflake_type.precision is not None and self.snowflake_type.scale is not None + return f"NUMBER({self.snowflake_type.precision}, {self.snowflake_type.scale})" + return self.snowflake_type_name + def to_destination_field(self) -> SnowflakeDestinationField: - return SnowflakeDestinationField(name=self.name, type=self.snowflake_type_name, is_nullable=self.nullable) + return SnowflakeDestinationField( + name=self.name, + type=self.snowflake_type_name, + is_nullable=self.nullable, + precision=self.snowflake_type.precision, + scale=self.snowflake_type.scale, + ) def with_new_arrow_type(self, new_type: pa.DataType) -> "SnowflakeField": return SnowflakeField(self.name, data_type_to_snowflake_type(new_type), new_type, self.nullable) @@ -933,6 +968,8 @@ async def get_table(self, table: SnowflakeTable) -> SnowflakeTable: field_metadata.name, FIELD_ID_TO_NAME[field_metadata.type_code], # type: ignore[arg-type] field_metadata.is_nullable, + field_metadata.precision, + field_metadata.scale, ) for field_metadata in metadata if field_metadata.name.lower() in record_batch_field_names @@ -965,7 +1002,7 @@ async def create_table(self, table: SnowflakeTable, exists_ok: bool = True) -> N won't fail if the table already exists. """ field_ddl = ", ".join( - f"{_quote_snowflake_identifier(field.name)} {field.snowflake_type_name}" for field in table + f"{_quote_snowflake_identifier(field.name)} {field.snowflake_type_ddl}" for field in table ) if_not_exists = "" diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.py index cdc6c58697b0..013be2ec6513 100644 --- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.py +++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_client.py @@ -157,6 +157,7 @@ async def test_table(snowflake_client, database, schema, table_name): fields=( SnowflakeField("id", SnowflakeType("INTEGER", False), pa.int64(), True), SnowflakeField("text", SnowflakeType("TEXT", False), pa.string(), True), + SnowflakeField("amount", SnowflakeType("NUMBER", False, 38, 2), pa.decimal128(38, 2), True), ), parents=(database, schema), ) diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_field.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_field.py new file mode 100644 index 000000000000..da3068ddda4b --- /dev/null +++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/test_field.py @@ -0,0 +1,15 @@ +import pyarrow as pa + +from products.batch_exports.backend.temporal.destinations.snowflake_batch_export import ( + SnowflakeDestinationField, + SnowflakeField, +) + + +def test_decimal_field_maps_to_number_and_reads_back_as_decimal() -> None: + field = SnowflakeField.from_arrow_field(pa.field("amount", pa.decimal128(38, 2))) + + assert field.snowflake_type_ddl == "NUMBER(38, 2)" + # Snowflake reports a 'NUMBER(38, 2)' column as 'FIXED' with precision 38 and scale 2. + assert SnowflakeField.from_destination_field(SnowflakeDestinationField("amount", "FIXED", True, 38, 2)) == field + assert SnowflakeField.from_destination_field(field.to_destination_field()) == field diff --git a/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py b/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py index 1162ab9fb0fe..7379e502f195 100644 --- a/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py +++ b/products/batch_exports/backend/tests/temporal/destinations/snowflake/utils.py @@ -100,6 +100,8 @@ class FakeMetadata(typing.NamedTuple): name: str type_code: int is_nullable: bool + precision: int | None = None + scale: int | None = None class FakeSnowflakeCursor: diff --git a/products/business_knowledge/backend/api/libraries.py b/products/business_knowledge/backend/api/libraries.py index 12c97f79c318..ab838188ba2b 100644 --- a/products/business_knowledge/backend/api/libraries.py +++ b/products/business_knowledge/backend/api/libraries.py @@ -1,5 +1,5 @@ from functools import cached_property -from typing import Any, cast +from typing import Any from uuid import UUID from django.core.exceptions import ValidationError as DjangoValidationError @@ -14,13 +14,12 @@ from posthog.api.mixins import ValidatedRequest, validated_request from posthog.api.routing import TeamAndOrgViewSetMixin -from posthog.models.user import User from posthog.permissions import APIScopePermission, PostHogFeatureFlagPermission from posthog.rate_limit import BurstRateThrottle, SustainedRateThrottle from products.access_control.backend.facade.user_access_control import UserAccessControl -from ..github_repos import has_github_repos_feature_flag, selected_repositories +from ..github_repos import selected_repositories from ..libraries import ( LibraryNotFound, LibraryRejected, @@ -123,10 +122,6 @@ def retrieve(self, request: Request, pk: str, **kwargs: Any) -> Response: def partial_update(self, request: ValidatedRequest, pk: str, **kwargs: Any) -> Response: library_id = _parse_library_id(pk) data = request.validated_data - if "repos" in data: - user = cast(User, request.user) - if not has_github_repos_feature_flag(self.team, str(user.distinct_id)): - raise exceptions.PermissionDenied(detail="GitHub repositories are not available for this project.") allowlist = self._allowlist() if "repos" in data else set() try: with transaction.atomic(): diff --git a/products/business_knowledge/backend/api/repositories.py b/products/business_knowledge/backend/api/repositories.py index 52333247f40f..e14de15c5cbe 100644 --- a/products/business_knowledge/backend/api/repositories.py +++ b/products/business_knowledge/backend/api/repositories.py @@ -3,7 +3,7 @@ from drf_spectacular.utils import OpenApiResponse, extend_schema from rest_framework import exceptions from rest_framework.decorators import action -from rest_framework.permissions import BasePermission, IsAuthenticated +from rest_framework.permissions import IsAuthenticated from rest_framework.request import Request from rest_framework.response import Response from rest_framework.viewsets import ViewSet @@ -15,7 +15,6 @@ from posthog.rate_limit import BurstRateThrottle, SustainedRateThrottle from products.business_knowledge.backend.github_repos import ( - GITHUB_REPOS_FLAG, GithubReposError, RepositoryFile, RepositorySearchOutcome, @@ -42,28 +41,12 @@ _WRITE_ACTIONS = frozenset({"connect", "disconnect", "select"}) -class BusinessKnowledgeGithubReposPermission(BasePermission): - """Both the product flag and the repository flag. The stock checker returns on the first hit.""" - - def has_permission(self, request: Request, view: Any) -> bool: - checker = PostHogFeatureFlagPermission() - for flag in ("product-business-knowledge", GITHUB_REPOS_FLAG): - view.posthog_feature_flag = flag - if not checker.has_permission(request, view): - self.message = checker.message - code = getattr(checker, "code", None) - if code is not None: - self.code = code - return False - return True - - class BusinessKnowledgeRepositoriesViewSet(TeamAndOrgViewSetMixin, ViewSet): scope_object = "business_knowledge" # Plain ViewSet: object permissions never run, so resource-level access is checked here. requires_resource_level_access = True - permission_classes = [IsAuthenticated, APIScopePermission, BusinessKnowledgeGithubReposPermission] - posthog_feature_flag = GITHUB_REPOS_FLAG + permission_classes = [IsAuthenticated, APIScopePermission, PostHogFeatureFlagPermission] + posthog_feature_flag = "product-business-knowledge" throttle_classes = [BurstRateThrottle, SustainedRateThrottle] pagination_class = None diff --git a/products/business_knowledge/backend/github_repos.py b/products/business_knowledge/backend/github_repos.py index aec31a3350c1..650835813bb2 100644 --- a/products/business_knowledge/backend/github_repos.py +++ b/products/business_knowledge/backend/github_repos.py @@ -14,7 +14,6 @@ from urllib.parse import quote from uuid import UUID -from django.conf import settings from django.core.cache import cache from django.db import connection, models, transaction from django.utils import timezone @@ -36,14 +35,12 @@ ) from posthog.models.team.team import Team from posthog.models.user import User -from posthog.permissions import posthog_feature_flag_enabled from .learning_settings import get_environment_business_knowledge_config from .models import TeamBusinessKnowledgeConfig logger = structlog.get_logger(__name__) -GITHUB_REPOS_FLAG = "business-knowledge-github-repos" MAX_GITHUB_REPOS = 100 MAX_SEARCH_HITS = 20 MAX_README_HITS = 5 @@ -122,21 +119,7 @@ class RepositoryFile: truncated: bool -def has_github_repos_feature_flag(team: Team, distinct_id: str) -> bool: - """Same person, organization and project context as the API permission, so the prompt and the tools agree. - - On in DEBUG so a local playground can call the tools. - """ - if settings.DEBUG: - return True - return posthog_feature_flag_enabled( - GITHUB_REPOS_FLAG, distinct_id, organization_id=team.organization_id, team_id=team.id - ) - - -def repository_tools_enabled(team: Team, distinct_id: str) -> bool: - if not has_github_repos_feature_flag(team, distinct_id): - return False +def repository_tools_enabled(team: Team) -> bool: integration_id, repos = selected_repositories(team) return integration_id is not None and bool(repos) diff --git a/products/business_knowledge/backend/sandbox.py b/products/business_knowledge/backend/sandbox.py index 9cbf0719ccb7..e0aeef003abd 100644 --- a/products/business_knowledge/backend/sandbox.py +++ b/products/business_knowledge/backend/sandbox.py @@ -13,7 +13,6 @@ from posthog.dataclasses import frozen from posthog.models.team.team import Team -from posthog.models.user import User from products.tasks.backend.facade import api as tasks_facade from products.tasks.backend.facade.contracts import CreatedTaskDTO, TaskRunDTO @@ -239,11 +238,6 @@ def open_sandbox_task_ids(*, team_id: int, user_id: int) -> set[UUID]: ) -def _distinct_id_for(user_id: int) -> str: - distinct_id = User.objects.filter(pk=user_id).values_list("distinct_id", flat=True).first() - return str(distinct_id) - - def start_sandbox_run( *, team: Team, @@ -277,9 +271,7 @@ def start_sandbox_run( created = tasks_facade.create_and_run_task( team=team, title=_title_for(question), - description=build_sandbox_prompt( - question, always_on, repo_tools=repository_tools_enabled(team, _distinct_id_for(user_id)) - ), + description=build_sandbox_prompt(question, always_on, repo_tools=repository_tools_enabled(team)), origin_product=origin, user_id=user_id, repository=None, diff --git a/products/business_knowledge/backend/temporal/learning/activities/analyze.py b/products/business_knowledge/backend/temporal/learning/activities/analyze.py index 772b5abff439..8bbdc75174f0 100644 --- a/products/business_knowledge/backend/temporal/learning/activities/analyze.py +++ b/products/business_knowledge/backend/temporal/learning/activities/analyze.py @@ -111,6 +111,11 @@ def __init__(self, code: str) -> None: self.code = code +class LearningOutputTruncated(LearningAnalysisError): + def __init__(self, stage: str) -> None: + super().__init__(f"{stage}_output_truncated") + + class _SupersessionUnavailable(Exception): """The conflicting source is not one this job may replace, so the candidate must not be published.""" @@ -248,7 +253,29 @@ def _invoke_structured_model( tool_choice={"type": "tool", "name": tool_name}, extra_headers={"X-PostHog-Privacy-Mode": "true"}, ) - return _parse_tool_result(response.content, output_model, stage) + try: + return _parse_tool_result(response.content, output_model, stage) + except (LearningAnalysisError, ValidationError) as error: + if response.stop_reason == "max_tokens": + logger.warning( + "business_knowledge.learning.output_truncated", + stage=stage, + team_id=team.id, + trace_id=trace_id, + max_tokens=max_tokens, + ) + raise LearningOutputTruncated(stage) from None + if isinstance(error, LearningAnalysisError): + content = response.content if isinstance(response.content, list) else [] + logger.error( # noqa: TRY400 — the error is re-raised, so Temporal records its traceback + "business_knowledge.learning.invalid_output", + stage=stage, + team_id=team.id, + trace_id=trace_id, + stop_reason=response.stop_reason, + block_types=[getattr(block, "type", type(block).__name__) for block in content], + ) + raise except LearningAnalysisError: raise except Exception as error: @@ -545,6 +572,7 @@ def _finish_without_knowledge( "low_confidence": "rejected_not_useful", "learned_cap_reached": "rejected_learned_cap", "stale_candidate": "rejected_stale_candidate", + "output_truncated": "rejected_output_truncated", }.get(rejection_code, "rejected_not_useful") _increment_counter(metric_outcome) logger.info( @@ -698,7 +726,11 @@ def _analyze(run: KnowledgeLearningRun, input: AnalyzeLearningEvidenceInput) -> _resolve_learning_user(run.team) trace_id = str(run.id) - extracted = _extract_candidate(run.team, bundle, trace_id) + try: + extracted = _extract_candidate(run.team, bundle, trace_id) + except LearningOutputTruncated: + # Sampling runs at temperature 0, so a retry truncates again. A reply this long is not concise knowledge. + return _finish_without_knowledge(run, rejection_code="output_truncated") rejection_code = _extraction_rejection(extracted) if rejection_code != "none": return _finish_without_knowledge(run, rejection_code=rejection_code) diff --git a/products/business_knowledge/backend/temporal/learning/schemas.py b/products/business_knowledge/backend/temporal/learning/schemas.py index a2d36372a34c..8a73ba56efc5 100644 --- a/products/business_knowledge/backend/temporal/learning/schemas.py +++ b/products/business_knowledge/backend/temporal/learning/schemas.py @@ -21,6 +21,7 @@ "already_completed", "learned_cap_reached", "stale_candidate", + "output_truncated", ] LearningResult = Literal["knowledge_created", "no_knowledge", "ineligible", "superseded"] ExtractionRejectionCode = Literal["none", "case_specific", "not_useful", "unsupported", "low_confidence"] diff --git a/products/business_knowledge/backend/tests/test_api.py b/products/business_knowledge/backend/tests/test_api.py index f7fc7d18afa7..1ae65221046b 100644 --- a/products/business_knowledge/backend/tests/test_api.py +++ b/products/business_knowledge/backend/tests/test_api.py @@ -446,8 +446,7 @@ def test_library_from_another_team_is_not_found(self, _ff) -> None: assert retrieve.status_code == status.HTTP_404_NOT_FOUND assert search.status_code == status.HTTP_404_NOT_FOUND - @patch("products.business_knowledge.backend.api.libraries.has_github_repos_feature_flag", return_value=True) - def test_parent_repo_replace_keeps_a_child_only_link(self, _ff, _repos_flag) -> None: + def test_parent_repo_replace_keeps_a_child_only_link(self, _ff) -> None: from posthog.models.integration import Integration from posthog.models.scoping import team_scope from posthog.models.team import Team diff --git a/products/business_knowledge/backend/tests/test_learning_analyzer.py b/products/business_knowledge/backend/tests/test_learning_analyzer.py index 46efc15452b8..281af3fcce71 100644 --- a/products/business_knowledge/backend/tests/test_learning_analyzer.py +++ b/products/business_knowledge/backend/tests/test_learning_analyzer.py @@ -577,6 +577,50 @@ def test_model_error_marks_run_failed_without_capturing_sensitive_error( client.with_options.assert_called_once_with(timeout=_MODEL_TIMEOUT_SECONDS, max_retries=0) assert entered.messages.create.call_args.kwargs["extra_headers"] == {"X-PostHog-Privacy-Mode": "true"} + @pytest.mark.parametrize( + "content", + [ + pytest.param([], id="no_tool_block"), + pytest.param( + [ + ToolUseBlock( + id="toolu_test", + name="ExtractedKnowledge", + input={"canonical_topic": "Refund policy"}, + type="tool_use", + ) + ], + id="partial_tool_block", + ), + ], + ) + def test_truncated_extraction_completes_without_knowledge_instead_of_failing( + self, team: Team, content: list[ToolUseBlock] + ) -> None: + run, input = _setup_sync(team) + provider = _Provider(EvidenceBundle(replies=("Refunds are available within 30 days.",))) + client, entered = _gateway_client() + truncated = MagicMock() + truncated.stop_reason = "max_tokens" + truncated.content = content + entered.messages.create.return_value = truncated + + with ( + patch(f"{_MODULE}.get_learning_provider", return_value=provider), + patch(f"{_MODULE}.build_ai_gateway_anthropic_client", return_value=client), + patch(f"{_MODULE}.logic.create_generated_knowledge_document") as publish, + ): + result = analyze_learning_evidence(input) + + run.refresh_from_db() + assert result.result == "no_knowledge" + assert result.rejection_code == "output_truncated" + assert run.result == LearningRunResult.NO_KNOWLEDGE + assert run.status == "completed" + assert run.error == "" + assert entered.messages.create.call_count == 1 + publish.assert_not_called() + def test_missing_gateway_config_fails_before_the_model(self, team: Team) -> None: run, input = _setup_sync(team) provider = _Provider(EvidenceBundle(replies=("Refunds are available within 30 days.",))) diff --git a/products/business_knowledge/frontend/scenes/sources/LibraryDialog.tsx b/products/business_knowledge/frontend/scenes/sources/LibraryDialog.tsx index 5766f0426c75..4b3f509f1c04 100644 --- a/products/business_knowledge/frontend/scenes/sources/LibraryDialog.tsx +++ b/products/business_knowledge/frontend/scenes/sources/LibraryDialog.tsx @@ -2,14 +2,11 @@ import { useActions, useValues } from 'kea' import { LemonModal } from '@posthog/lemon-ui' -import { useFeatureFlag } from 'lib/hooks/useFeatureFlag' - import { businessKnowledgeLogic } from './businessKnowledgeLogic' import { LibraryNameForm } from './LibraryNameForm' import { LibraryRepoForm } from './LibraryRepoForm' export function LibraryDialog(): JSX.Element | null { - const githubEnabled = useFeatureFlag('BUSINESS_KNOWLEDGE_GITHUB_REPOS') const { libraryDialog, libraryScope, libraries, savingLibrary } = useValues(businessKnowledgeLogic) const { closeLibraryDialog, createLibrary, renameLibrary, saveLibraryRepos } = useActions(businessKnowledgeLogic) const active = libraries.find((library) => library.id === libraryScope) @@ -44,7 +41,7 @@ export function LibraryDialog(): JSX.Element | null { width={480} footer={null} > - {libraryDialog === 'repos' && githubEnabled && active ? ( + {libraryDialog === 'repos' && active ? ( Rename - {githubEnabled && ( - } - onClick={() => openLibraryDialog('repos')} - data-attr="business-knowledge-library-repos" - > - Repositories - - )} + } + onClick={() => openLibraryDialog('repos')} + data-attr="business-knowledge-library-repos" + > + Repositories + {counts.join(' · ')} {selectedLibrary.is_default && . Sources learned from support tickets join this library.}

    - {githubEnabled && selectedLibrary.repos.length > 0 && ( + {selectedLibrary.repos.length > 0 && (
    {selectedLibrary.repos.map((repo) => ( }> diff --git a/products/business_knowledge/mcp/tools.yaml b/products/business_knowledge/mcp/tools.yaml index 7f96d1651361..337321f82786 100644 --- a/products/business_knowledge/mcp/tools.yaml +++ b/products/business_knowledge/mcp/tools.yaml @@ -126,7 +126,7 @@ tools: business-knowledge-repositories-search. Returns the file text and a permalink. Does not search file contents. Pass library_id to read a repository in that library, using the same id as document search and repository search. Omit library_id to read any selected repository. - feature_flag: business-knowledge-github-repos + feature_flag: product-business-knowledge business-knowledge-repositories-search: operation: business_knowledge_repositories_search enabled: true @@ -144,7 +144,7 @@ tools: holds a handbook or docs, search that repository for those words and read the file. Does not search inside file contents and does not call GitHub code search. Pass library_id to search repositories in that library, using the same id as document search and file read. Omit library_id to search every selected repository. - feature_flag: business-knowledge-github-repos + feature_flag: product-business-knowledge business-knowledge-sources-list: operation: business_knowledge_sources_list enabled: true diff --git a/products/cdp/backend/api/hog_function.py b/products/cdp/backend/api/hog_function.py index 8df10aeb1498..1e788be88e0c 100644 --- a/products/cdp/backend/api/hog_function.py +++ b/products/cdp/backend/api/hog_function.py @@ -646,6 +646,14 @@ def to_internal_value(self, data): raise serializers.ValidationError({"template_id": f"No template found for id '{data['template_id']}'"}) + # A test run of an unsaved function takes template_id from the caller, and the template decides + # which reserved functions the code may call, so it gets the same check as a create. + is_unsaved_test = ( + instance is None and self.context.get("view") and self.context["view"].action == "invocations" + ) + if is_unsaved_test: + self._validate_template_is_creatable(template) + if is_create: # Set defaults for new functions data["inputs_schema"] = data.get("inputs_schema") or [] diff --git a/products/cdp/backend/api/test/test_hog_function.py b/products/cdp/backend/api/test/test_hog_function.py index e68d3d566a74..5d3a830f61ec 100644 --- a/products/cdp/backend/api/test/test_hog_function.py +++ b/products/cdp/backend/api/test/test_hog_function.py @@ -2802,6 +2802,30 @@ def test_test_invocation_rejects_a_posthog_connection_input(self): assert "PostHog connection" in json.dumps(response.json()) mock_create_hog_invocation_test.assert_not_called() + def test_test_invocation_of_unsaved_function_cannot_borrow_a_hidden_templates_reserved_calls(self): + self._create_email_template() + + with patch( + "products.cdp.backend.api.hog_function.create_hog_invocation_test" + ) as mock_create_hog_invocation_test: + response = self.client.post( + f"/api/projects/{self.team.id}/hog_functions/new/invocations/", + data={ + "configuration": { + "type": "destination", + "name": "Sneaky", + "template_id": "template-email-test", + "hog": "sendEmail({'to': {'email': 'someone@example.com'}, 'subject': 'hi'})", + "inputs": {}, + }, + "mock_async_functions": False, + }, + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.json() + assert "is internal and cannot be used" in json.dumps(response.json()) + mock_create_hog_invocation_test.assert_not_called() + @parameterized.expand( [ ("errors_list", {"errors": ["Missing event"]}, ["Missing event"]), diff --git a/products/conversations/backend/ai/human_outcome.py b/products/conversations/backend/ai/human_outcome.py index 062b5b36b529..7591801f8ac9 100644 --- a/products/conversations/backend/ai/human_outcome.py +++ b/products/conversations/backend/ai/human_outcome.py @@ -72,6 +72,7 @@ def record_human_outcome(*, team_id: int, ticket_id: str, draft_message_id: str, latest_ai_draft_id = ( _ticket_comments(team_id=team_id, ticket_id=ticket_id) .filter(item_context__author_type="AI", item_context__is_private=True, deleted=False) + .filter(item_context__automated_update_key__isnull=True) .order_by("-created_at", "-id") .values_list("id", flat=True) .first() @@ -114,6 +115,7 @@ def maybe_record_human_outcome(*, team_id: int, ticket_id: str, comment_id: str, # to the customer, so a later human reply is a follow-up, not adoption of a draft. ai_note = ( comments.filter(item_context__author_type="AI", item_context__is_private=True, deleted=False) + .filter(item_context__automated_update_key__isnull=True) .filter(before_this) .order_by("-created_at", "-id") .first() diff --git a/products/conversations/backend/api/tests/test_ai_triage_api.py b/products/conversations/backend/api/tests/test_ai_triage_api.py index edf529138bed..4f8d9bd9c2d8 100644 --- a/products/conversations/backend/api/tests/test_ai_triage_api.py +++ b/products/conversations/backend/api/tests/test_ai_triage_api.py @@ -89,6 +89,21 @@ def test_rejects_an_outcome_for_an_old_draft(self) -> None: self.ticket.refresh_from_db() assert "human_outcome" not in self.ticket.ai_triage + def test_automated_update_does_not_replace_the_latest_ai_draft(self) -> None: + Comment.objects.create( + team=self.team, + scope="conversations_ticket", + item_id=str(self.ticket.id), + content="Pull request update", + item_context={"author_type": "AI", "is_private": True, "automated_update_key": "pr-123"}, + ) + + response = self.client.post(self.url, self._payload("used"), format="json") + + assert response.status_code == status.HTTP_202_ACCEPTED + self.ticket.refresh_from_db() + assert self.ticket.ai_triage["human_outcome"] == "used" + class TestAiTriageSourcesAPI(APIBaseTest): def setUp(self) -> None: diff --git a/products/conversations/backend/api/tests/test_signals.py b/products/conversations/backend/api/tests/test_signals.py index 7a00638d6ad1..a36a1f9c4833 100644 --- a/products/conversations/backend/api/tests/test_signals.py +++ b/products/conversations/backend/api/tests/test_signals.py @@ -150,6 +150,15 @@ def test_first_public_human_reply_records_human_outcome(self, mock_on_commit): self.ticket.refresh_from_db() assert self.ticket.ai_triage["human_outcome"] == "used" + def test_automated_update_does_not_replace_draft_for_human_outcome(self, mock_on_commit): + draft = "Add the snippet to the head of every page." + self._create_ai_message(draft) + self._create_ai_message("Pull request update", item_context={"automated_update_key": "pr-123"}) + self._create_team_message(draft) + + self.ticket.refresh_from_db() + assert self.ticket.ai_triage["human_outcome"] == "used" + def test_used_draft_then_edited_send_upgrades_human_outcome(self, mock_on_commit): draft = "Add the snippet to the head of every page, then reload to send a pageview." ai_draft = self._create_ai_message(draft) diff --git a/products/conversations/backend/api/tests/test_tickets.py b/products/conversations/backend/api/tests/test_tickets.py index 11d1188ae08a..e1ec39b8c269 100644 --- a/products/conversations/backend/api/tests/test_tickets.py +++ b/products/conversations/backend/api/tests/test_tickets.py @@ -3480,6 +3480,25 @@ def test_ai_feedback_rejects_invalid_rating(self, mock_on_commit): ) assert response.status_code == status.HTTP_400_BAD_REQUEST + @patch("products.conversations.backend.api.tickets.posthoganalytics.capture") + def test_ai_feedback_rejects_automated_update(self, mock_capture, mock_on_commit): + update = Comment.objects.create( + team=self.team, + scope="conversations_ticket", + item_id=str(self.ticket.id), + content="Pull request update", + item_context={"author_type": "AI", "is_private": True, "automated_update_key": "pr-123"}, + ) + + response = self.client.post( + self.url, + {"message_id": str(update.id), "rating": "good"}, + format="json", + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + mock_capture.assert_not_called() + class TestTicketAccessControl(APIBaseTest): """Resource- and object-level access control for support tickets (the `ticket` RBAC resource).""" diff --git a/products/conversations/backend/api/tickets.py b/products/conversations/backend/api/tickets.py index 6cfd36907aa4..84b7b9316da5 100644 --- a/products/conversations/backend/api/tickets.py +++ b/products/conversations/backend/api/tickets.py @@ -1933,6 +1933,22 @@ def ai_feedback(self, request, *args, **kwargs): serializer.is_valid(raise_exception=True) data = serializer.validated_data + try: + message_id = uuid.UUID(data["message_id"]) + except ValueError: + message_id = None + if ( + message_id + and Comment.objects.filter( + id=message_id, + team_id=self.team_id, + scope="conversations_ticket", + item_id=str(ticket.id), + item_context__automated_update_key__isnull=False, + ).exists() + ): + raise ValidationError({"message_id": "Automated updates cannot receive AI feedback."}) + ai_triage = ticket.ai_triage or {} trace_id = ai_triage.get("ai_trace_id") distinct_id = f"reviewer:{request.user.distinct_id}" diff --git a/products/conversations/backend/facade/api.py b/products/conversations/backend/facade/api.py index ed31ad9a2920..b63e1b47e2e3 100644 --- a/products/conversations/backend/facade/api.py +++ b/products/conversations/backend/facade/api.py @@ -59,10 +59,11 @@ Status, Ticket, ) -from products.conversations.backend.models.constants import WORKFLOW_AUTHOR_TYPE +from products.conversations.backend.models.constants import WORKFLOW_AUTHOR_TYPE, Channel from products.conversations.backend.services.messages import public_human_ticket_replies -from products.conversations.backend.slack import get_slack_client +from products.conversations.backend.slack import _is_internal_channel, get_slack_client from products.conversations.backend.support_slack import ( + automated_update_channel_allowed, get_support_slack_bot_token, supporthog_lacks_custom_identity_scope, ) @@ -441,6 +442,59 @@ def post_ticket_internal_note(team_id: int, ticket_id: str, content: str, *, ded return str(comment.id) +def post_ticket_automated_update( + team_id: int, ticket_id: str, *, public_content: str, private_content: str, dedupe_key: str +) -> str | None: + source_ticket = Ticket.objects.filter(team_id=team_id, id=ticket_id).select_related("team").first() + if source_ticket is None: + return None + checked_channel = source_ticket.slack_channel_id + send_to_slack = False + if ( + source_ticket.channel_source == Channel.SLACK + and checked_channel + and source_ticket.slack_thread_ts + and automated_update_channel_allowed(source_ticket.team, checked_channel) + ): + try: + send_to_slack = _is_internal_channel(get_slack_client(source_ticket.team), checked_channel) + except Exception: + logger.exception("Failed to check support Slack channel before automated update", ticket_id=ticket_id) + + with transaction.atomic(): + ticket = Ticket.objects.select_for_update().filter(team_id=team_id, id=ticket_id).first() + if ticket is None: + return None + if Comment.objects.filter( + team_id=team_id, + scope="conversations_ticket", + item_id=ticket_id, + item_context__automated_update_key=dedupe_key, + deleted=False, + ).exists(): + return None + send_to_slack = bool( + send_to_slack + and ticket.channel_source == Channel.SLACK + and ticket.slack_channel_id == checked_channel + and ticket.slack_thread_ts == source_ticket.slack_thread_ts + ) + comment = Comment.objects.create( + team_id=team_id, + scope="conversations_ticket", + item_id=ticket_id, + content=public_content if send_to_slack else private_content, + item_context={ + "author_type": "AI", + "is_private": not send_to_slack, + "automated_update_key": dedupe_key, + "internal_channel_only": send_to_slack, + **({"internal_note_key": dedupe_key} if not send_to_slack else {}), + }, + ) + return str(comment.id) + + def trigger_immediate_channel_summary( *, team_id: int, diff --git a/products/conversations/backend/models/constants.py b/products/conversations/backend/models/constants.py index 63fb9e56876c..76ce60614837 100644 --- a/products/conversations/backend/models/constants.py +++ b/products/conversations/backend/models/constants.py @@ -1,5 +1,7 @@ from django.db import models +AUTOMATED_UPDATE_SLACK_CHANNEL_IDS = "automated_update_slack_channel_ids" + class Channel(models.TextChoices): WIDGET = "widget", "Widget" diff --git a/products/conversations/backend/services/delivery.py b/products/conversations/backend/services/delivery.py index dd23a3f7d908..e9af5a3e795d 100644 --- a/products/conversations/backend/services/delivery.py +++ b/products/conversations/backend/services/delivery.py @@ -220,7 +220,7 @@ def _body_snapshot( include_images=False, organization_id=team.organization_id, ) - return { + snapshot = { "text": slack_text, "blocks": slack_blocks, "author_name": author.name, @@ -228,6 +228,9 @@ def _body_snapshot( "media_team_id": media_team_id, "images": _image_refs(comment.rich_content if isinstance(comment.rich_content, dict) else None), } + if isinstance(comment.item_context, dict) and comment.item_context.get("internal_channel_only") is True: + snapshot["internal_channel_only"] = True + return snapshot def _enqueue_failed_slack_body( diff --git a/products/conversations/backend/signals.py b/products/conversations/backend/signals.py index fb1519ae0201..1da8304dcf17 100644 --- a/products/conversations/backend/signals.py +++ b/products/conversations/backend/signals.py @@ -143,6 +143,14 @@ def update_ticket_on_message(sender, instance: Comment, created: bool, **kwargs) if not created: return + if ( + instance.created_by_id is None + and isinstance(instance.item_context, dict) + and instance.item_context.get("author_type") == "AI" + and instance.item_context.get("automated_update_key") + ): + return + # Capture values for closure (avoid referencing instance in deferred callback) team_id = instance.team_id item_id = instance.item_id diff --git a/products/conversations/backend/slack.py b/products/conversations/backend/slack.py index 02e233beaccd..69636db650ff 100644 --- a/products/conversations/backend/slack.py +++ b/products/conversations/backend/slack.py @@ -1665,7 +1665,7 @@ def ticket_unfurl(ticket: "Ticket", team: Team) -> dict: } -def _is_internal_channel(client: WebClient, channel: str) -> bool: +def _is_internal_channel(client: WebClient, channel: str, *, raise_on_error: bool = False) -> bool: """Whether `channel` is an ordinary channel of this workspace alone. Fails closed, twice over: a channel we cannot read is one we cannot prove is internal, and @@ -1684,6 +1684,8 @@ def _is_internal_channel(client: WebClient, channel: str) -> bool: response = client.conversations_info(channel=channel) except Exception: logger.warning("slack_support_unfurl_channel_lookup_failed", slack_channel_id=channel) + if raise_on_error: + raise return False info = response.get("channel") or {} if any(info.get(flag) for flag in _NON_INTERNAL_CHANNEL_FLAGS): diff --git a/products/conversations/backend/support_slack.py b/products/conversations/backend/support_slack.py index f3ede863a48e..19fd8ba4d568 100644 --- a/products/conversations/backend/support_slack.py +++ b/products/conversations/backend/support_slack.py @@ -11,6 +11,7 @@ from posthog.models.utils import mask_key_value from products.conversations.backend.models import TeamConversationsSlackConfig +from products.conversations.backend.models.constants import AUTOMATED_UPDATE_SLACK_CHANNEL_IDS if TYPE_CHECKING: from posthog.models.team.team import Team @@ -28,6 +29,13 @@ SUPPORT_SLACK_CUSTOM_IDENTITY_SCOPE = "chat:write.customize" +def automated_update_channel_allowed(team: "Team", channel_id: str) -> bool: + project_team = team.parent_team or team + settings = project_team.conversations_settings + channel_ids = settings.get(AUTOMATED_UPDATE_SLACK_CHANNEL_IDS) if isinstance(settings, dict) else None + return isinstance(channel_ids, list) and channel_id in channel_ids + + # Every SupportHog delivery is verified with the signing secret, so the read is cached and not made # once per request. A rotation reaches each process within five minutes. @cache_for(timedelta(minutes=5)) diff --git a/products/conversations/backend/tasks/slack.py b/products/conversations/backend/tasks/slack.py index 84ffe3590cfe..f47f0f9ffa01 100644 --- a/products/conversations/backend/tasks/slack.py +++ b/products/conversations/backend/tasks/slack.py @@ -83,6 +83,7 @@ NudgeClassifierVerdict, NudgeFunnelVerdict, SlackConfirmationNeedsRetry, + _is_internal_channel, capture_nudge_event, capture_support_event, create_ticket_from_confirmation, @@ -102,7 +103,11 @@ ticket_deep_link, ) -from ..support_slack import SUPPORT_SLACK_ALLOWED_HOST_SUFFIXES, supporthog_missing_file_scopes +from ..support_slack import ( + SUPPORT_SLACK_ALLOWED_HOST_SUFFIXES, + automated_update_channel_allowed, + supporthog_missing_file_scopes, +) logger = structlog.get_logger(__name__) SUPPORTHOG_EVENT_IDEMPOTENCY_TTL_SECONDS = 6 * 60 @@ -846,6 +851,9 @@ def _post_slack_body( "text": slack_text, "username": sender.username, } + if payload.get("internal_channel_only") is True: + message_kwargs["unfurl_links"] = False + message_kwargs["unfurl_media"] = False if client_msg_id: message_kwargs["client_msg_id"] = client_msg_id if sender.icon_url: @@ -928,6 +936,34 @@ def _load_slack_delivery_runtime(claim: DeliveryClaim) -> SlackDeliveryRuntime | def _deliver_slack_body(claim: DeliveryClaim, runtime: SlackDeliveryRuntime) -> None: + channel = str(runtime.route.get("channel") or "") + if runtime.payload.get("internal_channel_only") is True and not automated_update_channel_allowed( + runtime.team, channel + ): + _fail_claimed_delivery_part( + claim, + error_code="channel_not_allowlisted", + error="The Slack channel is not approved for automated updates", + ) + return + if runtime.payload.get("internal_channel_only") is True: + try: + is_internal = _is_internal_channel(runtime.client, channel, raise_on_error=True) + except Exception as exc: + _retry_delivery_claim( + claim, + error_code="channel_lookup_failed", + error="Could not check whether the Slack channel is internal", + retry_after_seconds=_retry_after_seconds(exc), + ) + return + if not is_internal: + _fail_claimed_delivery_part( + claim, + error_code="channel_not_internal", + error="The Slack channel is not confirmed as internal", + ) + return ts = _post_slack_body( client=runtime.client, team=runtime.team, diff --git a/products/conversations/backend/temporal/coordinator.py b/products/conversations/backend/temporal/coordinator.py index 7e2cf93d2f9a..bbb8442b3285 100644 --- a/products/conversations/backend/temporal/coordinator.py +++ b/products/conversations/backend/temporal/coordinator.py @@ -208,6 +208,7 @@ def _collect_eligible(lookback_minutes: int = TICKET_LOOKBACK_MINUTES) -> list[E team_id=team_id, scope="conversations_ticket", item_id__in=ticket_ids, + item_context__automated_update_key__isnull=True, ).values_list("item_id", "item_context__author_type", "created_at"): if author_type == "customer": prev = latest_customer_msg.get(item_id) diff --git a/products/conversations/backend/temporal/tests/test_coordinator.py b/products/conversations/backend/temporal/tests/test_coordinator.py index 37eb3518c48e..256e8cde9038 100644 --- a/products/conversations/backend/temporal/tests/test_coordinator.py +++ b/products/conversations/backend/temporal/tests/test_coordinator.py @@ -406,7 +406,7 @@ def _team_and_ticket(self, *, status: str = "pending", awaiting: bool = True): ticket.refresh_from_db() return team, ticket - def _comment(self, team, ticket, *, author_type: str, minutes_ago: int): + def _comment(self, team, ticket, *, author_type: str, minutes_ago: int, automated_update_key: str | None = None): from posthog.models.comment import Comment created = timezone.now() - timedelta(minutes=minutes_ago) @@ -415,7 +415,12 @@ def _comment(self, team, ticket, *, author_type: str, minutes_ago: int): scope="conversations_ticket", item_id=str(ticket.id), content=author_type, - item_context={"author_type": author_type, "is_private": False, "persist_as": "clarification"}, + item_context={ + "author_type": author_type, + "is_private": False, + "persist_as": "clarification", + **({"automated_update_key": automated_update_key} if automated_update_key else {}), + }, ) Comment.objects.filter(id=comment.id).update(created_at=created) from products.conversations.backend.models.ticket import Ticket as TicketModel @@ -424,6 +429,23 @@ def _comment(self, team, ticket, *, author_type: str, minutes_ago: int): ticket.refresh_from_db() return comment + @pytest.mark.django_db + @patch(f"{COORD_MODULE}._is_master_flag_enabled", return_value=True) + def test_automated_pr_update_does_not_answer_a_customer_followup(self, mock_master_flag): + team, ticket = self._team_and_ticket() + self._comment(team, ticket, author_type="customer", minutes_ago=10) + self._comment(team, ticket, author_type="AI", minutes_ago=5) + self._comment(team, ticket, author_type="customer", minutes_ago=3) + self._comment( + team, + ticket, + author_type="AI", + minutes_ago=1, + automated_update_key="signals_pr:report:pr", + ) + + assert [item.ticket_id for item in _collect_eligible()] == [str(ticket.id)] + @parameterized.expand( [ ( diff --git a/products/conversations/backend/tests/test_slack_delivery.py b/products/conversations/backend/tests/test_slack_delivery.py index 854148d17133..fb3b3ee364d8 100644 --- a/products/conversations/backend/tests/test_slack_delivery.py +++ b/products/conversations/backend/tests/test_slack_delivery.py @@ -99,6 +99,108 @@ def _create_reply(self, content: str = "Support reply") -> Comment: def _part(self) -> ConversationDeliveryPart: return ConversationDeliveryPart.objects.unscoped().select_related("delivery").get() + @parameterized.expand( + [ + ("shared", {"is_channel": True, "is_ext_shared": True}, True, False, "channel_not_internal"), + ("internal", {"is_channel": True}, True, True, ""), + ("approval_removed", {"is_channel": True}, False, False, "channel_not_allowlisted"), + ] + ) + @patch("products.conversations.backend.tasks.slack.get_slack_client") + def test_automated_update_checks_channel_before_delivery( + self, + _name: str, + channel_info: dict, + approved: bool, + expected_sent: bool, + expected_error: str, + mock_get_client: MagicMock, + ) -> None: + self.team.conversations_settings = { + **self.team.conversations_settings, + "automated_update_slack_channel_ids": ["C123"] if approved else [], + } + self.team.save(update_fields=["conversations_settings"]) + Comment.objects.create( + team=self.team, + scope="conversations_ticket", + item_id=str(self.ticket.id), + content="[PostHog bot] Pull request opened", + item_context={ + "author_type": "AI", + "is_private": False, + "automated_update_key": "signals_pr:report:pr", + "internal_channel_only": True, + }, + ) + part = self._part() + payload = part.payload + assert isinstance(payload, dict) + assert payload["internal_channel_only"] is True + client = MagicMock() + client.conversations_info.return_value = {"channel": channel_info} + client.chat_postMessage.return_value = {"ts": "1700.1"} + mock_get_client.return_value = client + + process_slack_delivery_part(str(part.id)) + + part.refresh_from_db() + if expected_sent: + assert part.status == ConversationDeliveryPart.Status.ACCEPTED + kwargs = client.chat_postMessage.call_args.kwargs + assert kwargs["unfurl_links"] is False + assert kwargs["unfurl_media"] is False + else: + assert part.status == ConversationDeliveryPart.Status.FAILED + assert part.last_error_code == expected_error + client.chat_postMessage.assert_not_called() + + @parameterized.expand( + [ + ("timeout", TimeoutError("Slack unavailable"), None), + ("rate_limited", _slack_api_error("ratelimited", retry_after="17", status_code=429), 17), + ] + ) + @patch("products.conversations.backend.tasks.slack.get_slack_client") + def test_automated_update_retries_failed_channel_lookup( + self, _name: str, error: Exception, retry_after: int | None, mock_get_client: MagicMock + ) -> None: + self.team.conversations_settings = { + **self.team.conversations_settings, + "automated_update_slack_channel_ids": ["C123"], + } + self.team.save(update_fields=["conversations_settings"]) + Comment.objects.create( + team=self.team, + scope="conversations_ticket", + item_id=str(self.ticket.id), + content="[PostHog bot] Pull request opened", + item_context={"author_type": "AI", "is_private": False, "internal_channel_only": True}, + ) + part = self._part() + client = MagicMock() + client.conversations_info.side_effect = [error, {"channel": {"is_channel": True}}] + client.chat_postMessage.return_value = {"ts": "1700.1"} + mock_get_client.return_value = client + + with time_machine.travel("2026-09-21 12:00:00", tick=False): + ConversationDeliveryPart.objects.unscoped().filter(id=part.id).update(due_at=timezone.now()) + process_slack_delivery_part(str(part.id)) + part.refresh_from_db() + assert part.status == ConversationDeliveryPart.Status.PENDING + assert part.last_error_code == "channel_lookup_failed" + assert part.due_at > timezone.now() + if retry_after is not None: + assert part.due_at == timezone.now() + timedelta(seconds=retry_after) + client.chat_postMessage.assert_not_called() + + ConversationDeliveryPart.objects.unscoped().filter(id=part.id).update(due_at=timezone.now()) + process_slack_delivery_part(str(part.id)) + + part.refresh_from_db() + assert part.status == ConversationDeliveryPart.Status.ACCEPTED + client.chat_postMessage.assert_called_once() + def test_enqueue_failure_rolls_back_the_comment(self) -> None: with ( self.assertRaises(RuntimeError), diff --git a/products/conversations/frontend/components/Chat/aiDraftAction.test.ts b/products/conversations/frontend/components/Chat/aiDraftAction.test.ts index b1bb5faf15bd..fcdc76abfb63 100644 --- a/products/conversations/frontend/components/Chat/aiDraftAction.test.ts +++ b/products/conversations/frontend/components/Chat/aiDraftAction.test.ts @@ -21,6 +21,7 @@ describe('aiDraftAction', () => { ['public auto-sent reply', { isPrivate: false, persistAs: 'reply' as const }, null], ['public clarifying question', { isPrivate: false, persistAs: 'clarification' as const }, null], ['human private note', { authorType: 'human' as const }, null], + ['automated private note', { automatedUpdate: true }, null], ])('%s', (_name, overrides, expected) => { expect(aiDraftAction({ ...base, ...overrides })).toBe(expected) }) diff --git a/products/conversations/frontend/components/Chat/aiDraftAction.ts b/products/conversations/frontend/components/Chat/aiDraftAction.ts index 62581edc392f..b96e43985dc2 100644 --- a/products/conversations/frontend/components/Chat/aiDraftAction.ts +++ b/products/conversations/frontend/components/Chat/aiDraftAction.ts @@ -18,9 +18,9 @@ export function firstClarifyingQuestion(questions: unknown): string | null { } export function aiDraftAction( - message: Pick + message: Pick ): AiDraftActionKind | null { - if (message.authorType !== 'AI' || !message.isPrivate) { + if (message.authorType !== 'AI' || !message.isPrivate || message.automatedUpdate) { return null } if (message.persistAs === 'clarification') { diff --git a/products/conversations/frontend/scenes/ticket/supportTicketSceneLogic.test.ts b/products/conversations/frontend/scenes/ticket/supportTicketSceneLogic.test.ts index bf8d8e5d32d3..360f7844f140 100644 --- a/products/conversations/frontend/scenes/ticket/supportTicketSceneLogic.test.ts +++ b/products/conversations/frontend/scenes/ticket/supportTicketSceneLogic.test.ts @@ -164,6 +164,19 @@ describe('supportTicketSceneLogic ai reply feedback', () => { }) }) + it('keeps feedback on the AI reply after an automated update', () => { + logic.actions.setMessages([ + makeAiComment('msg-ai-1', false), + { + ...makeAiComment('pr-update', false), + created_at: '2026-01-01T00:01:00Z', + item_context: { author_type: 'AI', is_private: false, automated_update_key: 'pr-123' }, + } as unknown as CommentType, + ]) + + expect(logic.values.latestAiMessage?.id).toBe('msg-ai-1') + }) + it('calls backend relay on good feedback', async () => { await expectLogic(logic, () => { logic.actions.submitAiReplyFeedback('msg-ai-1', 'good') @@ -348,6 +361,19 @@ describe('supportTicketSceneLogic chatMessages mapping', () => { expect(logic.values.latestAiDraftId).toBe('new-draft') }) + it('shows automated private notes without treating them as AI drafts', () => { + featureFlagLogic.actions.setFeatureFlags([], { [FEATURE_FLAGS.PRODUCT_SUPPORT_AI_NOTES]: false }) + logic.actions.setMessages([ + { + ...makeAiComment('pr-update'), + item_context: { author_type: 'AI', is_private: true, automated_update_key: 'pr-123' }, + } as unknown as CommentType, + ]) + + expect(logic.values.chatMessages).toEqual([expect.objectContaining({ id: 'pr-update', automatedUpdate: true })]) + expect(logic.values.latestAiDraftId).toBeNull() + }) + it('selects widget delivery statuses for public team messages', () => { logic.actions.setTicket({ ...makeTicket(), unread_customer_count: 1 }) logic.actions.setMessages([ diff --git a/products/conversations/frontend/scenes/ticket/supportTicketSceneLogic.ts b/products/conversations/frontend/scenes/ticket/supportTicketSceneLogic.ts index c39b85fa48b6..149a4106f1ff 100644 --- a/products/conversations/frontend/scenes/ticket/supportTicketSceneLogic.ts +++ b/products/conversations/frontend/scenes/ticket/supportTicketSceneLogic.ts @@ -281,7 +281,11 @@ export function getEmailReplyBlockedReason( * make the thread misrepresent the conversation. */ export function isAiPrivateNote(message: Pick): boolean { - return message.item_context?.author_type === 'AI' && message.item_context?.is_private === true + return ( + message.item_context?.author_type === 'AI' && + message.item_context?.is_private === true && + !message.item_context?.automated_update_key + ) } // Generated by kea-typegen. Update if you're an agent, ignore if you're human. @@ -1226,6 +1230,7 @@ export const supportTicketSceneLogic = kea([ createdBy: message.created_by, createdAt: message.created_at, isPrivate: message.item_context?.is_private || false, + automatedUpdate: !!message.item_context?.automated_update_key, version: message.version, emailDeliveryStatus: message.item_context?.email_delivery_status, fromZendesk: message.item_context?.from_zendesk === true, @@ -1301,7 +1306,7 @@ export const supportTicketSceneLogic = kea([ (s) => [s.chatMessages], (chatMessages: ChatMessage[]): ChatMessage | null => { for (let i = chatMessages.length - 1; i >= 0; i--) { - if (chatMessages[i].authorType === 'AI') { + if (chatMessages[i].authorType === 'AI' && !chatMessages[i].automatedUpdate) { return chatMessages[i] } } diff --git a/products/conversations/frontend/types.ts b/products/conversations/frontend/types.ts index 9c0eb580fdc2..30cf45427af8 100644 --- a/products/conversations/frontend/types.ts +++ b/products/conversations/frontend/types.ts @@ -234,6 +234,7 @@ export interface ChatMessage { createdBy?: MessageAuthor | null createdAt: string isPrivate?: boolean + automatedUpdate?: boolean /** Edit count from the comment row. 0 means never edited. */ version?: number emailDeliveryStatus?: EmailDeliveryStatus diff --git a/products/data_warehouse/backend/logic/external_data_source/notifications.py b/products/data_warehouse/backend/logic/external_data_source/notifications.py index 1be8f3e3a738..98af24cc1c63 100644 --- a/products/data_warehouse/backend/logic/external_data_source/notifications.py +++ b/products/data_warehouse/backend/logic/external_data_source/notifications.py @@ -24,13 +24,6 @@ # notified — the email communicates them in aggregate via the "+N more" line. MAX_SCHEMAS_PER_DIGEST_EMAIL = 30 -# Re-notify a schema that stays failed this long after its last email. A paused -# schema never runs again, so it produces no newer failed job to re-trigger the -# digest — without this it gets exactly one email, then silence for weeks while -# its data goes stale. The catch-up sweep runs daily, so it picks the schema up -# the first day past this age and stamps a fresh time, spacing reminders evenly. -RENOTIFY_STILL_FAILING_AFTER = dt.timedelta(days=7) - # A schema the user switched off is not a failure to chase: it will not run again, and nobody # asked it to. Its last status stays Failed for the syncs UI, so the digest cannot read status # alone. `auto_disabled_at` marks the other way syncing stops, where PostHog halted the schema @@ -40,31 +33,22 @@ SCHEMA_STOPPED_BY_USER = Q(should_sync=False, auto_disabled_at__isnull=True) -def get_team_ids_with_recent_sync_failures( - lookback: dt.timedelta = dt.timedelta(hours=26), - renotify_after: dt.timedelta = RENOTIFY_STILL_FAILING_AFTER, -) -> list[int]: +def get_team_ids_with_recent_sync_failures(lookback: dt.timedelta = dt.timedelta(hours=26)) -> list[int]: """Teams with still-failing schemas that need a digest. - Powers the daily catch-up digest. It flushes two groups: - - - Failures that the one-email-per-day block swallowed. A failure counts only - if it is newer than the schema's `last_error_notified_at` stamp, so failures - already covered by an earlier digest do not trigger a duplicate. - - Schemas that stay failed but produce no newer failed run — a paused schema - never syncs again. Their last email is older than `renotify_after`, so they - get a reminder instead of silence. - - Schemas the user switched off are left out entirely, per `SCHEMA_STOPPED_BY_USER`. + Powers the daily catch-up digest. It flushes failures that the one-email-per-day + block swallowed. A failure counts only if it is newer than the schema's + `last_error_notified_at` stamp, so failures already covered by an earlier digest + do not trigger a duplicate. A paused schema never runs again, so after its first + digest it stays out until a new run fails. Schemas the user switched off are left + out entirely, per `SCHEMA_STOPPED_BY_USER`. The lookback exceeds the 24h digest day on purpose: a failure just after the 10:00 UTC rollover, blocked because that digest day's email already went out, is 24h15m+ old by the next catch-up run — a 24h lookback would drop it forever for paused schemas. The stamp check above keeps the wider window duplicate-free. """ - now = dt.datetime.now(dt.UTC) - cutoff = now - lookback - renotify_cutoff = now - renotify_after + cutoff = dt.datetime.now(dt.UTC) - lookback # Drive from the schema side: the jobs table grows with every sync run and has # no index on (status, finished_at), so starting there would seq-scan it daily. # Schemas are one row each, and their jobs are reachable via the schema_id FK index. @@ -78,21 +62,18 @@ def get_team_ids_with_recent_sync_failures( .exclude(source__deleted=True) .exclude(SCHEMA_STOPPED_BY_USER) .filter(status=ExternalDataSchemaStatus.FAILED) - .filter(Exists(unnotified_failed_job) | Q(last_error_notified_at__lt=renotify_cutoff)) + .filter(Exists(unnotified_failed_job)) .values_list("team_id", flat=True) .distinct() ) -def notify_external_data_sync_failures( - team_id: int, renotify_after: dt.timedelta = RENOTIFY_STILL_FAILING_AFTER -) -> None: +def notify_external_data_sync_failures(team_id: int) -> None: """Email the team a digest of failing external data schemas that need a notification. - A schema is listed when it was never notified, has a failed run newer than its last - notification, or has stayed failed for longer than `renotify_after` since its last - email. The last case reminds the team about a paused schema, which never runs again - and so produces no newer failed run to re-trigger the digest on its own. Schemas of a + A schema is listed when it was never notified, or has a failed run newer than its last + notification. A paused schema never runs again, so it is listed once and then stays out + of later digests until someone turns syncing back on and a new run fails. Schemas of a deleted source, and schemas the user switched off (see `SCHEMA_STOPPED_BY_USER`), are excluded entirely. Runs inside the digest Celery task; exceptions are swallowed so a notification problem never crash-loops the task. Throttling to one email per team per @@ -100,7 +81,6 @@ def notify_external_data_sync_failures( scheduling this for every failed job is safe. """ try: - renotify_cutoff = dt.datetime.now(dt.UTC) - renotify_after newer_failed_job = ExternalDataJob.objects.filter( schema_id=OuterRef("id"), status=ExternalDataJobStatus.FAILED, @@ -111,11 +91,7 @@ def notify_external_data_sync_failures( .exclude(source__deleted=True) .exclude(SCHEMA_STOPPED_BY_USER) .filter(team_id=team_id, status=ExternalDataSchemaStatus.FAILED) - .filter( - Q(last_error_notified_at__isnull=True) - | Q(last_error_notified_at__lt=renotify_cutoff) - | Exists(newer_failed_job) - ) + .filter(Q(last_error_notified_at__isnull=True) | Exists(newer_failed_job)) .select_related("source") .order_by("name") ) diff --git a/products/data_warehouse/backend/logic/external_data_source/test/test_notifications.py b/products/data_warehouse/backend/logic/external_data_source/test/test_notifications.py index cf42bed816e0..f881fcf0756a 100644 --- a/products/data_warehouse/backend/logic/external_data_source/test/test_notifications.py +++ b/products/data_warehouse/backend/logic/external_data_source/test/test_notifications.py @@ -317,31 +317,38 @@ def test_already_notified_failure_renotifies_only_on_newer_run(self, run_offset, assert mock_sender.called == expected_sent @pytest.mark.parametrize( - "notified_age,expected_sent", + "should_sync,auto_disabled,sync_type_config,notified_age", [ - (dt.timedelta(days=8), True), # still failing past the window -> remind again - (dt.timedelta(days=6), False), # notified recently, no new run -> stay silent + (False, True, {}, dt.timedelta(days=8)), + (False, True, {}, dt.timedelta(days=90)), + (True, False, {"cdc_broken": {"reason": "auto_dropped_critical_lag"}}, dt.timedelta(days=8)), ], ) - def test_renotifies_still_failing_schema_without_a_newer_run(self, notified_age, expected_sent): - # A paused schema never runs again, so it produces no newer failed job. It must - # still get a reminder once its last email ages past the re-notify window. + def test_paused_schema_is_listed_only_once(self, should_sync, auto_disabled, sync_type_config, notified_age): + # A paused schema never runs again, so once a digest has listed it, later digests stay + # silent about it however long it stays failed. team, source = _create_team_and_source() - ExternalDataSchema.objects.create( + notified_at = dt.datetime.now(dt.UTC) - notified_age + schema = ExternalDataSchema.objects.create( name="Charge", team=team, source=source, status=ExternalDataSchema.Status.FAILED, - should_sync=False, - auto_disabled_at=dt.datetime.now(dt.UTC) - notified_age, + should_sync=should_sync, + auto_disabled_at=notified_at if auto_disabled else None, + sync_type_config=sync_type_config, latest_error="Invalid API key", - last_error_notified_at=dt.datetime.now(dt.UTC) - notified_age, + last_error_notified_at=notified_at, + ) + job = ExternalDataJob.objects.create( + team=team, pipeline=source, schema=schema, status=ExternalDataJob.Status.FAILED ) + ExternalDataJob.objects.filter(id=job.id).update(finished_at=notified_at - dt.timedelta(minutes=5)) with patch(SENDER_PATH) as mock_sender: notify_external_data_sync_failures(team.pk) - assert mock_sender.called == expected_sent + mock_sender.assert_not_called() class TestGetTeamIdsWithRecentSyncFailures: @@ -436,19 +443,19 @@ def test_includes_failure_blocked_just_after_digest_rollover(self): assert get_team_ids_with_recent_sync_failures() == [team.pk] - def test_includes_still_failing_schema_past_the_renotify_window(self): - # PostHog halted this schema itself, so it never runs again. No recent failed run - # (30h stale) and the last email is 8 days old: the team would drop out forever - # without the re-notify path. - team = self._create_schema_with_job( + @pytest.mark.parametrize("notified_age", [dt.timedelta(days=8), dt.timedelta(days=90)]) + def test_excludes_paused_schema_already_notified(self, notified_age): + # PostHog halted this schema itself, so it never runs again. Its one digest already went + # out, so the catch-up sweep must not pick it up again however old that email is. + self._create_schema_with_job( schema_status=ExternalDataSchema.Status.FAILED, - job_finished_at=dt.datetime.now(dt.UTC) - dt.timedelta(hours=30), - last_error_notified_at=dt.datetime.now(dt.UTC) - dt.timedelta(days=8), + job_finished_at=dt.datetime.now(dt.UTC) - notified_age - dt.timedelta(minutes=5), + last_error_notified_at=dt.datetime.now(dt.UTC) - notified_age, should_sync=False, - auto_disabled_at=dt.datetime.now(dt.UTC) - dt.timedelta(days=9), + auto_disabled_at=dt.datetime.now(dt.UTC) - notified_age, ) - assert get_team_ids_with_recent_sync_failures() == [team.pk] + assert get_team_ids_with_recent_sync_failures() == [] def test_only_returns_qualifying_teams(self): qualifying = self._create_schema_with_job( diff --git a/products/desktop/packages/core/src/inbox/reportActions.test.ts b/products/desktop/packages/core/src/inbox/reportActions.test.ts index 91445921461c..a974f6e08b0a 100644 --- a/products/desktop/packages/core/src/inbox/reportActions.test.ts +++ b/products/desktop/packages/core/src/inbox/reportActions.test.ts @@ -263,6 +263,28 @@ describe("buildDiscussReportPrompt", () => { }); describe("canCreateImplementationPr", () => { + it("does not offer a pull request for a scout self-improvement report without a repository", () => { + const report = { + id: "r", + status: "pending_input", + is_self_improvement_report: true, + repo_slug: null, + } as Partial as SignalReport; + + expect(canCreateImplementationPr(report)).toBe(false); + }); + + it("offers a pull request for a scout self-improvement report with a repository", () => { + const report = { + id: "r", + status: "pending_input", + is_self_improvement_report: true, + repo_slug: "posthog/posthog", + } as Partial as SignalReport; + + expect(canCreateImplementationPr(report)).toBe(true); + }); + it("a merged PR no longer blocks a fresh attempt, but an open one does", () => { const base = { id: "r", diff --git a/products/desktop/packages/core/src/inbox/reportActions.ts b/products/desktop/packages/core/src/inbox/reportActions.ts index c382b66d2667..9e82001b5e4a 100644 --- a/products/desktop/packages/core/src/inbox/reportActions.ts +++ b/products/desktop/packages/core/src/inbox/reportActions.ts @@ -17,6 +17,9 @@ export function canCreateImplementationPr( isTaskLookupPending?: boolean; } = {}, ): boolean { + if (report.is_self_improvement_report && !report.repo_slug) { + return false; + } if (context.hasLiveImplementationTask || context.isTaskLookupPending) { return false; } diff --git a/products/error_tracking/backend/facade/api.py b/products/error_tracking/backend/facade/api.py index 43d9084f8dfe..590bf8f34ce4 100644 --- a/products/error_tracking/backend/facade/api.py +++ b/products/error_tracking/backend/facade/api.py @@ -191,6 +191,7 @@ def get_issue_basics(team_id: int, issue_id: UUID | str) -> contracts.ErrorTrack description=issue.description, status=issue.status, severity=issue.severity, + created_at=issue.created_at, first_seen=getattr(issue, "first_seen", None), ) diff --git a/products/error_tracking/backend/facade/contracts.py b/products/error_tracking/backend/facade/contracts.py index a6bd0dc2875f..01b6f505b313 100644 --- a/products/error_tracking/backend/facade/contracts.py +++ b/products/error_tracking/backend/facade/contracts.py @@ -313,6 +313,7 @@ class ErrorTrackingIssueBasics: description: str | None status: str severity: str | None + created_at: datetime # Earliest across all fingerprints, merged ones included. first_seen: datetime | None diff --git a/products/error_tracking/backend/logic/__init__.py b/products/error_tracking/backend/logic/__init__.py index 2b1917bd46fc..9e3b9a03bcfc 100644 --- a/products/error_tracking/backend/logic/__init__.py +++ b/products/error_tracking/backend/logic/__init__.py @@ -99,7 +99,7 @@ def get_issue_basics(team_id: int, issue_id: UUID | str) -> ErrorTrackingIssue | return ( ErrorTrackingIssue.objects.with_first_seen() .filter(team_id=team_id, id=issue_id) - .only("id", "name", "description", "status") + .only("id", "name", "description", "status", "severity", "created_at") .first() ) diff --git a/products/error_tracking/backend/logic/issue_summaries.py b/products/error_tracking/backend/logic/issue_summaries.py index e32e0ab72474..cdaa0b7d4ec6 100644 --- a/products/error_tracking/backend/logic/issue_summaries.py +++ b/products/error_tracking/backend/logic/issue_summaries.py @@ -8,6 +8,8 @@ import json import time import hashlib +from collections.abc import Iterator +from contextlib import contextmanager from datetime import datetime, timedelta from enum import StrEnum from uuid import UUID, uuid4 @@ -19,7 +21,7 @@ from openai import OpenAI from openai.types.chat import ChatCompletionMessageParam from openai.types.shared_params import ResponseFormatJSONSchema -from prometheus_client import Counter +from prometheus_client import Counter, Histogram from pydantic import ValidationError from redis import Redis from redis.exceptions import RedisError @@ -37,8 +39,10 @@ from posthog.llm.gateway_client import build_openai_client, resolve_ai_gateway_config, team_distinct_id from posthog.llm.semantic_enrichment import extract_json_object from posthog.models import Team, User +from posthog.ph_client import ph_background_capture from posthog.redis import get_client from posthog.token_bucket import BucketDecision, BucketUnavailable, consume, peek, refund +from posthog.utils import get_instance_region from products.access_control.backend.facade.api import ( get_restricted_properties_with_group_type_index_for_team, @@ -104,6 +108,7 @@ ISSUE_SUMMARY_BUDGET_KEY_PREFIX = "error_tracking_issue_summary" ISSUE_SUMMARY_LOCK_KEY_PREFIX = "error_tracking_issue_summary_lock" ISSUE_SUMMARY_AI_PRODUCT = "error_tracking" +ISSUE_SUMMARY_TASK_EVENT = "error_tracking_issue_summary_generation" # The Go gateway keeps the prompt and the reply out of the AI observability project when this is set. PRIVACY_MODE_HEADER = "X-PostHog-Privacy-Mode" @@ -177,6 +182,36 @@ class IssueSummaryOutcome(StrEnum): labelnames=("outcome",), ) +ISSUE_SUMMARY_STAGE_SECONDS = Histogram( + "error_tracking_issue_summary_stage_seconds", + "Time spent in each issue summary generation stage", + labelnames=("stage",), + buckets=(0.05, 0.1, 0.25, 0.5, 1, 2, 4, 6, 8, 12, 20, 30, 60, 120), +) + + +class IssueSummaryTimings: + """Milliseconds per generation stage, keyed `_ms`. A stage that did not run has no entry. + + Stages nest, so do not add them up: `total` contains every stage except `queue`, and `inputs` contains + `latest_event`, `first_event` and `aggregates`. + """ + + def __init__(self) -> None: + self.ms: dict[str, int] = {} + + @contextmanager + def stage(self, name: str) -> Iterator[None]: + started = time.monotonic() + try: + yield + finally: + self.record(name, time.monotonic() - started) + + def record(self, name: str, seconds: float) -> None: + self.ms[f"{name}_ms"] = round(seconds * 1000) + ISSUE_SUMMARY_STAGE_SECONDS.labels(stage=name).observe(seconds) + @frozen class IssueSummaryResult: @@ -289,7 +324,7 @@ def _publish_generation(team_id: int, issue_id: UUID | str) -> None: transport_options={"socket_timeout": timeout, "socket_connect_timeout": timeout}, ) as connection: generate_error_tracking_issue_summary.apply_async( - args=(team_id, str(issue_id)), connection=connection, retry=False + args=(team_id, str(issue_id)), kwargs={"queued_at": time.time()}, connection=connection, retry=False ) @@ -312,13 +347,55 @@ def wait_for_issue_summary( time.sleep(min(WAIT_POLL_INTERVAL_SECONDS, remaining)) -def get_or_generate_issue_summary(team_id: int, issue_id: UUID | str) -> IssueSummaryResult: +def run_issue_summary_task(team_id: int, issue_id: UUID | str, queued_at: float | None) -> None: + """Generate in the background task, then record how long each stage took. + + `queued_at` is the publisher's wall clock, so the queue time includes any clock skew between the hosts. + """ + timings = IssueSummaryTimings() + if queued_at is not None: + timings.record("queue", max(time.time() - queued_at, 0)) + try: + result = get_or_generate_issue_summary(team_id, issue_id, timings) + except Exception: + _report_task(team_id, issue_id, IssueSummaryOutcome.UNEXPECTED_ERROR, has_summary=False, timings=timings) + raise + _report_task(team_id, issue_id, result.outcome, has_summary=result.summary is not None, timings=timings) + + +def _report_task( + team_id: int, issue_id: UUID | str, outcome: IssueSummaryOutcome, *, has_summary: bool, timings: IssueSummaryTimings +) -> None: + # The background client sends from its own thread, so the worker takes the next task without waiting for a flush. + try: + ph_background_capture()( + distinct_id=team_distinct_id(team_id), + event=ISSUE_SUMMARY_TASK_EVENT, + properties={ + "team_id": team_id, + "issue_id": str(issue_id), + # The client's `region` super property names the destination project and overwrites an event `region`. + "instance_region": get_instance_region(), + "outcome": outcome.value, + "has_summary": has_summary, + **timings.ms, + }, + ) + except Exception: + logger.warning("error_tracking_issue_summary_capture_failed", team_id=team_id, exc_info=True) + + +def get_or_generate_issue_summary( + team_id: int, issue_id: UUID | str, timings: IssueSummaryTimings | None = None +) -> IssueSummaryResult: """Return a stored summary, or generate one when none is stored or a stale one is due. Generation runs at most once at a time per issue. A failure keeps the stored summary. """ + timings = timings or IssueSummaryTimings() try: - result = _get_or_generate(team_id, issue_id) + with timings.stage("total"): + result = _get_or_generate(team_id, issue_id, timings) except Exception: ISSUE_SUMMARY_OUTCOMES.labels(outcome=IssueSummaryOutcome.UNEXPECTED_ERROR.value).inc() raise @@ -329,11 +406,12 @@ def get_or_generate_issue_summary(team_id: int, issue_id: UUID | str) -> IssueSu issue_id=str(issue_id), outcome=result.outcome.value, has_summary=result.summary is not None, + **timings.ms, ) return result -def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult: +def _get_or_generate(team_id: int, issue_id: UUID | str, timings: IssueSummaryTimings) -> IssueSummaryResult: stored = get_issue_summary(team_id, issue_id) if (blocked := generation_blocked(team_id, stored)) is not None: return IssueSummaryResult(outcome=blocked, summary=stored) @@ -342,7 +420,7 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult: return IssueSummaryResult(outcome=held, summary=stored) try: - result = _generate_claimed(team_id, issue_id) + result = _generate_claimed(team_id, issue_id, timings) except Exception: _settle_claim(team_id, issue_id, claim, IssueSummaryOutcome.UNEXPECTED_ERROR) raise @@ -350,7 +428,7 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult: return result -def _generate_claimed(team_id: int, issue_id: UUID | str) -> IssueSummaryResult: +def _generate_claimed(team_id: int, issue_id: UUID | str, timings: IssueSummaryTimings) -> IssueSummaryResult: # Another worker can finish between the first read and the claim. stored = get_issue_summary(team_id, issue_id) if not regeneration_due(stored, timezone.now()): @@ -364,7 +442,8 @@ def _generate_claimed(team_id: int, issue_id: UUID | str) -> IssueSummaryResult: team = Team.objects.get(id=team_id) try: - inputs = build_issue_summary_inputs(team, issue_id) + with timings.stage("inputs"): + inputs = build_issue_summary_inputs(team, issue_id, timings) except Exception as error: capture_exception(error, {"team_id": team_id, "issue_id": str(issue_id)}) logger.warning("error_tracking_issue_summary_inputs_failed", team_id=team_id, issue_id=str(issue_id)) @@ -390,7 +469,8 @@ def _generate_claimed(team_id: int, issue_id: UUID | str) -> IssueSummaryResult: return IssueSummaryResult(outcome=IssueSummaryOutcome.GATEWAY_UNCONFIGURED, summary=stored) try: - raw = _call_model(team_id, inputs.prompt) + with timings.stage("model"): + raw = _call_model(team_id, issue_id, inputs.prompt) except Exception as error: capture_exception(error, {"team_id": team_id, "issue_id": str(issue_id)}) logger.warning("error_tracking_issue_summary_model_failed", team_id=team_id, issue_id=str(issue_id)) @@ -401,7 +481,8 @@ def _generate_claimed(team_id: int, issue_id: UUID | str) -> IssueSummaryResult: logger.warning("error_tracking_issue_summary_invalid_output", team_id=team_id, issue_id=str(issue_id)) return IssueSummaryResult(outcome=IssueSummaryOutcome.INVALID_OUTPUT, summary=stored) - category = infer_issue_category(team_id, inputs.prompt) + with timings.stage("category"): + category = infer_issue_category(team_id, inputs.prompt) generated_at = timezone.now() try: summary, _ = ErrorTrackingIssueSummary.objects.for_team(team_id).update_or_create( @@ -485,7 +566,9 @@ def compute_input_hash(*, fingerprints: list[str], latest_release: str | None, o ) -def build_issue_summary_inputs(team: Team, issue_id: UUID | str) -> IssueSummaryInputs | IssueSummaryOutcome: +def build_issue_summary_inputs( + team: Team, issue_id: UUID | str, timings: IssueSummaryTimings +) -> IssueSummaryInputs | IssueSummaryOutcome: """Build the prompt and the input hash, or say why the issue cannot be summarised.""" issue = ErrorTrackingIssue.objects.with_first_seen().filter(team_id=team.id, id=issue_id).first() if issue is None: @@ -494,12 +577,13 @@ def build_issue_summary_inputs(team: Team, issue_id: UUID | str) -> IssueSummary now = timezone.now() event_properties: dict[str, object] = {} - for lookback_days in (ISSUE_SUMMARY_RECENT_EVENT_LOOKBACK_DAYS, ISSUE_SUMMARY_LATEST_EVENT_LOOKBACK_DAYS): - event_properties = _exception_event_properties( - team, issue_id, after=now - timedelta(days=lookback_days), before=now, latest=True - ) - if event_properties: - break + with timings.stage("latest_event"): + for lookback_days in (ISSUE_SUMMARY_RECENT_EVENT_LOOKBACK_DAYS, ISSUE_SUMMARY_LATEST_EVENT_LOOKBACK_DAYS): + event_properties = _exception_event_properties( + team, issue_id, after=now - timedelta(days=lookback_days), before=now, latest=True + ) + if event_properties: + break stack_trace = ( render_stacktrace(mark_in_app_frames(event_properties), ISSUE_SUMMARY_STACKTRACE_MAX_TOKENS) if event_properties @@ -510,15 +594,17 @@ def build_issue_summary_inputs(team: Team, issue_id: UUID | str) -> IssueSummary first_event_properties: dict[str, object] = {} if (window := first_event_window(first_seen)) is not None: - first_event_properties = _exception_event_properties( - team, - issue_id, - after=datetime.fromisoformat(window.after), - before=datetime.fromisoformat(window.before), - latest=False, - ) + with timings.stage("first_event"): + first_event_properties = _exception_event_properties( + team, + issue_id, + after=datetime.fromisoformat(window.after), + before=datetime.fromisoformat(window.before), + latest=False, + ) - aggregated_issue = _aggregated_issue(team, issue_id) + with timings.stage("aggregates"): + aggregated_issue = _aggregated_issue(team, issue_id) impact = build_impact(aggregated_issue) trend = build_trend(aggregated_issue, first_seen) or {} latest_release = extract_release(event_properties).get("version") @@ -665,11 +751,11 @@ def _aggregated_issue(team: Team, issue_id: UUID | str) -> dict[str, object]: } -def _call_model(team_id: int, prompt: str) -> str: +def _call_model(team_id: int, issue_id: UUID | str, prompt: str) -> str: client: OpenAI = build_openai_client( "django", ai_product=ISSUE_SUMMARY_AI_PRODUCT, - properties={"team_id": str(team_id)}, + properties={"team_id": str(team_id), "issue_id": str(issue_id)}, distinct_id=team_distinct_id(team_id), ) messages: list[ChatCompletionMessageParam] = [ diff --git a/products/error_tracking/backend/presentation/views/query.py b/products/error_tracking/backend/presentation/views/query.py index a5b7bcf3002a..891d4b2ac448 100644 --- a/products/error_tracking/backend/presentation/views/query.py +++ b/products/error_tracking/backend/presentation/views/query.py @@ -139,14 +139,27 @@ def summary_payload(summary: contracts.ErrorTrackingIssueSummary) -> dict[str, o class ErrorTrackingQueryViewSet(TeamAndOrgViewSetMixin, viewsets.ViewSet): scope_object = "error_tracking" - def _report_issue_read(self, request: ValidatedRequest, issue_id: str, endpoint: str, **properties: object) -> None: + def _report_issue_read( + self, + request: ValidatedRequest, + issue_basics: contracts.ErrorTrackingIssueBasics, + endpoint: str, + **properties: object, + ) -> None: # The frontend issue events see only the web app, so this is the one record of which issues # MCP, CLI and API callers read. A capture failure must not fail the read. + issue_id = str(issue_basics.id) try: report_user_action( cast(User, request.user), "error_tracking_issue_read", - {"issue_id": issue_id, "endpoint": endpoint, **properties}, + { + "issue_id": issue_id, + "endpoint": endpoint, + "issue_created_at": issue_basics.created_at.isoformat(), + "issue_severity": issue_basics.severity or None, + **properties, + }, team=self.team, organization=self.team.organization, request=request, @@ -264,6 +277,7 @@ def issue(self, request: ValidatedRequest, **kwargs: object) -> Response: return self._issue_response( request, issue_id, + issue_basics, payload, started_at=started_at, include_breakdown=include_breakdown, @@ -307,6 +321,7 @@ def issue(self, request: ValidatedRequest, **kwargs: object) -> Response: return self._issue_response( request, issue_id, + issue_basics, payload, started_at=started_at, include_breakdown=include_breakdown, @@ -317,6 +332,7 @@ def _issue_response( self, request: ValidatedRequest, issue_id: str, + issue_basics: contracts.ErrorTrackingIssueBasics, payload: dict[str, object], *, started_at: float, @@ -331,7 +347,7 @@ def _issue_response( payload["summary_status"] = summary_read.status.value self._report_issue_read( request, - issue_id, + issue_basics, "issue", include_breakdown=include_breakdown, include_sparkline=include_sparkline, @@ -441,7 +457,8 @@ def issue_events(self, request: ValidatedRequest, **kwargs: object) -> Response: issue_id = str(params["issueId"]) limit = cast(int, params.get("limit", 1)) offset = cast(int, params.get("offset", 0)) - if not facade_api.issue_exists_by_id(self.team.id, issue_id): + issue_basics = facade_api.get_issue_basics(self.team.id, issue_id) + if issue_basics is None: return Response(status=status.HTTP_404_NOT_FOUND) date_range = build_date_range(params.get("dateRange")) requested_includes = params.get("include") @@ -491,5 +508,5 @@ def issue_events(self, request: ValidatedRequest, **kwargs: object) -> Response: payload: dict[str, object] = {"results": results, "hasMore": has_more, "limit": limit, "offset": offset} if next_offset is not None: payload["nextOffset"] = next_offset - self._report_issue_read(request, issue_id, "issue_events") + self._report_issue_read(request, issue_basics, "issue_events") return Response(payload) diff --git a/products/error_tracking/backend/tasks/tasks.py b/products/error_tracking/backend/tasks/tasks.py index 81414b1979c7..6d365136674d 100644 --- a/products/error_tracking/backend/tasks/tasks.py +++ b/products/error_tracking/backend/tasks/tasks.py @@ -94,8 +94,8 @@ def dispatch_error_tracking_alert_deliveries(team_id: int, notifications: list[d max_retries=0, ) @with_team_scope() -def generate_error_tracking_issue_summary(team_id: int, issue_id: str) -> None: +def generate_error_tracking_issue_summary(team_id: int, issue_id: str, queued_at: float | None = None) -> None: # The generator imports the LLM client and HogQL, so keep them off every worker's boot path. - from products.error_tracking.backend.logic.issue_summaries import get_or_generate_issue_summary # noqa: PLC0415 + from products.error_tracking.backend.logic.issue_summaries import run_issue_summary_task # noqa: PLC0415 - get_or_generate_issue_summary(team_id, issue_id) + run_issue_summary_task(team_id, issue_id, queued_at) diff --git a/products/error_tracking/backend/tests/api/test_query_api.py b/products/error_tracking/backend/tests/api/test_query_api.py index dd62bf4ca4b6..52017e081b53 100644 --- a/products/error_tracking/backend/tests/api/test_query_api.py +++ b/products/error_tracking/backend/tests/api/test_query_api.py @@ -936,16 +936,36 @@ def test_issue_events_returns_404_for_foreign_issue(self) -> None: ( "issue", True, + ErrorTrackingIssue.Severity.HIGH, {"includeSparkline": True}, - {"include_breakdown": False, "include_sparkline": True, "summary_status": "unavailable"}, + { + "include_breakdown": False, + "include_sparkline": True, + "summary_status": "unavailable", + "issue_created_at": "2026-04-01T00:00:00+00:00", + "issue_severity": "high", + }, ), ( "issue", False, + None, {"includeSparkline": True}, - {"include_breakdown": False, "include_sparkline": True, "summary_status": "unavailable"}, + { + "include_breakdown": False, + "include_sparkline": True, + "summary_status": "unavailable", + "issue_created_at": "2026-04-01T00:00:00+00:00", + "issue_severity": None, + }, + ), + ( + "issue_events", + True, + ErrorTrackingIssue.Severity.HIGH, + {}, + {"issue_created_at": "2026-04-01T00:00:00+00:00", "issue_severity": "high"}, ), - ("issue_events", True, {}, {}), ] ) @time_machine.travel("2026-04-24T12:00:00Z", tick=False) @@ -953,10 +973,12 @@ def test_issue_reads_capture_an_event_for_existing_issues_only( self, endpoint: str, has_exceptions: bool, + severity: str | None, extra_body: dict[str, object], expected_properties: dict[str, object], ) -> None: - self.create_issue() + self.create_issue(severity=severity) + ErrorTrackingIssue.objects.filter(id=self.issue_id).update(created_at=datetime(2026, 4, 1, tzinfo=UTC)) if has_exceptions: self.create_exception_event() flush_persons_and_events() diff --git a/products/error_tracking/backend/tests/test_issue_summaries.py b/products/error_tracking/backend/tests/test_issue_summaries.py index ab15ae21d127..b577e7b96fb5 100644 --- a/products/error_tracking/backend/tests/test_issue_summaries.py +++ b/products/error_tracking/backend/tests/test_issue_summaries.py @@ -1,6 +1,7 @@ import json import signal import socket +from contextlib import nullcontext from datetime import UTC, datetime, timedelta from typing import Any @@ -27,6 +28,7 @@ from products.error_tracking.backend.logic.issue_summaries import ( IssueSummaryInputs, IssueSummaryOutcome, + IssueSummaryTimings, build_issue_summary_inputs, compute_input_hash, get_or_generate_issue_summary, @@ -130,7 +132,9 @@ def setUp(self) -> None: self.decide = MagicMock(return_value=_category_result("own_code")) self.enterContext(patch.object(team_flags, "feature_enabled_or_false", return_value=True)) self.enterContext(patch.object(issue_summaries, "resolve_ai_gateway_config", return_value=MagicMock())) - self.enterContext(patch.object(issue_summaries, "build_openai_client", return_value=self.client_mock)) + self.build_client = self.enterContext( + patch.object(issue_summaries, "build_openai_client", return_value=self.client_mock) + ) self.build_inputs = self.enterContext( patch.object(issue_summaries, "build_issue_summary_inputs", side_effect=lambda *_: self.inputs) ) @@ -179,6 +183,7 @@ def test_generates_and_stores_summary( assert request["extra_headers"] == {"X-PostHog-Privacy-Mode": "true"} assert request["response_format"]["json_schema"]["strict"] is True assert self.decide.call_args.args[0].privacy_mode is True + assert self.build_client.call_args.kwargs["properties"]["issue_id"] == str(self.issue.id) @parameterized.expand( [ @@ -193,6 +198,33 @@ def test_generates_when_redis_is_down(self, _name: str, target: str, patch_kwarg assert result.outcome == IssueSummaryOutcome.GENERATED + @parameterized.expand( + [ + ("generated", None, IssueSummaryOutcome.GENERATED, {"queue_ms", "total_ms", "inputs_ms", "model_ms"}), + ( + "unexpected_error", + OperationalError("db"), + IssueSummaryOutcome.UNEXPECTED_ERROR, + {"queue_ms", "total_ms"}, + ), + ] + ) + def test_task_reports_outcome_and_stage_timings( + self, _name: str, error: Exception | None, outcome: IssueSummaryOutcome, stages: set[str] + ) -> None: + capture = MagicMock() + self.enterContext(patch.object(issue_summaries, "ph_background_capture", return_value=capture)) + if error is not None: + self.enterContext(patch.object(issue_summaries, "get_issue_summary", side_effect=error)) + + with time_machine.travel(NOW, tick=False), self.assertRaises(type(error)) if error else nullcontext(): + issue_summaries.run_issue_summary_task(self.team.id, self.issue.id, NOW.timestamp() - 2) + + properties = capture.call_args.kwargs["properties"] + assert properties["outcome"] == outcome.value + assert properties["queue_ms"] == 2000 + assert stages <= {key for key in properties if key.endswith("_ms")} + @property def _budget_key(self) -> str: return f"{issue_summaries.ISSUE_SUMMARY_BUDGET_KEY_PREFIX}:{self.team.id}" @@ -334,6 +366,7 @@ def test_merge_keeps_target_summary_and_deletes_source_summary(self) -> None: def _start(self) -> tuple[bool, bool]: with patch.object(issue_summaries.generate_error_tracking_issue_summary, "apply_async") as apply_async: started = summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id, user=self.user) + self.publish_call = apply_async.call_args return started, apply_async.called @parameterized.expand( @@ -353,8 +386,13 @@ def test_start_generation_queues_only_when_it_can_generate( if lock_value is not None: issue_summaries.get_client().set(self._lock_key, lock_value, ex=60) - with patch.object(team_flags, "feature_enabled_or_false", return_value=flag): + with ( + patch.object(team_flags, "feature_enabled_or_false", return_value=flag), + time_machine.travel(NOW, tick=False), + ): assert self._start() == (started, queued) + if queued: + assert self.publish_call.kwargs["kwargs"] == {"queued_at": NOW.timestamp()} def test_start_generation_queues_when_redis_is_down(self) -> None: with patch.object(issue_summaries, "get_client", side_effect=RedisError("down")): @@ -585,9 +623,11 @@ def test_builds_prompt_from_latest_event_and_first_release(self) -> None: ) flush_persons_and_events() - inputs = build_issue_summary_inputs(self.team, issue.id) + timings = IssueSummaryTimings() + inputs = build_issue_summary_inputs(self.team, issue.id, timings) assert isinstance(inputs, IssueSummaryInputs) + assert {"latest_event_ms", "first_event_ms", "aggregates_ms"} <= timings.ms.keys() assert "Exception: TypeError: Cannot read properties of undefined" in inputs.prompt assert "\ndispatch in react-dom.js\n[in-app] addToCart in cart.ts line 42" in inputs.prompt assert "Handled: no" in inputs.prompt @@ -601,4 +641,6 @@ def test_builds_prompt_from_latest_event_and_first_release(self) -> None: def test_issue_without_exception_events_has_no_inputs(self) -> None: issue = ErrorTrackingIssue.objects.create(team=self.team, name="TypeError") - assert build_issue_summary_inputs(self.team, issue.id) == IssueSummaryOutcome.NO_EXCEPTION + assert ( + build_issue_summary_inputs(self.team, issue.id, IssueSummaryTimings()) == IssueSummaryOutcome.NO_EXCEPTION + ) diff --git a/products/experiments/frontend/components/ExperimentMetaBar/ExperimentRemainingTime.tsx b/products/experiments/frontend/components/ExperimentMetaBar/ExperimentRemainingTime.tsx index 2a9296bde038..6688e902d8dd 100644 --- a/products/experiments/frontend/components/ExperimentMetaBar/ExperimentRemainingTime.tsx +++ b/products/experiments/frontend/components/ExperimentMetaBar/ExperimentRemainingTime.tsx @@ -1,9 +1,13 @@ import { useValues } from 'kea' -import { IconCheck, IconGear, IconHourglass } from '@posthog/icons' +import { IconCheck, IconGear, IconHourglass, IconWarning } from '@posthog/icons' import { LemonButton, Tooltip } from '@posthog/lemon-ui' import { LemonProgressCircle } from 'lib/lemon-ui/LemonProgressCircle/LemonProgressCircle' +import { + MINIMUM_DETECTABLE_EFFECT_MAX, + roundUpDetectableEffect, +} from 'scenes/experiments/RunningTimeCalculator/calculations' import { runningTimeLogic } from 'scenes/experiments/RunningTimeCalculator/runningTimeLogic' import { Experiment } from '~/types' @@ -17,9 +21,8 @@ export function ExperimentRemainingTime({ experiment: Experiment onConfigure: () => void }): JSX.Element { - const { remainingDays, currentExposures, targetSampleSize, isComplete, isCalculating } = useValues( - runningTimeLogic({ experiment }) - ) + const { remainingDays, currentExposures, targetSampleSize, isComplete, isCalculating, outlook, targetEffect } = + useValues(runningTimeLogic({ experiment })) const launched = isLaunched(experiment) const showProgress = currentExposures !== null && targetSampleSize !== null @@ -37,6 +40,28 @@ export function ExperimentRemainingTime({ Target reached ) + } else if (outlook?.isOutOfReach) { + // A countdown of months helps nobody, so the header says what the experiment can detect instead. + const reachable = roundUpDetectableEffect(outlook.reachableEffect) + const isBeyondMaximum = reachable > MINIMUM_DETECTABLE_EFFECT_MAX + content = ( + + + + + {isBeyondMaximum + ? `Can detect only changes over ${MINIMUM_DETECTABLE_EFFECT_MAX}% within 4 weeks` + : `Can detect ${reachable.toLocaleString()}%+ changes within 4 weeks`} + + + + ) } else { content = ( diff --git a/products/experiments/frontend/modals/RunningTimeConfigModal/DetectableEffectTable.tsx b/products/experiments/frontend/modals/RunningTimeConfigModal/DetectableEffectTable.tsx new file mode 100644 index 000000000000..dadb13b739d3 --- /dev/null +++ b/products/experiments/frontend/modals/RunningTimeConfigModal/DetectableEffectTable.tsx @@ -0,0 +1,137 @@ +import { useActions, useValues } from 'kea' + +import { IconCheck, IconWarning } from '@posthog/icons' +import { LemonButton, LemonTable, LemonTag, Tooltip } from '@posthog/lemon-ui' + +import { dayjs } from 'lib/dayjs' +import { Label } from 'lib/ui/Label/Label' +import { + describeCheckpointDay, + describeUseButton, + formatDetectableEffect, +} from 'scenes/experiments/RunningTimeCalculator/calculations' +import { runningTimeLogic } from 'scenes/experiments/RunningTimeCalculator/runningTimeLogic' + +import { Experiment } from '~/types' + +export interface DetectableEffectTableProps { + experiment: Experiment +} + +function formatDate(date: dayjs.Dayjs): string { + return date.format(date.isSame(dayjs(), 'year') ? 'MMM D' : 'MMM D, YYYY') +} + +export function DetectableEffectTable({ experiment: experimentProp }: DetectableEffectTableProps): JSX.Element | null { + const logic = runningTimeLogic({ experiment: experimentProp }) + const { + outlook, + currentExposures, + isComplete, + dailyExposureRate, + estimateMetric, + canIncludeMoreUsers, + experiment, + } = useValues(logic) + const { selectCheckpoint } = useActions(logic) + + const startDate = experiment.start_date + if (!outlook || currentExposures === null || dailyExposureRate === null || !startDate) { + return null + } + + const dateAt = (days: number): dayjs.Dayjs => dayjs(startDate).add(days * 24, 'hour') + + return ( +
    + + checkpoint.key} + rowStatus={(checkpoint) => (checkpoint.isTarget ? 'highlighted' : null)} + rowClassName={(checkpoint) => (checkpoint.isTarget ? 'font-semibold' : null)} + columns={[ + { + title: 'When', + key: 'when', + // Takes the spare width, so that the number columns stay in place while the target row moves. + width: '100%', + render: (_, checkpoint) => ( + + {checkpoint.isTarget && isComplete && ( + + + + )} + {checkpoint.isTarget && outlook.isOutOfReach && ( + + + + )} + {checkpoint.kind === 'now' ? 'Today' : formatDate(dateAt(checkpoint.days))} + {`· ${describeCheckpointDay(checkpoint, { + startDate, + currentSampleSize: currentExposures, + })}`} + + ), + }, + { + title: 'Exposures', + key: 'exposures', + align: 'right', + render: (_, checkpoint) => {checkpoint.exposures.toLocaleString()}, + }, + { + title: 'Detectable effect', + key: 'detectable', + align: 'right', + render: (_, checkpoint) => {formatDetectableEffect(checkpoint.detectableEffect)}, + }, + { + key: 'set-target', + align: 'right', + width: 0, + render: (_, checkpoint) => { + if (checkpoint.isTarget) { + return ( + + Target + + ) + } + const { effect, disabledReason, tooltip } = describeUseButton(checkpoint) + return ( + selectCheckpoint(checkpoint.key, effect)} + data-attr="experiment-running-time-set-target" + > + Use + + ) + }, + }, + ]} + /> +
    + Based on + {estimateMetric?.name ?? 'the primary metric'} + + {`${estimateMetric ? `, the ${estimateMetric.isFirstOfSeveral ? 'first ' : ''}primary metric` : ''}.${ + estimateMetric?.hasOtherMetrics ? ' Other metrics can need more time.' : '' + } Projections assume about ${Math.round(dailyExposureRate).toLocaleString()} exposures a day, the average since launch.${ + outlook.isOutOfReach && canIncludeMoreUsers + ? ' To finish sooner, include more users in the experiment.' + : '' + }`} + +
    +
    + ) +} diff --git a/products/experiments/frontend/modals/RunningTimeConfigModal/MinimumDetectableEffectField.tsx b/products/experiments/frontend/modals/RunningTimeConfigModal/MinimumDetectableEffectField.tsx new file mode 100644 index 000000000000..f2a030229726 --- /dev/null +++ b/products/experiments/frontend/modals/RunningTimeConfigModal/MinimumDetectableEffectField.tsx @@ -0,0 +1,80 @@ +import { useEffect, useState } from 'react' + +import { LemonInput } from '@posthog/lemon-ui' + +import { LemonSlider } from 'lib/lemon-ui/LemonSlider' +import { Label } from 'lib/ui/Label/Label' +import { + MINIMUM_DETECTABLE_EFFECT_MAX, + MINIMUM_DETECTABLE_EFFECT_MIN, +} from 'scenes/experiments/RunningTimeCalculator/calculations' + +export interface MinimumDetectableEffectFieldProps { + value: number + onChange: (value: number) => void +} + +const isInRange = (value: number | undefined): value is number => + value !== undefined && value >= MINIMUM_DETECTABLE_EFFECT_MIN && value <= MINIMUM_DETECTABLE_EFFECT_MAX + +export function MinimumDetectableEffectField({ value, onChange }: MinimumDetectableEffectFieldProps): JSX.Element { + // The input keeps what the user types, so that an empty field or a partial number stays editable. Only a value in + // range reaches the estimate. + const [draft, setDraft] = useState(value) + useEffect(() => setDraft(value), [value]) + + const isDraftEmpty = draft === undefined || Number.isNaN(draft) + + const commitDraft = (): void => { + if (draft === undefined || Number.isNaN(draft)) { + setDraft(value) + } else if (!isInRange(draft)) { + const clamped = Math.min(MINIMUM_DETECTABLE_EFFECT_MAX, Math.max(MINIMUM_DETECTABLE_EFFECT_MIN, draft)) + setDraft(clamped) + onChange(clamped) + } + } + + return ( +
    + +
    +
    + +
    +
    + { + setDraft(next) + if (isInRange(next)) { + onChange(next) + } + }} + onBlur={commitDraft} + onPressEnter={commitDraft} + status={isDraftEmpty || isInRange(draft) ? 'default' : 'danger'} + min={MINIMUM_DETECTABLE_EFFECT_MIN} + max={MINIMUM_DETECTABLE_EFFECT_MAX} + step={1} + className="flex-1" + data-attr="experiment-running-time-mde-input" + /> + % +
    +
    + {!isDraftEmpty && !isInRange(draft) && ( +
    + {`Enter a value from ${MINIMUM_DETECTABLE_EFFECT_MIN} to ${MINIMUM_DETECTABLE_EFFECT_MAX}.`} +
    + )} +
    + ) +} diff --git a/products/experiments/frontend/modals/RunningTimeConfigModal/MinimumDetectableEffectHelp.tsx b/products/experiments/frontend/modals/RunningTimeConfigModal/MinimumDetectableEffectHelp.tsx new file mode 100644 index 000000000000..08a473155f71 --- /dev/null +++ b/products/experiments/frontend/modals/RunningTimeConfigModal/MinimumDetectableEffectHelp.tsx @@ -0,0 +1,14 @@ +export interface MinimumDetectableEffectHelpProps { + // The color too, so that the text matches the other hints where it sits. + className: string +} + +export function MinimumDetectableEffectHelp({ className }: MinimumDetectableEffectHelpProps): JSX.Element { + return ( +
    + The minimum detectable effect is the smallest change you want to detect, relative to the control. Smaller + changes are harder to tell apart from random noise, so they need more exposures. It affects only the running + time, not how the metrics are calculated. +
    + ) +} diff --git a/products/experiments/frontend/modals/RunningTimeConfigModal/RunningTimeConfigModal.tsx b/products/experiments/frontend/modals/RunningTimeConfigModal/RunningTimeConfigModal.tsx index 817ab2010a54..f6f3d2078765 100644 --- a/products/experiments/frontend/modals/RunningTimeConfigModal/RunningTimeConfigModal.tsx +++ b/products/experiments/frontend/modals/RunningTimeConfigModal/RunningTimeConfigModal.tsx @@ -1,9 +1,7 @@ import { useActions, useValues } from 'kea' -import { IconCheck } from '@posthog/icons' -import { LemonBanner, LemonButton, LemonInput, LemonModal, LemonSegmentedButton } from '@posthog/lemon-ui' +import { LemonBanner, LemonButton, LemonInput, LemonModal, LemonSegmentedButton, Spinner } from '@posthog/lemon-ui' -import { LemonSlider } from 'lib/lemon-ui/LemonSlider' import { Label } from 'lib/ui/Label/Label' import { ManualCalculatorMetricType } from 'scenes/experiments/RunningTimeCalculator/calculations' import { runningTimeLogic } from 'scenes/experiments/RunningTimeCalculator/runningTimeLogic' @@ -12,6 +10,10 @@ import { Experiment } from '~/types' import { isLaunched } from 'products/experiments/frontend/experimentStatus' +import { DetectableEffectTable } from './DetectableEffectTable' +import { MinimumDetectableEffectField } from './MinimumDetectableEffectField' +import { MinimumDetectableEffectHelp } from './MinimumDetectableEffectHelp' + export interface RunningTimeConfigModalProps { experiment: Experiment } @@ -48,23 +50,25 @@ export function RunningTimeConfigModal({ experiment: experimentProp }: RunningTi const { config, manualFormPreview, - currentExposures, - targetSampleSize, - dailyExposureRate, - remainingDays, isRunningTimeConfigModalOpen, experiment, isSaving, + outlook, + isCalculating, + automaticTargetFailed, + automaticTargetRequest, } = useValues(runningTimeLogic({ experiment: experimentProp })) - const { setConfig, save, cancel } = useActions(runningTimeLogic({ experiment: experimentProp })) + const { setConfig, save, cancel, loadAutomaticTarget } = useActions( + runningTimeLogic({ experiment: experimentProp }) + ) - const hasAutomaticData = remainingDays !== null const isPreLaunch = !isLaunched(experiment) + const effectField = setConfig({ mde })} /> return ( - + -

    {isPreLaunch ? 'Estimate running time' : 'Running time configuration'}

    +

    {isPreLaunch ? 'Estimate running time' : 'Running time'}
    @@ -85,11 +89,11 @@ export function RunningTimeConfigModal({ experiment: experimentProp }: RunningTi value={config.mode} onChange={(value) => setConfig({ mode: value as 'automatic' | 'manual' })} /> -
    - {config.mode === 'manual' - ? 'Enter your own values to calculate sample size and running time.' - : 'Uses actual experiment data to calculate estimates.'} -
    + {config.mode === 'manual' && ( +
    + Enter your own values to calculate sample size and running time. +
    + )}
    {config.mode === 'manual' ? ( @@ -144,35 +148,8 @@ export function RunningTimeConfigModal({ experiment: experimentProp }: RunningTi
    - -
    -
    - setConfig({ mde: Math.round(value * 10) / 10 })} - min={1} - max={100} - step={0.1} - /> -
    -
    - setConfig({ mde: Number(value) || 1 })} - min={1} - max={100} - step={0.1} - className="flex-1" - /> - % -
    -
    -
    - The smallest delta (change) you want to be able to measure with statistical - significance. Changing this setting does not impact the statistical analysis, only - the estimated runtime. Lower values require more data and longer run times. -
    + {effectField} +
    @@ -192,74 +169,36 @@ export function RunningTimeConfigModal({ experiment: experimentProp }: RunningTi ) : ( <> - {!hasAutomaticData ? ( - - Waiting for sufficient data to calculate time estimates. Need at least 1 day and 100 - exposures. - - ) : ( - <> -
    - -
    - {remainingDays === 0 ? ( - - - Complete - - ) : ( - `~${Math.ceil(remainingDays)} day${Math.ceil(remainingDays) !== 1 ? 's' : ''}` - )} -
    + {effectField} +
    + {!outlook && isCalculating ? ( +
    + + Calculating the running time…
    -
    - -
    - {currentExposures !== null && targetSampleSize !== null - ? `${currentExposures.toLocaleString()} / ${targetSampleSize.toLocaleString()} exposures` - : '—'} -
    -
    -
    - -
    - {dailyExposureRate !== null - ? `~${Math.round(dailyExposureRate).toLocaleString()} exposures/day` - : '—'} -
    -
    - - )} -
    - -
    -
    - setConfig({ mde: Math.round(value * 10) / 10 })} - min={1} - max={100} - step={0.1} - /> -
    -
    - setConfig({ mde: Number(value) || 1 })} - min={1} - max={100} - step={0.1} - className="flex-1" - /> - % -
    -
    -
    - The smallest delta (change) you want to be able to measure with statistical - significance. Changing this setting does not impact the statistical analysis, only - the estimated runtime. Lower values require more data and longer run times. -
    + ) : automaticTargetFailed ? ( + loadAutomaticTarget(automaticTargetRequest), + } + : undefined + } + > + Couldn't calculate the running time. + + ) : !outlook ? ( + + Not enough data to estimate the running time yet. It needs exposures, and some + activity on the primary metric, in each variant. + + ) : ( + + )} +
    )} diff --git a/products/experiments/skills/diagnosing-experiment-health/references/interpretation.md b/products/experiments/skills/diagnosing-experiment-health/references/interpretation.md index a8674f3f5212..c20f03253a89 100644 --- a/products/experiments/skills/diagnosing-experiment-health/references/interpretation.md +++ b/products/experiments/skills/diagnosing-experiment-health/references/interpretation.md @@ -339,6 +339,9 @@ real anomaly worth investigating with the experiment URL. **Behavior** `[HIGH]`. The duration display next to the experiment's status is an estimate of sample size. It reads "~N days left" while the exposures are below the target, and "Target reached" once they are at or above it. +When the target is more than four weeks away, it reads "Can detect X%+ changes within 4 weeks" with a warning icon instead. +X is the smallest change that the experiment can detect within four weeks at its current traffic, not the minimum detectable effect. +Above 100% it reads "Can detect only changes over 100% within 4 weeks". Older versions read "Complete". On an ended experiment "Complete" is the status tag: every ended experiment carries it, and the duration display is hidden. Read `status` first when the user says "complete". @@ -352,6 +355,8 @@ Read `status` first when the user says "complete". - In manual mode the target comes from the baseline the user typed, not from the results. - "Target reached" says that an effect of that size would probably have shown. It says nothing about significance, and it does not end the experiment. +- The running time settings list the exposures and the change that the experiment can detect today, at the next week marks, and on the date it reaches the target. + A change over 100% shows as ">100%".