diff --git a/package-lock.json b/package-lock.json index b2b92b9..8b12b3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,9 @@ "": { "name": "@tginternal/editor", "version": "1.17.0", + "dependencies": { + "entities": "^6.0.1" + }, "devDependencies": { "@codemirror/lang-javascript": "^6.2.1", "@codemirror/lint": "^6.4.2", @@ -131,6 +134,7 @@ "integrity": "sha512-CGOfOJqWjg2qW/Mb6zNsDm+u5vFQ8DxXfbM09z69p5Z6+mE1ikP2jUXw+j42Pf1XTYED2Rni5f95npYeuwMDQA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@babel/code-frame": "^7.29.0", "@babel/generator": "^7.29.0", @@ -775,6 +779,7 @@ "integrity": "sha512-WA0zdU7xfF10+5I3HhUUq3kqOx3KjqmtQ9lqZjfK7jtYk4G72YW9rezcSywpaUMCWOMlq+6E0pO1IWg1TNIhtg==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@codemirror/state": "^6.6.0", "crelt": "^1.0.6", @@ -843,7 +848,6 @@ "integrity": "sha512-L/B1lc/TViYk4DcpGxtAVbx0ZyiKM5ktoIyafGkH6zg/tj+mA+NE//aPYKG0k8kCHSHVJrpLpcAlOBEXQ3SavA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@emotion/memoize": "^0.9.0", "@emotion/sheet": "^1.4.0", @@ -921,8 +925,7 @@ "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.4.0.tgz", "integrity": "sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@emotion/styled": { "version": "11.14.1", @@ -977,8 +980,7 @@ "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.4.0.tgz", "integrity": "sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", @@ -2167,6 +2169,7 @@ "integrity": "sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@octokit/auth-token": "^6.0.0", "@octokit/graphql": "^9.0.3", @@ -3485,6 +3488,7 @@ "integrity": "sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~6.21.0" } @@ -3516,6 +3520,7 @@ "integrity": "sha512-z9VXpC7MWrhfWipitjNdgCauoMLRdIILQsAEV+ZesIzBq/oUlxk0m3ApZuMFCXdnS4U7KrI+l3WRUEGQ8K1QKw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@types/prop-types": "*", "csstype": "^3.2.2" @@ -3604,6 +3609,7 @@ "integrity": "sha512-tbsV1jPne5CkFQCgPBcDOt30ItF7aJoZL997JSF7MhGQqOeT3svWRYxiqlfA5RUdlHN6Fi+EI9bxqbdyAUZjYQ==", "dev": true, "license": "BSD-2-Clause", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "6.21.0", "@typescript-eslint/types": "6.21.0", @@ -3849,6 +3855,7 @@ "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -4256,6 +4263,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -5020,6 +5028,18 @@ "dev": true, "license": "MIT" }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/env-ci": { "version": "11.2.0", "resolved": "https://registry.npmjs.org/env-ci/-/env-ci-11.2.0.tgz", @@ -5280,6 +5300,7 @@ "deprecated": "This version is no longer supported. Please see https://eslint.org/version-support for other options.", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.6.1", @@ -6087,7 +6108,6 @@ "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", "dev": true, "license": "BSD-3-Clause", - "peer": true, "dependencies": { "react-is": "^16.7.0" } @@ -6607,6 +6627,7 @@ "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@jest/core": "^29.7.0", "@jest/types": "^29.6.3", @@ -7532,6 +7553,7 @@ "integrity": "sha512-8dD6FusOQSrpv9Z1rdNMdlSgQOIP880DHqnohobOmYLElGEqAL/JvxvuxZO16r4HtjTlfPRDC1hbvxC9dPN2nA==", "dev": true, "license": "MIT", + "peer": true, "bin": { "marked": "bin/marked.js" }, @@ -9701,6 +9723,7 @@ "dev": true, "inBundle": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -10512,6 +10535,7 @@ "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0" }, @@ -10525,6 +10549,7 @@ "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "loose-envify": "^1.1.0", "scheduler": "^0.23.2" @@ -10538,8 +10563,7 @@ "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/react-refresh": { "version": "0.17.0", @@ -10866,6 +10890,7 @@ "integrity": "sha512-WRgl5GcypwramYX4HV+eQGzUbD7UUbljVmS+5G1uMwX/wLgYuJAxGeerXJDMO2xshng4+FXqCgyB5QfClV6WjA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@semantic-release/commit-analyzer": "^13.0.1", "@semantic-release/error": "^4.0.0", @@ -12149,6 +12174,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -12284,6 +12310,7 @@ "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -12381,6 +12408,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -12580,6 +12608,7 @@ "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", diff --git a/package.json b/package.json index 735aa62..6731581 100644 --- a/package.json +++ b/package.json @@ -100,5 +100,9 @@ }, "publishConfig": { "access": "public" + }, + "//dependencies": "entities pinned to 6.x — latest dual CJS/ESM release; 7+ are ESM-only and break the CommonJS jest tests and the .cjs build", + "dependencies": { + "entities": "^6.0.1" } } diff --git a/src/parser/placeholders/findEntities.test.ts b/src/parser/placeholders/findEntities.test.ts new file mode 100644 index 0000000..7ff6eed --- /dev/null +++ b/src/parser/placeholders/findEntities.test.ts @@ -0,0 +1,52 @@ +import { findEntities } from "./findEntities"; + +describe("findEntities", () => { + it("finds a named entity", () => { + const result = findEntities("Activité"); + expect(result).toEqual([ + { decoded: "é", raw: "é", position: { start: 7, end: 15 } }, + ]); + }); + + it("finds a decimal numeric entity", () => { + const result = findEntities("café"); + expect(result[0].decoded).toEqual("é"); + expect(result[0].raw).toEqual("é"); + }); + + it("finds a hex numeric entity", () => { + const result = findEntities("café"); + expect(result[0].decoded).toEqual("é"); + expect(result[0].raw).toEqual("é"); + }); + + it("decodes nbsp", () => { + const result = findEntities("a b"); + expect(result[0].decoded).toEqual(" "); + }); + + it("decodes amp, lt, gt", () => { + expect(findEntities("&")[0].decoded).toEqual("&"); + expect(findEntities("<")[0].decoded).toEqual("<"); + expect(findEntities(">")[0].decoded).toEqual(">"); + }); + + it("ignores a bare ampersand", () => { + expect(findEntities("Tom & Jerry")).toEqual([]); + expect(findEntities("AT&T")).toEqual([]); + }); + + it("ignores an unrecognized reference", () => { + expect(findEntities("a¬arealentity;b")).toEqual([]); + }); + + it("requires the trailing semicolon", () => { + expect(findEntities("é no semicolon")).toEqual([]); + }); + + it("finds multiple entities", () => { + const result = findEntities("é-à"); + expect(result.map((e) => e.decoded)).toEqual(["é", "à"]); + expect(result[1].position.start).toEqual(9); + }); +}); diff --git a/src/parser/placeholders/findEntities.ts b/src/parser/placeholders/findEntities.ts new file mode 100644 index 0000000..299e9d0 --- /dev/null +++ b/src/parser/placeholders/findEntities.ts @@ -0,0 +1,33 @@ +import { decodeHTMLStrict } from "entities"; + +export type EntityInfoType = { + decoded: string; + raw: string; + position: { start: number; end: number }; +}; + +// Named (é), decimal (é) or hex (é) HTML character references. +// A trailing semicolon is required so a bare "&" in text (e.g. "Tom & Jerry") +// is never matched. +const ENTITY_REGEX = /&(#x[0-9a-fA-F]+|#[0-9]+|[a-zA-Z][a-zA-Z0-9]*);/g; + +export const findEntities = (text: string): EntityInfoType[] => { + const result: EntityInfoType[] = []; + ENTITY_REGEX.lastIndex = 0; + let match: RegExpExecArray | null; + while ((match = ENTITY_REGEX.exec(text)) !== null) { + const raw = match[0]; + const decoded = decodeHTMLStrict(raw); + // decodeHTMLStrict returns the input unchanged when it isn't a recognized + // reference — that's how we tell "&b;" apart from "é". + if (decoded === raw) { + continue; + } + result.push({ + decoded, + raw, + position: { start: match.index, end: match.index + raw.length }, + }); + } + return result; +}; diff --git a/src/parser/placeholders/getPlaceholders.test.ts b/src/parser/placeholders/getPlaceholders.test.ts index ce6a0bc..7851850 100644 --- a/src/parser/placeholders/getPlaceholders.test.ts +++ b/src/parser/placeholders/getPlaceholders.test.ts @@ -97,4 +97,29 @@ describe("get placeholders", () => { expect(first.normalizedValue).toEqual("{test}"); expect(first.position.start).toEqual(3); }); + + it("parse html entity", () => { + const placeholders = getPlaceholders("Activité"); + expect(placeholders![0].type).toEqual("entity"); + expect(placeholders![0].name).toEqual("é"); + expect(placeholders![0].normalizedValue).toEqual("é"); + }); + + it("ignores a bare ampersand", () => { + expect(getPlaceholders("Tom & Jerry")).toEqual([]); + }); + + it("orders entities and tags by position", () => { + const placeholders = getPlaceholders("&x"); + expect(placeholders!.map((p) => p.type)).toEqual([ + "entity", + "tagOpen", + "tagClose", + ]); + }); + + it("ignores entities inside a tag attribute", () => { + const placeholders = getPlaceholders('z'); + expect(placeholders!.map((p) => p.type)).toEqual(["tagOpen", "tagClose"]); + }); }); diff --git a/src/parser/placeholders/getPlaceholders.ts b/src/parser/placeholders/getPlaceholders.ts index d5102d3..801504e 100644 --- a/src/parser/placeholders/getPlaceholders.ts +++ b/src/parser/placeholders/getPlaceholders.ts @@ -13,6 +13,7 @@ import { import type { SyntaxNode, Tree } from "@lezer/common"; import { Placeholder } from "../types"; import { TagInfoType, findTags } from "./findTags"; +import { findEntities } from "./findEntities"; function getAllChildren(node: SyntaxNode) { const result: SyntaxNode[] = []; @@ -175,18 +176,39 @@ export const getPlaceholders = (input: string, nested?: boolean) => { case Text: case TextNested: { - findTags(getNodeText(node)).forEach((tagInfo) => { - return addPlaceholder( - placeholderFromTag({ - ...tagInfo, - position: { - start: tagInfo.position.start + node.from, - end: tagInfo.position.end + node.from, - }, - }) + const nodeText = getNodeText(node); + const tags = findTags(nodeText); + const shift = (position: { start: number; end: number }) => ({ + start: position.start + node.from, + end: position.end + node.from, + }); + + const nodePlaceholders: Placeholder[] = tags.map((tagInfo) => + placeholderFromTag({ ...tagInfo, position: shift(tagInfo.position) }) + ); + + findEntities(nodeText).forEach((entityInfo) => { + // skip entities living inside a tag (e.g. an attribute value) + const insideTag = tags.some( + (tag) => + entityInfo.position.start < tag.position.end && + entityInfo.position.end > tag.position.start ); + if (insideTag) { + return; + } + nodePlaceholders.push({ + type: "entity", + name: entityInfo.decoded, + normalizedValue: entityInfo.raw, + position: shift(entityInfo.position), + }); }); + nodePlaceholders + .sort((a, b) => a.position.start - b.position.start) + .forEach(addPlaceholder); + enter = false; break; } diff --git a/src/parser/placeholdersStyle.ts b/src/parser/placeholdersStyle.ts index ec5b467..45d5fcf 100644 --- a/src/parser/placeholdersStyle.ts +++ b/src/parser/placeholdersStyle.ts @@ -10,6 +10,7 @@ export type Placeholders = { variable: Placeholder; tag: Placeholder; variant: Placeholder; + entity: Placeholder; }; const DEFAULT_COLORS = { @@ -28,11 +29,16 @@ const DEFAULT_COLORS = { background: "#F0F2F4", text: "#4D5B6E", }, + entity: { + border: "#F9C4D6", + background: "#FCDEE9", + text: "#822343", + }, } satisfies Placeholders; type Props = { styled: (component: any) => any; - colors?: Placeholders; + colors?: Partial; component?: any; }; @@ -41,9 +47,10 @@ type Props = { */ export const generatePlaceholdersStyle = ({ styled, - colors = DEFAULT_COLORS, + colors: colorsProp, component = "div", }: Props): StyledComponent => { + const colors = { ...DEFAULT_COLORS, ...colorsProp }; return styled(component)` white-space: pre-wrap; & .placeholder-widget { @@ -92,6 +99,19 @@ export const generatePlaceholdersStyle = ({ color: ${colors.tag.text}; } + // entities render as the decoded glyph inline (not a pill), with a subtle + // tint so they stay readable mid-word while still being marked + & .placeholder-entity { + min-width: 0; + border: none; + border-radius: 3px; + padding: 0px 1px; + font-size: inherit; + vertical-align: baseline; + background-color: ${colors.entity.background}; + color: ${colors.entity.text}; + } + // in rtl mode, revert the placeholders direction &[dir="rtl"] .placeholder-tagOpen { border-radius: 0px 10px 10px 0px; diff --git a/src/parser/types.ts b/src/parser/types.ts index aa1d028..c5fe2f1 100644 --- a/src/parser/types.ts +++ b/src/parser/types.ts @@ -4,7 +4,7 @@ export type Position = { }; export type Placeholder = { - type: "variable" | "tagOpen" | "tagClose" | "tagSelfClosed" | "hash"; + type: "variable" | "tagOpen" | "tagClose" | "tagSelfClosed" | "hash" | "entity"; position: Position; name: string; error?: "missing_open_tag" | "missing_close_tag"; diff --git a/vite.config.ts b/vite.config.ts index dc575d4..4d13a30 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -18,7 +18,8 @@ export default defineConfig({ }, plugins: [ react(), - externalizeDeps(), + // bundle `entities` so consumers don't need it as a direct dependency + externalizeDeps({ except: ["entities"] }), { name: "build-parser", watchChange(id) {