diff --git a/.github/workflows/html-api-commoncrawl-comment.yml b/.github/workflows/html-api-commoncrawl-comment.yml new file mode 100644 index 0000000000000..68b48bbb783f1 --- /dev/null +++ b/.github/workflows/html-api-commoncrawl-comment.yml @@ -0,0 +1,152 @@ +# Manages the pull request side of html-api-commoncrawl.yml. +# +# Pull requests from forks cannot be written to by the workflows they trigger, so this removes the request label and +# posts the report from here instead. +name: HTML API CommonCrawl Comments + +on: + pull_request_target: + types: [ 'labeled' ] + branches: + - trunk + - '6.[4-9]' + - '[7-9].[0-9]' + paths: + - 'src/wp-includes/html-api/**' + - 'src/wp-includes/class-wp-token-map.php' + - 'src/wp-includes/kses.php' + - 'tools/html-api-commoncrawl/**' + - '.github/workflows/html-api-commoncrawl.yml' + - '.github/workflows/html-api-commoncrawl-comment.yml' + workflow_run: + workflows: [ 'HTML API CommonCrawl Analysis' ] + types: + - completed + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Removes the label which requested the analysis, so that it can be applied again to request another one. + remove-label: + name: Remove the analysis request label + runs-on: ubuntu-24.04 + permissions: + issues: write + pull-requests: write + timeout-minutes: 5 + if: ${{ github.event_name == 'pull_request_target' && github.event.label.name == 'html-api-commoncrawl' }} + steps: + - name: Remove the label + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + await github.rest.issues.removeLabel( { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.payload.pull_request.number, + name: 'html-api-commoncrawl', + } ); + + # Leaves the report on the pull request, replacing the report from any earlier analysis. + comment: + name: Comment with the analysis report + runs-on: ubuntu-24.04 + permissions: + actions: read + issues: write + pull-requests: write + timeout-minutes: 5 + if: > + github.event_name == 'workflow_run' && + github.event.workflow_run.conclusion == 'success' && + ( github.event.workflow_run.event == 'pull_request' || github.event.workflow_run.event == 'workflow_dispatch' ) + steps: + - name: Download the report + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const fs = require( 'fs' ); + + const artifacts = await github.rest.actions.listWorkflowRunArtifacts( { + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + } ); + + const artifact = artifacts.data.artifacts.find( ( artifact ) => artifact.name === 'html-api-commoncrawl-comment' ); + + const download = await github.rest.actions.downloadArtifact( { + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: artifact.id, + archive_format: 'zip', + } ); + + fs.writeFileSync( `${ process.env.RUNNER_TEMP }/comment.zip`, Buffer.from( download.data ) ); + + - name: Unzip the report + run: unzip "$RUNNER_TEMP/comment.zip" -d "$RUNNER_TEMP/comment" + + - name: Leave or replace the report comment + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const fs = require( 'fs' ); + const { owner, repo } = context.repo; + const run = context.payload.workflow_run; + const marker = ''; + + const issue_number = Number( fs.readFileSync( `${ process.env.RUNNER_TEMP }/comment/NR`, 'utf8' ) ); + const body = fs.readFileSync( `${ process.env.RUNNER_TEMP }/comment/report.md`, 'utf8' ); + + if ( ! Number.isInteger( issue_number ) || issue_number < 1 ) { + core.setFailed( 'The report does not name a pull request.' ); + return; + } + + const { data: pr } = await github.rest.pulls.get( { owner, repo, pull_number: issue_number } ); + + /* + * The report and the pull request number come from a workflow run which the pull request + * itself controls, so only comment on the pull request whose branch triggered the run. + * Manual runs can only be started by people with write access. + */ + if ( + run.event === 'pull_request' && + ( pr.head.repo?.full_name !== run.head_repository.full_name || pr.head.ref !== run.head_branch ) + ) { + core.setFailed( `Pull request #${ issue_number } did not trigger this workflow run.` ); + return; + } + + if ( pr.state !== 'open' ) { + core.info( 'The pull request has been closed. No comment will be left.' ); + return; + } + + // Search every page of comments for an earlier report, stopping at the first match. + let existing; + for await ( const { data: comments } of github.paginate.iterator( github.rest.issues.listComments, { + owner, + repo, + issue_number, + per_page: 100, + } ) ) { + existing = comments.find( ( comment ) => + comment.user.login === 'github-actions[bot]' && comment.body.startsWith( marker ) + ); + if ( existing ) { + break; + } + } + + if ( existing ) { + await github.rest.issues.updateComment( { owner, repo, comment_id: existing.id, body } ); + core.notice( `Replaced the report comment: ${ existing.html_url }` ); + return; + } + + const { data: comment } = await github.rest.issues.createComment( { owner, repo, issue_number, body } ); + core.notice( `Left the report comment: ${ comment.html_url }` ); diff --git a/.github/workflows/html-api-commoncrawl.yml b/.github/workflows/html-api-commoncrawl.yml new file mode 100644 index 0000000000000..3dff1d2be1bf5 --- /dev/null +++ b/.github/workflows/html-api-commoncrawl.yml @@ -0,0 +1,218 @@ +# Compares HTML API parsing performance and support, and wp_kses() performance, with and without a pull request, using +# documents from CommonCrawl. +# +# This takes tens of minutes, so it only runs on request: either by adding the `html-api-commoncrawl` label to a pull +# request, which is removed again as soon as it is added, or by running this workflow manually with a pull request +# number. The report is left as a comment on the pull request by `html-api-commoncrawl-comment.yml`. +name: HTML API CommonCrawl Analysis + +on: + # The HTML Processor was introduced in WordPress 6.4. + pull_request: + types: [ 'labeled' ] + branches: + - trunk + - '6.[4-9]' + - '[7-9].[0-9]' + paths: + - 'src/wp-includes/html-api/**' + - 'src/wp-includes/class-wp-token-map.php' + - 'src/wp-includes/kses.php' + - 'tools/html-api-commoncrawl/**' + - '.github/workflows/html-api-commoncrawl.yml' + - '.github/workflows/html-api-commoncrawl-comment.yml' + workflow_dispatch: + inputs: + pull-request: + description: 'Pull request number to analyze' + required: true + type: number + warcs: + description: 'How many WARC files to download from CommonCrawl, each with around 20,000 HTML documents' + required: false + type: number + default: 1 + seed: + description: 'Picks the WARC files: leave empty for a new random choice, or give the seed from an earlier report to analyze the same documents again' + required: false + type: string + default: '' + crawl: + description: 'CommonCrawl crawl to read, e.g. CC-MAIN-2026-43, or latest' + required: false + type: string + default: 'latest' + fetch-duration: + description: 'Longest time to spend fetching documents, e.g. 90s, 30m, 1h' + required: false + type: string + default: '30m' + rounds: + description: 'How many times to time each document with each checkout. Two processes per round run at once on each pair of vCPUs' + required: false + type: number + default: 2 + small-max-bytes: + description: 'Documents smaller than this many bytes are small' + required: false + type: number + default: 32768 + large-min-bytes: + description: 'Documents at least this many bytes are large' + required: false + type: number + default: 262144 + +# Cancels all previous workflow runs for the same pull request that have not completed. +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || inputs.pull-request }} + cancel-in-progress: true + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +env: + # The analysis keeps every HTML document in this many WARC files, unless the fetch duration passes first. + HTML_API_BENCH_WARCS: ${{ inputs.warcs || 1 }} + HTML_API_BENCH_SEED: ${{ inputs.seed || '' }} + HTML_API_BENCH_CRAWL: ${{ inputs.crawl || 'latest' }} + HTML_API_BENCH_FETCH_DURATION: ${{ inputs.fetch-duration || '30m' }} + HTML_API_BENCH_ROUNDS: ${{ inputs.rounds || 2 }} + HTML_API_BENCH_SMALL_MAX_BYTES: ${{ inputs.small-max-bytes || 32768 }} + HTML_API_BENCH_LARGE_MIN_BYTES: ${{ inputs.large-min-bytes || 262144 }} + +jobs: + # Runs the analysis and uploads the report for html-api-commoncrawl-comment.yml to post. + # + # Performs the following steps: + # - Checks out the pull request merged into its target branch, and the target branch beside it. + # - Sets up PHP. + # - Fetches every HTML document in WARC files picked at random from the latest CommonCrawl crawl. + # - Times the Tag Processor, the HTML Processor, and wp_kses() on each document with both checkouts, running the rounds + # of both checkouts at the same time, one per vCPU, so that both share the same load. + # - Records which documents each checkout supports, and why the others are unsupported, with both checkouts at once. + # - Writes the report. + # - Uploads the report and the pull request number. + analyze: + name: Analyze HTML API changes against CommonCrawl + runs-on: ubuntu-24.04 + permissions: + contents: read + timeout-minutes: 90 + if: ${{ github.event_name == 'workflow_dispatch' || github.event.label.name == 'html-api-commoncrawl' }} + env: + CC_ANALYZER: merge/tools/html-api-commoncrawl/cc-analyzer.phar + SCRIPTS: merge/tools/html-api-commoncrawl + steps: + - name: Checkout the pull request merged into its target branch + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && format( 'refs/pull/{0}/merge', inputs.pull-request ) || '' }} + path: merge + fetch-depth: 2 + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + persist-credentials: false + + # The merge commit's first parent is the target branch commit which the pull request is merged into. + - name: Checkout the target branch + id: checkouts + run: | + git -C merge worktree add --detach ../base HEAD^1 + echo "base-sha=$( git -C base rev-parse HEAD )" >> "$GITHUB_OUTPUT" + echo "merge-sha=$( git -C merge rev-parse HEAD )" >> "$GITHUB_OUTPUT" + + - name: Set up PHP + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # 2.37.2 + with: + php-version: '8.5' + coverage: none + extensions: curl, mbstring, pdo_sqlite, zlib + ini-values: memory_limit=1G + + # Streams whole WARC files, a handful of requests in all: CommonCrawl blocks runners which send many requests, such as + # one per record. Every HTML document in them is kept; the number of files only bounds the download. The files are + # picked at random from the crawl's whole list, around 100,000 files fetched over a couple of weeks, so that each run + # sees a different part of the web. The seed is reported, and running the workflow again with it reads the same files. When CommonCrawl + # asks to slow down, with HTTP 429 or 503, no request is sent until its Retry-After or the backoff has passed. A download + # which still fails is listed, and the analysis runs on the documents already fetched. The cache limit is raised so + # that pruning never runs during the fetch. + - name: Fetch documents from CommonCrawl + id: fetch + run: | + seed="${HTML_API_BENCH_SEED:-$(( (RANDOM << 15) | RANDOM ))}" + echo "seed=$seed" >> "$GITHUB_OUTPUT" + php "$CC_ANALYZER" --workspace "$RUNNER_TEMP/cc-analyzer" batch fetch corpus \ + --crawl "$HTML_API_BENCH_CRAWL" \ + --warc-order random \ + --seed "$seed" \ + --resource-types html \ + --limit all \ + --limit-warcs "$HTML_API_BENCH_WARCS" \ + --max-jobs "$HTML_API_BENCH_WARCS" \ + --http-retries 6 \ + --cache-max-bytes 17179869184 \ + --duration "$HTML_API_BENCH_FETCH_DURATION" \ + --progress text + + # Each round runs one process for each checkout, and as many rounds run at once as there are pairs of vCPUs: two on + # the standard runner. Both checkouts are timed under the same load, as on a busy server, and the step takes about + # as long as one round would on its own. Each process replays the batch separately, so the documents' times are + # independent. If one process fails, the others are stopped and the step fails. + - name: Time parsing with both checkouts + run: | + trap 'kill $( jobs -p ) 2> /dev/null || true' EXIT + rounds_at_once=$(( $( nproc ) / 2 )) + (( rounds_at_once > 0 )) || rounds_at_once=1 + for (( first = 1; first <= HTML_API_BENCH_ROUNDS; first += rounds_at_once )); do + last=$(( first + rounds_at_once - 1 )) + (( last <= HTML_API_BENCH_ROUNDS )) || last=$HTML_API_BENCH_ROUNDS + echo "Rounds ${first} to ${last}: base and merge, $(( 2 * ( last - first + 1 ) )) processes at once" + pids=() + for round in $( seq "$first" "$last" ); do + for checkout in base merge; do + HTML_API_BENCH_WP_SRC="$GITHUB_WORKSPACE/$checkout/src" \ + CC_ANALYZER_OUTPUT_DIR="$RUNNER_TEMP/results/$checkout/round-$round" \ + php "$CC_ANALYZER" --workspace "$RUNNER_TEMP/cc-analyzer" batch run corpus "$SCRIPTS/benchmark.php" & + pids+=( "$!" ) + done + done + for pid in "${pids[@]}"; do + wait "$pid" + done + done + + # Nothing is timed here, so both checkouts run at once. + - name: Record support with both checkouts + run: | + trap 'kill $( jobs -p ) 2> /dev/null || true' EXIT + pids=() + for checkout in base merge; do + HTML_API_BENCH_WP_SRC="$GITHUB_WORKSPACE/$checkout/src" \ + CC_ANALYZER_OUTPUT_DIR="$RUNNER_TEMP/results/$checkout/analysis" \ + php "$CC_ANALYZER" --workspace "$RUNNER_TEMP/cc-analyzer" batch run corpus "$SCRIPTS/analysis.php" & + pids+=( "$!" ) + done + for pid in "${pids[@]}"; do + wait "$pid" + done + + - name: Write the report + run: | + mkdir "$RUNNER_TEMP/comment" + echo "$PR_NUMBER" > "$RUNNER_TEMP/comment/NR" + php "$SCRIPTS/report.php" "$RUNNER_TEMP/results/base" "$RUNNER_TEMP/results/merge" > "$RUNNER_TEMP/comment/report.md" + env: + PR_NUMBER: ${{ github.event.pull_request.number || inputs.pull-request }} + HTML_API_BENCH_BASE_SHA: ${{ steps.checkouts.outputs.base-sha }} + HTML_API_BENCH_MERGE_SHA: ${{ steps.checkouts.outputs.merge-sha }} + HTML_API_BENCH_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + HTML_API_BENCH_SEED: ${{ steps.fetch.outputs.seed }} + + - name: Upload the report + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: html-api-commoncrawl-comment + path: ${{ runner.temp }}/comment/ + if-no-files-found: error + retention-days: 1 diff --git a/src/wp-includes/html-api/class-wp-html-tag-processor.php b/src/wp-includes/html-api/class-wp-html-tag-processor.php index 7e047c4808081..99ec65d5de277 100644 --- a/src/wp-includes/html-api/class-wp-html-tag-processor.php +++ b/src/wp-includes/html-api/class-wp-html-tag-processor.php @@ -1021,33 +1021,17 @@ private function base_class_next_token(): bool { return true; } - // Parse all of its attributes. - while ( $this->parse_next_attribute() ) { - continue; - } + $this->parse_all_attributes(); // Ensure that the tag closes before the end of the document. - if ( - self::STATE_INCOMPLETE_INPUT === $this->parser_state || - $this->bytes_already_parsed >= strlen( $this->html ) - ) { - // Does this appropriately clear state (parsed attributes)? - $this->parser_state = self::STATE_INCOMPLETE_INPUT; + if ( self::STATE_INCOMPLETE_INPUT === $this->parser_state ) { $this->bytes_already_parsed = $was_at; - return false; } - $tag_ends_at = strpos( $this->html, '>', $this->bytes_already_parsed ); - if ( false === $tag_ends_at ) { - $this->parser_state = self::STATE_INCOMPLETE_INPUT; - $this->bytes_already_parsed = $was_at; - - return false; - } - $this->parser_state = self::STATE_MATCHED_TAG; - $this->bytes_already_parsed = $tag_ends_at + 1; - $this->token_length = $this->bytes_already_parsed - $this->token_starts_at; + $this->parser_state = self::STATE_MATCHED_TAG; + ++$this->bytes_already_parsed; + $this->token_length = $this->bytes_already_parsed - $this->token_starts_at; /* * Certain tags require additional processing. The first-letter pre-check @@ -1485,28 +1469,15 @@ private function skip_rcdata( string $tag_name ): bool { continue; } - while ( $this->parse_next_attribute() ) { - continue; - } - - $at = $this->bytes_already_parsed; - if ( $at >= strlen( $this->html ) ) { - return false; - } - - if ( '>' === $html[ $at ] ) { - $this->bytes_already_parsed = $at + 1; - return true; - } + $this->parse_all_attributes(); - if ( $at + 1 >= strlen( $this->html ) ) { + // Ensure that the tag closes before the end of the document. + if ( self::STATE_INCOMPLETE_INPUT === $this->parser_state ) { return false; } - if ( '/' === $html[ $at ] && '>' === $html[ $at + 1 ] ) { - $this->bytes_already_parsed = $at + 2; - return true; - } + ++$this->bytes_already_parsed; + return true; } return false; @@ -1695,18 +1666,15 @@ private function skip_script_data(): bool { return false; } - while ( $this->parse_next_attribute() ) { - continue; - } + $this->parse_all_attributes(); - if ( $this->bytes_already_parsed >= $doc_length ) { + // Ensure that the tag closes before the end of the document. + if ( self::STATE_INCOMPLETE_INPUT === $this->parser_state ) { return false; } - if ( '>' === $html[ $this->bytes_already_parsed ] ) { - ++$this->bytes_already_parsed; - return true; - } + ++$this->bytes_already_parsed; + return true; } ++$at; @@ -2201,168 +2169,175 @@ private function parse_next_tag(): bool { } /** - * Parses the next attribute. + * Parses attributes exhaustively in a tag. * - * @since 6.2.0 - * @ignore + * After calling this function, the processor will either be at the tag-closing `>` or the + * parser state will be set to `self::STATE_INCOMPLETE_INPUT` if the end of the document is + * reached before attribute parsing finishes. * - * @return bool Whether an attribute was found. Returns `false` upon reaching the end - * of the tag (`>` or `/>`) or the end of the document. + * @since 7.3.0 + * @ignore */ - private function parse_next_attribute(): bool { + private function parse_all_attributes(): void { $doc_length = strlen( $this->html ); - // Skip whitespace and slashes. - $skipped_length = strspn( $this->html, " \t\f\r\n/", $this->bytes_already_parsed ); - $this->bytes_already_parsed += $skipped_length; - if ( $this->bytes_already_parsed >= $doc_length ) { - $this->parser_state = self::STATE_INCOMPLETE_INPUT; - - return false; - } - - /** - * This block serves two purposes: - * - * - A fast path for common tag-ending `>`. - * - A check for the self-closing flag which must appear as `/>`. - * - * In a tag like ``, `/` is the attribute value, not a self-closing - * flag. When it appears in this form, the parser has already consumed the - * attribute value, `$skipped_length` is 0, and this checks below correctly - * identify whether there is a self-closing flag. - * - * Note: Both start and end tags may have the self-closing flag. - */ - if ( '>' === $this->html[ $this->bytes_already_parsed ] ) { - if ( $skipped_length > 0 && '/' === $this->html[ $this->bytes_already_parsed - 1 ] ) { - $this->has_self_closing_flag = true; + while ( true ) { + // Skip whitespace and slashes. + $skipped_length = strspn( $this->html, " \t\f\r\n/", $this->bytes_already_parsed ); + $this->bytes_already_parsed += $skipped_length; + if ( $this->bytes_already_parsed >= $doc_length ) { + $this->parser_state = self::STATE_INCOMPLETE_INPUT; + break; } - return false; - } - - /* - * Treat the equal sign as a part of the attribute - * name if it is the first encountered byte. - * - * @link https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-name-state - */ - $name_length = '=' === $this->html[ $this->bytes_already_parsed ] - ? 1 + strcspn( $this->html, "=/> \t\f\r\n", $this->bytes_already_parsed + 1 ) - : strcspn( $this->html, "=/> \t\f\r\n", $this->bytes_already_parsed ); - - // No attribute, just tag closer. - if ( 0 === $name_length || $this->bytes_already_parsed + $name_length >= $doc_length ) { - return false; - } - - $attribute_start = $this->bytes_already_parsed; - $attribute_name = substr( $this->html, $attribute_start, $name_length ); - $this->bytes_already_parsed += $name_length; - if ( $this->bytes_already_parsed >= $doc_length ) { - $this->parser_state = self::STATE_INCOMPLETE_INPUT; - return false; - } - - $this->skip_whitespace(); - if ( $this->bytes_already_parsed >= $doc_length ) { - $this->parser_state = self::STATE_INCOMPLETE_INPUT; + /** + * This block serves two purposes: + * + * - A fast path for common tag-ending `>`. + * - A check for the self-closing flag which must appear as `/>`. + * + * In a tag like ``, `/` is the attribute value, not a self-closing + * flag. When it appears in this form, the parser has already consumed the + * attribute value, `$skipped_length` is 0, and this checks below correctly + * identify whether there is a self-closing flag. + * + * Note: Both start and end tags may have the self-closing flag. + */ + if ( '>' === $this->html[ $this->bytes_already_parsed ] ) { + if ( $skipped_length > 0 && '/' === $this->html[ $this->bytes_already_parsed - 1 ] ) { + $this->has_self_closing_flag = true; + } + break; + } - return false; - } + /* + * Treat the equal sign as a part of the attribute + * name if it is the first encountered byte. + * + * @link https://html.spec.whatwg.org/multipage/parsing.html#before-attribute-name-state + */ + $name_length = '=' === $this->html[ $this->bytes_already_parsed ] + ? 1 + strcspn( $this->html, "=/> \t\f\r\n", $this->bytes_already_parsed + 1 ) + : strcspn( $this->html, "=/> \t\f\r\n", $this->bytes_already_parsed ); - $has_value = '=' === $this->html[ $this->bytes_already_parsed ]; - if ( $has_value ) { - ++$this->bytes_already_parsed; - $this->skip_whitespace(); - if ( $this->bytes_already_parsed >= $doc_length ) { + if ( $this->bytes_already_parsed + $name_length >= $doc_length ) { $this->parser_state = self::STATE_INCOMPLETE_INPUT; - - return false; + break; } - switch ( $this->html[ $this->bytes_already_parsed ] ) { - case "'": - case '"': - $quote = $this->html[ $this->bytes_already_parsed ]; - $value_start = $this->bytes_already_parsed + 1; - $end_quote_at = strpos( $this->html, $quote, $value_start ); - $end_quote_at = false === $end_quote_at ? $doc_length : $end_quote_at; - $value_length = $end_quote_at - $value_start; - $attribute_end = $end_quote_at + 1; - $this->bytes_already_parsed = $attribute_end; - break; + $attribute_start = $this->bytes_already_parsed; + /** + * Attribute name. + * + * The name is non-empty because whitespace and slashes are skipped, `>` ends parsing, + * and a leading `=` is included. PHPStan cannot infer this, so the type is annotated below. + * + * @var non-empty-string $attribute_name + */ + $attribute_name = substr( $this->html, $attribute_start, $name_length ); + $this->bytes_already_parsed += $name_length; - default: - $value_start = $this->bytes_already_parsed; - $value_length = strcspn( $this->html, "> \t\f\r\n", $value_start ); - $attribute_end = $value_start + $value_length; - $this->bytes_already_parsed = $attribute_end; + // There is no whitespace to skip when `=` follows the name. + if ( '=' !== $this->html[ $this->bytes_already_parsed ] ) { + $this->skip_whitespace(); + if ( $this->bytes_already_parsed >= $doc_length ) { + $this->parser_state = self::STATE_INCOMPLETE_INPUT; + break; + } } - } else { - $value_start = $this->bytes_already_parsed; - $value_length = 0; - $attribute_end = $attribute_start + $name_length; - } - if ( $attribute_end >= $doc_length ) { - $this->parser_state = self::STATE_INCOMPLETE_INPUT; + $has_value = '=' === $this->html[ $this->bytes_already_parsed ]; + if ( $has_value ) { + ++$this->bytes_already_parsed; - return false; - } + // There is no whitespace to skip when a quote follows `=`. + $quote = $this->html[ $this->bytes_already_parsed ] ?? ''; + if ( '"' !== $quote && "'" !== $quote ) { + $this->skip_whitespace(); + if ( $this->bytes_already_parsed >= $doc_length ) { + $this->parser_state = self::STATE_INCOMPLETE_INPUT; + break; + } + $quote = $this->html[ $this->bytes_already_parsed ]; + } - if ( $this->is_closing_tag ) { - return true; - } + switch ( $quote ) { + case "'": + case '"': + $value_start = $this->bytes_already_parsed + 1; + $end_quote_at = strpos( $this->html, $quote, $value_start ); + $end_quote_at = false === $end_quote_at ? $doc_length : $end_quote_at; + $value_length = $end_quote_at - $value_start; + $attribute_end = $end_quote_at + 1; + $this->bytes_already_parsed = $attribute_end; + break; + + default: + $value_start = $this->bytes_already_parsed; + $value_length = strcspn( $this->html, "> \t\f\r\n", $value_start ); + $attribute_end = $value_start + $value_length; + $this->bytes_already_parsed = $attribute_end; + } + } else { + $value_start = $this->bytes_already_parsed; + $value_length = 0; + $attribute_end = $attribute_start + $name_length; + } - /* - * > There must never be two or more attributes on - * > the same start tag whose names are an ASCII - * > case-insensitive match for each other. - * - HTML 5 spec - * - * @link https://html.spec.whatwg.org/multipage/syntax.html#attributes-2:ascii-case-insensitive - * - * The tokenizer replaces U+0000 NULL bytes in - * attribute names with U+FFFD. - * - * @link https://html.spec.whatwg.org/#attribute-name-state - */ - $comparable_name = strtolower( str_replace( "\x00", "\u{FFFD}", $attribute_name ) ); - - // If an attribute is listed many times, only use the first declaration and ignore the rest. - if ( ! isset( $this->attributes[ $comparable_name ] ) ) { - $this->attributes[ $comparable_name ] = new WP_HTML_Attribute_Token( - $attribute_name, - $value_start, - $value_length, - $attribute_start, - $attribute_end - $attribute_start, - ! $has_value - ); + if ( $attribute_end >= $doc_length ) { + $this->parser_state = self::STATE_INCOMPLETE_INPUT; + break; + } - return true; - } + if ( $this->is_closing_tag ) { + continue; + } - /* - * Track the duplicate attributes so if we remove it, all disappear together. - * - * While `$this->duplicated_attributes` could always be stored as an `array()`, - * which would simplify the logic here, storing a `null` and only allocating - * an array when encountering duplicates avoids needless allocations in the - * normative case of parsing tags with no duplicate attributes. - */ - $duplicate_span = new WP_HTML_Span( $attribute_start, $attribute_end - $attribute_start ); - if ( null === $this->duplicate_attributes ) { - $this->duplicate_attributes = array( $comparable_name => array( $duplicate_span ) ); - } elseif ( ! isset( $this->duplicate_attributes[ $comparable_name ] ) ) { - $this->duplicate_attributes[ $comparable_name ] = array( $duplicate_span ); - } else { - $this->duplicate_attributes[ $comparable_name ][] = $duplicate_span; - } + /* + * > There must never be two or more attributes on + * > the same start tag whose names are an ASCII + * > case-insensitive match for each other. + * - HTML 5 spec + * + * @link https://html.spec.whatwg.org/multipage/syntax.html#attributes-2:ascii-case-insensitive + * + * The tokenizer replaces U+0000 NULL bytes in + * attribute names with U+FFFD. + * + * @link https://html.spec.whatwg.org/#attribute-name-state + */ + $comparable_name = strtolower( str_replace( "\x00", "\u{FFFD}", $attribute_name ) ); + + // If an attribute is listed many times, only use the first declaration and ignore the rest. + if ( ! isset( $this->attributes[ $comparable_name ] ) ) { + $this->attributes[ $comparable_name ] = new WP_HTML_Attribute_Token( + $attribute_name, + $value_start, + $value_length, + $attribute_start, + $attribute_end - $attribute_start, + ! $has_value + ); + continue; + } - return true; + /* + * Track the duplicate attributes so if we remove it, all disappear together. + * + * While `$this->duplicated_attributes` could always be stored as an `array()`, + * which would simplify the logic here, storing a `null` and only allocating + * an array when encountering duplicates avoids needless allocations in the + * normative case of parsing tags with no duplicate attributes. + */ + $duplicate_span = new WP_HTML_Span( $attribute_start, $attribute_end - $attribute_start ); + if ( null === $this->duplicate_attributes ) { + $this->duplicate_attributes = array( $comparable_name => array( $duplicate_span ) ); + } elseif ( ! isset( $this->duplicate_attributes[ $comparable_name ] ) ) { + $this->duplicate_attributes[ $comparable_name ] = array( $duplicate_span ); + } else { + $this->duplicate_attributes[ $comparable_name ][] = $duplicate_span; + } + } } /** diff --git a/tools/html-api-commoncrawl/.gitattributes b/tools/html-api-commoncrawl/.gitattributes new file mode 100644 index 0000000000000..374d62b9c7ee4 --- /dev/null +++ b/tools/html-api-commoncrawl/.gitattributes @@ -0,0 +1 @@ +cc-analyzer.phar binary diff --git a/tools/html-api-commoncrawl/README.md b/tools/html-api-commoncrawl/README.md new file mode 100644 index 0000000000000..90be6ba1f1ed1 --- /dev/null +++ b/tools/html-api-commoncrawl/README.md @@ -0,0 +1,77 @@ +# HTML API CommonCrawl analysis + +Compares how the HTML API parses real-world documents with and without a pull request. It fetches every HTML document in a WARC file picked at random from the latest [CommonCrawl](https://commoncrawl.org/) crawl, then replays the same documents through both checkouts to measure: + +- how long the Tag Processor and the HTML Processor take to parse each document, and how long `wp_kses( $html, 'post' )` takes on it, for all documents and for small, medium, and large ones; +- which documents the HTML Processor does not support, and why it stopped on them, with a few example documents for each reason. Each example shows the token the HTML Processor stopped on, the token with up to 30 characters (extended grapheme clusters) of the document on either side, its byte offsets in the document, and the breadcrumbs of open elements at that point. + +The [`HTML API CommonCrawl Analysis`](../../.github/workflows/html-api-commoncrawl.yml) workflow runs this for a pull request when: +- the `html-api-commoncrawl` label is added to the pull request, if it changes the HTML API or `kses.php`. The label is removed straight away, so add it again to run another analysis. +- or it is run manually from the Actions tab with a pull request number. + +The report is left as a comment on the pull request. Each new analysis replaces the previous comment. + +## Files + +- `cc-analyzer.phar`: [cc-analyzer](#rebuilding-cc-analyzerphar), which fetches documents from CommonCrawl into a local batch, and replays a batch through an analysis script. It streams whole WARC files and keeps their HTML responses. That takes only a handful of requests: CommonCrawl blocks runners which send many, such as one per record. The files are picked at random from the crawl's whole list of around 100,000, fetched over a couple of weeks, so each analysis sees a different part of the web. One file already holds documents from nearly 20,000 hosts. It honors CommonCrawl's rate limiting, and if a download still fails, it lists the failure and the analysis runs on the documents it has. +- `bootstrap.php`: loads the HTML API, and what `wp_kses()` needs, from the `src/` directory in `HTML_API_BENCH_WP_SRC`, without loading WordPress. It presents a UTF-8 site at example.com. When a change adds a file to the HTML API, add it to the list of files here. +- `benchmark.php`: an analysis script which times the Tag Processor, the HTML Processor, and `wp_kses( $html, 'post' )` on each document and writes `benchmark.csv`. +- `analysis.php`: an analysis script which records whether the HTML Processor supports each document, and where it stopped on those it does not, and writes `analysis.csv`. +- `report.php`: compares the results from two checkouts and prints the Markdown report. + +## Running locally + +This needs PHP 8.5, as used by the workflow, with the `curl`, `pdo_sqlite`, and `zlib` extensions. Each analysis script runs in a separate process for each checkout, because PHP cannot load two versions of the HTML API into one process. + +```sh +# Fetch 1,000 documents, or as many as can be fetched in 5 minutes. The workflow uses --limit all instead, to keep every +# HTML document in the WARC files it downloads. +php -d memory_limit=1G tools/html-api-commoncrawl/cc-analyzer.phar --workspace /tmp/cc batch fetch corpus \ + --resource-types html --warc-order random --limit 1000 --limit-warcs 1 --max-jobs 1 --http-retries 6 --duration 5m --progress text + +# Time each document three times with each checkout, and record support once. +for checkout in base merge; do + for round in 1 2 3; do + HTML_API_BENCH_WP_SRC="/path/to/$checkout/src" CC_ANALYZER_OUTPUT_DIR="/tmp/results/$checkout/round-$round" \ + php -d memory_limit=1G tools/html-api-commoncrawl/cc-analyzer.phar --workspace /tmp/cc \ + batch run corpus tools/html-api-commoncrawl/benchmark.php + done + + HTML_API_BENCH_WP_SRC="/path/to/$checkout/src" CC_ANALYZER_OUTPUT_DIR="/tmp/results/$checkout/analysis" \ + php -d memory_limit=1G tools/html-api-commoncrawl/cc-analyzer.phar --workspace /tmp/cc \ + batch run corpus tools/html-api-commoncrawl/analysis.php +done + +php tools/html-api-commoncrawl/report.php /tmp/results/base /tmp/results/merge > report.md +``` + +A batch can be replayed as many times as needed without fetching it again. + +The workflow runs these replays at the same time, one process for each round of each checkout, as many rounds at once as there are pairs of vCPUs. On the standard 4-vCPU runner, both rounds of both checkouts run together, so timing takes about as long as one round. Both checkouts are timed under the same load, as on a busy server. A vCPU may be one of a core's two hardware threads, so each process can run slower than it would alone, and the times are best compared with each other rather than read on their own. Recording support for both checkouts also runs at once. + +## Settings + +The workflow reads these environment variables. When it is run manually, each one can be set from an input of the same name. + +| Variable | Default | Meaning | +| --- | --- | --- | +| `HTML_API_BENCH_WARCS` | `1` | How many WARC files to download. Every HTML document in them is analyzed; each file holds around 20,000. | +| `HTML_API_BENCH_SEED` | random | Picks the WARC files. The report gives the seed it used; give it again, with the same crawl, to analyze the same documents. | +| `HTML_API_BENCH_CRAWL` | `latest` | The CommonCrawl crawl to read, such as `CC-MAIN-2026-43`. | +| `HTML_API_BENCH_FETCH_DURATION` | `30m` | Longest time to spend fetching. Fetching stops at whichever limit is reached first. | +| `HTML_API_BENCH_ROUNDS` | `2` | How many times to time each document with each checkout. Each document's time is its median. | +| `HTML_API_BENCH_SMALL_MAX_BYTES` | `32768` | Documents smaller than this are small. | +| `HTML_API_BENCH_LARGE_MIN_BYTES` | `262144` | Documents at least this large are large. Everything between is medium. | + +## Rebuilding `cc-analyzer.phar` + +`cc-analyzer.phar` was built from cc-analyzer commit `1bca572` (branch `index-sampled-fetch`) with `tools/build-phar.php`. Before building, the default HTTP identity was removed so that requests carry only a generic `User-Agent` and no `From` header: + +- `src/Http/CurlStreamClient.php`: `DEFAULT_USER_AGENT` set to `'cc-analyzer/0.1 PHP-cURL'` and `DEFAULT_CONTACT_HEADER` set to `''`. +- `src/Cli/Application.php`: the `--user-agent` and `--contact-email` defaults set to the same values, and the help text to match. + +To rebuild, make the same changes in a copy of cc-analyzer and run: + +```sh +php -d phar.readonly=0 tools/build-phar.php --output /path/to/wordpress-develop/tools/html-api-commoncrawl/cc-analyzer.phar +``` diff --git a/tools/html-api-commoncrawl/analysis.php b/tools/html-api-commoncrawl/analysis.php new file mode 100644 index 0000000000000..d554f9acfacb0 --- /dev/null +++ b/tools/html-api-commoncrawl/analysis.php @@ -0,0 +1,138 @@ + 2 * $max_bytes ) { + $text = substr( $text, 0, $max_bytes ) . '…' . substr( $text, -$max_bytes ); + } + + return mb_scrub( $text, 'UTF-8' ); +} + +/** + * Returns the extended grapheme clusters of a document next to a byte offset. + * + * @param string $html Document. + * @param int $offset Byte offset of the boundary, at the start or end of a token. + * @param bool $before Whether to take the clusters before the offset, rather than after it. + * @return string Up to HTML_API_BENCH_CONTEXT_GRAPHEMES clusters, marked with `…` on the side where the document continues. + */ +function html_api_bench_context( $html, $offset, $before ) { + $start = $before ? max( 0, $offset - HTML_API_BENCH_CONTEXT_BYTES ) : $offset; + $window = mb_scrub( substr( $html, $start, $before ? $offset - $start : HTML_API_BENCH_CONTEXT_BYTES ), 'UTF-8' ); + $count = HTML_API_BENCH_CONTEXT_GRAPHEMES; + $text = 1 === preg_match( $before ? "/\X{0,{$count}}\z/u" : "/\A\X{0,{$count}}/u", $window, $matches ) ? $matches[0] : ''; + + if ( $before ) { + return ( $start > 0 || strlen( $text ) < strlen( $window ) ? '…' : '' ) . $text; + } + + return $text . ( $offset + HTML_API_BENCH_CONTEXT_BYTES < strlen( $html ) || strlen( $text ) < strlen( $window ) ? '…' : '' ); +} + +$csv = fopen( "{$output_dir}/analysis.csv", 'wb' ); +fputcsv( $csv, array( 'key', 'url', 'bytes', 'last_error', 'reason', 'line', 'token_name', 'incomplete', 'token_at', 'token_length', 'token_text', 'breadcrumbs', 'context_before', 'context_after' ), ',', '"', '' ); + +return static function ( $document ) use ( $csv ) { + $processor = WP_HTML_Processor::create_full_parser( $document->body ); + while ( $processor->next_token() ) { + continue; + } + + $last_error = $processor->get_last_error(); + $reason = ''; + $line = ''; + $token_name = ''; + $token_at = ''; + $token_length = ''; + $token_text = ''; + $breadcrumbs = ''; + $before = ''; + $after = ''; + + if ( WP_HTML_Processor::ERROR_UNSUPPORTED === $last_error ) { + $exception = $processor->get_unsupported_exception(); + $reason = $exception->getMessage(); + $line = $exception->getTrace()[0]['line']; + $token_name = $exception->token_name; + $token_at = $exception->token_at; + $token_length = strlen( $exception->token ); + $token_text = html_api_bench_excerpt( $exception->token ); + $breadcrumbs = implode( ' > ', $exception->stack_of_open_elements ); + $before = html_api_bench_context( $document->body, $token_at, true ); + $after = html_api_bench_context( $document->body, $token_at + $token_length, false ); + } elseif ( null !== $last_error ) { + $reason = $last_error; + $breadcrumbs = implode( ' > ', $processor->get_breadcrumbs() ?? array() ); + } + + fputcsv( + $csv, + array( + "{$document->inputStateKey}|{$document->recordId}", + $document->targetUri, + strlen( $document->body ), + $last_error ?? '', + $reason, + $line, + $token_name, + $processor->paused_at_incomplete_token() ? '1' : '0', + $token_at, + $token_length, + $token_text, + $breadcrumbs, + $before, + $after, + ), + ',', + '"', + '' + ); + + html_api_bench_progress( 'support' ); +}; diff --git a/tools/html-api-commoncrawl/benchmark.php b/tools/html-api-commoncrawl/benchmark.php new file mode 100644 index 0000000000000..d1ec1eac7a88a --- /dev/null +++ b/tools/html-api-commoncrawl/benchmark.php @@ -0,0 +1,89 @@ +Warm & up

One

Two
Three
'; +for ( $i = 0; $i < 100; $i++ ) { + $processor = new WP_HTML_Tag_Processor( $warm_up ); + while ( $processor->next_token() ) { + continue; + } + + $processor = WP_HTML_Processor::create_full_parser( $warm_up ); + while ( $processor->next_token() ) { + continue; + } + + wp_kses( $warm_up, 'post' ); +} + +return static function ( $document ) use ( $csv ) { + $html = $document->body; + + $started_at = hrtime( true ); + $tag = new WP_HTML_Tag_Processor( $html ); + while ( $tag->next_token() ) { + continue; + } + $tag_ns = hrtime( true ) - $started_at; + + // Free the Tag Processor here so that its destruction is not timed with the HTML Processor. + unset( $tag ); + + $started_at = hrtime( true ); + $processor = WP_HTML_Processor::create_full_parser( $html ); + while ( $processor->next_token() ) { + continue; + } + $html_ns = hrtime( true ) - $started_at; + + // Free the HTML Processor here so that its destruction is not timed with wp_kses(). + unset( $processor ); + + $started_at = hrtime( true ); + $kses = wp_kses( $html, 'post' ); + $kses_ns = hrtime( true ) - $started_at; + + unset( $kses ); + + fputcsv( + $csv, + array( + "{$document->inputStateKey}|{$document->recordId}", + $document->targetUri, + strlen( $html ), + $tag_ns, + $html_ns, + $kses_ns, + ), + ',', + '"', + '' + ); + + html_api_bench_progress( 'timing' ); +}; diff --git a/tools/html-api-commoncrawl/bootstrap.php b/tools/html-api-commoncrawl/bootstrap.php new file mode 100644 index 0000000000000..1fa712770e6c3 --- /dev/null +++ b/tools/html-api-commoncrawl/bootstrap.php @@ -0,0 +1,153 @@ + report.md + * + * Environment variables: + * + * HTML_API_BENCH_SMALL_MAX_BYTES Documents smaller than this are "small". Default 32768. + * HTML_API_BENCH_LARGE_MIN_BYTES Documents at least this large are "large". Default 262144. + * HTML_API_BENCH_BASE_SHA Commit of the target branch, shown in the report. + * HTML_API_BENCH_MERGE_SHA Commit of the PR merged into the target branch, shown in the report. + * HTML_API_BENCH_RUN_URL Link to the workflow run, shown in the report. + * HTML_API_BENCH_SEED Seed which picked the WARC files, shown in the report so that they can be read again. + * + * @package WordPress + */ + +const HTML_API_BENCH_REPORT_MARKER = ''; +const HTML_API_BENCH_EXAMPLES = 10; + +// How many example documents to show for each reason the HTML Processor stopped. +const HTML_API_BENCH_REASON_EXAMPLES = 3; + +// The smallest share of documents a size bin at either end of the chart needs to be drawn. +const HTML_API_BENCH_CHART_MIN_BIN_SHARE = 0.02; + +$small_max_bytes = (int) ( getenv( 'HTML_API_BENCH_SMALL_MAX_BYTES' ) ? getenv( 'HTML_API_BENCH_SMALL_MAX_BYTES' ) : 32768 ); +$large_min_bytes = (int) ( getenv( 'HTML_API_BENCH_LARGE_MIN_BYTES' ) ? getenv( 'HTML_API_BENCH_LARGE_MIN_BYTES' ) : 262144 ); + +/** + * Reads a CSV file with a header row into a list of associative arrays. + * + * @param string $path Path to the CSV file. + * @return array[] Rows keyed by column name. + */ +function read_csv( $path ) { + $handle = fopen( $path, 'rb' ); + $header = fgetcsv( $handle, null, ',', '"', '' ); + $rows = array(); + $row = fgetcsv( $handle, null, ',', '"', '' ); + while ( false !== $row ) { + $rows[] = array_combine( $header, $row ); + $row = fgetcsv( $handle, null, ',', '"', '' ); + } + fclose( $handle ); + + return $rows; +} + +/** + * Returns the median of a list of numbers. + * + * @param array $values Numbers. + * @return float Median. + */ +function median( $values ) { + sort( $values ); + $count = count( $values ); + $middle = intdiv( $count, 2 ); + + return 0 === $count % 2 ? ( $values[ $middle - 1 ] + $values[ $middle ] ) / 2 : $values[ $middle ]; +} + +/** + * Returns the given percentile of a list of numbers using the nearest-rank method. + * + * @param array $values Numbers. + * @param int $percentile Percentile from 1 to 100. + * @return float Value at the percentile. + */ +function percentile( $values, $percentile ) { + sort( $values ); + + return $values[ max( 0, (int) ceil( $percentile / 100 * count( $values ) ) - 1 ) ]; +} + +/** + * Loads one checkout's results, keyed by document. + * + * @param string $dir Results directory. + * @return array[] Each document's URL, size, median parse times, and support details. + */ +function load_results( $dir ) { + $timings = array(); + foreach ( glob( "{$dir}/round-*/benchmark.csv" ) as $path ) { + foreach ( read_csv( $path ) as $row ) { + $timings[ $row['key'] ]['url'] = $row['url']; + $timings[ $row['key'] ]['bytes'] = (int) $row['bytes']; + $timings[ $row['key'] ]['tag_ns'][] = (int) $row['tag_ns']; + $timings[ $row['key'] ]['html_ns'][] = (int) $row['html_ns']; + $timings[ $row['key'] ]['kses_ns'][] = (int) $row['kses_ns']; + } + } + + $documents = array(); + foreach ( read_csv( "{$dir}/analysis/analysis.csv" ) as $row ) { + $timing = $timings[ $row['key'] ]; + if ( $timing['url'] !== $row['url'] ) { + throw new Exception( "Benchmark and analysis disagree on the URL for {$row['key']}." ); + } + + $documents[ $row['key'] ] = array( + 'url' => $row['url'], + 'bytes' => $timing['bytes'], + 'tag_ns' => median( $timing['tag_ns'] ), + 'html_ns' => median( $timing['html_ns'] ), + 'kses_ns' => median( $timing['kses_ns'] ), + 'supported' => '' === $row['last_error'], + 'reason' => $row['reason'], + 'line' => $row['line'], + 'incomplete' => '1' === $row['incomplete'], + 'token_at' => $row['token_at'], + 'token_length' => $row['token_length'], + 'token_text' => $row['token_text'], + 'breadcrumbs' => $row['breadcrumbs'], + 'before' => $row['context_before'], + 'after' => $row['context_after'], + ); + } + + if ( count( $documents ) !== count( $timings ) ) { + throw new Exception( "Benchmark and analysis cover different documents in {$dir}." ); + } + + return $documents; +} + +/** + * Formats a duration in nanoseconds with a unit suited to its size. + * + * @param float $ns Duration in nanoseconds. + * @param bool $signed Whether to always show the sign. + * @return string Formatted duration. + */ +function format_ns( $ns, $signed = false ) { + $sign = $ns < 0 ? '−' : ( $signed && $ns > 0 ? '+' : '' ); + $ns = abs( $ns ); + + if ( $ns >= 1e9 ) { + return sprintf( '%s%.2f s', $sign, $ns / 1e9 ); + } + if ( $ns >= 1e6 ) { + return sprintf( '%s%.2f ms', $sign, $ns / 1e6 ); + } + if ( $ns >= 1e3 ) { + return sprintf( '%s%.1f µs', $sign, $ns / 1e3 ); + } + + return sprintf( '%s%d ns', $sign, $ns ); +} + +/** + * Formats a percentage change. + * + * @param float $percent Percentage. + * @return string Formatted percentage. + */ +function format_percent( $percent ) { + return sprintf( '%s%.2f%%', $percent < 0 ? '−' : ( $percent > 0 ? '+' : '' ), abs( $percent ) ); +} + +/** + * Formats a byte count. + * + * @param int $bytes Byte count. + * @return string Formatted size. + */ +function format_bytes( $bytes ) { + if ( $bytes >= 1048576 ) { + return sprintf( '%.1f MiB', $bytes / 1048576 ); + } + if ( $bytes >= 1024 ) { + return sprintf( '%d KiB', round( $bytes / 1024 ) ); + } + + return "{$bytes} B"; +} + +/** + * Formats a count with its share of a total. + * + * @param int $count Count. + * @param int $total Total. + * @return string Formatted count. + */ +function format_share( $count, $total ) { + return 0 === $total ? '0' : sprintf( '%s (%.1f%%)', number_format( $count ), 100 * $count / $total ); +} + +/** + * Formats a signed count. + * + * @param int $count Count. + * @return string Formatted count. + */ +function format_delta( $count ) { + return $count > 0 ? '+' . number_format( $count ) : ( $count < 0 ? '−' . number_format( -$count ) : '0' ); +} + +/** + * Formats text from a crawled document for safe display as inline code in a Markdown table. + * + * @param string $text Text to display. + * @param int $max_length Longest text to show before truncating. + * @return string Inline code. + */ +function format_code( $text, $max_length = 100 ) { + if ( mb_strlen( $text ) > $max_length ) { + $text = mb_substr( $text, 0, $max_length - 1 ) . '…'; + } + + return '`' . str_replace( array( '`', '|', "\n", "\r" ), array( "'", '\|', ' ', ' ' ), $text ) . '`'; +} + +/** + * Formats the bytes of a document which the HTML Processor stopped on, from where they start to where they end. + * + * @param array $result Analysis result for one document. + * @return string Byte range, or an empty string if it is not known. + */ +function format_bytes_span( $result ) { + if ( '' === $result['token_at'] ) { + return ''; + } + + return number_format( (int) $result['token_at'] ) . '–' . number_format( (int) $result['token_at'] + (int) $result['token_length'] ); +} + +/** + * Formats breadcrumbs as inline code, keeping the innermost elements of a deep stack. + * + * @param string $breadcrumbs Element names from the outermost to the innermost, separated by ` > `. + * @param int $max_count Most elements to show. + * @return string Inline code, or an empty string if there are no breadcrumbs. + */ +function format_breadcrumbs( $breadcrumbs, $max_count = 6 ) { + if ( '' === $breadcrumbs ) { + return ''; + } + + $elements = explode( ' > ', $breadcrumbs ); + if ( count( $elements ) > $max_count ) { + $elements = array( '…', ...array_slice( $elements, -$max_count ) ); + } + + return format_code( implode( ' > ', $elements ), 120 ); +} + +/** + * Formats the token the HTML Processor stopped on with the text around it in the document. + * + * A long token is shortened in the middle, so that the text on either side of it stays in view. + * + * @param array $result Analysis result for one document. + * @return string Inline code, or an empty string if the token is not known. + */ +function format_context( $result ) { + if ( '' === $result['token_at'] ) { + return ''; + } + + $token = $result['token_text']; + if ( mb_strlen( $token ) > 40 ) { + $token = mb_substr( $token, 0, 20 ) . '…' . mb_substr( $token, -19 ); + } + + return format_code( $result['before'] . $token . $result['after'], 400 ); +} + +/** + * Formats where the HTML Processor stopped on a document: the token, the text around it, its bytes, and the breadcrumbs. + * + * @param array $result Analysis result for one document. + * @return string[] Table cells. + */ +function failure_cells( $result ) { + return array( + '' === $result['token_text'] ? '' : format_code( $result['token_text'], 60 ), + format_context( $result ), + format_bytes_span( $result ), + format_breadcrumbs( $result['breadcrumbs'] ), + ); +} + +/** + * Renders a Markdown table. + * + * @param string[] $header Column headings. + * @param array[] $rows Table cells. + * @return string Markdown table. + */ +function table( $header, $rows ) { + $lines = array(); + $lines[] = '| ' . implode( ' | ', $header ) . ' |'; + $lines[] = '|' . str_repeat( ' --- |', count( $header ) ); + foreach ( $rows as $row ) { + $lines[] = '| ' . implode( ' | ', $row ) . ' |'; + } + + return implode( "\n", $lines ) . "\n"; +} + +/** + * Renders a table comparing parse times for each size group. + * + * @param array[] $pairs Documents, each with `base` and `merge` results. + * @param array $groups Group names mapped to a predicate on document size. + * @param string $parser One of `tag_ns`, `html_ns`, or `kses_ns`. + * @return string Markdown table. + */ +function timing_table( $pairs, $groups, $parser ) { + $rows = array(); + foreach ( $groups as $label => $in_group ) { + $base_total = 0; + $merge_total = 0; + $deltas = array(); + $percents = array(); + foreach ( $pairs as $pair ) { + if ( ! $in_group( $pair['base']['bytes'] ) ) { + continue; + } + + $base = $pair['base'][ $parser ]; + $merge = $pair['merge'][ $parser ]; + $base_total += $base; + $merge_total += $merge; + $deltas[] = $merge - $base; + $percents[] = 100 * ( $merge - $base ) / $base; + } + + if ( array() === $deltas ) { + $rows[] = array( $label, '0', '', '', '', '', '', '', '', '' ); + continue; + } + + $rows[] = array( + $label, + number_format( count( $deltas ) ), + format_ns( $base_total ), + format_ns( $merge_total ), + format_ns( $merge_total - $base_total, true ), + '**' . format_percent( 100 * ( $merge_total - $base_total ) / $base_total ) . '**', + format_ns( array_sum( $deltas ) / count( $deltas ), true ), + format_ns( median( $deltas ), true ), + format_percent( median( $percents ) ), + format_ns( percentile( $deltas, 95 ), true ), + ); + } + + return table( + array( 'Group', 'Docs', 'Target total', 'PR total', 'Δ total', 'Δ total %', 'Mean Δ/doc', 'Median Δ/doc', 'Median Δ%/doc', 'p95 Δ/doc' ), + $rows + ); +} + +/** + * Renders a Mermaid chart of median parse time by document size for both checkouts. + * + * Documents are binned by size in powers of two, with every document under 2 KiB in the first bin. + * Bins at either end with too few documents for a meaningful median are left out, as otherwise + * a handful of very large documents sets the scale and hides the differences everywhere else. + * + * @param array[] $pairs Documents, each with `base` and `merge` results. + * @param string $parser One of `tag_ns`, `html_ns`, or `kses_ns`. + * @param string $title Chart title. + * @return string Mermaid code block. + */ +function size_chart( $pairs, $parser, $title ) { + $bins = array(); + foreach ( $pairs as $pair ) { + $bin = max( 10, (int) floor( log( max( 1, $pair['base']['bytes'] ), 2 ) ) ); + $bins[ $bin ]['base'][] = $pair['base'][ $parser ] / 1e6; + $bins[ $bin ]['merge'][] = $pair['merge'][ $parser ] / 1e6; + } + ksort( $bins ); + + $min_count = HTML_API_BENCH_CHART_MIN_BIN_SHARE * count( $pairs ); + $left_out = 0; + foreach ( array( 'array_key_first', 'array_key_last' ) as $end ) { + while ( count( $bins[ $end( $bins ) ]['base'] ) < $min_count ) { + $left_out += count( $bins[ $end( $bins ) ]['base'] ); + unset( $bins[ $end( $bins ) ] ); + } + } + + $labels = array(); + $base = array(); + $merge = array(); + foreach ( $bins as $bin => $times ) { + $labels[] = 10 === $bin ? '"0"' : ( $bin < 20 ? '"' . 2 ** ( $bin - 10 ) . 'K"' : '"' . 2 ** ( $bin - 20 ) . 'M"' ); + $base[] = round( median( $times['base'] ), 3 ); + $merge[] = round( median( $times['merge'] ), 3 ); + } + + return implode( + "\n", + array( + '```mermaid', + '%%{init: {"themeVariables": {"xyChart": {"plotColorPalette": "#2271b1, #d63638"}}}}%%', + 'xychart-beta', + " title \"{$title}\"", + ' x-axis "Document size (bytes, at least)" [' . implode( ', ', $labels ) . ']', + ' y-axis "Median time (ms)"', + ' line [' . implode( ', ', $base ) . ']', + ' line [' . implode( ', ', $merge ) . ']', + '```', + '', + 'Blue: target branch. Red: PR merged into the target branch.' . ( + $left_out > 0 + ? sprintf( ' Sizes at either end with fewer than %d%% of documents are not drawn (%s documents).', 100 * HTML_API_BENCH_CHART_MIN_BIN_SHARE, number_format( $left_out ) ) + : '' + ) . '', + '', + ) + ); +} + +/** + * Renders the documents whose parse time grew the most. + * + * @param array[] $pairs Documents, each with `base` and `merge` results. + * @param string $parser One of `tag_ns`, `html_ns`, or `kses_ns`. + * @return string Markdown table. + */ +function slowdowns_table( $pairs, $parser ) { + usort( + $pairs, + static function ( $a, $b ) use ( $parser ) { + return ( $b['merge'][ $parser ] - $b['base'][ $parser ] ) <=> ( $a['merge'][ $parser ] - $a['base'][ $parser ] ); + } + ); + + $rows = array(); + foreach ( array_slice( $pairs, 0, HTML_API_BENCH_EXAMPLES ) as $pair ) { + $base = $pair['base'][ $parser ]; + $merge = $pair['merge'][ $parser ]; + $rows[] = array( + format_code( $pair['base']['url'] ), + format_bytes( $pair['base']['bytes'] ), + format_ns( $base ), + format_ns( $merge ), + format_ns( $merge - $base, true ), + format_percent( 100 * ( $merge - $base ) / $base ), + ); + } + + return table( array( 'URL', 'Size', 'Target', 'PR', 'Δ', 'Δ%' ), $rows ); +} + +/** + * Renders example documents whose support changed. + * + * Where the HTML Processor stopped is shown for the PR, or for the target + * branch when the PR supports the document. + * + * @param array[] $pairs Documents, each with `base` and `merge` results. + * @return string Markdown table. + */ +function examples_table( $pairs ) { + $rows = array(); + foreach ( array_slice( $pairs, 0, HTML_API_BENCH_EXAMPLES ) as $pair ) { + $rows[] = array( + format_code( $pair['base']['url'] ), + format_bytes( $pair['base']['bytes'] ), + $pair['base']['supported'] ? 'supported' : format_code( "L{$pair['base']['line']}: {$pair['base']['reason']}" ), + $pair['merge']['supported'] ? 'supported' : format_code( "L{$pair['merge']['line']}: {$pair['merge']['reason']}" ), + ...failure_cells( $pair['merge']['supported'] ? $pair['base'] : $pair['merge'] ), + ); + } + + return table( array( 'URL', 'Size', 'Target', 'PR', 'Token', 'Context', 'Bytes', 'Breadcrumbs' ), $rows ); +} + +/** + * Renders a few example documents for each reason the HTML Processor stopped. + * + * Examples come from the PR, or from the target branch for a reason only it + * stops for. They are the smallest documents, which are the easiest to inspect, + * taking at most one from each host. Each shows the token the HTML Processor + * stopped on, the text around it, its bytes in the document, and the + * breadcrumbs at that point. + * + * @param array[] $reasons Reasons, in report order, each with `base` and `merge` counts, call sites, and documents. + * @param callable $source_url Returns the link to `class-wp-html-processor.php` at a commit, or null. + * @param string $base_sha Commit of the target branch. + * @param string $merge_sha Commit of the PR merged into the target branch. + * @return string Markdown table. + */ +function reason_examples_table( $reasons, $source_url, $base_sha, $merge_sha ) { + $rows = array(); + foreach ( $reasons as $reason => $sides ) { + $side = isset( $sides['merge']['documents'] ) ? 'merge' : 'base'; + $documents = $sides[ $side ]['documents']; + $url = $source_url( 'merge' === $side ? $merge_sha : $base_sha ); + usort( + $documents, + static function ( $a, $b ) { + return $a['bytes'] <=> $b['bytes']; + } + ); + + $hosts = array(); + foreach ( $documents as $document ) { + $host = strtolower( (string) parse_url( $document['url'], PHP_URL_HOST ) ); + if ( isset( $hosts[ $host ] ) ) { + continue; + } + + $hosts[ $host ] = true; + $line = $document['line']; + $rows[] = array( + 1 === count( $hosts ) ? format_code( $reason, 120 ) : '', + format_code( $document['url'] ), + format_bytes( $document['bytes'] ), + '' === $line ? '' : ( $url ? "[L{$line}]({$url}#L{$line})" : "L{$line}" ) . ( 'base' === $side ? ' (target)' : '' ), + ...failure_cells( $document ), + ); + + if ( count( $hosts ) === HTML_API_BENCH_REASON_EXAMPLES ) { + break; + } + } + } + + return table( array( 'Reason', 'URL', 'Size', 'Call site', 'Token', 'Context', 'Bytes', 'Breadcrumbs' ), $rows ); +} + +$base_results = load_results( $argv[1] ); +$merge_results = load_results( $argv[2] ); + +$pairs = array(); +foreach ( $base_results as $key => $base ) { + $pairs[ $key ] = array( + 'base' => $base, + 'merge' => $merge_results[ $key ], + ); +} + +if ( count( $pairs ) !== count( $merge_results ) ) { + throw new Exception( 'The target and PR results cover different documents.' ); +} + +$supported_in_both = array_filter( + $pairs, + static function ( $pair ) { + return $pair['base']['supported'] && $pair['merge']['supported']; + } +); + +$groups = array( + 'All' => static function () { + return true; + }, + 'Small (< ' . format_bytes( $small_max_bytes ) . ')' => static function ( $bytes ) use ( $small_max_bytes ) { + return $bytes < $small_max_bytes; + }, + 'Medium' => static function ( $bytes ) use ( $small_max_bytes, $large_min_bytes ) { + return $bytes >= $small_max_bytes && $bytes < $large_min_bytes; + }, + 'Large (≥ ' . format_bytes( $large_min_bytes ) . ')' => static function ( $bytes ) use ( $large_min_bytes ) { + return $bytes >= $large_min_bytes; + }, +); + +$rounds = count( glob( "{$argv[1]}/round-*/benchmark.csv" ) ); +preg_match( '~/crawl-data/([^/]+)/~', array_key_first( $pairs ), $crawl ); +$base_sha = getenv( 'HTML_API_BENCH_BASE_SHA' ); +$merge_sha = getenv( 'HTML_API_BENCH_MERGE_SHA' ); +$run_url = getenv( 'HTML_API_BENCH_RUN_URL' ); +$seed = getenv( 'HTML_API_BENCH_SEED' ); + +// The WARC files the documents came from, named by the part of each document's key before its record range. +$warcs = array(); +foreach ( array_keys( $pairs ) as $key ) { + if ( preg_match( '~/(crawl-data/[^#|]+\.warc\.gz)~', $key, $warc ) ) { + $warcs[ $warc[1] ] = true; + } +} +ksort( $warcs ); + +$report = HTML_API_BENCH_REPORT_MARKER . "\n"; +$report .= "## HTML API CommonCrawl analysis\n\n"; +$report .= sprintf( + "Parsed %s documents from CommonCrawl `%s` with the target branch%s and with this PR merged into it%s. %s. Negative deltas mean the PR is faster.%s\n\n", + number_format( count( $pairs ) ), + $crawl[1], + $base_sha ? ' (`' . substr( $base_sha, 0, 10 ) . '`)' : '', + $merge_sha ? ' (`' . substr( $merge_sha, 0, 10 ) . '`)' : '', + 1 === $rounds ? 'Each document was timed once' : "Each time is the median of {$rounds} rounds", + $run_url ? " [Workflow run]({$run_url})." : '' +); +$report .= sprintf( + "

%s\n\n%s\n%s
\n\n", + 1 === count( $warcs ) ? 'The WARC file read' : sprintf( 'The %d WARC files read', count( $warcs ) ), + implode( '', array_map( static fn ( $path ) => "- `{$path}`\n", array_keys( $warcs ) ) ), + false !== $seed && '' !== $seed + ? sprintf( "%s picked at random with seed `%s`. Run the workflow manually with this seed and crawl `%s` to analyze the same documents again.\n\n", 1 === count( $warcs ) ? 'It was' : 'They were', $seed, $crawl[1] ) + : '' +); + +$report .= "### HTML Processor parse time\n\n"; +$report .= "All documents, including those where parsing stopped early on unsupported markup:\n\n"; +$report .= timing_table( $pairs, $groups, 'html_ns' ) . "\n"; +$report .= sprintf( "Documents supported by both the target branch and the PR (%s):\n\n", number_format( count( $supported_in_both ) ) ); +$report .= timing_table( $supported_in_both, $groups, 'html_ns' ) . "\n"; +$report .= size_chart( $pairs, 'html_ns', 'HTML Processor: median parse time by document size' ) . "\n"; +$report .= "
Largest HTML Processor slowdowns\n\n" . slowdowns_table( $pairs, 'html_ns' ) . "\n
\n\n"; + +$report .= "### Tag Processor parse time\n\n"; +$report .= timing_table( $pairs, $groups, 'tag_ns' ) . "\n"; +$report .= size_chart( $pairs, 'tag_ns', 'Tag Processor: median parse time by document size' ) . "\n"; +$report .= "
Largest Tag Processor slowdowns\n\n" . slowdowns_table( $pairs, 'tag_ns' ) . "\n
\n\n"; + +$report .= "### wp_kses() time\n\n"; +$report .= "Time for `wp_kses( \$html, 'post' )`. Only the files it needs are loaded, without WordPress's default filters such as those on `pre_kses`, so times differ from a full site.\n\n"; +$report .= timing_table( $pairs, $groups, 'kses_ns' ) . "\n"; +$report .= size_chart( $pairs, 'kses_ns', 'wp_kses(): median time by document size' ) . "\n"; +$report .= "
Largest wp_kses() slowdowns\n\n" . slowdowns_table( $pairs, 'kses_ns' ) . "\n
\n\n"; + +$report .= "### HTML Processor support\n\n"; +$rows = array(); +foreach ( $groups as $label => $in_group ) { + $count = 0; + $base_unsupported = 0; + $merge_unsupported = 0; + $base_incomplete = 0; + $merge_incomplete = 0; + foreach ( $pairs as $pair ) { + if ( ! $in_group( $pair['base']['bytes'] ) ) { + continue; + } + + ++$count; + $base_unsupported += $pair['base']['supported'] ? 0 : 1; + $merge_unsupported += $pair['merge']['supported'] ? 0 : 1; + $base_incomplete += $pair['base']['incomplete'] ? 1 : 0; + $merge_incomplete += $pair['merge']['incomplete'] ? 1 : 0; + } + + $rows[] = array( + $label, + number_format( $count ), + format_share( $base_unsupported, $count ), + format_share( $merge_unsupported, $count ), + '**' . format_delta( $merge_unsupported - $base_unsupported ) . '**', + number_format( $base_incomplete ), + number_format( $merge_incomplete ), + ); +} +$report .= table( array( 'Group', 'Docs', 'Unsupported (target)', 'Unsupported (PR)', 'Δ', 'Incomplete (target)', 'Incomplete (PR)' ), $rows ) . "\n"; +$report .= "Incomplete documents end inside a token, usually because the crawler truncated them. They are not counted as unsupported.\n\n"; + +$reasons = array(); +foreach ( $pairs as $pair ) { + foreach ( array( 'base', 'merge' ) as $side ) { + $result = $pair[ $side ]; + if ( $result['supported'] ) { + continue; + } + + $reasons[ $result['reason'] ][ $side ]['count'] = ( $reasons[ $result['reason'] ][ $side ]['count'] ?? 0 ) + 1; + $reasons[ $result['reason'] ][ $side ]['documents'][] = $result; + if ( '' !== $result['line'] ) { + $reasons[ $result['reason'] ][ $side ]['lines'][ $result['line'] ] = ( $reasons[ $result['reason'] ][ $side ]['lines'][ $result['line'] ] ?? 0 ) + 1; + } + } +} +uasort( + $reasons, + static function ( $a, $b ) { + return max( $b['base']['count'] ?? 0, $b['merge']['count'] ?? 0 ) <=> max( $a['base']['count'] ?? 0, $a['merge']['count'] ?? 0 ); + } +); + +/* + * Links call sites to the source when running on GitHub Actions, which provides + * the repository. Both the target and the merge commits are in the repository. + */ +$source_url = static function ( $sha ) { + $server = getenv( 'GITHUB_SERVER_URL' ); + $repository = getenv( 'GITHUB_REPOSITORY' ); + + return $server && $repository && $sha + ? "{$server}/{$repository}/blob/{$sha}/src/wp-includes/html-api/class-wp-html-processor.php" + : null; +}; +$format_lines = static function ( $lines, $url ) { + ksort( $lines ); + $parts = array(); + foreach ( $lines as $line => $count ) { + $parts[] = $url ? "[L{$line}]({$url}#L{$line}) ({$count})" : "L{$line} ({$count})"; + } + + return implode( ', ', $parts ); +}; + +$rows = array(); +foreach ( $reasons as $reason => $sides ) { + $base_count = $sides['base']['count'] ?? 0; + $merge_count = $sides['merge']['count'] ?? 0; + $rows[] = array( + format_code( $reason, 120 ), + number_format( $base_count ), + number_format( $merge_count ), + format_delta( $merge_count - $base_count ), + $format_lines( $sides['base']['lines'] ?? array(), $source_url( $base_sha ) ), + $format_lines( $sides['merge']['lines'] ?? array(), $source_url( $merge_sha ) ), + ); +} +$report .= "Unsupported documents by the reason the HTML Processor stopped. Call sites are lines in `class-wp-html-processor.php` of each checkout.\n\n"; +$report .= table( array( 'Reason', 'Target', 'PR', 'Δ', 'Target call sites', 'PR call sites' ), $rows ) . "\n"; +if ( array() !== $reasons ) { + $report .= "
Example documents for each reason\n\n" . reason_examples_table( $reasons, $source_url, $base_sha, $merge_sha ) . "\n
\n\n"; +} + +$newly_unsupported = array_values( + array_filter( + $pairs, + static function ( $pair ) { + return $pair['base']['supported'] && ! $pair['merge']['supported']; + } + ) +); +$newly_supported = array_values( + array_filter( + $pairs, + static function ( $pair ) { + return ! $pair['base']['supported'] && $pair['merge']['supported']; + } + ) +); +$changed_reason = array_values( + array_filter( + $pairs, + static function ( $pair ) { + return ! $pair['base']['supported'] && ! $pair['merge']['supported'] && $pair['base']['reason'] !== $pair['merge']['reason']; + } + ) +); + +foreach ( + array( + 'Newly unsupported with this PR' => $newly_unsupported, + 'Newly supported with this PR' => $newly_supported, + 'Unsupported for a different reason with this PR' => $changed_reason, + ) as $label => $changed +) { + $report .= sprintf( "- **%s:** %s\n", $label, number_format( count( $changed ) ) ); +} +$report .= "\n"; + +foreach ( + array( + 'Newly unsupported examples' => $newly_unsupported, + 'Newly supported examples' => $newly_supported, + 'Different-reason examples' => $changed_reason, + ) as $label => $changed +) { + if ( array() !== $changed ) { + $report .= "
{$label}\n\n" . examples_table( $changed ) . "\n
\n\n"; + } +} + +// GitHub rejects comments longer than this. +if ( mb_strlen( $report ) > 65536 ) { + throw new Exception( sprintf( 'The report is %d characters, which is too long for a GitHub comment.', mb_strlen( $report ) ) ); +} + +echo $report;