Skip to content

Commit e18bcc8

Browse files
author
Nedas
committed
Merge remote-tracking branch 'origin/main' into feature/switching-to-translated
# Conflicts: # README.md # build-docs.py
2 parents f7a4abe + bcd1bb8 commit e18bcc8

590 files changed

Lines changed: 10982 additions & 2623 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/deploy.yml‎

Lines changed: 131 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,80 @@ on:
99
branches:
1010
- main
1111

12+
# Every job except close_pull_request_job carries the same `if`: run on pushes
13+
# to main and on open PRs, but only build the Crowdin `localization` branch
14+
# once a review is requested. GitHub Actions has no way to share a job
15+
# condition, so the expression is repeated verbatim; keep the copies in sync.
16+
#
17+
# Job layout:
18+
# lint_job, doctest_job independent checks on the repo sources
19+
# build_job full multi-language site build
20+
# link_check_job needs the built site from build_job
21+
# deploy_job waits for all of the above, so a failing check
22+
# blocks the deploy (production and PR previews)
1223
jobs:
24+
lint_job:
25+
if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
26+
runs-on: ubuntu-latest
27+
name: Lint build scripts
28+
steps:
29+
- uses: actions/checkout@v4
30+
- name: Set up Python
31+
uses: actions/setup-python@v5
32+
with:
33+
python-version: '3.11'
34+
- name: Set up uv
35+
# `./run scripts check` runs ruff and mypy through uvx.
36+
uses: astral-sh/setup-uv@v10.0.1
37+
- name: Install shell tooling
38+
run: |
39+
sudo apt-get update
40+
command -v shellcheck >/dev/null || sudo apt-get install -y shellcheck
41+
command -v shfmt >/dev/null || sudo apt-get install -y shfmt
42+
shellcheck --version | head -2
43+
shfmt --version
44+
- name: Check scripts
45+
# ruff lint + format diff + mypy --strict on the qualified Python
46+
# sources (pyproject.toml); shellcheck + shfmt diff on the run scripts.
47+
run: ./run scripts check
48+
49+
doctest_job:
50+
if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
51+
runs-on: ubuntu-latest
52+
name: Doctest C# code blocks
53+
env:
54+
# Optional repository secret: a direct download URL for the Linux x64
55+
# Tabular Editor CLI archive (te-linux-x64.tar.gz). The CLI is gated
56+
# behind sign-in, so there is no public URL to fetch it from. When the
57+
# secret is unset, only the te-free annotation validation runs.
58+
TE_CLI_URL: ${{ secrets.TE_CLI_DOWNLOAD_URL }}
59+
steps:
60+
- uses: actions/checkout@v4
61+
- name: Set up Python
62+
uses: actions/setup-python@v5
63+
with:
64+
python-version: '3.11'
65+
- name: Validate doctest annotations
66+
# Grammar and coverage of the ```csharp {compile} / {run ...} fences
67+
# in every annotated content/*.md; no te needed.
68+
run: ./run doctest validate
69+
- name: Install Tabular Editor CLI
70+
if: env.TE_CLI_URL != ''
71+
run: |
72+
mkdir -p "$RUNNER_TEMP/te"
73+
curl -sSfL "$TE_CLI_URL" | tar -xz -C "$RUNNER_TEMP/te"
74+
chmod +x "$RUNNER_TEMP/te/te"
75+
echo "$RUNNER_TEMP/te" >> "$GITHUB_PATH"
76+
"$RUNNER_TEMP/te/te" --version
77+
- name: Run doctests
78+
if: env.TE_CLI_URL != ''
79+
# Compile and execute every annotated block against a throwaway model
80+
# and diff its Output() against the documented **Output** fence.
81+
run: ./run doctest
82+
- name: Doctest execution skipped
83+
if: env.TE_CLI_URL == ''
84+
run: echo "::notice title=Doctests not executed::Set the TE_CLI_DOWNLOAD_URL repository secret to a te-linux-x64.tar.gz download URL to compile and run the annotated C# blocks in CI. Only annotation validation ran."
85+
1386
build_job:
1487
if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
1588
runs-on: windows-latest
@@ -48,17 +121,73 @@ jobs:
48121
Write-Host "API files generated: $((Get-ChildItem content/api/*.yml).Count) YAML files"
49122
shell: pwsh
50123
- name: Build all documentation
51-
run: python build-docs.py --all --skip-gen
124+
# English DocFX warnings fail the build; es/zh warnings do not, so they
125+
# are collected into a markdown report for the job summary below.
126+
run: python build-docs.py --all --skip-gen --warnings-report build-warnings.md
127+
env:
128+
PYTHONUNBUFFERED: '1' # keep the per-language banners in order with DocFX's own output in the log
129+
- name: Publish DocFX warnings
130+
if: always()
131+
shell: bash
132+
run: |
133+
if [ -f build-warnings.md ]; then
134+
cat build-warnings.md >> "$GITHUB_STEP_SUMMARY"
135+
else
136+
echo '## DocFX build warnings' >> "$GITHUB_STEP_SUMMARY"
137+
echo 'No report was produced (the build stopped before any language was built).' >> "$GITHUB_STEP_SUMMARY"
138+
fi
52139
- name: Upload build artifact
53140
uses: actions/upload-artifact@v4
54141
with:
55142
name: site
56143
path: _site
57144
retention-days: 7
58145

146+
link_check_job:
147+
needs: build_job
148+
runs-on: ubuntu-latest
149+
name: Check links
150+
# A full run fetches every unique external URL once (a few thousand), so
151+
# it takes minutes; the cap keeps a stalled host from holding the deploy.
152+
timeout-minutes: 30
153+
steps:
154+
- uses: actions/checkout@v4
155+
- name: Set up Python
156+
uses: actions/setup-python@v5
157+
with:
158+
python-version: '3.11'
159+
- name: Download build artifact
160+
uses: actions/download-artifact@v4
161+
with:
162+
name: site
163+
path: _site
164+
- name: Check links
165+
# Internal (own-site) links are errors and fail this job: local files,
166+
# anchors, and old root-style paths (those are resolved against the
167+
# live site's redirects). External links are warnings only: they are
168+
# listed in the report and never fail the run, since third-party
169+
# outages and bot-blocking are outside our control.
170+
shell: bash
171+
run: ./run check-links stats 2>&1 | tee link-check-report.txt
172+
- name: Publish link report
173+
# The report (including the external URLs to verify by hand) goes to
174+
# the job summary, so warnings are visible without opening the log.
175+
if: always()
176+
shell: bash
177+
run: |
178+
{
179+
echo '## Link check report'
180+
echo
181+
echo 'Internal broken links fail this job. External URLs listed under "WARNINGS" are informational; verify them by hand.'
182+
echo
183+
echo '```'
184+
head -c 900000 link-check-report.txt
185+
echo '```'
186+
} >> "$GITHUB_STEP_SUMMARY"
187+
59188
deploy_job:
60189
if: (github.head_ref != 'localization' && github.event_name == 'push') || (github.head_ref != 'localization' && github.event_name == 'pull_request' && github.event.action != 'closed') || (github.head_ref == 'localization' && github.event.action == 'review_requested')
61-
needs: build_job
190+
needs: [build_job, lint_job, doctest_job, link_check_job]
62191
runs-on: ubuntu-latest
63192
name: Deploy to Azure
64193
steps:

‎README.md‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ swa start _site
6969
| `--skip-api` | Reuse existing API metadata in content/api, ~30-40% faster (local markdown iteration only, requires `--serve`/`--lang`; never for testing/CI/CD/releases) |
7070
| `--permissive` | Don't treat English DocFX warnings as build failures (for local iteration; full/CI builds stay strict) |
7171
| `--sync` | Copy English over missing/outdated translations for a local build. Fallback copies only: never commit them. The translation status file is not touched |
72+
| `--warnings-report PATH` | Write a markdown summary of the DocFX warnings per built language to `PATH` (CI appends it to the job summary); written even when the build fails |
7273

7374
## What the Build Script Does
7475

@@ -84,6 +85,38 @@ swa start _site
8485
10. **Generates SWA config** - Creates `staticwebapp.config.json` for Azure Static Web Apps routing
8586
11. **Generates the sitemap index** - Runs `gen_sitemap_index.py` last, once every per-language sitemap exists, to write the site-wide sitemap index and `robots.txt`
8687

88+
# Continuous integration
89+
90+
Every pull request against `main` (and every push to `main`) runs the workflow in [`.github/workflows/deploy.yml`](.github/workflows/deploy.yml).
91+
The deploy step waits for all checks, so a failing check blocks both the production deploy and the PR preview site.
92+
93+
| Job | What it runs | Fails when |
94+
|-----|--------------|------------|
95+
| Lint build scripts | `./run scripts check`: ruff, ruff format, mypy `--strict` on the qualified Python sources (see `pyproject.toml`); shellcheck and shfmt on `run` and `build_scripts/run_scripts/*.sh` | any lint, type, or formatting finding |
96+
| Doctest C# code blocks | `./run doctest validate` always; `./run doctest` (compile, run, and compare `**Output**` blocks) when a te CLI is available, see below | malformed annotation; with te: compile error, runtime error, or output mismatch |
97+
| Build Documentation | `python build-docs.py --all --skip-gen --warnings-report build-warnings.md` on Windows, uploads `_site`; the per-language DocFX warning report goes to the job summary | DocFX build failure or English DocFX warnings |
98+
| Check links | `python build_scripts/check_links.py validate stats` against the built `_site` | any broken **internal** link: missing file, missing anchor, or an old root-style path that no longer redirects on the live site |
99+
| Deploy to Azure | Azure Static Web Apps upload | only after all of the above pass |
100+
101+
**External links are warnings only.**
102+
The link check fetches every unique external URL once, but a bad status or a network error on a third-party site never fails the run:
103+
the site is outside our control, and outages, bot-blocking, and link rot should not stop docs from shipping.
104+
The full report, including the list of external URLs to verify by hand, is attached to the job summary of the "Check links" job.
105+
Fix or remove genuinely dead external links as part of normal maintenance.
106+
107+
**Translation build warnings are visible but not blocking.**
108+
Only English DocFX warnings fail the build; the es and zh builds are Crowdin-managed translations and may carry warnings such as broken bookmarks or missing includes.
109+
The "Build Documentation" job summary lists every DocFX warning per language, so translation problems can be spotted and fed back without blocking a deploy.
110+
111+
**Executing the doctests needs the te CLI.**
112+
The CLI download is gated behind sign-in, so the workflow cannot fetch it from a public URL.
113+
Set the repository secret `TE_CLI_DOWNLOAD_URL` to a direct download URL for `te-linux-x64.tar.gz` (for example a private blob with a SAS token);
114+
the doctest job then installs it and runs the full `./run doctest`.
115+
Without the secret, the job validates the annotations only and posts a notice.
116+
Use a te build aligned with the current Tabular Editor 3 release, otherwise the compare step reports false drift.
117+
118+
Run the same checks locally before pushing: `./run scripts check`, `./run doctest`, and `./run build` followed by `./run check-links`.
119+
87120
# Project Structure
88121

89122
```

‎build-docs.py‎

Lines changed: 96 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@
2222
(~30-40% faster). LOCAL markdown iteration ONLY, with --serve/--lang;
2323
never for testing, CI/CD, or release builds.
2424
--permissive Don't fail the English build on DocFX warnings (local iteration)
25+
--warnings-report PATH
26+
Write a markdown summary of the DocFX warnings per built language
27+
to PATH (for CI job summaries); written even when the build fails
2528
"""
2629

2730
import argparse
@@ -35,21 +38,58 @@
3538

3639

3740
_DOCFX_WARNING_RE = re.compile(r': warning ', re.IGNORECASE)
41+
_ANSI_RE = re.compile(r'\x1b\[[0-9;]*m')
3842

43+
# Language -> DocFX warning diagnostics seen while building it. Filled by
44+
# build_language(); rendered by write_warnings_report() for --warnings-report.
45+
BUILD_WARNINGS: dict[str, list[str]] = {}
3946

40-
def run_command(cmd: list[str], description: str, check: bool = True, fail_on_warnings: bool = False) -> int:
47+
48+
def _use_utf8_stdio() -> None:
49+
"""Make stdout/stderr UTF-8 with replacement for unencodable characters.
50+
51+
DocFX echoes translated headings and file names (e.g. Chinese) in its
52+
diagnostics, which this script re-prints while streaming. When stdout is
53+
redirected on Windows (CI logs, `| tee`), Python defaults to the ANSI code
54+
page and print() raises UnicodeEncodeError on them, aborting the build over
55+
a log line. A real console is already UTF-16/UTF-8 on Windows, so this only
56+
changes the redirected case.
57+
"""
58+
for stream in (sys.stdout, sys.stderr):
59+
if hasattr(stream, "reconfigure"):
60+
stream.reconfigure(encoding="utf-8", errors="replace")
61+
62+
63+
def _clean_warning_line(line: str) -> str:
64+
"""Strip colour codes and the absolute working-directory prefix from a DocFX diagnostic."""
65+
text = _ANSI_RE.sub('', line).rstrip()
66+
cwd = str(Path.cwd())
67+
return text.replace(cwd + os.sep, '').replace(cwd + '/', '')
68+
69+
70+
def run_command(
71+
cmd: list[str],
72+
description: str,
73+
check: bool = True,
74+
fail_on_warnings: bool = False,
75+
warnings: list[str] | None = None,
76+
) -> int:
4177
"""Run a command and return exit code.
4278
4379
If fail_on_warnings=True, streams output line-by-line, counts DocFX warning
4480
diagnostics (lines matching ': warning '), and returns exit code 1 if any
4581
are found — even when the process itself exits 0.
82+
83+
If `warnings` is a list, output is streamed the same way and every warning
84+
diagnostic is appended to it (colour codes and the working directory
85+
stripped) for the --warnings-report summary; the exit code is unaffected.
4686
"""
4787
print(f"\n{'='*60}")
4888
print(f" {description}")
4989
print(f"{'='*60}")
5090
print(f"Running: {' '.join(cmd)}\n")
5191

52-
if fail_on_warnings:
92+
if fail_on_warnings or warnings is not None:
5393
warning_count = 0
5494
process = subprocess.Popen(
5595
cmd,
@@ -64,13 +104,15 @@ def run_command(cmd: list[str], description: str, check: bool = True, fail_on_wa
64104
print(line, end='', flush=True)
65105
if _DOCFX_WARNING_RE.search(line):
66106
warning_count += 1
107+
if warnings is not None:
108+
warnings.append(_clean_warning_line(line))
67109
process.wait()
68110

69111
if check and process.returncode != 0:
70112
print(f"Error: Command failed with exit code {process.returncode}")
71113
return process.returncode
72114

73-
if warning_count > 0:
115+
if fail_on_warnings and warning_count > 0:
74116
print(f"\nError: DocFX produced {warning_count} warning(s). Failing build.")
75117
return 1
76118

@@ -239,13 +281,49 @@ def build_language(lang: str, sync: bool = False, skip_api: bool = False, permis
239281
# `docfx build` skips API metadata regeneration and reuses the existing
240282
# content/api/*.yml (the _apiSource DLLs don't change between content edits),
241283
# which is ~30-40% faster; bare `docfx` regenerates metadata then builds.
284+
BUILD_WARNINGS[lang] = []
242285
return run_docfx(
243286
[*(["build"] if skip_api else []), config_path],
244287
f"Building {lang} documentation",
245-
fail_on_warnings=(lang == "en" and not permissive)
288+
fail_on_warnings=(lang == "en" and not permissive),
289+
warnings=BUILD_WARNINGS[lang],
246290
)
247291

248292

293+
_REPORT_MAX_LINES = 200 # per language; keeps the job summary well under GitHub's 1 MiB cap
294+
295+
296+
def write_warnings_report(path: Path) -> None:
297+
"""Write a markdown summary of the DocFX warnings collected per built language.
298+
299+
Meant for CI job summaries (append the file to $GITHUB_STEP_SUMMARY). English
300+
warnings already fail the build, but the localized builds only warn, so this
301+
is where translation warnings become visible without reading the whole log.
302+
"""
303+
lines = ["## DocFX build warnings", ""]
304+
if not BUILD_WARNINGS:
305+
lines += ["No language build ran.", ""]
306+
else:
307+
lines += ["| Language | Warnings |", "|----------|---------:|"]
308+
lines += [f"| {lang} | {len(found)} |" for lang, found in BUILD_WARNINGS.items()]
309+
lines += [
310+
"",
311+
"English warnings fail the build. Other languages are translations managed outside "
312+
"this repo, so their warnings are listed here but never block a deploy.",
313+
"",
314+
]
315+
for lang, found in BUILD_WARNINGS.items():
316+
if not found:
317+
continue
318+
lines += [f"<details><summary>{lang}: {len(found)} warning(s)</summary>", "", "```text"]
319+
lines += found[:_REPORT_MAX_LINES]
320+
if len(found) > _REPORT_MAX_LINES:
321+
lines.append(f"... {len(found) - _REPORT_MAX_LINES} more; see the build log")
322+
lines += ["```", "", "</details>", ""]
323+
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
324+
print(f"Warnings report written to {path}")
325+
326+
249327
def copy_languages_manifest() -> int:
250328
"""Copy languages.json to _site/ root for runtime access."""
251329
manifest_src = Path("metadata/languages.json")
@@ -340,6 +418,7 @@ def fix_xref_in_api() -> int:
340418

341419

342420
def main() -> int:
421+
_use_utf8_stdio()
343422
parser = argparse.ArgumentParser(
344423
description="Build documentation for one or more languages",
345424
formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -354,9 +433,20 @@ def main() -> int:
354433
parser.add_argument("--skip-api", action="store_true", help="LOCAL markdown iteration only (requires --serve/--lang): reuse existing content/api, ~30-40%% faster. NEVER for testing/CI/CD/releases")
355434
parser.add_argument("--permissive", action="store_true", help="Don't treat English DocFX warnings as build failures (for local iteration; keep full/CI builds strict)")
356435
parser.add_argument("--sync", action="store_true", help="Copy English over missing/outdated translations for a local build (fallback copies only; never commit them)")
357-
436+
parser.add_argument("--warnings-report", metavar="PATH", help="Write a markdown summary of DocFX warnings per built language to PATH (for CI job summaries); written even when the build fails")
437+
358438
args = parser.parse_args()
359-
439+
440+
try:
441+
return _build(args)
442+
finally:
443+
# Written even after a failed language build, so CI still gets the partial picture.
444+
if args.warnings_report:
445+
write_warnings_report(Path(args.warnings_report))
446+
447+
448+
def _build(args: argparse.Namespace) -> int:
449+
"""Run the build described by the parsed command line; returns the exit code."""
360450
# List available languages
361451
if args.list:
362452
langs = get_available_languages()

‎build_scripts/README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ Both need `te` on PATH.
4444

4545
Current state only validates semantic bridge docs.
4646
The code block annotations can be used in any doc.
47+
CI runs `./run doctest validate` on every PR, and the full `./run doctest` when the `TE_CLI_DOWNLOAD_URL` repository secret provides a te CLI; see [Continuous integration](../README.md#continuous-integration).
4748

4849
#### `te_script_runner.py` -- generic runner
4950

@@ -183,6 +184,7 @@ Fragment/text failures keep their `#anchor` / `:~:text=` in that list (the bare
183184
the fragment is what broke); a wholly unreachable URL is listed bare.
184185
185186
Exit codes: `1` if any internal (own-site) reference is broken, else `0`. External warnings never fail the run, so third-party link rot will not break CI.
187+
CI runs `validate stats` against the built site on every PR and attaches the report to the job summary; see [Continuous integration](../README.md#continuous-integration).
186188
187189
Options to modify output:
188190

0 commit comments

Comments
 (0)