Pronunciation dictionaries

.gitignore

@@ -21,3 +21,6 @@ docs/*.zip
 *.obj
 *.exe
 __pycache__/
+
+# macOS
+.DS_Store

LICENSING.md

@@ -29,11 +29,29 @@ the copyright.
 | `doubletalk/nvda/**` | NVDA add-on driver, manifest, build script (our code) | BSD-3-Clause | David Sexton |
 | `doubletalk/mame/i86.*`, `i186.*`, `i86inline.h` | Vendored MAME 8086/80186 + 80C188EB CPU core | BSD-3-Clause | Carl; **Christopher Toth** (I80C188EB / EB Peripheral Control Block additions) |
 | `doubletalk/mame/endianness.h` | Vendored MAME utility | BSD-3-Clause | Aaron Giles, Vas Crabb |
+| `doubletalk/rcdict/rcdict*.{c,h}`, `example.dict` | Pronunciation-dictionary layer, shared verbatim with its upstream project | BSD-3-Clause | see the file headers |
+| `doubletalk/rcdict/remimu.h` | Vendored single-header regex engine | **CC0 / public domain** | wareya |
 | `docs/**`, `notes/**`, `*.md` | Our documentation and research notes | BSD-3-Clause | David Sexton (except `notes/investigation-audio-path.md`: **Christopher Toth**) |
 | `doubletalkpc.bin` (the ROM) | DoubleTalk PC firmware | **Proprietary — not included** | RC Systems, Inc. |
 
 Every source file carries a matching `license:` / `copyright-holders:` header
-(MAME-style `// license:` for C/C++, `# license:` for scripts/manifests).
+(MAME-style `// license:` for C/C++, `# license:` for scripts/manifests); the
+`doubletalk/rcdict/**` files use SPDX identifiers instead, because they are
+shared byte-for-byte with another repository and must read the same in both.
+
+### About `doubletalk/rcdict/**`
+
+These files are mirrored from their upstream project and must not be edited
+here — a sync check there fails if the two copies differ. They are BSD-3-Clause
+and depend on nothing beyond libc, which is exactly what lets them serve both
+this project and a GPL-3 one. **The sharing only works in that direction:**
+BSD-3 code can be linked into a GPL-3 program, so nothing under `rcdict/` may
+ever acquire a GPL header, or a dependency on anything outside this tree.
+
+`remimu.h` is a third-party regex engine released under CC0, i.e. placed in the
+public domain. CC0 imposes no conditions, so it adds nothing to what a
+redistributor of this project has to do; it is listed for completeness and
+because attribution is polite rather than required.
 
 ## Original project code — BSD-3-Clause
 

doubletalk/Makefile

@@ -3,9 +3,15 @@
 # Standalone DoubleTalk PC emulator (vendored MAME 80C188EB core + shim)
 CXX ?= g++
 CXXFLAGS ?= -O2 -g
-CXXFLAGS += -std=c++20 -Ishim -Imame -Wall -Wno-unused-variable
+CXXFLAGS += -std=c++20 -Ishim -Imame -I. -Wall -Wno-unused-variable
+CC ?= cc
+CFLAGS ?= -O2 -g
 
-OBJS = build/emu.o build/i86.o build/i186.o build/doubletalk_board.o build/dtalk.o
+# rcdict is mirrored verbatim from its upstream project (BSD-3-Clause, libc
+# only); it is what makes dtalk_say apply a pronunciation dictionary. Do not
+# edit it here - the sync runs upstream, and its check belongs in CI.
+OBJS = build/emu.o build/i86.o build/i186.o build/doubletalk_board.o build/dtalk.o \
+       build/rcdict.o build/rcdict_regex.o
 
 all: build/dtalk_cli build/libdtalk.a
 
@@ -27,6 +33,16 @@ build/doubletalk_board.o: doubletalk_board.cpp doubletalk_board.h shim/emu.h | b
 build/dtalk.o: dtalk.cpp dtalk.h doubletalk_board.h shim/emu.h | build
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
+# rcdict is C, not C++, and builds with nothing but its own directory on the
+# include path -- that self-containment is the whole point of it.
+build/rcdict.o: rcdict/rcdict.c rcdict/rcdict.h | build
+	$(CC) $(CFLAGS) -std=c99 -Wall -Wextra -Ircdict -c $< -o $@
+
+# remimu.h is vendored (CC0) and does not compile clean under -Wall; the noise
+# is all in its disabled debug paths, so quiet just this object.
+build/rcdict_regex.o: rcdict/rcdict_regex.c rcdict/remimu.h | build
+	$(CC) $(CFLAGS) -std=c99 -w -Ircdict -c $< -o $@
+
 build/libdtalk.a: $(OBJS) | build
 	ar rcs $@ $(OBJS)
 
@@ -52,23 +68,44 @@ clean:
 # rather than probing which alternative the builder happens to have selected.
 # This project's own code uses no threads; the dependency comes entirely
 # from libstdc++'s posix build.
-MINGW_FLAGS = -std=c++20 -O2 -Ishim -Imame -Wall -DDTALK_DLL -DDTALK_BUILD \
+MINGW_FLAGS = -std=c++20 -O2 -Ishim -Imame -I. -Wall -DDTALK_DLL -DDTALK_BUILD \
 	-static -static-libgcc -static-libstdc++
 WIN_SRCS = shim/emu.cpp mame/i86.cpp mame/i186.cpp doubletalk_board.cpp dtalk.cpp
-WIN_DEPS = shim/emu.h dtalk.h doubletalk_board.h $(WIN_SRCS)
+WIN_DEPS = shim/emu.h dtalk.h doubletalk_board.h rcdict/rcdict.h $(WIN_SRCS)
+
+# rcdict is C and has to be COMPILED as C: handing rcdict.c to g++ compiles it
+# as C++, where its designated initializers and implicit void* conversions are
+# errors. So each DLL is two compilers' output linked together - the C++
+# emulator, and rcdict built by the matching gcc. Warnings are off for
+# rcdict_regex.c alone, because vendored remimu.h is not clean under -Wall.
+#
+# Leaving rcdict out of these two rules was a silent failure worth naming: the
+# DLL linked, dtalk_set_dictionary was exported, and the driver simply had no
+# way to build a dictionary to hand it.
+MINGW_CFLAGS = -std=c99 -O2 -Ircdict
 
 win32: build/win32/dtalk.dll
 win64: build/win64/dtalk64.dll
 windows: win32 win64
 
-build/win32/dtalk.dll: $(WIN_DEPS)
+build/win32/dtalk.dll: $(WIN_DEPS) rcdict/rcdict.c rcdict/rcdict_regex.c
 	mkdir -p build/win32
-	i686-w64-mingw32-g++ $(MINGW_FLAGS) -shared \
-		-o $@ $(WIN_SRCS) -Wl,--out-implib,build/win32/dtalk.lib
-
-build/win64/dtalk64.dll: $(WIN_DEPS)
+	i686-w64-mingw32-gcc $(MINGW_CFLAGS) -Wall -Wextra -c rcdict/rcdict.c \
+		-o build/win32/rcdict.o
+	i686-w64-mingw32-gcc $(MINGW_CFLAGS) -w -c rcdict/rcdict_regex.c \
+		-o build/win32/rcdict_regex.o
+	i686-w64-mingw32-g++ $(MINGW_FLAGS) -shared -o $@ $(WIN_SRCS) \
+		build/win32/rcdict.o build/win32/rcdict_regex.o \
+		-Wl,--out-implib,build/win32/dtalk.lib
+
+build/win64/dtalk64.dll: $(WIN_DEPS) rcdict/rcdict.c rcdict/rcdict_regex.c
 	mkdir -p build/win64
-	x86_64-w64-mingw32-g++ $(MINGW_FLAGS) -shared \
-		-o $@ $(WIN_SRCS) -Wl,--out-implib,build/win64/dtalk64.lib
+	x86_64-w64-mingw32-gcc $(MINGW_CFLAGS) -Wall -Wextra -c rcdict/rcdict.c \
+		-o build/win64/rcdict.o
+	x86_64-w64-mingw32-gcc $(MINGW_CFLAGS) -w -c rcdict/rcdict_regex.c \
+		-o build/win64/rcdict_regex.o
+	x86_64-w64-mingw32-g++ $(MINGW_FLAGS) -shared -o $@ $(WIN_SRCS) \
+		build/win64/rcdict.o build/win64/rcdict_regex.o \
+		-Wl,--out-implib,build/win64/dtalk64.lib
 
 .PHONY: all clean win32 win64 windows

doubletalk/dtalk.cpp

@@ -6,6 +6,7 @@
 #include "dtalk.h"
 
 #include "doubletalk_board.h"
+#include "rcdict/rcdict.h"
 
 #include <cmath>
 #include <deque>
@@ -224,6 +225,7 @@ struct dtalk
 	u64 samples_dropped = 0;           // grid samples pulled/carried but never delivered (dtalk_stop)
 	s64 idle_stable = 0;               // consecutive idle cycles observed
 	int rate_boost = 0;                // current boost level (0 = authentic)
+	rcdict_options dict{};             // pronunciation rules; profile always set
 	u16 rate_orig[RATE_REGION_HI - RATE_REGION_LO + 1]; // pristine period words
 	bool rate_saved = false;
 
@@ -352,6 +354,7 @@ extern "C" {
 dtalk *dtalk_create(const void *rom, size_t rom_size)
 {
 	dtalk *dt = new dtalk;
+	rcdict_options_init(&dt->dict, &rcdict_doubletalk_pc);
 	if (!dt->board.load_rom(static_cast<const u8 *>(rom), rom_size))
 	{
 		delete dt;
@@ -417,10 +420,75 @@ void dtalk_queue(dtalk *dt, const void *bytes, size_t len)
 	dt->idle_stable = 0;
 }
 
+/* --- pronunciation dictionary ---------------------------------------------
+ *
+ * rcdict/ is mirrored verbatim from upstream and must not be edited here; it
+ * is BSD-3-Clause and depends on nothing but libc, which is what lets the same
+ * sources serve more than one engine. All that happens here is that dtalk_say
+ * runs its text through the rules first.
+ *
+ * The RC8650 datasheet's Table 5 is captioned "DoubleTalk Phoneme Symbols", so
+ * the phoneme set, the Table 6 modifiers, the D/T/C mode commands, the Ctrl-A
+ * command character and the nI index markers are the same on this card; what
+ * differs is carried by rcdict_doubletalk_pc.
+ */
+
+void dtalk_set_dictionary(dtalk *dt, const rcdict *d, int inline_phonemes)
+{
+	if (!dt) return;
+	dt->dict.dict = d;                 // borrowed; the caller keeps ownership
+	dt->dict.inline_phonemes = inline_phonemes ? 1 : 0;
+}
+
+size_t dtalk_expand(dtalk *dt, const char *in, size_t inlen,
+                    char *out, size_t outcap)
+{
+	if (!dt || !in) return 0;
+	return rcdict_expand(&dt->dict, in, inlen, out, outcap, nullptr, nullptr);
+}
+
+/* Construction, re-exported for the DLL. See the comment in dtalk.h for why
+ * these exist at all -- nothing here is more than a forwarding call. */
+
+rcdict *dtalk_dict_new(void)
+{
+	return rcdict_new(&rcdict_doubletalk_pc);
+}
+
+void dtalk_dict_free(rcdict *d)
+{
+	rcdict_free(d);
+}
+
+int dtalk_dict_add_file(rcdict *d, const char *path)
+{
+	if (!d || !path) return 0;
+	return rcdict_add_file(d, path, nullptr, nullptr);
+}
+
+size_t dtalk_dict_rule_count(const rcdict *d)
+{
+	return d ? rcdict_rule_count(d) : 0;
+}
+
 void dtalk_say(dtalk *dt, const char *text)
 {
-	dtalk_queue(dt, text, std::strlen(text));
+	const size_t len = std::strlen(text);
 	const u8 cr = 0x0d;
+
+	// No dictionary and no escape is the common case, and stays allocation
+	// free.
+	if (!dt->dict.dict && !dt->dict.inline_phonemes) {
+		dtalk_queue(dt, text, len);
+		dtalk_queue(dt, &cr, 1);
+		return;
+	}
+
+	const size_t need = rcdict_expand(&dt->dict, text, len, nullptr, 0,
+	                                  nullptr, nullptr);
+	std::vector<char> buf(need + 1);
+	rcdict_expand(&dt->dict, text, len, buf.data(), need + 1, nullptr, nullptr);
+	dtalk_queue(dt, buf.data(), need);
 	dtalk_queue(dt, &cr, 1);
 }
 

doubletalk/dtalk.h

@@ -78,9 +78,67 @@ DTALK_API uint8_t dtalk_lpc_status(dtalk *dt);
  * card RDY-gated as it accepts them while dtalk_synth() runs. */
 DTALK_API void dtalk_queue(dtalk *dt, const void *bytes, size_t len);
 
-/* Convenience: queue text followed by CR. */
+/* Convenience: queue text followed by CR. The pronunciation dictionary, if
+ * one is set, is applied first. */
 DTALK_API void dtalk_say(dtalk *dt, const char *text);
 
+/* --- pronunciation dictionary -------------------------------------------- */
+
+/* Substitute pronunciations into text on its way to the card: respellings,
+ * phonemes through the card's own phoneme mode (Ctrl-A D), or embedded
+ * commands. The rules, the file format and the loaders belong to rcdict, which
+ * is mirrored verbatim from upstream -- include rcdict/rcdict.h to build one.
+ * All of it is optional and off until asked for.
+ *
+ * The dictionary is BORROWED: it must outlive the instance, or be replaced
+ * with NULL before it is freed. inline_phonemes switches on the "[[K AE T]]"
+ * escape in ordinary text, which is off by default because "[[" is wiki link
+ * syntax and a user reading a wiki must not lose text to it. */
+struct rcdict;
+DTALK_API void dtalk_set_dictionary(dtalk *dt, const struct rcdict *d,
+                                    int inline_phonemes);
+
+/* Run the dictionary over text WITHOUT queueing it, returning the number of
+ * bytes the result needs (not counting a terminating NUL); a value >= outcap
+ * means it was truncated and the call should be repeated bigger.
+ *
+ * For callers that split long text into utterances themselves: one word can
+ * expand into forty characters of phonemes plus the mode switches, so
+ * splitting first and expanding after can push a piece past what the card will
+ * take. Expand, then split. */
+DTALK_API size_t dtalk_expand(dtalk *dt, const char *in, size_t inlen,
+                              char *out, size_t outcap);
+
+/* Building a dictionary, for callers that cannot reach rcdict's own symbols.
+ *
+ * A C caller that links libdtalk.a should just include rcdict/rcdict.h and use
+ * rcdict_new/rcdict_add_file directly -- these add nothing. They exist for the
+ * DLL: dtalk.h marks its entry points __declspec(dllexport), which turns off
+ * mingw's export-everything default, so rcdict's own symbols are not in the
+ * DLL's export table and a ctypes caller cannot see them. Re-exporting the
+ * four calls a host actually needs is a great deal less crude than exporting
+ * every symbol in the image, and it keeps the export marker out of rcdict --
+ * which has to stay free of anything Windows- or project-specific if it is to
+ * go on being shared verbatim with its other host.
+ *
+ * dtalk_dict_new also settles the profile (rcdict_doubletalk_pc) here rather
+ * than making the caller name it. That is the one piece of the construction a
+ * host has no business choosing.
+ *
+ * The result is BORROWED by dtalk_set_dictionary: detach it with NULL before
+ * dtalk_dict_free. */
+DTALK_API struct rcdict *dtalk_dict_new(void);
+DTALK_API void dtalk_dict_free(struct rcdict *d);
+
+/* Append one file's rules, returning how many were added (0 on any failure:
+ * unreadable file, or a file whose every line was rejected). Rules are
+ * first-match-wins in load order across all files, so a later file can only
+ * add -- to override an earlier one it has to be loaded first. */
+DTALK_API int dtalk_dict_add_file(struct rcdict *d, const char *path);
+
+/* Total rules held, across every file added. */
+DTALK_API size_t dtalk_dict_rule_count(const struct rcdict *d);
+
 /* Immediate stop (Ctrl-X / DTLK_CLEAR, written un-gated per the manual):
  * drops the host-side queue, stops speech, flushes the card's buffer, and
  * discards pending audio and index marks. Synthesizer settings persist. */

doubletalk/nvda/README.md

@@ -57,3 +57,72 @@ Index commands become Ctrl-A nI markers (rolling 0–99 mapped back to NVDA's
 index values); the emulator reports each marker with its exact output-sample
 position, which the driver converts to `synthIndexReached` notifications as
 playback passes it. Cancel writes the card's own Ctrl-X clear command.
+
+## Pronunciation dictionaries
+
+The driver applies `rcdict`, a pronunciation layer shared byte-for-byte with
+another engine: respellings, phonemes through the card's own phoneme mode (Ctrl-A
+D), or embedded commands, matched by word, substring or regular expression.
+The format is documented in `../rcdict/rcdict.h`, with a worked example in
+`../rcdict/example.dict`.
+
+**[The dictionary format is documented in `DICTIONARY-GUIDE.md`](../rcdict/DICTIONARY-GUIDE.md)** - a self-contained guide written for users rather than for this repository, and the thing to hand to anyone who asks how to write one.
+
+Dictionaries are `.dict` files. They are arranged from **NVDA menu >
+Preferences > Settings > DoubleTalk PC dictionaries**, which has a list of the
+files in load order and five buttons:
+
+| Button | Key | What it does |
+|---|---|---|
+| Add... | `Alt+D` | Adds a `.dict` file to the list, wherever it is |
+| Remove | `Alt+R` | Takes it out of the list |
+| Move up | `Alt+U` | Raises its priority |
+| Move down | `Alt+N` | Lowers it |
+| Reload dictionaries | `Alt+L` | Re-reads the files, so editing one does not mean restarting the synthesizer |
+
+`Alt+F` moves to the list itself.
+
+**Files are read where they are.** Add records where a file is; it does not copy
+it. So the dictionary you edit is the one the synthesizer reads: save it in your
+editor, press Reload, and the change is live. Keep them wherever suits - a
+folder of your own, a synced drive, a checkout under version control.
+
+A reference to a file that is not there right now (an unplugged drive, a share
+not mounted yet) stays in the list, marked *not found*, and is skipped until it
+comes back. Removing it is your decision, not the add-on's.
+
+**Order is the whole point of the list.** Rules are tried from the top down and
+the first match wins, so a file higher in the list overrides one below it - a
+later file can only *add*. The order lives in NVDA's configuration, not in the
+filenames.
+
+**There is also a folder, for anyone who just wants somewhere to put one.**
+`%APPDATA%\nvda\doubletalkpc\` is scanned as well: a `.dict` file dropped in
+there works with no configuration at all, appended after everything the list
+names, in sorted order. It is how dictionaries worked before there was a panel
+and it still works that way - the panel is how you take control of what beats
+what. Its files show in the list under their bare names, and because the folder
+is scanned rather than listed, Remove on one of them has to delete it; the panel
+says so and asks first. Remove on anything else only unlists it.
+
+The list is applied when the Settings dialog is closed with OK or Apply, or
+immediately by the Reload button.
+
+The substitution is made before the utterance is queued, never inside
+`dtalk_say` - this driver queues its own bytes with `dtalk_queue`, so anything
+applied further down would never fire for NVDA.
+
+## Testing
+
+    make -C ../..          # the tests build a shared libdtalk from build/*.o
+    tests/run.sh
+
+`tests/` stubs NVDA's API (`nvdastub.py`), stages a copy of the driver package
+with the host's shared library standing in for `dtalk64.dll`, and drives the
+real `speak()`. It covers the dictionary path: that files are found and loaded
+in the right order, that a substitution reaches the queued bytes, that the
+Ctrl-A prefix and any index markers come through untouched, that the utterance
+still ends in the CR without which the card says nothing at all, and - the
+check the others rest on - that the card really renders different audio with a
+dictionary loaded than without. `test_dictfiles.py` covers the load-order rules
+on their own, with no emulator involved.