test: add Unicode boundary and composite type tests (23 tests)

tests/test_unicode.rs (11 tests): non-Latin scripts, homoglyphs, NFD
preservation, non-BMP emoji, embedded null byte

tests/test_composites.rs (12 tests): Option<Vec<T>>, Vec<Option<T>>,
HashMap<String,Option<T>>, newtype structs, tuple structs, enum variants

Author: lefty

tests/test_composites.rs

@@ -0,0 +1,189 @@
+use std::collections::HashMap;
+
+use pyo3::prelude::*;
+use pythonize::{depythonize, pythonize};
+use serde::{Deserialize, Serialize};
+
+// FR-013: Color enum at module scope — shared across three test functions
+// (one exception to the "inside function" rule).
+#[derive(Serialize, Deserialize, Debug, PartialEq)]
+enum Color {
+    Red,                    // unit variant
+    Rgb(u8, u8, u8),        // tuple variant
+    Named { name: String }, // struct variant
+}
+
+// ---------------------------------------------------------------------------
+// FR-008 — Option<Vec<i64>> round-trip
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_option_vec_none_round_trip() {
+    Python::attach(|py| {
+        let val: Option<Vec<i64>> = None;
+        let result = pythonize(py, &val).and_then(|o| depythonize::<Option<Vec<i64>>>(&o));
+        // BASELINE: None round-trips as None.
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_option_vec_some_empty_round_trip() {
+    Python::attach(|py| {
+        let val: Option<Vec<i64>> = Some(vec![]);
+        let result = pythonize(py, &val).and_then(|o| depythonize::<Option<Vec<i64>>>(&o));
+        // BASELINE: Some(vec![]) round-trips as Some(vec![]).
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_option_vec_some_values_round_trip() {
+    Python::attach(|py| {
+        let val: Option<Vec<i64>> = Some(vec![1, 2, 3]);
+        let result = pythonize(py, &val).and_then(|o| depythonize::<Option<Vec<i64>>>(&o));
+        // BASELINE: Some([1, 2, 3]) round-trips intact.
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+// ---------------------------------------------------------------------------
+// FR-009 — Vec<Option<i64>> round-trip
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_vec_option_with_nones_round_trip() {
+    Python::attach(|py| {
+        let val: Vec<Option<i64>> = vec![Some(1i64), None, Some(3)];
+        let result = pythonize(py, &val).and_then(|o| depythonize::<Vec<Option<i64>>>(&o));
+        // BASELINE: None entries are preserved at their indices; no index shift.
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_vec_option_all_nones_round_trip() {
+    Python::attach(|py| {
+        let val: Vec<Option<i64>> = vec![None::<i64>, None, None];
+        let result = pythonize(py, &val).and_then(|o| depythonize::<Vec<Option<i64>>>(&o));
+        // BASELINE: all-None vec round-trips as three None entries.
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+// ---------------------------------------------------------------------------
+// FR-010 — HashMap<String, Option<i64>> null value semantics
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_hashmap_string_option_none_value_round_trip() {
+    Python::attach(|py| {
+        let mut val: HashMap<String, Option<i64>> = HashMap::new();
+        val.insert("key".to_string(), None::<i64>);
+        val.insert("other".to_string(), Some(42i64));
+        let result =
+            pythonize(py, &val).and_then(|o| depythonize::<HashMap<String, Option<i64>>>(&o));
+        // FR-010: characterising whether None dict value survives round-trip.
+        // BASELINE: None value is preserved under "key"; not dropped or transmuted.
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_struct_option_field_explicit_none_round_trip() {
+    Python::attach(|py| {
+        #[derive(Serialize, Deserialize, Debug, PartialEq)]
+        struct MaybeVal {
+            name: String,
+            count: Option<i64>,
+        }
+
+        let val = MaybeVal {
+            name: "test".to_string(),
+            count: None,
+        };
+        let result = pythonize(py, &val).and_then(|o| depythonize::<MaybeVal>(&o));
+        // FR-010: explicit None field vs missing key — asserting observed behaviour.
+        // BASELINE: count: None round-trips; pythonize emits explicit null, not absent key.
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+// ---------------------------------------------------------------------------
+// FR-012 — Newtype struct and 2-tuple struct round-trip
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_newtype_struct_round_trip() {
+    Python::attach(|py| {
+        #[derive(Serialize, Deserialize, Debug, PartialEq)]
+        struct Wrapper(i64);
+
+        let val = Wrapper(42i64);
+        let result = pythonize(py, &val).and_then(|o| depythonize::<Wrapper>(&o));
+        // BASELINE: newtype struct serialises as its inner value (42);
+        // depythonize reconstructs Wrapper(42) from the raw integer.
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_tuple_struct_round_trip() {
+    Python::attach(|py| {
+        #[derive(Serialize, Deserialize, Debug, PartialEq)]
+        struct Pair(i64, String);
+
+        let val = Pair(7i64, String::from("hello"));
+        let result = pythonize(py, &val).and_then(|o| depythonize::<Pair>(&o));
+        // BASELINE: tuple struct serialises as a Python list [7, "hello"];
+        // depythonize reconstructs Pair(7, "hello") from the list.
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+// ---------------------------------------------------------------------------
+// FR-013 — Enum variant representations
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_enum_unit_variant_round_trip() {
+    Python::attach(|py| {
+        let val = Color::Red;
+        let py_obj = pythonize(py, &val).expect("pythonize Color::Red failed");
+        let repr = py_obj.repr().unwrap().to_string();
+        eprintln!("FR-013 unit variant repr: {repr}");
+        let result: Result<Color, _> = depythonize(&py_obj);
+        // BASELINE: Color::Red serialises as the Python string 'Red' (observed repr: 'Red').
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_enum_tuple_variant_round_trip() {
+    Python::attach(|py| {
+        let val = Color::Rgb(255, 128, 0);
+        let py_obj = pythonize(py, &val).expect("pythonize Color::Rgb failed");
+        let repr = py_obj.repr().unwrap().to_string();
+        eprintln!("FR-013 tuple variant repr: {repr}");
+        let result: Result<Color, _> = depythonize(&py_obj);
+        // BASELINE: Color::Rgb(255,128,0) serialises as {'Rgb': (255, 128, 0)} —
+        // value is a Python *tuple*, not a list (observed repr: {'Rgb': (255, 128, 0)}).
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_enum_struct_variant_round_trip() {
+    Python::attach(|py| {
+        let val = Color::Named {
+            name: String::from("crimson"),
+        };
+        let py_obj = pythonize(py, &val).expect("pythonize Color::Named failed");
+        let repr = py_obj.repr().unwrap().to_string();
+        eprintln!("FR-013 struct variant repr: {repr}");
+        let result: Result<Color, _> = depythonize(&py_obj);
+        // BASELINE: Color::Named serialises as {'Named': {'name': 'crimson'}}
+        // (observed repr: {'Named': {'name': 'crimson'}}).
+        assert_eq!(result.unwrap(), val);
+    });
+}

tests/test_unicode.rs

@@ -0,0 +1,146 @@
+use pyo3::prelude::*;
+use pythonize::{depythonize, pythonize};
+
+// ---------------------------------------------------------------------------
+// Case 1 — Non-Latin script round-trips
+// Normative: String values containing non-Latin scripts must round-trip
+// byte-exactly.  These are not characterisation tests.
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_string_cyrillic_round_trip() {
+    Python::attach(|py| {
+        let val = String::from("Привет"); // U+041F U+0440 U+0438 U+0432 U+0435 U+0442
+        let result = pythonize(py, &val).and_then(|py_val| depythonize::<String>(&py_val));
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_string_cjk_round_trip() {
+    Python::attach(|py| {
+        let val = String::from("你好"); // U+4F60 U+597D
+        let result = pythonize(py, &val).and_then(|py_val| depythonize::<String>(&py_val));
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_string_arabic_round_trip() {
+    Python::attach(|py| {
+        let val = String::from("مرحبا"); // U+0645 U+0631 U+062D U+0628 U+0627
+        let result = pythonize(py, &val).and_then(|py_val| depythonize::<String>(&py_val));
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_string_devanagari_round_trip() {
+    Python::attach(|py| {
+        let val = String::from("नमस्ते"); // U+0928 U+092E U+0938 U+094D U+0924 U+0947
+        let result = pythonize(py, &val).and_then(|py_val| depythonize::<String>(&py_val));
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_string_hebrew_round_trip() {
+    Python::attach(|py| {
+        let val = String::from("שלום"); // U+05E9 U+05DC U+05D5 U+05DD
+        let result = pythonize(py, &val).and_then(|py_val| depythonize::<String>(&py_val));
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+// ---------------------------------------------------------------------------
+// Case 2 — Homoglyphs are preserved as their original codepoint
+// BASELINE: Python does not normalise homoglyphs; each codepoint round-trips
+// to itself with no cross-codepoint equality.
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_string_latin_capital_a_round_trip() {
+    Python::attach(|py| {
+        let val = String::from("\u{0041}"); // Latin capital A (U+0041)
+        let result = pythonize(py, &val).and_then(|py_val| depythonize::<String>(&py_val));
+        // BASELINE: round-trip preserves U+0041; Python does not normalise to another A-homoglyph.
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_string_cyrillic_capital_a_round_trip() {
+    Python::attach(|py| {
+        // Cyrillic capital А (U+0410) — visually identical to Latin A (U+0041)
+        let val = String::from("\u{0410}");
+        let result = pythonize(py, &val).and_then(|py_val| depythonize::<String>(&py_val));
+        // BASELINE: round-trip preserves U+0410; Python does not rewrite to U+0041.
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+#[test]
+fn test_string_fullwidth_latin_capital_a_round_trip() {
+    Python::attach(|py| {
+        let val = String::from("\u{FF21}"); // Fullwidth Latin capital Ａ (U+FF21)
+        let result = pythonize(py, &val).and_then(|py_val| depythonize::<String>(&py_val));
+        // BASELINE: round-trip preserves U+FF21; Python does not rewrite to U+0041.
+        assert_eq!(result.unwrap(), val);
+    });
+}
+
+// ---------------------------------------------------------------------------
+// Case 3 — NFD is preserved, no silent NFC recomposition
+// Confirmed baseline: CPython's PyString::new / to_cow() are length-aware;
+// no Unicode normalisation is applied.  NFD in → NFD out.
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_string_nfd_combining_acute_preserved() {
+    Python::attach(|py| {
+        // NFD: 'e' (U+0065, 1 byte) + combining acute accent (U+0301, 2 bytes) = 3 bytes.
+        // Do NOT use the precomposed "é" (U+00E9, NFC, 2 bytes) — that would be trivial.
+        let val = String::from("e\u{0301}");
+        let result = pythonize(py, &val).and_then(|py_val| depythonize::<String>(&py_val));
+        // BASELINE (confirmed 2026-04-04): CPython preserves NFD; no NFC recomposition.
+        assert!(result.is_ok());
+        let back = result.unwrap();
+        assert_eq!(back, "e\u{0301}");
+        assert_eq!(back.len(), 3); // 3 bytes in UTF-8; NFC "é" (U+00E9) would be 2
+    });
+}
+
+// ---------------------------------------------------------------------------
+// Case 4 — Non-BMP emoji round-trip
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_string_non_bmp_emoji_round_trip() {
+    Python::attach(|py| {
+        // Non-BMP: 4 UTF-8 bytes; Python 3 str handles this natively.
+        let val = String::from("🦀"); // U+1F980, Rust str len = 4
+        let result = pythonize(py, &val).and_then(|py_val| depythonize::<String>(&py_val));
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap(), "🦀");
+    });
+}
+
+// ---------------------------------------------------------------------------
+// Case 5 — Null byte survives round-trip
+// Confirmed baseline: pythonize uses length-aware CPython APIs; embedded null
+// is not a string terminator.
+// ---------------------------------------------------------------------------
+
+#[test]
+fn test_string_embedded_null_round_trip() {
+    Python::attach(|py| {
+        let val = String::from("hello\x00world");
+        let result = pythonize(py, &val).and_then(|py_val| depythonize::<String>(&py_val));
+        // BASELINE (confirmed 2026-04-04): pythonize uses length-aware CPython APIs;
+        // embedded null is not a string terminator.
+        // Caveat: truncation could occur if an intermediate consumer uses
+        // PyUnicode_AsUTF8 (no-size variant) — not pythonize's bug.
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap(), "hello\x00world");
+    });
+}