From 06f2f68d6389c8e587d1b9bca70f21ae364dc02a Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:34:01 +0200 Subject: [PATCH 01/12] docs: fill in several missing todos and undocumented functions Add more ReST headings within the source headers so that the API docs are better organized within the ToC. --- docs/index.rst | 10 ++-- include/asdf/core/datatype.h | 24 +++++++++ include/asdf/core/ndarray.h | 78 ++++++++++++++++++++++++++---- include/asdf/error.h | 5 ++ include/asdf/extension.h | 6 +-- include/asdf/file.h | 94 +++++++++++++++++++++++++++++++----- include/asdf/value.h | 93 ++++++++++++++++++++++++++++++----- 7 files changed, 270 insertions(+), 40 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 8d3ba672..907190e9 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -51,9 +51,13 @@ headers. api/asdf/yaml.h -.. todo:: - - Document lower-level APIs. +Beyond the high-level `asdf_file_t` interface, libasdf also exposes a +lower-level, event-based API for streaming through an ASDF file without +building the whole tree in memory. It is built around the parser in +``asdf/parser.h``, which emits a sequence of `asdf_event_t` values (defined in +``asdf/event.h``), and the corresponding emitter in ``asdf/emitter.h`` for +writing. These are intended for advanced use cases and are not yet fully +documented; most applications should prefer the high-level API described above. Resources diff --git a/include/asdf/core/datatype.h b/include/asdf/core/datatype.h index 5789c7c9..cdb8bc81 100644 --- a/include/asdf/core/datatype.h +++ b/include/asdf/core/datatype.h @@ -17,6 +17,14 @@ ASDF_BEGIN_DECLS #define ASDF_CORE_DATATYPE_TAG ASDF_CORE_TAG_PREFIX "datatype-1.0.0" +/** + * .. _datatype-types: + * + * Types + * ----- + */ + + /** * Enum for basic ndarray scalar datatypes * @@ -117,6 +125,14 @@ typedef struct asdf_datatype asdf_datatype_t; ASDF_DECLARE_EXTENSION(datatype, asdf_datatype_t); +/** + * .. _datatype-strings: + * + * String conversion + * ----------------- + */ + + /** * Parse an ASDF scalar datatype and return the corresponding `asdf_scalar_datatype_t` * @@ -151,6 +167,14 @@ ASDF_EXPORT asdf_scalar_datatype_t asdf_scalar_datatype_from_string(const char * ASDF_EXPORT const char *asdf_scalar_datatype_to_string(asdf_scalar_datatype_t datatype); +/** + * .. _datatype-sizes: + * + * Datatype sizes + * -------------- + */ + + /** * Get the size of an `asdf_datatype_t` in bytes * diff --git a/include/asdf/core/ndarray.h b/include/asdf/core/ndarray.h index a9617ae4..311393ee 100644 --- a/include/asdf/core/ndarray.h +++ b/include/asdf/core/ndarray.h @@ -56,6 +56,14 @@ ASDF_BEGIN_DECLS #define ASDF_CORE_NDARRAY_TAG ASDF_CORE_TAG_PREFIX "ndarray-1.1.0" +/** + * .. _ndarray-types: + * + * Types + * ----- + */ + + /** * Error codes returned by some functions that read ndarray data */ @@ -113,6 +121,14 @@ typedef struct asdf_ndarray asdf_ndarray_t; #endif +/** + * .. _ndarray-get: + * + * Getting ndarrays + * ---------------- + */ + + // NOTE: For now I don't see any good way to generate docstrings for functions // generated by ASDF_DECLARE_EXTENSION, so we just insert them manually for // now. Should be possible with a custom extension to Sphinx (or Hawkmoth) @@ -161,27 +177,47 @@ typedef struct asdf_ndarray asdf_ndarray_t; ASDF_DECLARE_EXTENSION(ndarray, asdf_ndarray_t); -/** ndarray methods */ +/** + * .. _ndarray-data-access: + * + * Data access + * ----------- + */ /** - * Return a pointer to the ndarray data + * Return a pointer to the ndarray's element data + * + * If the array is stored in a compressed block a buffer containing the + * decompressed data is returned (see also :ref:`compression`). + * The data is presented in the array's source datatype and byte order; + * use `asdf_ndarray_read_all` or the ``asdf_ndarray_read_tile_*`` + * functions to read it converted to a host datatype and native byte order. * - * ..todo:: + * The returned pointer is owned by ``ndarray`` and remains valid until the + * ndarray (or its file) is destroyed; the caller must not free it. * - * Finish documenting me. + * :param ndarray: An `asdf_ndarray_t *` + * :param size: If non-``NULL``, receives the size of the data in bytes + * :return: A pointer to the (decompressed) data, or ``NULL`` on error -- for + * example if the array has no associated data block (such as an inline + * array) or if decompression failed */ ASDF_EXPORT const void *asdf_ndarray_data(asdf_ndarray_t *ndarray, size_t *size); /** - * Return a pointer to the raw (compressed, in the case of compressed arrays) ndarray - * data + * Return a pointer to the raw ndarray data, without decompressing it * - * On non-compressed arrays this is equivalent to `asdf_ndarray_data`. + * For arrays stored in a compressed block this returns the still-compressed + * bytes exactly as they appear in the file. For uncompressed arrays it is + * equivalent to `asdf_ndarray_data`. * - * ..todo:: + * As with `asdf_ndarray_data`, the returned pointer is owned by ``ndarray`` + * and must not be freed by the caller. * - * Finish documenting me. + * :param ndarray: An `asdf_ndarray_t *` + * :param size: If non-``NULL``, receives the size of the raw data in bytes + * :return: A pointer to the raw data, or ``NULL`` on error */ ASDF_EXPORT const void *asdf_ndarray_data_raw(asdf_ndarray_t *ndarray, size_t *size); @@ -205,6 +241,14 @@ ASDF_EXPORT uint64_t asdf_ndarray_size(const asdf_ndarray_t *ndarray); ASDF_EXPORT uint64_t asdf_ndarray_nbytes(const asdf_ndarray_t *ndarray); +/** + * .. _ndarray-alloc: + * + * Allocating data buffers + * ----------------------- + */ + + /** * Allocate heap memory large enough to store the data for the ndarray * @@ -248,6 +292,14 @@ ASDF_EXPORT void *asdf_ndarray_data_alloc_temp(asdf_file_t *file, asdf_ndarray_t ASDF_EXPORT void asdf_ndarray_data_dealloc(asdf_ndarray_t *ndarray); +/** + * .. _ndarray-storage: + * + * Storage and compression + * ----------------------- + */ + + /** * Set the compression method to use for the ndarray data when writing * @@ -311,6 +363,14 @@ ASDF_EXPORT void asdf_ndarray_storage_set(asdf_ndarray_t *ndarray, asdf_array_st ASDF_EXPORT asdf_block_t *asdf_ndarray_block(asdf_ndarray_t *ndarray); +/** + * .. _ndarray-read: + * + * Reading array data + * ------------------ + */ + + /** * Read the full ndarray, copying into the provided buffer (or allocating a * destination buffer if ``dst = NULL``) diff --git a/include/asdf/error.h b/include/asdf/error.h index 794a8216..ded0d613 100644 --- a/include/asdf/error.h +++ b/include/asdf/error.h @@ -13,6 +13,11 @@ ASDF_BEGIN_DECLS +/** + * Error codes + * ----------- + */ + /** * Error codes set on an `asdf_file_t` or other context. * diff --git a/include/asdf/extension.h b/include/asdf/extension.h index d69e8857..e9398033 100644 --- a/include/asdf/extension.h +++ b/include/asdf/extension.h @@ -132,7 +132,7 @@ ASDF_EXPORT void asdf_tag_destroy(asdf_tag_t *tag); } -/** +/* * Auto-generated helper to clone extension types * * Extension types may optionally not implement the copy method, in which case a shallow @@ -157,7 +157,7 @@ ASDF_EXPORT void asdf_tag_destroy(asdf_tag_t *tag); } -/** +/* * Helper to clone a NULL-terminated array of pointers to an extension object * * For example, clones an `asdf_history_entry_t **` array. @@ -184,7 +184,7 @@ ASDF_EXPORT void asdf_tag_destroy(asdf_tag_t *tag); } -/** Auto-generated helper to free extension type objects */ +/* Auto-generated helper to free extension type objects */ #define ASDF_EXT_DEFINE_DESTROY(extname, type) \ ASDF_EXPORT void asdf_##extname##_destroy(type *object) { \ if (!object) \ diff --git a/include/asdf/file.h b/include/asdf/file.h index 3c21f1e9..fe2e38f1 100644 --- a/include/asdf/file.h +++ b/include/asdf/file.h @@ -28,6 +28,14 @@ ASDF_BEGIN_DECLS +/** + * .. _file-handles: + * + * File handles + * ------------ + */ + + /** * An opaque struct representing an open ASDF file handle * @@ -47,12 +55,13 @@ typedef struct asdf_file asdf_file_t; /** - * Options for decompression mode, for use with - * :c:type:`asdf_config_t` - * - * .. todo:: + * Options for block decompression mode, for use with the ``decomp.mode`` field + * of `asdf_config_t` * - * Document modes + * This controls *when* compressed binary block data is decompressed: eagerly + * (all at once the first time a block is accessed) or lazily (decompressed in + * page-sized chunks on demand, where supported). See :ref:`compression` for + * details, and the individual `asdf_block_decomp_mode_t` members below. * * .. todo:: * @@ -810,24 +819,70 @@ ASDF_EXPORT bool asdf_is_double(asdf_file_t *file, const char *path); ASDF_EXPORT asdf_value_err_t asdf_get_double(asdf_file_t *file, const char *path, double *out); /** + * .. _file-extension-getters: + * * Extension object getters * ------------------------ + */ + +/** + * These functions are the generic forms behind the per-extension + * ``asdf_is_`` / ``asdf_get_`` / ``asdf_set_`` + * helpers (such as ``asdf_get_ndarray``) that each registered extension + * generates. They take an explicit `asdf_extension_t *`, which can be looked + * up for a registered tag with ``asdf_extension_get``, and are useful when the + * extension type is only known at runtime. See :ref:`extensions` for more on + * the extension mechanism. + */ + +/** + * Check whether the value at ``path`` is of the given extension type * - * .. todo:: - * - * Needs :ref:`extensions` documentation. + * :param file: The `asdf_file_t *` for the file + * :param path: The :ref:`yaml-pointer` to the value + * :param ext: The `asdf_extension_t *` describing the extension type + * :return: ``true`` if a value exists at ``path`` and matches the extension + * type, otherwise ``false`` */ ASDF_EXPORT bool asdf_is_extension_type(asdf_file_t *file, const char *path, asdf_extension_t *ext); /** - * .. todo:: + * Get the value at ``path`` deserialized into the given extension type * - * Needs :ref:`extensions` documentation. + * On success the deserialized extension object is written through ``out``; the + * concrete type of ``*out`` is the C type associated with the extension (for + * example ``asdf_ndarray_t *`` for the ndarray extension). The caller owns the + * returned object and must release it with the extension's corresponding + * ``asdf__destroy`` function. + * + * :param file: The `asdf_file_t *` for the file + * :param path: The :ref:`yaml-pointer` to the value + * :param ext: The `asdf_extension_t *` describing the extension type + * :param out: Receives the deserialized extension object on success + * :return: `ASDF_VALUE_OK` if the value exists and is of the extension type, + * otherwise `ASDF_VALUE_ERR_NOT_FOUND` or `ASDF_VALUE_ERR_TYPE_MISMATCH` */ ASDF_EXPORT asdf_value_err_t asdf_get_extension_type(asdf_file_t *file, const char *path, asdf_extension_t *ext, void **out); +/** + * Serialize an extension object into the tree at ``path`` + * + * The object pointed to by ``obj`` is serialized according to ``ext`` and + * written into the tree at ``path``, creating any intermediate mappings as + * needed and replacing any existing value already at that path. + * + * Typically it is better to use the higher level `asdf_set_` + * functions, (e.g. ``asdf_set_ndarray``) which are type-safe wrappers around + * this function, and don't require looking up the extension. + * + * :param file: The `asdf_file_t *` for the file + * :param path: The :ref:`yaml-pointer` at which to write the value + * :param obj: Pointer to the extension object to serialize + * :param ext: The `asdf_extension_t *` describing the extension type + * :return: `ASDF_VALUE_OK` on success, otherwise an `asdf_value_err_t` error + */ ASDF_EXPORT asdf_value_err_t asdf_set_extension_type( asdf_file_t *file, const char *path, const void *obj, asdf_extension_t *ext); @@ -838,9 +893,22 @@ ASDF_EXPORT asdf_value_err_t asdf_set_extension_type( * Writing values * -------------- * - * .. todo:: - * - * Needs general explanation, to do in #114 + * The ``asdf_set_`` family is the high-level counterpart to the + * ``asdf_get_`` getters: each writes a value into the ASDF metadata tree + * at a given :ref:`yaml-pointer` path. Every setter takes the `asdf_file_t *` + * as its first argument, the path as its second, and the value to write as the + * remaining argument(s). + * + * Any intermediate mappings named in the path that do not yet exist are + * created automatically, and an existing value already at the path is + * replaced. Each function returns `ASDF_VALUE_OK` on success or an + * `asdf_value_err_t` error code. + * + * ``asdf_set_value`` inserts a pre-built generic `asdf_value_t *`; + * ``asdf_set_string`` takes an explicit byte length while ``asdf_set_string0`` + * expects a NUL-terminated string; ``asdf_set_null`` takes no value argument. + * The remaining scalar variants accept the corresponding C type directly. See + * :ref:`writing` for a narrative guide. */ ASDF_EXPORT asdf_value_err_t diff --git a/include/asdf/value.h b/include/asdf/value.h index 9feefd8d..89a5ad34 100644 --- a/include/asdf/value.h +++ b/include/asdf/value.h @@ -18,6 +18,14 @@ ASDF_BEGIN_DECLS +/** + * .. _value-types: + * + * Types + * ----- + */ + + /** * Opaque struct representing a generic value from the ASDF tree */ @@ -173,6 +181,14 @@ typedef enum { typedef struct asdf_file asdf_file_t; +/** + * .. _value-generic: + * + * Generic values + * -------------- + */ + + /** * Free memory held by an `asdf_value_t` * @@ -232,7 +248,12 @@ typedef struct asdf_file asdf_file_t; /** Get the `asdf_file_t *` handle to the file to which a value belongs */ ASDF_EXPORT asdf_file_t *asdf_value_file(asdf_value_t *value); -/** Mappings */ +/** + * .. _value-mappings: + * + * Mappings + * -------- + */ /** * Opaque struct represening a mapping value @@ -353,9 +374,10 @@ ASDF_EXPORT void asdf_mapping_iter_destroy(asdf_mapping_iter_t *iter); * Values in the RHS mapping are *copied* during the update process, so the * RHS mapping remains valid. * - * .. todo:: - * - * Finish documenting me. + * :param mapping: The `asdf_mapping_t *` to update in place (the LHS) + * :param update: The `asdf_mapping_t *` whose entries are merged in (the RHS); + * it is left unmodified + * :return: ``ASDF_VALUE_OK`` on success, otherwise an `asdf_value_err_t` error */ ASDF_EXPORT asdf_value_err_t asdf_mapping_update(asdf_mapping_t *mapping, asdf_mapping_t *update); @@ -371,11 +393,24 @@ ASDF_EXPORT asdf_value_err_t asdf_mapping_update(asdf_mapping_t *mapping, asdf_m ASDF_EXPORT asdf_value_t *asdf_mapping_pop(asdf_mapping_t *mapping, const char *key); /** - * Set values on mappings + * Set values on a mapping * - * .. todo:: + * ``asdf_mapping_set`` inserts an existing generic `asdf_value_t *`; ownership + * of ``value`` transfers to the mapping on success. * - * Document the rest of these. + * The ``asdf_mapping_set_`` variants construct a new value from a C + * scalar and insert it in one step. ``asdf_mapping_set_string`` takes an + * explicit byte length; ``asdf_mapping_set_string0`` expects a NUL-terminated + * string. ``asdf_mapping_set_null`` takes no value argument. All other + * scalar variants accept the corresponding C type directly. + * + * ``asdf_mapping_set_mapping`` and ``asdf_mapping_set_sequence`` insert a + * nested container, transferring ownership of the supplied container to the + * parent mapping on success. + * + * In every case, if ``key`` already exists in the mapping its previous value + * is replaced. All functions return ``ASDF_VALUE_OK`` on success or an + * `asdf_value_err_t` error code on failure. */ /** @@ -433,7 +468,12 @@ ASDF_EXPORT asdf_value_err_t asdf_mapping_set_sequence(asdf_mapping_t *mapping, const char *key, asdf_sequence_t *value); -/** Sequences */ +/** + * .. _value-sequences: + * + * Sequences + * --------- + */ /** * Opaque struct representing a sequence value @@ -697,6 +737,14 @@ ASDF_EXPORT asdf_sequence_t *asdf_sequence_of_double( ASDF_EXPORT asdf_value_t *asdf_sequence_pop(asdf_sequence_t *sequence, int index); +/** + * .. _value-containers: + * + * Generic containers + * ------------------ + */ + + /** * Iterator handle for traversing either a mapping or a sequence * @@ -774,7 +822,12 @@ ASDF_EXPORT bool asdf_value_is_container(asdf_value_t *value); ASDF_EXPORT int asdf_container_size(asdf_value_t *container); -/** Extension-related functions */ +/** + * .. _value-extensions: + * + * Extension types + * --------------- + */ // Forward declaration for asdf_extension_t typedef struct _asdf_extension asdf_extension_t; @@ -820,7 +873,12 @@ asdf_value_as_extension_type(asdf_value_t *value, const asdf_extension_t *ext, v ASDF_EXPORT asdf_value_t *asdf_value_of_extension_type( asdf_file_t *file, const void *obj, const asdf_extension_t *ext); -/** Generic value functions */ +/** + * .. _value-type-generic: + * + * Type-generic access + * ------------------- + */ /** * Given an arbitrary `asdf_value_type_t` enum member, check if the value has @@ -850,7 +908,13 @@ ASDF_EXPORT asdf_value_err_t asdf_value_as_type(asdf_value_t *value, asdf_value_type_t type, void *out); -/* Scalar-related definitions */ +/** + * .. _value-scalars: + * + * Scalars + * ------- + */ + ASDF_EXPORT bool asdf_value_is_string(asdf_value_t *value); ASDF_EXPORT asdf_value_err_t asdf_value_as_string(asdf_value_t *value, const char **out, size_t *out_len); @@ -912,7 +976,12 @@ ASDF_EXPORT bool asdf_value_is_double(asdf_value_t *value); ASDF_EXPORT asdf_value_err_t asdf_value_as_double(asdf_value_t *value, double *out); ASDF_EXPORT asdf_value_t *asdf_value_of_double(asdf_file_t *file, double value); -/** Tree traversal functions */ +/** + * .. _value-traversal: + * + * Tree traversal + * -------------- + */ /** * Type definition for predicate functions used in `asdf_value_find` and From 9311fc74c2d80b193d48b1d45c60c4466cc43cf4 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Thu, 25 Jun 2026 16:44:06 +0200 Subject: [PATCH 02/12] docs: make per-page ToC's more manageable in the API docs --- docs/api/asdf/core/datatype.h.rst | 2 ++ docs/api/asdf/core/ndarray.h.rst | 2 ++ docs/api/asdf/emitter.h.rst | 2 ++ docs/api/asdf/error.h.rst | 2 ++ docs/api/asdf/file.h.rst | 2 ++ docs/api/asdf/value.h.rst | 2 ++ docs/api/asdf/yaml.h.rst | 2 ++ docs/index.rst | 2 +- include/asdf/file.h | 2 +- 9 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docs/api/asdf/core/datatype.h.rst b/docs/api/asdf/core/datatype.h.rst index feeb7525..c920bed4 100644 --- a/docs/api/asdf/core/datatype.h.rst +++ b/docs/api/asdf/core/datatype.h.rst @@ -1,3 +1,5 @@ +:tocdepth: 2 + asdf/core/datatype.h ==================== diff --git a/docs/api/asdf/core/ndarray.h.rst b/docs/api/asdf/core/ndarray.h.rst index 38219639..c6676f4e 100644 --- a/docs/api/asdf/core/ndarray.h.rst +++ b/docs/api/asdf/core/ndarray.h.rst @@ -1,3 +1,5 @@ +:tocdepth: 2 + asdf/core/ndarray.h =================== diff --git a/docs/api/asdf/emitter.h.rst b/docs/api/asdf/emitter.h.rst index b841ac51..a2e33a7d 100644 --- a/docs/api/asdf/emitter.h.rst +++ b/docs/api/asdf/emitter.h.rst @@ -1,3 +1,5 @@ +:tocdepth: 2 + asdf/emitter.h ================ diff --git a/docs/api/asdf/error.h.rst b/docs/api/asdf/error.h.rst index f87a57ab..3906e017 100644 --- a/docs/api/asdf/error.h.rst +++ b/docs/api/asdf/error.h.rst @@ -1,3 +1,5 @@ +:tocdepth: 2 + asdf/error.h ============ diff --git a/docs/api/asdf/file.h.rst b/docs/api/asdf/file.h.rst index f7a58c08..8ce78d72 100644 --- a/docs/api/asdf/file.h.rst +++ b/docs/api/asdf/file.h.rst @@ -1,3 +1,5 @@ +:tocdepth: 2 + .. _file.h: asdf/file.h diff --git a/docs/api/asdf/value.h.rst b/docs/api/asdf/value.h.rst index 52293cac..ba001664 100644 --- a/docs/api/asdf/value.h.rst +++ b/docs/api/asdf/value.h.rst @@ -1,3 +1,5 @@ +:tocdepth: 2 + asdf/value.h ============ diff --git a/docs/api/asdf/yaml.h.rst b/docs/api/asdf/yaml.h.rst index b87a6800..25770df3 100644 --- a/docs/api/asdf/yaml.h.rst +++ b/docs/api/asdf/yaml.h.rst @@ -1,3 +1,5 @@ +:tocdepth: 2 + asdf/yaml.h =========== diff --git a/docs/index.rst b/docs/index.rst index 907190e9..0acb4d77 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,7 +44,7 @@ Additional less commonly used APIs can be used by including the relevant headers. .. toctree:: - :maxdepth: 2 + :maxdepth: 1 api/asdf/emitter.h api/asdf/extension.h diff --git a/include/asdf/file.h b/include/asdf/file.h index fe2e38f1..409f70d5 100644 --- a/include/asdf/file.h +++ b/include/asdf/file.h @@ -873,7 +873,7 @@ asdf_get_extension_type(asdf_file_t *file, const char *path, asdf_extension_t *e * written into the tree at ``path``, creating any intermediate mappings as * needed and replacing any existing value already at that path. * - * Typically it is better to use the higher level `asdf_set_` + * Typically it is better to use the higher level ``asdf_set_`` * functions, (e.g. ``asdf_set_ndarray``) which are type-safe wrappers around * this function, and don't require looking up the extension. * From 68fccf39275bfb07305ecf2dfb9b977af778df2c Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Thu, 25 Jun 2026 17:46:07 +0200 Subject: [PATCH 03/12] docs: new page dedicated entirely to NDarrays gh-202 --- docs/index.rst | 1 + docs/usage/ndarrays.rst | 335 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 336 insertions(+) create mode 100644 docs/usage/ndarrays.rst diff --git a/docs/index.rst b/docs/index.rst index 0acb4d77..6505d8a4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,7 @@ Usage usage/opening usage/values usage/writing + usage/ndarrays usage/extensions usage/examples diff --git a/docs/usage/ndarrays.rst b/docs/usage/ndarrays.rst new file mode 100644 index 00000000..297d2a77 --- /dev/null +++ b/docs/usage/ndarrays.rst @@ -0,0 +1,335 @@ +.. _working-with-ndarrays: + +Working with ndarrays +===================== + +The :ref:`overview ` introduced the **ndarray** -- a typed, +multi-dimensional array whose bulk numeric data lives in a binary block while +its shape, datatype, and byte order are described by metadata in the YAML tree. +This page is a practical guide to *doing things* with ndarrays in libasdf: +reading them out of a file, copying and converting their data (including +sub-array "tiles"), and building new arrays to write back out. + +Everything here is declared in ``asdf/core/ndarray.h`` (pulled in by the +umbrella ``asdf.h``) with supporting datatype definitions in +``asdf/core/datatype.h``. + + +.. _ndarray-struct: + +The ``asdf_ndarray_t`` structure +-------------------------------- + +An ndarray is represented by an `asdf_ndarray_t`. When *reading*, the library +heap-allocates one for you and fills in the metadata; when *writing*, you fill +one in yourself, typically on the stack. The publicly visible fields are: + +.. list-table:: + :header-rows: 1 + :widths: 20 30 50 + + * - Field + - Type + - Meaning + * - ``ndim`` + - ``uint32_t`` + - Number of dimensions. + * - ``shape`` + - ``const uint64_t *`` + - Length of each dimension; an array of ``ndim`` values. + * - ``datatype`` + - `asdf_datatype_t` + - Element datatype (see :ref:`ndarray-datatypes` below). + * - ``byteorder`` + - `asdf_byteorder_t` + - Byte order of the stored data (``ASDF_BYTEORDER_LITTLE`` or + ``ASDF_BYTEORDER_BIG``). + * - ``source`` + - ``size_t`` + - Index of the binary block holding the data. + * - ``offset`` + - ``uint64_t`` + - Optional byte offset into the block where the data begins. + * - ``strides`` + - ``const int64_t *`` + - Optional per-dimension strides; ``NULL`` means C-contiguous. + +.. note:: + + Only a subset of full ndarray functionality is implemented so far. In + particular ``complex`` and ``float16`` datatypes, string (``ascii`` / + ``ucs4``) and structured datatypes, masks, and arbitrarily strided data are + not yet fully supported for reading. See the + `asdf/core/ndarray.h `__ + header for the current status. + + +.. _ndarray-reading: + +Reading an ndarray from a file +------------------------------ + +If you know the :ref:`yaml-pointer` path to an array, read it with +`asdf_get_ndarray`: + +.. code:: c + + asdf_ndarray_t *array = NULL; + + if (asdf_get_ndarray(file, "data", &array) != ASDF_VALUE_OK) { + fprintf(stderr, "no ndarray at /data\n"); + return 1; + } + + printf("%u-dimensional array\n", array->ndim); + for (uint32_t dim = 0; dim < array->ndim; dim++) + printf(" axis %u: %" PRIu64 " elements\n", dim, array->shape[dim]); + +The library allocates ``array``; release it with `asdf_ndarray_destroy` when +you are done: + +.. code:: c + + asdf_ndarray_destroy(array); + +If you do not know the path in advance you can search the tree for the first +ndarray with `asdf_value_find` and the ``asdf_value_is_ndarray`` predicate, then +cast the matching value with `asdf_value_as_ndarray`: + +.. code:: c + + asdf_value_t *root = asdf_get_value(file, ""); + asdf_value_t *found = asdf_value_find(root, asdf_value_is_ndarray); + + asdf_ndarray_t *array = NULL; + if (found && asdf_value_as_ndarray(found, &array) == ASDF_VALUE_OK) { + /* ... */ + } + + asdf_value_destroy(found); + asdf_value_destroy(root); + +See :ref:`values` for more on generic value handles and tree traversal. + + +.. _ndarray-data: + +Accessing the raw data +---------------------- + +The quickest way to reach the element data is `asdf_ndarray_data`, which returns +a pointer to the (decompressed) bytes and optionally their size: + +.. code:: c + + size_t nbytes = 0; + const void *data = asdf_ndarray_data(array, &nbytes); + +This pointer is owned by ``array`` and must not be freed. The data is in the +array's *source* datatype and byte order, exactly as stored -- no conversion is +performed. Two helpers describe the array's extent without touching the data: + +* `asdf_ndarray_size` -- total number of elements (the product of the shape). +* `asdf_ndarray_nbytes` -- total size of the data in bytes. + +`asdf_ndarray_data_raw` is a lower-level variant that, for compressed arrays, +returns the still-compressed bytes without decompressing them; for uncompressed +arrays it is equivalent to `asdf_ndarray_data`. + +Because `asdf_ndarray_data` exposes the data in its original layout, you are +responsible for honoring ``byteorder`` and ``datatype`` yourself. When you want +the data converted to a convenient host type, use the reading functions in the +next section instead. + + +.. _ndarray-read-convert: + +Reading data with conversion +---------------------------- + +`asdf_ndarray_read_all` copies the whole array into a buffer, converting it to +the host's native byte order and, optionally, to a different numeric datatype: + +.. code:: c + + double *values = NULL; + asdf_ndarray_err_t err = + asdf_ndarray_read_all(array, ASDF_DATATYPE_FLOAT64, (void **)&values); + + if (err == ASDF_NDARRAY_OK) { + uint64_t n = asdf_ndarray_size(array); + /* values[0 .. n-1] are now native-endian doubles */ + free(values); + } + +Passing ``NULL`` for the destination (as above, via a pointer whose value is +``NULL``) asks the library to allocate a buffer of the right size; the caller +then owns that memory and must ``free()`` it. Alternatively, pre-allocate a +buffer of `asdf_ndarray_nbytes` (or the appropriate size for the converted +type) and pass its address. Pass `ASDF_DATATYPE_SOURCE` as the destination +datatype to keep the array's original element type and only normalize byte +order. + +Reading tiles +~~~~~~~~~~~~~ + +Often you only need a rectangular sub-region ("tile") of a large array. +`asdf_ndarray_read_tile_ndim` reads an arbitrary N-dimensional cutout given an +``origin`` and a ``shape``, each an array of ``ndim`` values: + +.. code:: c + + /* A 5x5 tile anchored at the array origin of a 2-D array */ + uint64_t origin[2] = {0, 0}; + uint64_t shape[2] = {5, 5}; + + float *tile = NULL; + asdf_ndarray_err_t err = asdf_ndarray_read_tile_ndim( + array, origin, shape, ASDF_DATATYPE_FLOAT32, (void **)&tile); + + if (err == ASDF_NDARRAY_OK) { + /* tile holds 25 native-endian floats in row-major order */ + free(tile); + } + +The same buffer-ownership and datatype-conversion rules as +`asdf_ndarray_read_all` apply: pass ``NULL`` to have a buffer allocated for you, +or your own buffer to read into, and ``ASDF_DATATYPE_SOURCE`` to keep the source +type. A tile that extends past the bounds of the array yields +`ASDF_NDARRAY_ERR_OUT_OF_BOUNDS`. + +For the common two-dimensional case, `asdf_ndarray_read_tile_2d` offers a +friendlier signature taking ``x``, ``y``, ``width``, and ``height`` directly +(plus an optional ``plane_origin`` selecting a plane of a higher-dimensional +array): + +.. code:: c + + float *tile = NULL; + asdf_ndarray_read_tile_2d( + array, /* x= */ 10, /* y= */ 20, /* width= */ 8, /* height= */ 8, + /* plane_origin= */ NULL, ASDF_DATATYPE_FLOAT32, (void **)&tile); + + +.. _ndarray-datatypes: + +Datatypes and byte order +------------------------ + +An element datatype is described by `asdf_datatype_t`, whose ``type`` field is +one of the `asdf_scalar_datatype_t` enums -- for example `ASDF_DATATYPE_UINT8`, +`ASDF_DATATYPE_INT32`, `ASDF_DATATYPE_FLOAT32`, or `ASDF_DATATYPE_FLOAT64`. +For simple numeric arrays setting ``type`` is all that is required; +`asdf_datatype_size` then reports the size of a single element in bytes. + +Byte order is given by `asdf_byteorder_t`: `ASDF_BYTEORDER_LITTLE` or +`ASDF_BYTEORDER_BIG` for explicit endianness, or `ASDF_BYTEORDER_DEFAULT` to +let the library choose when writing. `asdf_scalar_datatype_from_string` and +`asdf_scalar_datatype_to_string` convert between the enumerators and their ASDF +string names (e.g. ``"float64"``). + + +.. _ndarray-writing: + +Building an ndarray to write +---------------------------- + +For writing you stack-allocate an `asdf_ndarray_t` and fill in the fields that +describe the array, then allocate a data buffer for its contents with +`asdf_ndarray_data_alloc`: + +.. code:: c + + const uint64_t shape[] = {128, 128}; + asdf_ndarray_t nd = { + .datatype = (asdf_datatype_t){.type = ASDF_DATATYPE_FLOAT32}, + .byteorder = ASDF_BYTEORDER_LITTLE, + .ndim = 2, + .shape = shape, + }; + + float *data = asdf_ndarray_data_alloc(&nd); + for (int idx = 0; idx < 128 * 128; idx++) + data[idx] = (float)idx; + + asdf_set_ndarray(file, "image", &nd); + +`asdf_ndarray_data_alloc` allocates a correctly sized buffer on the heap based +on the array's shape and datatype. After the file has been written, release it +with `asdf_ndarray_data_dealloc` (safe to call after `asdf_close`): + +.. code:: c + + asdf_write_to(file, "out.asdf"); + asdf_close(file); + asdf_ndarray_data_dealloc(&nd); + +.. note:: + + When building an ndarray *inside* an extension's serialize callback, use + `asdf_ndarray_data_alloc_temp` instead -- its buffer is freed automatically + once the write completes. + +See :ref:`writing` for the surrounding file-writing workflow. + + +.. _ndarray-storage-modes: + +Storage modes +------------- + +By default an array's data is written into a binary block ("internal" +storage). `asdf_ndarray_storage_set` selects a different +`asdf_array_storage_t` mode: + +* ``ASDF_ARRAY_STORAGE_INTERNAL`` -- data is written to a binary block in the + same file (the default). +* ``ASDF_ARRAY_STORAGE_INLINE`` -- data is serialized directly into the YAML + tree as a nested sequence. This is convenient for very small arrays but a + warning is logged above a configurable element-count threshold. +* ``ASDF_ARRAY_STORAGE_EXTERNAL`` -- reserved for data stored in a separate + file; not yet supported. + +.. code:: c + + asdf_ndarray_storage_set(&nd, ASDF_ARRAY_STORAGE_INLINE); + +`asdf_ndarray_storage` reports the mode that will be used for an array when +writing it out. If the array was ready from an existing file, it will also +report the storage format that was used in that file. + + +.. _compression: + +Compression +----------- + +Internal block data can be compressed. Call `asdf_ndarray_compression_set` +on the array *before* passing it to ``asdf_set_ndarray`` (or `asdf_write_to`): + +.. code:: c + + asdf_ndarray_compression_set(&nd, "lz4"); + asdf_set_ndarray(file, "image", &nd); + +The compression string must name one of the compressors built into libasdf: +``"zlib"``, ``"bzp2"``, or ``"lz4"``. Pass ``NULL`` or an empty string to +request no compression. Internally this is a thin wrapper around +`asdf_block_compression_set`, which operates on the raw `asdf_block_t` and can +be used for finer control. + +When *reading* compressed arrays, the block is decompressed on demand the first +time you access its data (for example via `asdf_ndarray_data` or any of the +reading functions). How and when that happens is controlled through the +``decomp`` options of `asdf_config_t` (passed to `asdf_open_ex`): + +* **Mode** (`asdf_block_decomp_mode_t`): ``ASDF_BLOCK_DECOMP_MODE_EAGER`` + decompresses the whole block at once, while ``ASDF_BLOCK_DECOMP_MODE_LAZY`` + decompresses in page-sized chunks on demand where supported (recent Linux via + ``userfaultfd``), falling back to eager decompression otherwise. + ``ASDF_BLOCK_DECOMP_MODE_AUTO`` lets the library choose. +* **Memory limits**: ``max_memory_bytes`` and ``max_memory_threshold`` bound how + much decompressed data is kept in memory before the library spills to a + temporary file on disk (``tmp_dir``). + +For most use cases the defaults are appropriate and no configuration is needed. From dbeb4c01171772746e9fba5668193dfee44405d4 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Thu, 25 Jun 2026 18:50:23 +0200 Subject: [PATCH 04/12] docs: add new dedicated page on compression This consolidates previously scattered information about compression handling into a single page; much better. --- docs/index.rst | 1 + docs/usage/compression.rst | 167 +++++++++++++++++++++++++++++++++++++ docs/usage/ndarrays.rst | 34 ++------ docs/usage/opening.rst | 119 +++++--------------------- docs/usage/writing.rst | 10 +-- 5 files changed, 197 insertions(+), 134 deletions(-) create mode 100644 docs/usage/compression.rst diff --git a/docs/index.rst b/docs/index.rst index 6505d8a4..02a92962 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,6 +19,7 @@ Usage usage/values usage/writing usage/ndarrays + usage/compression usage/extensions usage/examples diff --git a/docs/usage/compression.rst b/docs/usage/compression.rst new file mode 100644 index 00000000..31b4e1a6 --- /dev/null +++ b/docs/usage/compression.rst @@ -0,0 +1,167 @@ +.. _compression: + +Compression +=========== + +ASDF can compress the bulk data of an ndarray's binary block, trading CPU time +for a smaller file. Compression applies only to data stored in *internal* +blocks (see :ref:`ndarray-storage-modes`); inline data lives in the YAML tree +and is never compressed. + +libasdf ships with three compressors, named by the strings ASDF uses to +identify them on disk: + +* ``"zlib"`` -- the zlib/DEFLATE algorithm; broad compatibility. +* ``"bzp2"`` -- bzip2; typically smaller output, but slower. +* ``"lz4"`` -- LZ4; very fast, with more modest compression ratios; see + :ref:`compression-lz4` below for more details. + +.. _compression-writing: + +Compressing on write +-------------------- + +By default ndarray data is written uncompressed. To enable compression, call +`asdf_ndarray_compression_set` on the ndarray *before* passing it to +``asdf_set_ndarray`` (or `asdf_write_to`): + +.. code:: c + + asdf_ndarray_compression_set(&nd, "lz4"); + asdf_set_ndarray(file, "image", &nd); + +The compression string must name one of the compressors built into libasdf +listed above. Pass ``NULL`` or an empty string to explicitly request no +compression. + +`asdf_ndarray_compression_set` is a thin wrapper around the lower-level +`asdf_block_compression_set`, which operates on the raw `asdf_block_t` +associated with a block. For the vast majority of use cases the ndarray-level +shortcut is all you need. + + +.. _compression-reading: + +Decompression on read +--------------------- + +When *reading* a compressed array, its block is decompressed on demand the +first time you access the data -- for example via `asdf_ndarray_data` or any of +the reading functions (see :ref:`ndarray-data`). `asdf_ndarray_data_raw` +instead returns the still-compressed bytes without decompressing them. + +How and when decompression happens is governed by the ``decomp`` options of +`asdf_config_t`, passed to `asdf_open_ex` (see :ref:`configuration`): + +.. code:: + + asdf_config_t config = { + .decomp = { + .mode = ASDF_BLOCK_DECOMP_MODE_EAGER, + .max_memory_bytes = 1073741824, + .max_memory_threshold = 0.8, + .chunk_size = 409600, + .tmp_dir = "/var/tmp" + } + }; + +For most use cases the defaults are appropriate and no configuration is needed. +The two things that most deserve explanation are the handling of the *size* of +the decompressed data, and the decompression *mode*. + +Compressed data size +^^^^^^^^^^^^^^^^^^^^ + +An array containing sparsely populated data may be very small compressed, but +explode significantly when decompressed. + +By *default* decompression is performed entirely in-memory. For most files on +modern systems with significant RAM and swap space this won't be an issue. +However, libasdf also has the option to decompress to a temporary file on disk +(effectively a temporary pagefile). This may be useful if the file contains +very large arrays that do not fit in system memory. To control this behavior +you can use one or both of the +:c:member:`max_memory_bytes ` and +:c:member:`max_memory_threshold ` +options. + +The former sets a maximum number of *bytes* of decompressed data above which to +use decompression to disk. The latter sets a percentage (from ``0.0`` to +``1.0``) of total system memory above which to enable this behavior. If both +are specified, then the lower value of the two is applied as the absolute +threshold. + +Most users won't need these settings but they are there in case you do. + +By default this will write the file to your system's ``TMPDIR`` (typically +``/tmp`` or ``/var/tmp``). It also understands the environment variable +``ASDF_TMPDIR`` to use as the default for all ASDF files read with libasdf. + +.. warning:: + + However, many systems use a RAM-based filesystem like + `tmpfs `_ to back their temporary + directory, which also renders this feature meaningless. If you are + sure you definitely need this for large file support, you can either + pass the :c:member:`tmp_dir ` option to also + specify a specific disk-backed directory to use for the temp file. + +Currently every individual `asdf_file_t*` handle does its own decompression +separately, though a future option might be to allow multiple `asdf_file_t*` +to share the same decompressed data pages. + +Decompression mode +^^^^^^^^^^^^^^^^^^ + +In most cases, compressed block data is decompressed eagerly by default when +using either `ASDF_BLOCK_DECOMP_MODE_AUTO` or `ASDF_BLOCK_DECOMP_MODE_EAGER`. +This means that as soon as the decompressed data is needed the full block is +decompressed. + +However, there is also experimental support for `ASDF_BLOCK_DECOMP_MODE_LAZY` +(currently on supported Linux versions *only*). This allows blocks to be +decompressed one or more pages at a time on an as-needed basis, and works +totally transparently. + +For zlib and bzip2 compression this is mostly only useful if you want to access +just bytes early in the block, as these algorithms can only be decompressed +sequentially (and the ASDF Standard does not currently define a scheme for +tiled compression). If you need the entire block data it will all be +decompressed anyways. This can still be useful even in that case if one is +taking chunks of the data and processing them sequentially. + +By default lazy compression decompresses one system page at a time. However, +it may be more efficient to use a larger value--this can be controlled with the +`asdf_config_t.chunk_size` setting (in bytes). This will always be +automatically rounded up to the nearest page size. + +.. warning:: + + Due to technical limitations, lazy decompression does *not* work with + disk-backed decompression, and the memory threshold options are ignored. + If you really need both the best way is to ensure a sufficiently large + pagefile available on your system, and to let the kernel manage swapping. + See your system's documentation for the best way to create and manage + a pagefile. + + +.. _compression-lz4: + +A note on LZ4 +------------- + +The ``"lz4"`` compression is little peculiar. It is not formally part of the +ASDF Standard, but it has been supported by the Python +`asdf `_ library for some time, so libasdf +supports it for interoperability. Rather than compressing the whole block +as a single stream, the ASDF LZ4 format stores the data as a series of +sequential chunks, each of which can be decompressed independently. + +In principle this allows for more efficient *random* access to compressed data: +because an index can be built mapping each chunk to its range of decompressed +bytes, only the chunks actually needed have to be decompressed. libasdf does +not yet take advantage of this -- LZ4 blocks are currently decompressed +sequentially like the other codecs -- but true random-access decompression is +planned for a future version (see `GitHub issue #90 +`_). + diff --git a/docs/usage/ndarrays.rst b/docs/usage/ndarrays.rst index 297d2a77..df73dd68 100644 --- a/docs/usage/ndarrays.rst +++ b/docs/usage/ndarrays.rst @@ -217,8 +217,8 @@ Datatypes and byte order ------------------------ An element datatype is described by `asdf_datatype_t`, whose ``type`` field is -one of the `asdf_scalar_datatype_t` enums -- for example `ASDF_DATATYPE_UINT8`, -`ASDF_DATATYPE_INT32`, `ASDF_DATATYPE_FLOAT32`, or `ASDF_DATATYPE_FLOAT64`. +one of the `asdf_scalar_datatype_t` enums -- for example ``ASDF_DATATYPE_UINT8``, +``ASDF_DATATYPE_INT32``, ``ASDF_DATATYPE_FLOAT32``, or ``ASDF_DATATYPE_FLOAT64``. For simple numeric arrays setting ``type`` is all that is required; `asdf_datatype_size` then reports the size of a single element in bytes. @@ -298,38 +298,18 @@ storage). `asdf_ndarray_storage_set` selects a different writing it out. If the array was ready from an existing file, it will also report the storage format that was used in that file. - -.. _compression: +.. _ndarray-compression: Compression ----------- -Internal block data can be compressed. Call `asdf_ndarray_compression_set` -on the array *before* passing it to ``asdf_set_ndarray`` (or `asdf_write_to`): +Internal block data can be compressed. Call `asdf_ndarray_compression_set` on +the array *before* passing it to ``asdf_set_ndarray`` (or `asdf_write_to`): .. code:: c asdf_ndarray_compression_set(&nd, "lz4"); asdf_set_ndarray(file, "image", &nd); -The compression string must name one of the compressors built into libasdf: -``"zlib"``, ``"bzp2"``, or ``"lz4"``. Pass ``NULL`` or an empty string to -request no compression. Internally this is a thin wrapper around -`asdf_block_compression_set`, which operates on the raw `asdf_block_t` and can -be used for finer control. - -When *reading* compressed arrays, the block is decompressed on demand the first -time you access its data (for example via `asdf_ndarray_data` or any of the -reading functions). How and when that happens is controlled through the -``decomp`` options of `asdf_config_t` (passed to `asdf_open_ex`): - -* **Mode** (`asdf_block_decomp_mode_t`): ``ASDF_BLOCK_DECOMP_MODE_EAGER`` - decompresses the whole block at once, while ``ASDF_BLOCK_DECOMP_MODE_LAZY`` - decompresses in page-sized chunks on demand where supported (recent Linux via - ``userfaultfd``), falling back to eager decompression otherwise. - ``ASDF_BLOCK_DECOMP_MODE_AUTO`` lets the library choose. -* **Memory limits**: ``max_memory_bytes`` and ``max_memory_threshold`` bound how - much decompressed data is kept in memory before the library spills to a - temporary file on disk (``tmp_dir``). - -For most use cases the defaults are appropriate and no configuration is needed. +Compressed arrays are transparently decompressed when read back. See +:ref:`compression` for the full list of supported compressors. diff --git a/docs/usage/opening.rst b/docs/usage/opening.rst index ad32d697..cffe701e 100644 --- a/docs/usage/opening.rst +++ b/docs/usage/opening.rst @@ -109,102 +109,23 @@ filled in with the default settings. in defaults), so it's safe to point to a local variable. Any option in the user-provided config left as ``0`` will be filled in from the -default configuration (though this may change in the future). - -Currently, the only documented configuration options that may be of interest to -end users are the options for controlling the behavior of *decompression* of -compressed blocks. For example: - -.. code:: - - asdf_config_t config = { - .decomp = { - mode = ASDF_BLOCK_DECOMP_MODE_EAGER, - max_memory_bytes = 1073741824, - max_memory_threshold = 0.8, - chunk_size = 409600, - tmp_dir = "/var/tmp" - } - }; - -Two things that deserve explanation here are handling of the *size* of -the decompressed data, and the decompression *mode*. - -Compressed data size -^^^^^^^^^^^^^^^^^^^^ - -An array containing sparsely populated data may be very small compressed, but -explode significantly when decompressed. - -.. todo:: - - Document better how to inspect the compressed vs. decompressed sizes of a - block. - -By *default* decompression is performed entirely in-memory. For most files on -modern systems with significant RAM and swap space this won't be an issue. -However, libasdf also has the option to decompress to a temporary file on disk -(effectively a temporary pagefile). To control this behavior you can use one -or both of the -:c:member:`max_memory_bytes ` and -:c:member:`max_memory_threshold ` -options. - -The former sets a maximum number of *bytes* of decompressed data above which to -use decompression to disk. The latter sets a percentage (from ``0.0`` to -``1.0``) of total system memory above which to enable this behavior. If both -are specified, then the lower value of the two is applied as the absolute -threshold. - -Most users won't need these settings but they are there in case you do. - -By default this will write the file to your system's ``TMPDIR`` (typically -``/tmp`` or ``/var/tmp``). It also understands the environment variable -``ASDF_TMPDIR`` to use as the default for all ASDF files read with libasdf. - -.. warning:: - - However, many systems use a RAM-based filesystem like - `tmpfs `_ to back their temporary - directory, which also renders this feature meaningless. If you are - sure you definitely need this for large file support, you can either - pass the :c:member`tmp_dir ` -- eager, lazy, or automatic + decompression (see `asdf_block_decomp_mode_t`). +* :c:member:`max_memory_bytes ` and + :c:member:`max_memory_threshold ` -- + thresholds above which decompression spills to a temporary file on disk. +* :c:member:`chunk_size ` -- chunk size for lazy + decompression. +* :c:member:`tmp_dir ` -- directory for the on-disk + decompression temporary file. + +These are described in detail, along with the trade-offs between the different +decompression modes, in :ref:`compression`. The remaining ``parser``, +``emitter``, and ``log`` sub-structs are lower-level and not yet documented for +general use. diff --git a/docs/usage/writing.rst b/docs/usage/writing.rst index 76b186e6..806e074a 100644 --- a/docs/usage/writing.rst +++ b/docs/usage/writing.rst @@ -294,11 +294,5 @@ By default ndarray data is written uncompressed. To enable compression, call asdf_ndarray_compression_set(&nd, "lz4"); asdf_set_ndarray(file, "image", &nd); -The compression string must name one of the compressors built into libasdf. -The three supported values are ``"zlib"``, ``"bzp2"``, and ``"lz4"``. Pass -``NULL`` or an empty string to explicitly request no compression. - -`asdf_ndarray_compression_set` is a thin wrapper around the lower-level -`asdf_block_compression_set`, which operates on the raw `asdf_block_t` -associated with a block. For the vast majority of use cases the ndarray-level -shortcut is all you need. +See :ref:`compression` for the list of supported compressors and for more +detail on how compression and decompression work. From cedf4b264e5570230474532afe4750d2746ed9ae Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Thu, 25 Jun 2026 19:38:33 +0200 Subject: [PATCH 05/12] docs: fix numerous typos and formatting errors Fill in a bit more missing docstrings. gh-202 --- docs/usage/examples.rst | 4 +-- docs/usage/extensions.rst | 24 +++++++++++++----- docs/usage/overview.rst | 28 ++++++++++----------- docs/usage/values.rst | 17 +++++++------ include/asdf/core/datatype.h | 8 +++--- include/asdf/core/ndarray.h | 47 ++++++++++++++++++++++++------------ include/asdf/file.h | 22 ++++++++--------- include/asdf/value.h | 44 ++++++++++++++++++++++----------- 8 files changed, 121 insertions(+), 73 deletions(-) diff --git a/docs/usage/examples.rst b/docs/usage/examples.rst index 6104265c..dd1fef38 100644 --- a/docs/usage/examples.rst +++ b/docs/usage/examples.rst @@ -35,7 +35,7 @@ the ASDF tree, as well as extract block data. Inline comments provide further e // The simplest way to read metadata from the file is with the // `asdf_get_*` family of functions // They all return a value by pointer argument and return an - // `asdf_value_error_t` + // `asdf_value_err_t` // For example you can read a string from the metadata like: const char *software = NULL; @@ -67,7 +67,7 @@ the ASDF tree, as well as extract block data. Inline comments provide further e // Functions like `asdf_get_meta` that return into a double-pointer to a // struct allocate memory for that structure automatically. - // The all have a corresponding `asdf__destroy` function. + // They all have a corresponding `asdf__destroy` function. // The plan is to track these on the file object (issue #34) to make // memory management easier and cleaner, but for now you have to free // them manually when you're done with them. This is good practice in any diff --git a/docs/usage/extensions.rst b/docs/usage/extensions.rst index ae53401f..98eec094 100644 --- a/docs/usage/extensions.rst +++ b/docs/usage/extensions.rst @@ -6,7 +6,7 @@ Extending libasdf with extension types .. note:: The extension mechanism in libasdf is not fully stable and subject to - change, thought it is already possible to write third-party extensions. + change, though it is already possible to write third-party extensions. The extension mechanism allows providing custom code for handling tagged values in the ASDF tree, and converting it to a user-defined custom data @@ -42,10 +42,10 @@ tag in the YAML file. There is some open discussion about allowing implicit type conversion as well, if the value can be successfully deserialized. Though this is - generally againt the spirit of ASDF and YAML, where the semantic meaning + generally against the spirit of ASDF and YAML, where the semantic meaning carried by tags is considered important. -libasdf includes builtin extension types for many of the core ASDF types, +libasdf includes built-in extension types for many of the core ASDF types, including: * :ref:`core/asdf ` @@ -139,7 +139,7 @@ For this example we need to write a few pieces of code: static asdf_version_t asdf_foo_version = { .version = "1.0.0", .major = 1 - } + }; static asdf_software_t asdf_foo_software = { .name = "foo", @@ -159,16 +159,28 @@ or ``NULL`` if parsing the value failed: static asdf_value_err_t asdf_foo_deserialize(asdf_value_t *value, const void *userdata, void **out) { const char **foo_val = NULL; asdf_value_err_t err = asdf_value_as_string(value, &foo_val); + static const char *foo_prefix = "foo:"; if (ASDF_VALUE_OK != err) return err; + size_t prefix_len = strlen(foo_prefix); + char *buf = malloc(prefix_len + foo_len + 1); + + if (!buf) + return ASDF_VALUE_ERR_OOM; + + memcpy(buf, foo_prefix, prefix_len); + memcpy(buf + prefix_len, foo_val, foo_len); + buf[prefix_len + foo_len] = '\0'; + asdf_foo_t *foo = malloc(sizeof(asdf_foo_t)); if (!foo) { + free(buf); return ASDF_VALUE_ERR_OOM; } - foo->foo = strdup(*foo_val); + foo->foo = (const char *)buf; *out = foo; return ASDF_VALUE_OK; } @@ -197,7 +209,7 @@ generic YAML value generated from the extension value: if (!foo->foo) return NULL; - size_t prefix_len = strlen(foo_prefix); + size_t prefix_len = strlen("foo:"); size_t len = strlen(foo->foo); if (len < prefix_len) diff --git a/docs/usage/overview.rst b/docs/usage/overview.rst index a47e5a3b..f0f318ba 100644 --- a/docs/usage/overview.rst +++ b/docs/usage/overview.rst @@ -6,8 +6,8 @@ High-level overview To answer that, it helps to understand what an ASDF file is. At its core, an ASDF file is a human-readable **YAML document** with optional -binary data blocks attached. The YAML portion—called the -:ref:`tree `— describes the structure and metadata of the file. +binary data blocks attached. The YAML portion--called the +:ref:`tree `--describes the structure and metadata of the file. The binary blocks that may follow typically store large numerical arrays, images, or other data that would be inefficient to represent directly in YAML. @@ -16,7 +16,7 @@ the tree provides the structure and metadata, while the blocks hold the raw data. The tree itself usually follows a particular *schema* that defines the expected keys, value types, and overall organization. -All data in an ASDF file—whether metadata or large arrays—is accessed through +All data in an ASDF file--whether metadata or large arrays--is accessed through this tree. Conceptually, you can think of the file as one big nested mapping structure (like a Python :py:obj:`dict`) containing values of many different kinds. @@ -26,21 +26,21 @@ The basic building blocks A YAML document contains three main types of values: -* Mappings—collection of key/value pairs (aka hash maps, dictionaries, etc.) -* Sequences—ordered collections (aka arrays, lists, etc.) -* Scalars—simple values such as numbers, strings, or booleans +* Mappings--collection of key/value pairs (aka hash maps, dictionaries, etc.) +* Sequences--ordered collections (aka arrays, lists, etc.) +* Scalars--simple values such as numbers, strings, or booleans .. note:: - YAML itself does not prescribe strict types for scalar values—it treats them + YAML itself does not prescribe strict types for scalar values--it treats them effectively as plain strings. However, the `YAML Core Schema`_ defines a common set of interpretations (e.g., numbers, booleans, and null), and most high-level languages such as Python, JavaScript, etc. that implement YAML parsers adhere to this convention. - libasdf adheres to this schema as well when reading and writing scalar, - though it also :c:func:`possible ` to access values as raw - scalars. + libasdf adheres to this schema as well when reading and writing scalars, + though it is also :c:func:`possible ` to access values as + raw scalars. Beyond the core types: tags =========================== @@ -49,8 +49,8 @@ In addition to these core YAML types, ASDF supports values that represent complex or domain-specific objects. This is achieved using **YAML tags**, which associate a value with a particular type definition known to the software. -Tags allow arbitrary objects—such as coordinate systems, physical units, or -n-dimensional arrays—to be serialized and deserialized in a structured way. +Tags allow arbitrary objects--such as coordinate systems, physical units, or +n-dimensional arrays--to be serialized and deserialized in a structured way. For example, a tag might tell libasdf that a particular mapping should be interpreted as an ``ndarray`` instead of a plain dictionary. @@ -69,7 +69,7 @@ restore instances of custom classes. ASDF was originally designed around a Python reference implementation, and while the format itself is language-independent, it retains this spirit of structured object serialization. One of the most common tagged objects is the **ndarray**, -which provides efficient storage for numerical array data—described next. +which provides efficient storage for numerical array data--described next. .. _ndarrays: @@ -96,7 +96,7 @@ the binary data. From the point of view of the ASDF file format, a binary block is just a contiguous sequence of bytes with no intrinsic meaning or structure. It is the corresponding ``ndarray`` metadata in the tree that gives those bytes their -shape and semantic content—turning them into a structured numerical array. +shape and semantic content--turning them into a structured numerical array. This separation between structure (in YAML) and data (in binary blocks) is one of the key design principles of ASDF. It allows the format to combine human diff --git a/docs/usage/values.rst b/docs/usage/values.rst index e7f0ad9f..75c37b1b 100644 --- a/docs/usage/values.rst +++ b/docs/usage/values.rst @@ -19,17 +19,20 @@ Example .. code:: c - char **author = NULL; + char *author = NULL; asdf_value_err_t err = asdf_get_string0(file, "metadata/author", &author); - switch (err): + switch (err) { case ASDF_VALUE_OK: printf("the file's author is: %s\n", author); + break; case ASDF_VALUE_ERR_NOT_FOUND: fprintf(stderr, "the expected value metadata/author was not found\n"); + break; case ASDF_VALUE_ERR_TYPE_MISMATCH: fprintf(stderr, "the value metadata/author is expected to be a string\n"); + break; default: fprintf(stderr, "an unknown error occurred reading metadata/author\n"); @@ -123,7 +126,7 @@ Value types ----------- libasdf supports reading scalar values from the YAML Core Schema into C-native -datatypes. The set of support types is in `asdf_value_type_t`. +datatypes. The set of supported types is in `asdf_value_type_t`. Strings ^^^^^^^ @@ -135,7 +138,7 @@ of the string. Booleans ^^^^^^^^ -Boolean values can be read with `asdf_get_bool` and supports values like +Boolean values can be read with `asdf_get_bool` which supports values like "true/false" and also exceptionally treats the integers 0 and 1 as booleans when read as bools. @@ -150,19 +153,19 @@ interoperability purposes (but does define a separate big integer type via tags). If you try to read an integer into too-small of an integer type (e.g. you try -to read ``123456789`` with `asdf_get_uint8()`, or likewise if you try to read +to read ``123456789`` with `asdf_get_uint8()`), or likewise if you try to read a negative integer as an unsigned type, the respective ``asdf_get_<[u]int>`` call returns `ASDF_VALUE_ERR_OVERFLOW`. So unless you have a very specific application where some value is always -expected to be a small integer, your safest best is to just use +expected to be a small integer, your safest bet is to just use `asdf_get_int64`. In fact, the general `asdf_get_int` is just an alias for this. Floating point numbers ^^^^^^^^^^^^^^^^^^^^^^ -Floats work effectively the same as with integers, but only supports C 32-bit +Floats work effectively the same as with integers, but only support C 32-bit floats or 64-bit doubles. Here it is also the safest bet to just use `asdf_get_double` unless you are expecting relatively low-precision values. diff --git a/include/asdf/core/datatype.h b/include/asdf/core/datatype.h index cdb8bc81..c8a8eba6 100644 --- a/include/asdf/core/datatype.h +++ b/include/asdf/core/datatype.h @@ -93,9 +93,9 @@ typedef enum { * should not be explicitly written (just use the default) */ ASDF_BYTEORDER_DEFAULT = 0, - /** Litle-endian **/ + /** Big-endian */ ASDF_BYTEORDER_BIG = '>', - /** Big-endian **/ + /** Little-endian */ ASDF_BYTEORDER_LITTLE = '<' } asdf_byteorder_t; @@ -136,7 +136,7 @@ ASDF_DECLARE_EXTENSION(datatype, asdf_datatype_t); /** * Parse an ASDF scalar datatype and return the corresponding `asdf_scalar_datatype_t` * - * :param s: Null-terminated string + * :param dtype: Null-terminated string * :return: The corresponding `asdf_scalar_datatype_t` or `ASDF_DATATYPE_UNKNOWN` * * .. note:: @@ -197,7 +197,7 @@ ASDF_EXPORT uint64_t asdf_datatype_size(asdf_datatype_t *datatype); * Get the size in bytes of a scalar (numeric) ndarray element for a given * `asdf_scalar_datatype_t` * - * :param type: A member of `asdf_datatype_t` + * :param type: A member of `asdf_scalar_datatype_t` * :return: Size in bytes of a single element of that datatype, or ``-1`` for * non-scalar datatypes (for the present purposes strings are not considered * scalars, only numeric datatypes) diff --git a/include/asdf/core/ndarray.h b/include/asdf/core/ndarray.h index 311393ee..d4600b49 100644 --- a/include/asdf/core/ndarray.h +++ b/include/asdf/core/ndarray.h @@ -6,32 +6,33 @@ * * Support is not yet fully complete. What is implemented: * - * It can provide direct access to the raw data of an ndarray (via a - * ``void *``); users must use - * the metadata provided in the `asdf_ndarray_t` struct to interpret the data. - * However, data can also be copied as tiles using the - * `asdf_ndarray_read_tile_ndim` and `asdf_ndarray_read_tile_2d` functions. - * * * ASDF internal block sources * * All int and most float data types (other data types can be read but are * not fully implemented) * * :c:member:`shape `, - * :c:member:`byeorder `, and + * :c:member:`byteorder `, and * :c:member:`offset ` * * :c:member:`strides ` are partially supported * + * It can provide direct access to the raw data of an ndarray (via a + * ``void *``); users must use + * the metadata provided in the `asdf_ndarray_t` struct to interpret the data. + * However, data can also be copied as tiles using the + * `asdf_ndarray_read_tile_ndim` and `asdf_ndarray_read_tile_2d` functions. + * + * * What is not yet supported: * * * Shape containing '*' * * Reading ``complex64`` or ``complex128``, or ``float16`` datatypes * * Reading string datatypes (``ascii`` or ``ucs4``) * * Reading structured datatypes (the datatypes are parsed but there is are - * no APIs yet for interpreted structured array data + * no APIs yet for interpreting structured array data) * * Reading arbitrarily strided data * * Masks are not parsed or used at all, whether simple mask values or mask * arrays (though if present a warning is logged indicating lack of support) * - * The current limitations are purely aritificial--it is so that we can rapidly + * The current limitations are purely artificial--it is so that we can rapidly * develop the minimal viable product needed to make ASDF ndarray data * available in common use cases. * @@ -75,9 +76,19 @@ typedef enum { * bounds of the ndarray */ ASDF_NDARRAY_ERR_OUT_OF_BOUNDS, + /** A memory error occurred (typically out-of-memory) */ ASDF_NDARRAY_ERR_OOM, + /** + * An invalid argument was passed to the function (e.g. incorrect number + * of shape dimensions) + */ ASDF_NDARRAY_ERR_INVAL, + /** An overflow occurred, particularly when reading integer values */ ASDF_NDARRAY_ERR_OVERFLOW, + /** + * Some ndarray data elements could not be converted to the requested + * output datatype + */ ASDF_NDARRAY_ERR_CONVERSION, } asdf_ndarray_err_t; @@ -89,7 +100,7 @@ typedef enum { * This is the main object through which ndarrays are used. They can be * retrieved via `asdf_get_ndarray` and `asdf_value_as_ndarray`. The library * allocates memory for this data structure which must be freed by the user with - * `asdf_ndarray_destroy` when no-longer needed.. + * `asdf_ndarray_destroy` when no-longer needed. * * For convenience some basic fields are made public for now, though this may * not be ABI-stable in future releases. @@ -103,7 +114,7 @@ typedef struct { const uint64_t *shape; /** The datatype of the array as represented by `asdf_datatype_t` */ asdf_datatype_t datatype; - /** The byteorder of the array data where appliable */ + /** The byteorder of the array data where applicable */ asdf_byteorder_t byteorder; /** Optional offset into the binary block where the array data begins */ uint64_t offset; @@ -169,7 +180,7 @@ typedef struct asdf_ndarray asdf_ndarray_t; * * Release datastructures and memory allocated for an `asdf_ndarray_t` * - * :param value: The `asdf_ndarray_t *` + * :param ndarray: The `asdf_ndarray_t *` */ // clang-format on @@ -310,7 +321,7 @@ ASDF_EXPORT void asdf_ndarray_data_dealloc(asdf_ndarray_t *ndarray); * :param compression: String representing the compressor to use (e.g. "bzp2") * if any, or NULL or the empty string to set no compression * :return: Non-zero if the compression could not be set (e.g. invalid/unknown - * compressor; use `asdf_error` to check the error code + * compressor); use `asdf_error` to check the error code */ ASDF_EXPORT int asdf_ndarray_compression_set(asdf_ndarray_t *ndarray, const char *compression); @@ -387,6 +398,8 @@ ASDF_EXPORT asdf_block_t *asdf_ndarray_block(asdf_ndarray_t *ndarray); * the exact number of bytes in the source ndarray, or `NULL` to indicate * that a buffer should be allocated. In the latter case the caller is * responsible for freeing the allocated buffer. + * :return: An `asdf_ndarray_err_t`; either `ASDF_NDARRAY_OK` if the data read + * successfully; otherwise the relevant error code. */ ASDF_EXPORT asdf_ndarray_err_t asdf_ndarray_read_all(asdf_ndarray_t *ndarray, asdf_scalar_datatype_t dst_t, void **dst); @@ -403,7 +416,7 @@ asdf_ndarray_read_all(asdf_ndarray_t *ndarray, asdf_scalar_datatype_t dst_t, voi * :c:member:`ndim ` * :param shape: The shape of the tile to read--an array of size * :c:member:`ndim ` - * :param: dst_t: The output datatype, if conversion from the source array's + * :param dst_t: The output datatype, if conversion from the source array's * datatype to the output datatype is possible * * Currently, if no conversion is possible it will just copy the tile data @@ -414,6 +427,8 @@ asdf_ndarray_read_all(asdf_ndarray_t *ndarray, asdf_scalar_datatype_t dst_t, voi * the exact number of bytes in the output tile based on shape and datatype, * or `NULL` to indicate that a buffer should be allocated. In the latter * case the caller is responsible for freeing the allocated buffer. + * :return: An `asdf_ndarray_err_t`; either `ASDF_NDARRAY_OK` if the data read + * successfully; otherwise the relevant error code. */ ASDF_EXPORT asdf_ndarray_err_t asdf_ndarray_read_tile_ndim( asdf_ndarray_t *ndarray, @@ -434,7 +449,7 @@ ASDF_EXPORT asdf_ndarray_err_t asdf_ndarray_read_tile_ndim( * :param plane_origin: If the source array is greater than 2-dimensional, the * ``ndim - 2`` array of plane coordinates--may be `NULL` if either the source * array is 2-D or otherwise the outer-most plane is used - * :param: dst_t: The output datatype, if conversion from the source array's + * :param dst_t: The output datatype, if conversion from the source array's * datatype to the output datatype is possible * * Currently, if no conversion is possible it will just copy the tile data @@ -445,6 +460,8 @@ ASDF_EXPORT asdf_ndarray_err_t asdf_ndarray_read_tile_ndim( * the exact number of bytes in the output tile based on shape and datatype, * or `NULL` to indicate that a buffer should be allocated. In the latter * case the caller is responsible for freeing the allocated buffer. + * :return: An `asdf_ndarray_err_t`; either `ASDF_NDARRAY_OK` if the data read + * successfully; otherwise the relevant error code. */ ASDF_EXPORT asdf_ndarray_err_t asdf_ndarray_read_tile_2d( asdf_ndarray_t *ndarray, diff --git a/include/asdf/file.h b/include/asdf/file.h index 409f70d5..0c669ee7 100644 --- a/include/asdf/file.h +++ b/include/asdf/file.h @@ -47,7 +47,7 @@ typedef struct asdf_file asdf_file_t; /** - * .. _int file-configuration: + * .. _file-configuration: * * Configuration * ------------- @@ -707,9 +707,9 @@ ASDF_EXPORT bool asdf_is_null(asdf_file_t *file, const char *path); * integer type that can hold that value. For example the number ``42`` is * typed as `ASDF_VALUE_UINT8`. * - * However, integer up-casting to larger integer types. Downcasting that would - * cause an overflow is not allowed. For example ``42`` can be cast to an - * ``int16``, but ``-42`` cannot be cast to a ``uint16``. + * However, integer up-casting to larger integer types is allowed + * Downcasting that would cause an overflow is not allowed. For example ``42`` + * can be cast to an ``int16``, but ``-42`` cannot be cast to a ``uint16``. * * .. note:: * @@ -720,7 +720,7 @@ ASDF_EXPORT bool asdf_is_null(asdf_file_t *file, const char *path); * may also be `ASDF_VALUE_ERR_OVERFLOW` if the value is an integer that is too * large to represent in the requested type. * - * Big integers (greater than ``UINT64_MAX`` or less than ``INT64_MAX``) are + * Big integers (greater than ``UINT64_MAX`` or less than ``INT64_MIN``) are * not supported--in fact the ASDF Standard expressly * `forbids `_ writing them to ASDF files. Nevertheless * it could be supported in the future if the need arises. In fact, @@ -733,7 +733,7 @@ ASDF_EXPORT bool asdf_is_null(asdf_file_t *file, const char *path); * size * * :param file: The `asdf_file_t *` for the file - * :param path: The :ref:`yaml-pointer` to the bool + * :param path: The :ref:`yaml-pointer` to the int * :return: `true` if the value is an integer, `false` if it is another * type of value or if no value exists at that path. */ @@ -1030,7 +1030,7 @@ ASDF_EXPORT const char *asdf_block_compression(asdf_block_t *block); * :param compression: String representing the compressor to use (e.g. "bzp2") * if any, or NULL or the empty string to set no compression * :return: Non-zero if the compression could not be set (e.g. invalid/unknown - * compressor; use `asdf_error` to check the error code + * compressor); use `asdf_error` to check the error code */ ASDF_EXPORT int asdf_block_compression_set(asdf_block_t *block, const char *compression); @@ -1066,8 +1066,8 @@ ASDF_EXPORT const unsigned char *asdf_block_checksum(asdf_block_t *block); * Add and document option to automatically verify checksums. * * :param block: The `asdf_block_t *` handle - * :param computed: Optional pointer to a `uint8_t` buffer to receive the computed - * MD5 digest on return + * :param expected: Optional pointer to a `uint8_t` buffer to receive the + * computed MD5 digest on return * :return: True if the checksum is valid */ ASDF_EXPORT bool asdf_block_checksum_verify( @@ -1079,7 +1079,7 @@ ASDF_EXPORT bool asdf_block_checksum_verify( * * :param block: The `asdf_block_t *` handle * :param size: Optional `size_t *` into which the size of the block data is - * is returned + * returned * :return: A `void *` to the block data */ ASDF_EXPORT const void *asdf_block_data(asdf_block_t *block, size_t *size); @@ -1096,7 +1096,7 @@ ASDF_EXPORT const void *asdf_block_data(asdf_block_t *block, size_t *size); * * :param block: The `asdf_block_t *` handle * :param size: Optional `size_t *` into which the size of the block data is - * is returned + * returned * :return: A `void *` to the block data */ ASDF_EXPORT const void *asdf_block_data_raw(asdf_block_t *block, size_t *size); diff --git a/include/asdf/value.h b/include/asdf/value.h index 89a5ad34..5057a8c1 100644 --- a/include/asdf/value.h +++ b/include/asdf/value.h @@ -53,8 +53,8 @@ typedef enum { * A string * * A scalar is considered a string if it is explicitly quoted with single- - * or double-quotes, or uses a literal or folded scalar represenation style - * in the YAML document, or any other scalar that cannot be coecered to one + * or double-quotes, or uses a literal or folded scalar representation style + * in the YAML document, or any other scalar that cannot be coerced to one * of the other types defined in the `YAML Core Schema`_. */ ASDF_VALUE_STRING, @@ -222,14 +222,26 @@ ASDF_EXPORT asdf_value_t *asdf_value_clone(asdf_value_t *value); */ ASDF_EXPORT asdf_value_type_t asdf_value_get_type(asdf_value_t *value); + +/** + * Get full :ref:`yaml-pointer` path of an `asdf_value_t` + * + * If the value was created manually and not yet attached to a file, this + * will return ``NULL``. + * + * :param value: The `asdf_value_t *` handle + * :return: The path as a string (this is owned by the `asdf_value_t *` and + * must not be freed) or ``NULL`` if the value does not yet have a path in + * the document. + */ ASDF_EXPORT const char *asdf_value_path(asdf_value_t *value); /** * Get the parent value, if any, of the given value * * :param value: The `asdf_value_t *` handle - * :return: An `asdf_value_t *` of its parent mapping or sequence; if the input value - * is the root node, or is not an attached value, the parent is NULL + * :return: An `asdf_value_t *` of its parent mapping or sequence; if the input + * value is the root node, or is not an attached value, the parent is NULL */ ASDF_EXPORT asdf_value_t *asdf_value_parent(asdf_value_t *value); @@ -239,7 +251,10 @@ ASDF_EXPORT asdf_value_t *asdf_value_parent(asdf_value_t *value); */ ASDF_EXPORT const char *asdf_value_type_string(asdf_value_type_t type); -/* Return the value's tag if it has an *explicit* tag (implict tags are not returned) */ +/** + * Return the value's tag if it has an *explicit* tag (implicit tags are not + * returned) + */ ASDF_EXPORT const char *asdf_value_tag(asdf_value_t *value); // Forward-declaration needed for `asdf_value_file` @@ -262,7 +277,7 @@ ASDF_EXPORT asdf_file_t *asdf_value_file(asdf_value_t *value); * * By design, it is safe to cast a an `asdf_value_t *` to * `asdf_mapping_t *` so long as the value has been checked to be a - * sequence. + * mapping. */ typedef struct asdf_mapping asdf_mapping_t; @@ -418,9 +433,9 @@ ASDF_EXPORT asdf_value_t *asdf_mapping_pop(asdf_mapping_t *mapping, const char * * * If the key already exists in the mapping its value will be overwritten * - * :params mapping: Handle to the `asdf_mapping_t` to update - * :params key: The key at which to insert the value - * :params value: Generic `asdf_value_t *` to insert into the mapping + * :param mapping: Handle to the `asdf_mapping_t` to update + * :param key: The key at which to insert the value + * :param value: Generic `asdf_value_t *` to insert into the mapping * :return: `ASDF_VALUE_OK` on success or another `asdf_value_err_t` */ ASDF_EXPORT asdf_value_err_t @@ -731,7 +746,7 @@ ASDF_EXPORT asdf_sequence_t *asdf_sequence_of_double( * * :param sequence: The `asdf_sequence_t *` handle * :param index: The index of the item to remove; negative indices are also supported (e.g. - * ``asdf_sequence_pop(sequence, -1)`` removes the last item + * ``asdf_sequence_pop(sequence, -1)`` removes the last item) * :return: The generic `asdf_value_t *` of the removed value if any, or NULL */ ASDF_EXPORT asdf_value_t *asdf_sequence_pop(asdf_sequence_t *sequence, int index); @@ -816,8 +831,9 @@ ASDF_EXPORT bool asdf_value_is_container(asdf_value_t *value); /** * Generic container size * - * :param value: `asdf_value_t *` containing a mapping or a sequence - * :return: The size of the mapping or container, or -1 if the value is not a container type + * :param container: `asdf_value_t *` containing a mapping or a sequence + * :return: The size of the mapping or sequence, or -1 if the value is not a + * container type */ ASDF_EXPORT int asdf_container_size(asdf_value_t *container); @@ -837,7 +853,7 @@ typedef struct _asdf_extension asdf_extension_t; * * .. note:: * - * This is usually wrapped by some helper utility named like ``asdf_value_is_``, + * This is usually wrapped by some helper utility named like ``asdf_value_is_``, * such as ``asdf_value_is_ndarray``. * * But that is equivalent to running: @@ -855,7 +871,7 @@ ASDF_EXPORT bool asdf_value_is_extension_type(asdf_value_t *value, const asdf_ex * * .. note:: * - * This is usually wrapped by some helper utility named like ``asdf_value_as_``, + * This is usually wrapped by some helper utility named like ``asdf_value_as_``, * such as ``asdf_value_as_ndarray``. * * But that is equivalent to running: From c5433ba1b134b4560e0ba2eb7ab67959ed295067 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Fri, 26 Jun 2026 15:49:33 +0200 Subject: [PATCH 06/12] docs: add documentation of the command-line tool Set Sphinx up to generate a man page from it (manually mantaining a man page in roff is pure pain). The generated page is checked into the repository, however, so that it can easily be included in release tarballs. This means has to be manually re-generated when updating the CLI docs which, fortunately, shouldn't happen that often. Shame there's not also some way I could automatically extract this page directly from the code. Of course it could be done with a custom Sphinx extension, but not worth the effort. gh-202 --- Makefile.am | 5 + README.rst | 6 ++ docs/Makefile.am | 18 ++++ docs/conf.py | 14 ++- docs/index.rst | 12 +++ docs/man/asdf.1 | 240 +++++++++++++++++++++++++++++++++++++++++++++ docs/usage/cli.rst | 177 +++++++++++++++++++++++++++++++++ src/CMakeLists.txt | 7 ++ src/main.c | 7 +- 9 files changed, 482 insertions(+), 4 deletions(-) create mode 100644 docs/man/asdf.1 create mode 100644 docs/usage/cli.rst diff --git a/Makefile.am b/Makefile.am index 4db22778..d39b0866 100644 --- a/Makefile.am +++ b/Makefile.am @@ -105,6 +105,11 @@ asdf_SOURCES = src/main.c asdf_CFLAGS = $(AM_CFLAGS) $(FYAML_CFLAGS) $(STATGRAB_CFLAGS) $(LZ4_CFLAGS) $(CODE_COVERAGE_CFLAGS) asdf_LDFLAGS = $(CODE_COVERAGE_LDFLAGS) asdf_LDADD = third_party/libstc.la libasdf.la $(ARGP_LIBS) $(ASDF_LIBS) + +# The man page is generated from docs/usage/cli.rst with Sphinx but is checked +# in (see that file) so that building a release tarball does not require Sphinx. +# dist_man_MANS both distributes it and installs it to $(mandir)/man1. +dist_man_MANS = docs/man/asdf.1 endif # ASDF_BUILD_TOOL diff --git a/README.rst b/README.rst index 44577dbb..d0e7c42f 100644 --- a/README.rst +++ b/README.rst @@ -27,6 +27,12 @@ extract binary block data, as well as typed getters for metadata in the ASDF tre It also features an extension mechanism (still nascent) for reading ASDF schemas, including the core schemas such as ``core/ndarray-`` into C-native datastructures. +libasdf additionally installs a companion command-line tool, ``asdf``: a wrapper around the +library providing utilities for inspecting and extracting data from ASDF files. Its +capabilities are currently modest but will be expanded in the future; see the +`command-line tool documentation `__ +for details. + Getting Started --------------- diff --git a/docs/Makefile.am b/docs/Makefile.am index d395959f..9e2eeecd 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -16,8 +16,11 @@ EXTRA_DIST = \ environment.yml \ index.rst \ links.rst \ + usage/cli.rst \ + usage/compression.rst \ usage/examples.rst \ usage/extensions.rst \ + usage/ndarrays.rst \ usage/opening.rst \ usage/overview.rst \ usage/values.rst \ @@ -26,3 +29,18 @@ EXTRA_DIST = \ _static/images/favicon.ico \ _static/images/logo-dark-mode.png \ _static/images/logo-light-mode.png + + +# Regenerate the checked-in asdf(1) man page from usage/cli.rst. The man page +# (man/asdf.1) is committed to the repository so that building a release tarball +# or installing the tool does not require Sphinx; run this target after editing +# usage/cli.rst, then commit the updated man/asdf.1. +if BUILD_DOCS +man-page: $(srcdir)/man/asdf.1 + +$(srcdir)/man/asdf.1: $(srcdir)/usage/cli.rst $(srcdir)/conf.py + $(SPHINXBUILD) -b man $(srcdir) $(builddir)/_build/man + $(MKDIR_P) $(srcdir)/man + cp $(builddir)/_build/man/asdf.1 $@ +endif +.PHONY: man-page diff --git a/docs/conf.py b/docs/conf.py index 055d53eb..b5f55e60 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -51,7 +51,19 @@ def read_config_h() -> tuple[str, str, str]: )] # -- Options for manual page output -------------------------------------------- -man_pages = [("index", project.lower(), project + " Documentation", [author], 1)] +# Each tuple is (source doc, name, description, authors, manual section). The +# ``asdf`` entry generates an asdf(1) man page from the command-line tool page; +# build it with ``make man-page`` in this directory. +man_pages = [ + ("index", project.lower(), project + " Documentation", [author], 1), + ( + "usage/cli", + "asdf", + "command-line utility for inspecting and extracting data from ASDF files", + [author], + 1, + ), +] todo_include_todos = True diff --git a/docs/index.rst b/docs/index.rst index 02a92962..02b0c473 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -62,6 +62,18 @@ writing. These are intended for advanced use cases and are not yet fully documented; most applications should prefer the high-level API described above. +Command-line tool +================= + +libasdf installs a companion command-line program, ``asdf``, for inspecting +and extracting data from ASDF files directly from the shell. + +.. toctree:: + :maxdepth: 2 + + usage/cli + + Resources ========= diff --git a/docs/man/asdf.1 b/docs/man/asdf.1 new file mode 100644 index 00000000..6f884125 --- /dev/null +++ b/docs/man/asdf.1 @@ -0,0 +1,240 @@ +'\" t +.\" Man page generated from reStructuredText +.\" by the Docutils 0.22.4 manpage writer. +. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.TH "ASDF" "1" "Jun 26, 2026" "0.1" "libasdf" +.SH NAME +asdf \- command-line utility for inspecting and extracting data from ASDF files +.sp +The \fBasdf\fP command\-line tool is a companion utility installed alongside +libasdf. It is a thin wrapper around the library offering a handful of +sub\-commands for inspecting ASDF files and extracting their data. Its +capabilities are currently modest but will be expanded in future releases. +.sp +Each tool provided by \fBasdf\fP is run as a separate sub\-command. +.SH SYNOPSIS +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +asdf COMMAND [ARGS...] +.EE +.UNINDENT +.UNINDENT +.sp +Run \fBasdf COMMAND \-\-help\fP for detailed help on any sub\-command. The +available sub\-commands are: +.TS +box center; +l|l. +T{ +Command +T} T{ +Description +T} +_ +T{ +info +T} T{ +Print a rendering of an ASDF tree +T} +_ +T{ +dd +T} T{ +Dump data from an ASDF binary block +T} +_ +T{ +events +T} T{ +Print the event stream from the ASDF parser (for debugging) +T} +_ +T{ +verify\-checksums +T} T{ +Verify binary block MD5 checksums +T} +.TE +.sp +The global \fBasdf\fP command itself accepts \fB\-h\fP / \fB\-\-help\fP and \fB\-\-usage\fP\&. +.SH ASDF INFO +.sp +Print a human\-readable rendering of an ASDF file\(aqs YAML tree, and optionally +information about its binary blocks. +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +asdf info [OPTIONS] FILENAME +.EE +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-tree +Do not show the tree. +.UNINDENT +.INDENT 0.0 +.TP +.B \-b, \-\-blocks +Show information about the file\(aqs binary blocks. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-verify\-checksums +Verify the MD5 checksum of each block (implies \fB\-b\fP). +.UNINDENT +.sp +For example, to print the tree of a file together with its block information: +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +asdf info \-\-blocks observation.asdf +.EE +.UNINDENT +.UNINDENT +.SH ASDF DD +.sp +Dump the raw bytes of a single binary block to a file or to standard output, +named either by block index or by the tree path of an ndarray that references +it. Exactly one of \fB\-\-block\fP or \fB\-\-ndarray\fP must be given. +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +asdf dd [OPTIONS] INPUT [OUTPUT|\-] +.EE +.UNINDENT +.UNINDENT +.sp +\fBINPUT\fP is the ASDF file to read. \fBOUTPUT\fP is the destination file; a +single \fB\-\fP writes the data to standard output. +.INDENT 0.0 +.TP +.B \-b N, \-\-block N +Index of the block to extract (starting from \fB0\fP). +.UNINDENT +.INDENT 0.0 +.TP +.B \-n PATH, \-\-ndarray PATH +Tree path \%<#\:yaml-pointer> of an ndarray whose underlying block should +be extracted. +.UNINDENT +.INDENT 0.0 +.TP +.B \-r, \-\-raw +For a compressed block, dump the raw compressed bytes rather than +decompressing them first. Has no effect on uncompressed blocks. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c N, \-\-chunk\-size N +Read/write the data in chunks of \fBN\fP bytes. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-lazy\-decompression +Disable lazy (page\-fault\-driven) decompression; decompress eagerly instead. +Intended mainly for debugging. +.UNINDENT +.sp +For example, to extract the data of the ndarray at \fB/data\fP into a file: +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +asdf dd \-\-ndarray data observation.asdf data.bin +.EE +.UNINDENT +.UNINDENT +.SH ASDF EVENTS +.sp +Print the low\-level event stream produced by the ASDF parser as it walks a +file. This is primarily a debugging aid for libasdf itself and exposes the +event\-based parsing API (see libasdf \- The ASDF C Library \%<#\:asdf>). +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +asdf events [OPTIONS] FILENAME +.EE +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Show extra information about each event. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-no\-yaml +Do not produce YAML stream events (emit only the ASDF\-structural events). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-cap\-tree +Buffer the YAML tree and print it. This corresponds to the parser\(aqs +tree\-buffering option and is mainly useful for debugging. +.UNINDENT +.SH ASDF VERIFY-CHECKSUMS +.sp +Verify the MD5 checksum recorded in each binary block header against the +checksum computed from the block\(aqs data. The exit status is non\-zero if any +block fails verification. +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +asdf verify\-checksums [OPTIONS] FILENAME +.EE +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Print the checksum of every block, whether or not it verifies. Without this +option the command is quiet on success and reports only mismatches. +.UNINDENT +.sp +\fBNote:\fP +.INDENT 0.0 +.INDENT 3.5 +Checksum verification requires that libasdf was built with MD5 support +(via \fBlibbsd\fP). Without it, blocks are always reported as valid. +.UNINDENT +.UNINDENT +.SH Author +The ASDF Developers +.SH Copyright +2026, The ASDF Developers +.\" End of generated man page. diff --git a/docs/usage/cli.rst b/docs/usage/cli.rst new file mode 100644 index 00000000..63ad4575 --- /dev/null +++ b/docs/usage/cli.rst @@ -0,0 +1,177 @@ +.. This page is the source for the asdf(1) man page (docs/man/asdf.1), which is + generated with Sphinx but checked into the repository so that building a + release tarball or installing the tool does not require Sphinx in the + toolchain. After editing this page, regenerate the committed man page by + running ``make man-page`` from the docs build directory, then commit the + updated docs/man/asdf.1. + +.. _cli: + +asdf +==== + +The ``asdf`` command-line tool is a companion utility installed alongside +libasdf. It is a thin wrapper around the library offering a handful of +sub-commands for inspecting ASDF files and extracting their data. Its +capabilities are currently modest but will be expanded in future releases. + +Each tool provided by ``asdf`` is run as a separate sub-command. + +Synopsis +-------- + +.. code:: text + + asdf COMMAND [ARGS...] + +Run ``asdf COMMAND --help`` for detailed help on any sub-command. The +available sub-commands are: + +.. list-table:: + :header-rows: 1 + :widths: 25 75 + + * - Command + - Description + * - :ref:`info ` + - Print a rendering of an ASDF tree + * - :ref:`dd ` + - Dump data from an ASDF binary block + * - :ref:`events ` + - Print the event stream from the ASDF parser (for debugging) + * - :ref:`verify-checksums ` + - Verify binary block MD5 checksums + +The global ``asdf`` command itself accepts ``-h`` / ``--help`` and ``--usage``. + + +.. _cli-info: + +asdf info +--------- + +.. program:: asdf info + +Print a human-readable rendering of an ASDF file's YAML tree, and optionally +information about its binary blocks. + +.. code:: text + + asdf info [OPTIONS] FILENAME + +.. option:: --no-tree + + Do not show the tree. + +.. option:: -b, --blocks + + Show information about the file's binary blocks. + +.. option:: --verify-checksums + + Verify the MD5 checksum of each block (implies :option:`-b`). + +For example, to print the tree of a file together with its block information:: + + asdf info --blocks observation.asdf + + +.. _cli-dd: + +asdf dd +------- + +.. program:: asdf dd + +Dump the raw bytes of a single binary block to a file or to standard output, +named either by block index or by the tree path of an ndarray that references +it. Exactly one of :option:`--block` or :option:`--ndarray` must be given. + +.. code:: text + + asdf dd [OPTIONS] INPUT [OUTPUT|-] + +``INPUT`` is the ASDF file to read. ``OUTPUT`` is the destination file; a +single ``-`` writes the data to standard output. + +.. option:: -b N, --block N + + Index of the block to extract (starting from ``0``). + +.. option:: -n PATH, --ndarray PATH + + :ref:`Tree path ` of an ndarray whose underlying block should + be extracted. + +.. option:: -r, --raw + + For a compressed block, dump the raw compressed bytes rather than + decompressing them first. Has no effect on uncompressed blocks. + +.. option:: -c N, --chunk-size N + + Read/write the data in chunks of ``N`` bytes. + +.. option:: --no-lazy-decompression + + Disable lazy (page-fault-driven) decompression; decompress eagerly instead. + Intended mainly for debugging. + +For example, to extract the data of the ndarray at ``/data`` into a file:: + + asdf dd --ndarray data observation.asdf data.bin + + +.. _cli-events: + +asdf events +----------- + +.. program:: asdf events + +Print the low-level event stream produced by the ASDF parser as it walks a +file. This is primarily a debugging aid for libasdf itself and exposes the +event-based parsing API (see :ref:`asdf`). + +.. code:: text + + asdf events [OPTIONS] FILENAME + +.. option:: -v, --verbose + + Show extra information about each event. + +.. option:: --no-yaml + + Do not produce YAML stream events (emit only the ASDF-structural events). + +.. option:: --cap-tree + + Buffer the YAML tree and print it. This corresponds to the parser's + tree-buffering option and is mainly useful for debugging. + + +.. _cli-verify-checksums: + +asdf verify-checksums +--------------------- + +.. program:: asdf verify-checksums + +Verify the MD5 checksum recorded in each binary block header against the +checksum computed from the block's data. The exit status is non-zero if any +block fails verification. + +.. code:: text + + asdf verify-checksums [OPTIONS] FILENAME + +.. option:: -v, --verbose + + Print the checksum of every block, whether or not it verifies. Without this + option the command is quiet on success and reports only mismatches. + +.. note:: + + Checksum verification requires that libasdf was built with MD5 support + (via ``libbsd``). Without it, blocks are always reported as valid. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d68d6d8a..8867d771 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -140,4 +140,11 @@ install(TARGETS asdf-cli DESTINATION ${CMAKE_INSTALL_BINDIR} ) +# Install the tool's man page. It is generated from docs/usage/cli.rst with +# Sphinx but checked in (see that file) so that building/installing does not +# require Sphinx. +install(FILES ${CMAKE_SOURCE_DIR}/docs/man/asdf.1 + DESTINATION ${CMAKE_INSTALL_MANDIR}/man1 +) + diff --git a/src/main.c b/src/main.c index d854b767..7cb984db 100644 --- a/src/main.c +++ b/src/main.c @@ -63,9 +63,10 @@ static struct argp_option global_options[] = { /* The pre-doc (text before \v) is the static description. The help filter * appends the sub-command listing to it dynamically. The post-doc (text * after \v) is the static footer. */ -static char doc[] = "asdf -- Commandline utilities for managing ASDF files." - "\v" - "Run 'asdf COMMAND --help' for more information on a sub-command."; +static char + doc[] = "asdf -- Command-line utilities for inspecting and extracting data from ASDF files" + "\v" + "Run 'asdf COMMAND --help' for more information on a sub-command."; // info subcommand From 82c73998f55a15c0dd648aff2e798294f9cca624 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Fri, 26 Jun 2026 15:56:13 +0200 Subject: [PATCH 07/12] fix: small bug in `asdf dd` missing output argument handling Caught while looking at the code to write the CLI documentation. --- src/main.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main.c b/src/main.c index 7cb984db..88aade71 100644 --- a/src/main.c +++ b/src/main.c @@ -172,6 +172,8 @@ static error_t parse_dd_opt(int key, char *arg, struct argp_state *state) { if (!(args->has_block_index || args->ndarray_path) || (args->has_block_index && args->ndarray_path)) argp_error(state, "'dd' requires only one of --block or --ndarray"); + if (!args->input || !args->output) + argp_error(state, "'dd' requires INPUT and OUTPUT arguments."); break; default: return ARGP_ERR_UNKNOWN; From e36b638118fb7d3d5639b20c5c5a2228ec7fb524 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Fri, 26 Jun 2026 16:01:41 +0200 Subject: [PATCH 08/12] build: option to enable/disable building the CLI tool in cmake Same as in the autotools build, but was previously omitted. Mention the flag in the readme. --- README.rst | 1 + cmake/ASDFDependencies.cmake | 3 ++- cmake/ASDFOptions.cmake | 3 +++ src/CMakeLists.txt | 33 ++++++++++++++++----------------- 4 files changed, 22 insertions(+), 18 deletions(-) diff --git a/README.rst b/README.rst index d0e7c42f..2e8602b5 100644 --- a/README.rst +++ b/README.rst @@ -298,6 +298,7 @@ Clone the repository and build the project as follows:: cmake .. \ -D ENABLE_TESTING=[YES/NO] \ -D ENABLE_TESTING_SHELL=[YES/NO] \ + -D ENABLE_TOOL=[YES/NO] \ -D ENABLE_ASAN=[YES/NO] \ -D FYAML_NO_PKGCONFIG=[YES/NO] \ # If YES \ diff --git a/cmake/ASDFDependencies.cmake b/cmake/ASDFDependencies.cmake index fead6505..2233cfff 100644 --- a/cmake/ASDFDependencies.cmake +++ b/cmake/ASDFDependencies.cmake @@ -73,7 +73,8 @@ else() endif() -if(APPLE) +# argp is only needed by the command-line tool +if(ENABLE_TOOL AND APPLE) option(ARGP_NO_PKGCONFIG NO) if(ARGP_NO_PKGCONFIG) set(ARGP_LIBRARIES "argp") diff --git a/cmake/ASDFOptions.cmake b/cmake/ASDFOptions.cmake index 104a1581..3f19790f 100644 --- a/cmake/ASDFOptions.cmake +++ b/cmake/ASDFOptions.cmake @@ -11,6 +11,9 @@ set(ASDF_DEBUG OFF CACHE BOOL "Enable DEBUG code") # Additional feature flags option(ENABLE_STATIC "Build as a static library" OFF) +# Build the asdf command-line tool (depends on argp) +option(ENABLE_TOOL "Build the asdf command-line tool" ON) + # Documentation option(ENABLE_DOCS OFF) if (ENABLE_DOCS) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8867d771..043207f5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -122,23 +122,22 @@ install(TARGETS libasdf_static endif() # Generate CLI program -project(asdf-cli) -add_executable(asdf-cli main.c) -target_link_libraries(asdf-cli PRIVATE libasdf stc ${ARGP_LIBRARIES}) -target_link_directories(asdf-cli PRIVATE ${ARGP_LIBDIR}) -target_include_directories(asdf-cli PRIVATE - ${STC_DIR}/include - ${CMAKE_BINARY_DIR}/include - ${CMAKE_CURRENT_BINARY_DIR} - ${ARGP_INCLUDEDIR} - ${FYAML_INCLUDEDIR} -) - -# Set CLI program's name on-disk -set_target_properties(asdf-cli ${PROJECT_NAME} PROPERTIES OUTPUT_NAME "asdf") -install(TARGETS asdf-cli - DESTINATION ${CMAKE_INSTALL_BINDIR} -) +if(ENABLE_TOOL) + project(asdf-cli) + add_executable(asdf-cli main.c) + target_link_libraries(asdf-cli PRIVATE libasdf stc ${ARGP_LIBRARIES}) + target_link_directories(asdf-cli PRIVATE ${ARGP_LIBDIR}) + target_include_directories(asdf-cli PRIVATE + ${STC_DIR}/include + ${CMAKE_BINARY_DIR}/include + ${CMAKE_CURRENT_BINARY_DIR} + ${ARGP_INCLUDEDIR} + ) + set_target_properties(asdf-cli ${PROJECT_NAME} PROPERTIES OUTPUT_NAME "asdf") + install(TARGETS asdf-cli + DESTINATION ${CMAKE_INSTALL_BINDIR} + ) +endif() # Install the tool's man page. It is generated from docs/usage/cli.rst with # Sphinx but checked in (see that file) so that building/installing does not From 65f36c44f3b83e56093496b19bc9b0e9b6027d92 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Fri, 26 Jun 2026 16:24:51 +0200 Subject: [PATCH 09/12] build: still more fixes for sys/endian.h on macOS, now in cmake Since some versions of Apple's SDK now seem to include sys/endian.h check for that first. Still, don't redefine endianness macros if they are already defined, but the configure checks somehow failed. --- cmake/ASDFConfig.cmake | 4 ++-- src/compat/endian.h | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmake/ASDFConfig.cmake b/cmake/ASDFConfig.cmake index a5420612..1857a6ce 100644 --- a/cmake/ASDFConfig.cmake +++ b/cmake/ASDFConfig.cmake @@ -26,10 +26,10 @@ check_include_file(sys/endian.h HAVE_SYS_ENDIAN_H) if(HAVE_ENDIAN_H) set(ENDIAN_H endian.h) -elseif(HAVE_MACHINE_ENDIAN_H) - set(ENDIAN_H machine/endian.h) elseif(HAVE_SYS_ENDIAN_H) set(ENDIAN_H sys/endian.h) +elseif(HAVE_MACHINE_ENDIAN_H) + set(ENDIAN_H machine/endian.h) endif() macro(check_endian_decl func) diff --git a/src/compat/endian.h b/src/compat/endian.h index a8746626..53955948 100644 --- a/src/compat/endian.h +++ b/src/compat/endian.h @@ -26,7 +26,7 @@ #endif #if !HAVE_DECL_BE64TOH -#ifdef __APPLE__ +#if (defined(__APPLE__) && !defined(be64toh)) #include #define be64toh(x) OSSwapBigToHostInt64(x) #else @@ -36,7 +36,7 @@ #if !HAVE_DECL_BE32TOH -#ifdef __APPLE__ +#if (defined(__APPLE__) && !defined(be32toh)) #include #define be32toh(x) OSSwapBigToHostInt32(x) #else @@ -46,7 +46,7 @@ #if !HAVE_DECL_HTOBE16 -#ifdef __APPLE__ +#if (defined(__APPLE__) && !defined(htobe16)) #include #define htobe16(x) OSSwapHostToBigInt16(x) #else @@ -56,7 +56,7 @@ #if !HAVE_DECL_HTOBE32 -#ifdef __APPLE__ +#if (defined(__APPLE__) && !defined(htobe32)) #include #define htobe32(x) OSSwapHostToBigInt32(x) #else @@ -66,7 +66,7 @@ #if !HAVE_DECL_HTOBE64 -#ifdef __APPLE__ +#if (defined(__APPLE__) && !defined(htobe64)) #include #define htobe64(x) OSSwapHostToBigInt64(x) #else @@ -76,7 +76,7 @@ #if !HAVE_DECL_LE32TOH -#ifdef __APPLE__ +#if (defined(__APPLE__) && !defined(le32toh)) #include #define le32toh(x) OSSwapLittleToHostInt32(x) #else @@ -86,7 +86,7 @@ #if !HAVE_DECL_HTOLE32 -#ifdef __APPLE__ +#if (defined(__APPLE__) && !defined(htole32)) #include #define htole32(x) OSSwapHostToLittleInt32(x) #else From 2a0b15cb66e63378833064eba19b233626960aac Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:36:40 +0200 Subject: [PATCH 10/12] build: fix build-time logging configuration in cmake While working on updating the documentation for the logging interface, I noticed that the logging configuration was never really properly carried over from the autotools build (always unconditionally defined `ASDF_LOG_ENABLED` and set the default log level to `TRACE` which is undesirable). This fixes up the logging configuration options in cmake and documents them better. gh-202 --- README.rst | 19 +++++++++++++++++++ cmake/ASDFOptions.cmake | 33 +++++++++++++++++++++++++++++---- config.h.cmake | 8 ++++---- 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 2e8602b5..61437886 100644 --- a/README.rst +++ b/README.rst @@ -315,6 +315,25 @@ If doing a system install, as usual it's recommended to install to ``/usr/local` by providing ``-DCMAKE_INSTALL_PREFIX=/usr/local`` when running ``cmake``. Or, if you have a ``${HOME}/.local`` you can set the prefix there, etc. +Logging +^^^^^^^ + +libasdf can emit diagnostic log messages, controlled by the following options: + +- ``-D ENABLE_LOG=[YES/NO]`` -- compile libasdf's internal log statements into + the library (default ``YES``). When ``NO`` they compile to nothing. +- ``-D ENABLE_LOG_COLOR=[YES/NO]`` -- colorize log output (default ``YES``). +- ``-D LOG_DEFAULT=LEVEL`` -- the default runtime log level, used when none is + set explicitly; one of ``TRACE``, ``DEBUG``, ``INFO``, ``WARN`` (the + default), ``ERROR``, ``FATAL``, or ``NONE``. +- ``-D LOG_MIN=LEVEL`` -- the compile-time minimum level; messages below it are + compiled out entirely (default ``TRACE``). + +At runtime the default level can also be overridden through the +``ASDF_LOG_LEVEL`` environment variable. See the +`logging documentation `__ +for details. + Notes ^^^^^ diff --git a/cmake/ASDFOptions.cmake b/cmake/ASDFOptions.cmake index 3f19790f..c55f2d0a 100644 --- a/cmake/ASDFOptions.cmake +++ b/cmake/ASDFOptions.cmake @@ -1,11 +1,36 @@ # Configure options for ASDF library include(AddressAnalyzer) -if(ASDF_LOG_ENABLED) - option(ASDF_LOG_COLOR "Enable colored log output" ON) +# Logging (mirrors the Autotools --enable-logging / --enable-log-color / +# --with-log-default / --with-log-min options) +set(_ASDF_LOG_LEVELS TRACE DEBUG INFO WARN ERROR FATAL NONE) + +# Compile libasdf's own log statements into the library. When OFF, +# ASDF_LOG_ENABLED is left undefined and the log statements compile to nothing. +option(ENABLE_LOG "Compile in libasdf's internal log statements" ON) +if(ENABLE_LOG) + set(ASDF_LOG_ENABLED ON) +endif() + +# Colorized log output +option(ENABLE_LOG_COLOR "Enable colored log output" ON) +if(ENABLE_LOG_COLOR) + set(ASDF_LOG_COLOR ON) +endif() + +# Default runtime log level: the threshold used when none is set explicitly via +# asdf_config_t or the ASDF_LOG_LEVEL environment variable. +set(LOG_DEFAULT WARN CACHE STRING "Default runtime log level (one of ${_ASDF_LOG_LEVELS})") +if(NOT LOG_DEFAULT IN_LIST _ASDF_LOG_LEVELS) + message(FATAL_ERROR "LOG_DEFAULT must be one of: ${_ASDF_LOG_LEVELS}") +endif() + +# Compile-time minimum log level: messages below this level are compiled out. +set(LOG_MIN TRACE CACHE STRING "Compile-time minimum log level (one of ${_ASDF_LOG_LEVELS})") +if(NOT LOG_MIN IN_LIST _ASDF_LOG_LEVELS) + message(FATAL_ERROR "LOG_MIN must be one of: ${_ASDF_LOG_LEVELS}") endif() -set(ASDF_DEFAULT_LOG_LEVEL TRACE CACHE STRING "One of TRACE|DEBUG|INFO|WARN|ERROR|FATAL|NONE") -set(ASDF_LOG_MIN_LEVEL TRACE CACHE STRING "One of TRACE|DEBUG|INFO|WARN|ERROR|FATAL|NONE") + set(ASDF_DEBUG OFF CACHE BOOL "Enable DEBUG code") # Additional feature flags diff --git a/config.h.cmake b/config.h.cmake index bdbe5e7a..a600d0b7 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -2,16 +2,16 @@ #define CONFIG_H /* Enable color output in logs */ -#define ASDF_LOG_COLOR @ASDF_LOG_COLOR@ +#cmakedefine ASDF_LOG_COLOR /* Default runtime log level */ -#define ASDF_LOG_DEFAULT_LEVEL ASDF_LOG_@ASDF_LOG_MIN_LEVEL@ +#define ASDF_LOG_DEFAULT_LEVEL ASDF_LOG_@LOG_DEFAULT@ /* Enable logging */ -#define ASDF_LOG_ENABLED @ASDF_LOG_ENABLED@ +#cmakedefine ASDF_LOG_ENABLED /* Compile-time minimum log level */ -#define ASDF_LOG_MIN_LEVEL ASDF_LOG_@ASDF_LOG_MIN_LEVEL@ +#define ASDF_LOG_MIN_LEVEL ASDF_LOG_@LOG_MIN@ /* Name of package */ #define PACKAGE "@PACKAGE_NAME@" From e471ae25df651f88a8c2979a507f78125dd4a867 Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Tue, 30 Jun 2026 14:41:55 +0200 Subject: [PATCH 11/12] docs: better document the logging configuration and API The configuration options are useful to know about for end-users, especially if they want to disable logging entirely or change the log level. The logging API is probably mostly of interest for extension authors, though in principle could also be used by anyone writing a program that uses libasdf -- since it gives them a logging API for free. gh-202 --- docs/Makefile.am | 1 + docs/api/asdf/log.h.rst | 8 ++++ docs/index.rst | 1 + docs/usage/opening.rst | 50 ++++++++++++++++++-- include/asdf/file.h | 1 + include/asdf/log.h | 102 +++++++++++++++++++++++++++++++++++++--- 6 files changed, 152 insertions(+), 11 deletions(-) create mode 100644 docs/api/asdf/log.h.rst diff --git a/docs/Makefile.am b/docs/Makefile.am index 9e2eeecd..b6a284ea 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -11,6 +11,7 @@ EXTRA_DIST = \ api/asdf/error.h.rst \ api/asdf/extension.h.rst \ api/asdf/file.h.rst \ + api/asdf/log.h.rst \ api/asdf/value.h.rst \ api/asdf/yaml.h.rst \ environment.yml \ diff --git a/docs/api/asdf/log.h.rst b/docs/api/asdf/log.h.rst new file mode 100644 index 00000000..b9efa5cc --- /dev/null +++ b/docs/api/asdf/log.h.rst @@ -0,0 +1,8 @@ +:tocdepth: 2 + +.. _log.h: + +asdf/log.h +========== + +.. autodoc:: include/asdf/log.h diff --git a/docs/index.rst b/docs/index.rst index 02b0c473..529b8254 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -50,6 +50,7 @@ headers. api/asdf/emitter.h api/asdf/extension.h + api/asdf/log.h api/asdf/yaml.h diff --git a/docs/usage/opening.rst b/docs/usage/opening.rst index cffe701e..cdcf4091 100644 --- a/docs/usage/opening.rst +++ b/docs/usage/opening.rst @@ -111,7 +111,7 @@ filled in with the default settings. Any option in the user-provided config left as ``0`` will be filled in from the default configuration. -Currently the only configuration options documented for end users are those +The configuration options most likely to be of interest to end users are those under ``decomp``, which control the behavior of *decompression* of compressed blocks: @@ -126,6 +126,48 @@ blocks: decompression temporary file. These are described in detail, along with the trade-offs between the different -decompression modes, in :ref:`compression`. The remaining ``parser``, -``emitter``, and ``log`` sub-structs are lower-level and not yet documented for -general use. +decompression modes, in :ref:`compression`. + +The ``log`` sub-struct (an `asdf_log_cfg_t`) controls libasdf's diagnostic +logging for the file -- the verbosity level, the destination stream, and the +formatting; see :ref:`logging` below. The remaining ``parser`` and ``emitter`` +sub-structs are lower-level and not yet documented for general use. + + +.. _logging: + +Logging +------- + +libasdf can emit diagnostic log messages associated with an open file. Logging +is configured per file through the ``log`` field of `asdf_config_t` (an +`asdf_log_cfg_t`), so different files may log independently. For example, to +send informational and higher-severity messages to ``stderr``: + +.. code:: c + + asdf_config_t config = { + .log = {.level = ASDF_LOG_INFO} + }; + asdf_file_t *file = asdf_open_ex("observation.asdf", "r", &config); + +Messages are emitted only if their severity meets the configured +`asdf_log_level_t` threshold. Any field left zero-initialized takes a default: +the destination defaults to ``stderr``, the included fields to +`ASDF_LOG_FIELD_ALL`, and the level to the value of the ``ASDF_LOG_LEVEL`` +environment variable (one of ``NONE``, ``TRACE``, ``DEBUG``, ``INFO``, +``WARN``, ``ERROR``, ``FATAL``), or ``WARN`` if it is unset. This makes it +possible to raise the log level of an application without recompiling it:: + + ASDF_LOG_LEVEL=debug ./my-program observation.asdf + +.. note:: + + libasdf's own internal log statements are compiled in only when the library + is built with logging enabled (the default; disable with ``-DENABLE_LOG=NO`` + under CMake or ``--disable-logging`` under the Autotools build). The logging + API itself -- `asdf_file_log` and the `ASDF_LOG` macro -- is always available + for extension authors to emit their own messages through the same + configuration. + +See :ref:`log.h` for the full logging API. diff --git a/include/asdf/file.h b/include/asdf/file.h index 0c669ee7..9fdda323 100644 --- a/include/asdf/file.h +++ b/include/asdf/file.h @@ -99,6 +99,7 @@ typedef struct { /** Low-level emitter configuration; see `asdf_emitter_cfg_t` */ asdf_emitter_cfg_t emitter; + /** Logging configuration; see ``asdf_log_cfg_t`` */ asdf_log_cfg_t log; /** Decompression options */ diff --git a/include/asdf/log.h b/include/asdf/log.h index a6178555..307756ae 100644 --- a/include/asdf/log.h +++ b/include/asdf/log.h @@ -1,3 +1,30 @@ +/** + * .. _asdf/log.h: + * + * Logging configuration and the logging API. + * + * libasdf emits diagnostic log messages associated with each open file. Every + * `asdf_file_t` carries a logging configuration (an `asdf_log_cfg_t`), which can + * be provided up front through the ``log`` field of `asdf_config_t` when opening + * a file with `asdf_open_ex`. Messages at or above the configured + * `asdf_log_level_t` are written to the configured stream (``stderr`` by + * default), optionally colorized and with a configurable set of prefix fields. + * + * If no level is set explicitly the default is taken from the ``ASDF_LOG_LEVEL`` + * environment variable (one of ``NONE``, ``TRACE``, ``DEBUG``, ``INFO``, + * ``WARN``, ``ERROR``, or ``FATAL``, case-insensitive), falling back to + * ``WARN``. + * + * The library's own internal log statements are compiled in only when libasdf + * is built with logging enabled (the default; disable with ``-DENABLE_LOG=NO`` + * under CMake or ``--disable-logging`` under the Autotools build). The public + * `asdf_file_log` function and `ASDF_LOG` macro are provided for extension + * authors who wish to emit messages through the same configuration; see + * :ref:`extensions`. + */ + +// + #ifndef ASDF_LOG_H #define ASDF_LOG_H @@ -7,16 +34,30 @@ #include +/** + * Severity levels for log messages, in increasing order of severity + * + * A configured level acts as a threshold: only messages at that level or higher + * are emitted. `ASDF_LOG_NONE` disables logging entirely. + */ typedef enum { + /** Emit no messages */ ASDF_LOG_NONE = 0, + /** Very fine-grained tracing messages */ ASDF_LOG_TRACE, + /** Debugging messages */ ASDF_LOG_DEBUG, + /** Informational messages */ ASDF_LOG_INFO, + /** Warnings about recoverable problems */ ASDF_LOG_WARN, + /** Errors */ ASDF_LOG_ERROR, + /** Fatal, unrecoverable errors */ ASDF_LOG_FATAL, } asdf_log_level_t; +/** The number of distinct `asdf_log_level_t` values */ #define ASDF_LOG_NUM_LEVELS (ASDF_LOG_FATAL + 1) @@ -28,11 +69,24 @@ typedef enum { X(ASDF_LOG_FIELD_MSG, 4) +/** Bitmask selecting every available log field */ #define ASDF_LOG_FIELD_ALL \ (ASDF_LOG_FIELD_LEVEL | ASDF_LOG_FIELD_PACKAGE | ASDF_LOG_FIELD_FILE | ASDF_LOG_FIELD_LINE | \ ASDF_LOG_FIELD_MSG) +/** + * Flags selecting which fields the standard log formatter includes in each + * line + * + * Combine these as a bitmask in `asdf_log_cfg_t.fields`. The available flags + * are ``ASDF_LOG_FIELD_LEVEL`` (the severity), ``ASDF_LOG_FIELD_PACKAGE`` (the + * originating package/library), ``ASDF_LOG_FIELD_FILE`` and + * ``ASDF_LOG_FIELD_LINE`` (the source location), and ``ASDF_LOG_FIELD_MSG`` + * (the message text itself). `ASDF_LOG_FIELD_ALL` selects them all. + * + * Log formatting is not otherwise customizable yet. + */ typedef enum { // clang-format off #define X(flag, bit) flag = (1UL << (bit)), @@ -42,20 +96,33 @@ typedef enum { } asdf_log_field_t; +/** A bitmask of `asdf_log_field_t` flags */ typedef uint64_t asdf_log_fields_t; +/** + * Per-file logging configuration + * + * Pass one of these as the ``log`` field of `asdf_config_t` to `asdf_open_ex` + * to control logging for a file. Any field left zero-initialized is filled in + * with a default: the stream defaults to ``stderr``, the level to the value of + * the ``ASDF_LOG_LEVEL`` environment variable (or ``ASDF_LOG_WARN``), and the + * fields to `ASDF_LOG_FIELD_ALL`. + */ typedef struct { - FILE *stream; /* Currently just defaults to stderr */ + /** Destination stream for log output (defaults to ``stderr``) */ + FILE *stream; + /** Minimum severity to emit; see `asdf_log_level_t` */ asdf_log_level_t level; /** * Basic configuration of what fields are included in the standard log * formatter * * Log formatting is not fully customizable yet but specific fields may be - * enabled / disabled. + * enabled / disabled; see `asdf_log_field_t`. */ asdf_log_fields_t fields; + /** Disable colorized output even when the build supports it */ bool no_color; } asdf_log_cfg_t; @@ -67,10 +134,20 @@ typedef struct asdf_file asdf_file_t; ASDF_BEGIN_DECLS /** - * Public logging API for extension authors. + * Emit a log message associated with a file's logging configuration * - * Log a message associated with the given file's log configuration. - * Use asdf_value_file(value) to get the file pointer from a value. + * This is the public logging entry point for extension authors. The message + * is formatted like ``printf`` and emitted only if ``level`` meets the + * threshold configured for ``file``. In most cases the `ASDF_LOG` macro is + * more convenient, as it fills in the source file and line automatically. + * + * :param file: The `asdf_file_t *` whose log configuration to use; obtain it + * from an `asdf_value_t` with `asdf_value_file` if needed + * :param level: The severity of the message; see `asdf_log_level_t` + * :param src_file: Source file name to report (e.g. ``__FILE__``) + * :param lineno: Source line number to report (e.g. ``__LINE__``) + * :param fmt: A ``printf``-style format string + * :param ...: Arguments for ``fmt`` */ ASDF_EXPORT void asdf_file_log( const asdf_file_t *file, @@ -80,9 +157,20 @@ ASDF_EXPORT void asdf_file_log( const char *fmt, ...); +/** + * Convenience wrapper around `asdf_file_log` that supplies ``__FILE__`` and + * ``__LINE__`` automatically + * + * Expands to nothing unless libasdf is built with logging enabled (the + * ``ASDF_LOG_ENABLED`` macro defined), so log statements can be left in place + * with no overhead in a non-logging build. + * + * :param file: The `asdf_file_t *` whose log configuration to use + * :param level: The severity of the message; see `asdf_log_level_t` + * :param ...: A ``printf``-style format string followed by its arguments + */ #ifdef ASDF_LOG_ENABLED -#define ASDF_LOG(file, level, ...) \ - asdf_file_log((file), (level), __FILE__, __LINE__, __VA_ARGS__) +#define ASDF_LOG(file, level, ...) asdf_file_log((file), (level), __FILE__, __LINE__, __VA_ARGS__) #else #define ASDF_LOG(file, level, ...) ((void)0) #endif From 9cd154a96cfd38981babe57d471b0fda8294677d Mon Sep 17 00:00:00 2001 From: "E. Madison Bray" <676149+embray@users.noreply.github.com> Date: Tue, 30 Jun 2026 16:09:59 +0200 Subject: [PATCH 12/12] docs: fill in several more missing and incomplete API documentations gh-202 --- docs/conf.py | 1 + docs/usage/extensions.rst | 5 -- include/asdf/core/datatype.h | 28 ++++++++ include/asdf/extension.h | 87 +++++++++++++++++++++-- include/asdf/file.h | 5 ++ include/asdf/log.h | 6 +- include/asdf/value.h | 129 +++++++++++++++++++++++++++++++++++ 7 files changed, 250 insertions(+), 11 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index b5f55e60..2d2a82e8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -120,6 +120,7 @@ def read_config_h() -> tuple[str, str, str]: ('c:identifier', 'asdf_history_entry_t'), ('c:identifier', 'asdf_parser_cfg_t'), ('c:identifier', 'asdf_tag_t'), + ('c:identifier', 'asdf_version_t'), ] # Add intersphinx mappings diff --git a/docs/usage/extensions.rst b/docs/usage/extensions.rst index 98eec094..981c3f7d 100644 --- a/docs/usage/extensions.rst +++ b/docs/usage/extensions.rst @@ -106,11 +106,6 @@ represented as some more complex structure: foo: !tests/foo-1.0.0 foo ... -.. todo:: - - The ``core/complex`` tag might be a useful example to point to here, but we - haven't implemented it yet. - For this example we need to write a few pieces of code: * A struct representing for our "foo" type (an extension object may also be diff --git a/include/asdf/core/datatype.h b/include/asdf/core/datatype.h index c8a8eba6..1bffc833 100644 --- a/include/asdf/core/datatype.h +++ b/include/asdf/core/datatype.h @@ -41,19 +41,33 @@ ASDF_BEGIN_DECLS typedef enum { /** Reserved for invalid/unsupported datatypes */ ASDF_DATATYPE_UNKNOWN = 0, + /** Signed 8-bit integer */ ASDF_DATATYPE_INT8, + /** Unsigned 8-bit integer */ ASDF_DATATYPE_UINT8, + /** Signed 16-bit integer */ ASDF_DATATYPE_INT16, + /** Unsigned 16-bit integer */ ASDF_DATATYPE_UINT16, + /** Signed 32-bit integer */ ASDF_DATATYPE_INT32, + /** Unsigned 32-bit integer */ ASDF_DATATYPE_UINT32, + /** Signed 64-bit integer */ ASDF_DATATYPE_INT64, + /** Unsigned 64-bit integer */ ASDF_DATATYPE_UINT64, + /** 16-bit IEEE float (not yet fully supported) */ ASDF_DATATYPE_FLOAT16, + /** 32-bit IEEE float */ ASDF_DATATYPE_FLOAT32, + /** 64-bit IEEE float */ ASDF_DATATYPE_FLOAT64, + /** 64-bit complex (pair of 32-bit floats; not yet fully supported) */ ASDF_DATATYPE_COMPLEX64, + /** 128-bit complex (pair of 64-bit floats; not yet fully supported) */ ASDF_DATATYPE_COMPLEX128, + /** 8-bit boolean */ ASDF_DATATYPE_BOOL8, /** ASCII text datatype */ ASDF_DATATYPE_ASCII, @@ -105,13 +119,27 @@ typedef struct asdf_datatype asdf_datatype_t; struct asdf_datatype { + /** The scalar type of the datatype (or its elements, for compound types) */ asdf_scalar_datatype_t type; + /** + * Size in bytes of a single element of this datatype + * + * May be left ``0`` for built-in numeric datatypes, in which case + * `asdf_datatype_size` computes and fills it in. For string datatypes + * (`ASDF_DATATYPE_ASCII` / `ASDF_DATATYPE_UCS4`) it must be set explicitly. + */ uint64_t size; + /** Optional name of the datatype (e.g. a field name in a compound type) */ const char *name; + /** Byte order of the elements; see `asdf_byteorder_t` */ asdf_byteorder_t byteorder; + /** Number of dimensions, for a sub-array datatype (``0`` for scalars) */ uint32_t ndim; + /** Shape of the sub-array, an array of ``ndim`` values, if any */ const uint64_t *shape; + /** Number of fields, for a compound/structured datatype */ uint32_t nfields; + /** Array of ``nfields`` member datatypes, for a compound datatype */ const asdf_datatype_t *fields; }; diff --git a/include/asdf/extension.h b/include/asdf/extension.h index e9398033..e90ca005 100644 --- a/include/asdf/extension.h +++ b/include/asdf/extension.h @@ -18,10 +18,24 @@ typedef struct { } asdf_tag_t; +/** + * Metadata describing a piece of software involved in producing an ASDF file + * + * This corresponds to the ``core/software`` schema and is used to record any + * software that produced or contributed to a file -- including libasdf itself, + * which records its own `asdf_software_t` in the file's ``asdf_library`` + * metadata. It is most relevant to extension authors, who pass a pointer to + * one of these to `ASDF_REGISTER_EXTENSION` to document the software that + * implements their extension. See :ref:`extensions`. + */ typedef struct { + /** Name of the software */ const char *name; + /** Version of the software; see ``asdf_version_t`` */ const asdf_version_t *version; + /** Optional author of the software */ const char *author; + /** Optional homepage URL for the software */ const char *homepage; } asdf_software_t; @@ -57,17 +71,41 @@ struct _asdf_extension { typedef struct _asdf_extension asdf_extension_t; +/** + * Register an extension with the library + * + * This is normally called automatically by the constructor that + * `ASDF_REGISTER_EXTENSION` generates, and rarely needs to be called directly. + * + * :param ext: The `asdf_extension_t *` to register + */ ASDF_EXPORT void asdf_extension_register(asdf_extension_t *ext); + +/** + * Look up a registered extension by its tag + * + * :param file: The `asdf_file_t *` for the file + * :param tag: The tag string the extension was registered under + * :return: The matching `asdf_extension_t *`, or ``NULL`` if no extension is + * registered for ``tag`` + */ ASDF_EXPORT const asdf_extension_t *asdf_extension_get(asdf_file_t *file, const char *tag); /** - * Parse a tag string of the form "name" or "name-version" into an - * asdf_tag_t. Returns NULL on OOM. The caller owns the result - * and must free it with asdf_tag_destroy. + * Parse a tag string of the form ``"name"`` or ``"name-version"`` into an + * ``asdf_tag_t`` + * + * :param tag: The tag string to parse + * :return: A newly allocated ``asdf_tag_t *`` owned by the caller (free it with + * `asdf_tag_destroy`), or ``NULL`` on failure */ ASDF_EXPORT asdf_tag_t *asdf_tag_parse(const char *tag); -/** Free a tag returned by asdf_tag_parse. */ +/** + * Free a tag returned by `asdf_tag_parse` + * + * :param tag: The ``asdf_tag_t *`` to free + */ ASDF_EXPORT void asdf_tag_destroy(asdf_tag_t *tag); @@ -196,6 +234,35 @@ ASDF_EXPORT void asdf_tag_destroy(asdf_tag_t *tag); } +/** + * Define and register an extension type + * + * Generates the full family of public accessors for an extension type and a + * constructor that registers the extension with libasdf automatically at load + * time. Place it in a single ``.c`` file; use `ASDF_DECLARE_EXTENSION` in a + * header to expose the generated functions to other translation units. See + * :ref:`extensions` for a complete walkthrough. + * + * The generated functions are named after ``extname``: ``asdf_get_``, + * ``asdf_set_``, ``asdf_is_``, ``asdf_value_is_``, + * ``asdf_value_as_``, ``asdf_value_of_``, + * ``asdf__clone``, ``asdf__array_clone``, and + * ``asdf__destroy``. + * + * :param extname: Base name for the generated functions (need not match the + * C ``type``) + * :param tag: The YAML tag string the extension is registered for + * :param type: The C type the extension deserializes to (e.g. ``asdf_foo_t``) + * :param software: Pointer to an `asdf_software_t` describing the software that + * implements the extension + * :param serialize: A ``asdf_extension_serialize_t`` callback, or ``NULL`` + * :param deserialize: A ``asdf_extension_deserialize_t`` callback + * :param copy: A ``asdf_extension_copy_t`` deep-copy callback, or ``NULL`` for + * a shallow copy + * :param dealloc: A ``asdf_extension_dealloc_t`` callback to free the type + * :param userdata: Optional ``void *`` passed through to the callbacks, or + * ``NULL`` + */ #define ASDF_REGISTER_EXTENSION( \ extname, tag, type, software, serialize, deserialize, copy, dealloc, userdata) \ ASDF_EXT_DEFINE(extname, tag, software, serialize, deserialize, copy, dealloc, userdata); \ @@ -214,7 +281,17 @@ ASDF_EXPORT void asdf_tag_destroy(asdf_tag_t *tag); } -/* Provides declarations for auto-generated extension type APIs, for use in headers */ +/** + * Declare the public API generated by `ASDF_REGISTER_EXTENSION` + * + * Place this in a header to expose an extension's generated functions (the + * ``asdf_get_``/``asdf_set_``/etc. family listed under + * `ASDF_REGISTER_EXTENSION`) to other translation units. ``extname`` and + * ``type`` must match those passed to `ASDF_REGISTER_EXTENSION`. + * + * :param extname: Base name used for the generated functions + * :param type: The C type the extension deserializes to + */ #define ASDF_DECLARE_EXTENSION(extname, type) \ ASDF_EXPORT asdf_value_err_t asdf_value_as_##extname(asdf_value_t *value, type **out); \ ASDF_EXPORT bool asdf_value_is_##extname(asdf_value_t *value); \ diff --git a/include/asdf/file.h b/include/asdf/file.h index 9fdda323..dcae5be0 100644 --- a/include/asdf/file.h +++ b/include/asdf/file.h @@ -215,6 +215,11 @@ static asdf_file_t *asdf_open_mem(const void *buf, size_t size); * Opens an ASDF file for reading * * Equivalent to `asdf_open`. + * + * :param filename: A null-terminated string containing the local filesystem + * path to open + * :param mode: Currently must always be just ``"r"`` + * :return: An `asdf_file_t *`, or ``NULL`` on error */ static inline asdf_file_t *asdf_open_file(const char *filename, const char *mode) { return asdf_open_file_ex(filename, mode, NULL); diff --git a/include/asdf/log.h b/include/asdf/log.h index 307756ae..2fca081d 100644 --- a/include/asdf/log.h +++ b/include/asdf/log.h @@ -69,7 +69,11 @@ typedef enum { X(ASDF_LOG_FIELD_MSG, 4) -/** Bitmask selecting every available log field */ +/** + * Bitmask selecting every available log field + * + * See `asdf_log_field_t`. + */ #define ASDF_LOG_FIELD_ALL \ (ASDF_LOG_FIELD_LEVEL | ASDF_LOG_FIELD_PACKAGE | ASDF_LOG_FIELD_FILE | ASDF_LOG_FIELD_LINE | \ ASDF_LOG_FIELD_MSG) diff --git a/include/asdf/value.h b/include/asdf/value.h index 5057a8c1..05dc1a15 100644 --- a/include/asdf/value.h +++ b/include/asdf/value.h @@ -126,6 +126,7 @@ typedef enum { * `ASDF_VALUE_ERR_TYPE_MISMATCH`. */ typedef enum { + /** An unknown or unspecified error occurred */ ASDF_VALUE_ERR_UNKNOWN = -2, /** @@ -211,6 +212,9 @@ ASDF_EXPORT void asdf_value_destroy(asdf_value_t *value); * The new cloned value is not initially attached to the YAML tree, so * modification to its underlying value is not reflected in the YAML; it * may be inserted elsewhere in the tree later. + * + * :param value: The `asdf_value_t *` to clone + * :return: A new deep-copied `asdf_value_t *`, or ``NULL`` on failure */ ASDF_EXPORT asdf_value_t *asdf_value_clone(asdf_value_t *value); @@ -281,13 +285,80 @@ ASDF_EXPORT asdf_file_t *asdf_value_file(asdf_value_t *value); */ typedef struct asdf_mapping asdf_mapping_t; +/** Return ``true`` if ``value`` holds a YAML mapping */ ASDF_EXPORT bool asdf_value_is_mapping(asdf_value_t *value); + +/** + * Return the number of key/value entries in ``mapping`` + * + * :param mapping: The `asdf_mapping_t *` to query + * :return: The number of entries currently in the mapping + */ ASDF_EXPORT int asdf_mapping_size(asdf_mapping_t *mapping); + +/** + * Obtain a typed `asdf_mapping_t *` view of a generic value + * + * On success writes the `asdf_mapping_t *` into ``*out`` and returns + * ``ASDF_VALUE_OK``. Returns an error code and leaves ``*out`` unchanged if + * ``value`` is not a mapping. + * + * :param value: The generic `asdf_value_t *` to inspect + * :param out: Receives the `asdf_mapping_t *` on success + * :return: ``ASDF_VALUE_OK`` on success, otherwise an `asdf_value_err_t` error + */ ASDF_EXPORT asdf_value_err_t asdf_value_as_mapping(asdf_value_t *value, asdf_mapping_t **out); + +/** + * Return the generic `asdf_value_t *` view of a mapping + * + * The returned pointer shares ownership with ``mapping``; the caller must not + * free it independently. + * + * :param mapping: The `asdf_mapping_t *` to wrap + * :return: The same object as a generic `asdf_value_t *` + */ ASDF_EXPORT asdf_value_t *asdf_value_of_mapping(asdf_mapping_t *mapping); + +/** + * Create a new, empty mapping attached to ``file`` + * + * The caller owns the returned mapping and must eventually release it with + * `asdf_mapping_destroy`, or transfer ownership by inserting it into a parent + * mapping or sequence. Returns ``NULL`` on allocation failure. + * + * :param file: The `asdf_file_t *` that will own the mapping + * :return: A new `asdf_mapping_t *`, or ``NULL`` on failure + */ ASDF_EXPORT asdf_mapping_t *asdf_mapping_create(asdf_file_t *file); + +/** + * Set the YAML node style used when serializing ``mapping`` + * + * :param mapping: The `asdf_mapping_t *` to modify + * :param style: The desired `asdf_yaml_node_style_t` (e.g. block or flow) + */ ASDF_EXPORT void asdf_mapping_set_style(asdf_mapping_t *mapping, asdf_yaml_node_style_t style); + +/** + * Deep-copy a mapping + * + * The returned mapping is not attached to the tree; it may be inserted + * elsewhere later or released with `asdf_mapping_destroy`. + * + * :param mapping: The `asdf_mapping_t *` to clone + * :return: A new `asdf_mapping_t *`, or ``NULL`` on failure + */ ASDF_EXPORT asdf_mapping_t *asdf_mapping_clone(asdf_mapping_t *mapping); + +/** + * Free a mapping and all values it contains + * + * Must not be called on a mapping that has been inserted into a parent mapping + * or sequence -- ownership transfers at that point. + * + * :param mapping: The `asdf_mapping_t *` to free + */ ASDF_EXPORT void asdf_mapping_destroy(asdf_mapping_t *mapping); /** @@ -825,6 +896,7 @@ ASDF_EXPORT bool asdf_container_iter_next(asdf_container_iter_t **iter); */ ASDF_EXPORT void asdf_container_iter_destroy(asdf_container_iter_t *iter); +/** Return ``true`` if ``value`` holds a container (a mapping or a sequence) */ ASDF_EXPORT bool asdf_value_is_container(asdf_value_t *value); @@ -862,6 +934,11 @@ typedef struct _asdf_extension asdf_extension_t; * * const asdf_extension_t *ext = asdf_extension_get(file, "tag:..."); * asdf_value_is_extension_type(value, ext); + * + * :param value: The `asdf_value_t *` handle + * :param ext: The `asdf_extension_t *` describing the extension type + * :return: ``true`` if ``value`` is of the given extension type, otherwise + * ``false`` */ ASDF_EXPORT bool asdf_value_is_extension_type(asdf_value_t *value, const asdf_extension_t *ext); @@ -881,11 +958,30 @@ ASDF_EXPORT bool asdf_value_is_extension_type(asdf_value_t *value, const asdf_ex * asdf_ndarray_t *ndarray = NULL; * const asdf_extension_t *ext = asdf_extension_get(file, "tag:..."); * asdf_value_as_extension_type(value, ext, &ndarray); + * + * :param value: The `asdf_value_t *` handle + * :param ext: The `asdf_extension_t *` describing the extension type + * :param out: Receives the deserialized extension object on success; the + * concrete type of ``*out`` is the C type associated with the extension + * :return: `ASDF_VALUE_OK` if the value is of the extension type, otherwise + * `ASDF_VALUE_ERR_TYPE_MISMATCH` */ ASDF_EXPORT asdf_value_err_t asdf_value_as_extension_type(asdf_value_t *value, const asdf_extension_t *ext, void **out); +/** + * Construct a generic `asdf_value_t *` by serializing an extension object + * + * The low-level counterpart to the per-extension ``asdf_value_of_`` + * helpers; serializes ``obj`` according to ``ext`` into a new value owned by + * ``file``. + * + * :param file: The `asdf_file_t *` that will own the returned value + * :param obj: Pointer to the extension object to serialize + * :param ext: The `asdf_extension_t *` describing the extension type + * :return: A new `asdf_value_t *`, or ``NULL`` on failure + */ ASDF_EXPORT asdf_value_t *asdf_value_of_extension_type( asdf_file_t *file, const void *obj, const asdf_extension_t *ext); @@ -904,6 +1000,10 @@ ASDF_EXPORT asdf_value_t *asdf_value_of_extension_type( * * For checking against a specific extension type it's still necessary to use * `asdf_value_is_extension_type` + * + * :param value: The `asdf_value_t *` handle + * :param type: The `asdf_value_type_t` to test against + * :return: ``true`` if ``value`` is of the given type, otherwise ``false`` */ ASDF_EXPORT bool asdf_value_is_type(asdf_value_t *value, asdf_value_type_t type); @@ -919,6 +1019,12 @@ ASDF_EXPORT bool asdf_value_is_type(asdf_value_t *value, asdf_value_type_t type) * * For getting the value of a specific extension type it's still necessary to use * `asdf_value_as_extension_type` + * + * :param value: The `asdf_value_t *` handle + * :param type: The `asdf_value_type_t` to coerce the value to + * :param out: Address of the output destination, whose concrete type + * corresponds to ``type`` + * :return: `ASDF_VALUE_OK` on success, otherwise an `asdf_value_err_t` error */ ASDF_EXPORT asdf_value_err_t asdf_value_as_type(asdf_value_t *value, asdf_value_type_t type, void *out); @@ -931,6 +1037,29 @@ asdf_value_as_type(asdf_value_t *value, asdf_value_type_t type, void *out); * ------- */ +/** + * The functions below are the `asdf_value_t`-level scalar accessors, and come + * in three families for each scalar type (``string``, ``scalar``, ``bool``, + * ``null``, the sized integer types, ``float``, and ``double``): + * + * * ``asdf_value_is_`` -- return ``true`` if the value is of that type + * (following the `YAML Core Schema`_ interpretation; see :ref:`values`). + * * ``asdf_value_as_`` -- coerce the value to the corresponding C type, + * writing it through an output pointer and returning an `asdf_value_err_t` + * (``ASDF_VALUE_OK`` on success, ``ASDF_VALUE_ERR_TYPE_MISMATCH`` if the + * value is not of that type, and ``ASDF_VALUE_ERR_OVERFLOW`` for numeric + * values that do not fit the requested type). + * * ``asdf_value_of_`` -- construct a new `asdf_value_t *` from a C + * value, for insertion into the tree; it is owned by ``file``. + * + * These are the same operations performed by the file-level ``asdf_get_`` + * and ``asdf_set_`` functions, but acting directly on a generic + * `asdf_value_t *` rather than looking the value up by path. The ``string`` + * and ``scalar`` accessors have ``0``-suffixed variants + * (e.g. ``asdf_value_as_string0``) that return a NUL-terminated string instead + * of a pointer/length pair. + */ + ASDF_EXPORT bool asdf_value_is_string(asdf_value_t *value); ASDF_EXPORT asdf_value_err_t asdf_value_as_string(asdf_value_t *value, const char **out, size_t *out_len);