diff --git a/README.rst b/README.rst index 750df4ba..44577dbb 100644 --- a/README.rst +++ b/README.rst @@ -35,7 +35,8 @@ This returns an ``asdf_file_t *`` which is your main interface to the ASDF file. When done with the file make sure to call ``asdf_close`` to free resources: .. code:: c - :name: test-open-close-file + :test: test-open-close-file + :fixture: cube.asdf #include #include @@ -57,120 +58,186 @@ When done with the file make sure to call ``asdf_close`` to free resources: return 0; } -The following more complete example demonstrates how to read different metadata out of -the ASDF tree, as well as extract block data. Inline comments provide further explanation: +The next example demonstrates how to write some simple metadata and an ndarray +to a new file: .. code:: c - :name: test-read-metadata-ndarray + :test: test-write-file + :fixture: temp:out.asdf + + #include + + int main(int argc, char **argv) { + const char *filename = "out.asdf"; + + if (argc > 1) + filename = argv[1]; + + // open a "NULL" file for writing + asdf_file_t *file = asdf_open(NULL); + + // assign a string to the "name" key of the ASDF tree + asdf_set_string0(file, "name", "Dennis Richie"); + + // assign a numeric value to the "foo" key + asdf_set_int64(file, "foo", 42); + + // construct 2 arrays containing numeric values + uint64_t N = 100; + + asdf_ndarray_t sequence = { + .ndim = 1, + .shape = (uint64_t[]){N}, + .datatype = {.type = ASDF_DATATYPE_UINT64} + }; + uint8_t *sequence_data = asdf_ndarray_data_alloc(&sequence); + + asdf_ndarray_t squares = { + .ndim = 1, + .shape = (uint64_t[]){N}, + .datatype = {.type = ASDF_DATATYPE_UINT64} + }; + uint64_t *squares_data = asdf_ndarray_data_alloc(&squares); + + for (uint64_t idx = 0; idx < N; idx++) { + sequence_data[idx] = idx; + squares_data[idx] = idx * idx; + }; + + // assign the "sequence" array to the "sequence" key + asdf_set_ndarray(file, "sequence", &sequence); + + // nest the "squares" array under a parent "powers" key + asdf_set_ndarray(file, "powers/squares", &squares); + + // write the ASDF file to disk + asdf_write_to(file, filename); + + // clean up allocations + asdf_ndarray_data_dealloc(&sequence); + asdf_ndarray_data_dealloc(&squares); + asdf_close(file); + return 0; + } + +With libasdf installed on your system (see :ref:`development`) you can compile +and run this test like: + +.. code:: sh + + $ gcc asdf-write.c -o asdf-write -lasdf + $ ./asdf-write + +It should produce an output file at ``out.asdf`` which you can inspect by hand. +The YAML portion of the ASDF file should contain: + +.. code:: yaml + + #ASDF 1.0.0 + #ASDF_STANDARD 1.6.0 + %YAML 1.1 + %TAG ! tag:stsci.edu:asdf/ + --- !core/asdf-1.1.0 + asdf_library: !core/software-1.0.0 + name: libasdf + version: 0.1.0a2 + author: The libasdf Developers + homepage: https://github.com/asdf-format/libasdf + name: Dennis Richie + foo: 42 + sequence: !core/ndarray-1.1.0 + source: 0 + datatype: uint64 + shape: [ + 100 + ] + byteorder: little + powers: + squares: !core/ndarray-1.1.0 + source: 1 + datatype: uint64 + shape: [ + 100 + ] + byteorder: little + ... + + +The next example shows how to read back in the same file: + +.. code:: c + :test: test-read-file + :fixture: test-write-file.asdf #include #include #include int main(int argc, char **argv) { - if (argc < 2) { - fprintf(stderr, "Usage: %s filename\n", argv[0]); - return 1; - } - const char *filename = argv[1]; + const char *filename = "out.asdf"; - // The mode string "r" is required and is the only currently-supported mode + if (argc > 1) + filename = argv[1]; + + // open the ASDF file for reading asdf_file_t *file = asdf_open(filename, "r"); - - if (file == NULL) { - fprintf(stderr, "error opening the ASDF file\n"); + if (!file) { + fprintf(stderr, "Failed to open the file: %s\n", asdf_error(file)); return 1; } - - // The simplest way to read metadata from the file is with the - // `asdf_get_*` family of functions - // They all return a value by pointer argument and return an - // `asdf_value_error_t` - // For example you can read a string from the metadata like: - - const char *software = NULL; - // Returns a 0-terminated string into *software. - asdf_value_err_t err = asdf_get_string0(file, "asdf_library/author", &software); - - if (err == ASDF_VALUE_OK) { - printf("software: %s\n", software); - } - - // Other errors could be e.g. ASDF_VALUE_ERR_NOT_FOUND if the key doesn't - // exist, or ASDF_VALUE_ERR_TYPE_MISMATCH if it's not a string. - - // There are also extensions registered for some (not all yet) of the - // core schemas. Objects defined by extension schemas (identified by - // their YAML tags) also have corresponding asdf_get_ functions: - asdf_meta_t *meta = NULL; - - // This reads the top-level core/asdf-1.0.0 schema - err = asdf_get_meta(file, "/", &meta); - if (err == ASDF_VALUE_OK) { - if (meta->history.entries[0]) { - // This is a NULL-terminated array of asdf_history_entry_t* - printf("first history entry: %s\n", meta->history.entries[0]->description); - } - } - - // Functions like `asdf_get_meta` that return into a double-pointer to a - // struct allocate memory for that structure automatically. - // The all have a corresponding `asdf__destroy` function. - // The plan is to track these on the file object (issue #34) to make - // memory management easier and cleaner, but for now you have to free - // them manually when you're done with them. This is good practice in any - // case. - asdf_meta_destroy(meta); - - // ndarrays work no differently; this reads an ndarray named "cube". - asdf_ndarray_t *ndarray = NULL; - err = asdf_get_ndarray(file, "cube", &ndarray); - if (err != ASDF_VALUE_OK) { - fprintf(stderr, "error reading ndarray metadata: %d\n", err); - return 1; + + // read and print the string stored under "name" + const char *name = NULL; + if (asdf_get_string0(file, "name", &name) == ASDF_VALUE_OK) { + printf("name: %s\n", name); } - - printf("number of data dimensions: %d\n", ndarray->ndim); - - // Get just a raw pointer to the ndarray data block (if uncompressed). - // Optionally returns the size in bytes as well - size_t size = 0; - const void *data = asdf_ndarray_data(ndarray, &size); - - if (data == NULL) { - fprintf(stderr, "error reading ndarray data\n"); - return 1; + + // read and print the numeric value stored under "foo" + int64_t foo = 0; + if (asdf_get_int64(file, "foo", &foo) == ASDF_VALUE_OK) { + printf("foo: %li\n", foo); } - - // Slightly more useful is the asdf_ndarray_read_tile_ functions. - // They can copy the data, including converting endianness into a tile - // buffer. If an existing buffer is not passed it will allocate one of - // the correct size to hold the data. The user is responsible for - // freeing the buffer. - - // Read a 10x10x10 cube - const uint64_t origin[3] = {0, 0, 0}; - const uint64_t shape[3] = {10, 10, 10}; - void *tile = NULL; - asdf_ndarray_err_t array_err = asdf_ndarray_read_tile_ndim( - ndarray, - origin, - shape, - ASDF_DATATYPE_SOURCE, - &tile - ); - - if (array_err != ASDF_NDARRAY_OK) { - fprintf(stderr, "error reading ndarray: %d\n", array_err); - return 1; + + // read the "squares" array nested under the "powers" key + asdf_ndarray_t *squares = NULL; + uint64_t *squares_data = NULL; + if (asdf_get_ndarray(file, "powers/squares", &squares) == ASDF_VALUE_OK) { + if (asdf_ndarray_read_all(squares, ASDF_DATATYPE_UINT64, (void **)&squares_data) == ASDF_NDARRAY_OK) { + // print the sum of the squares array + uint64_t nelem = asdf_ndarray_size(squares); + uint64_t sum = 0; + for (uint64_t idx = 0; idx < nelem; idx++) { + sum += squares_data[idx]; + } + printf("sum of squares values: %li\n", sum); + } } - - free(tile); - asdf_ndarray_destroy(ndarray); + + // clean up allocations + free(squares_data); + asdf_ndarray_destroy(squares); asdf_close(file); return 0; } +Likewise compile and run the example with the output from the previous program: + +.. code:: sh + + $ gcc asdf-read.c -o asdf-read -lasdf + $ ./asdf-read + +This should output:: + + name: Dennis Richie + foo: 42 + sum of squares values: 328350 + +Additional examples can be found in the +`libasdf documentation `__. + + +.. _development: Development =========== diff --git a/docs/Makefile.am b/docs/Makefile.am index 441f61cd..d395959f 100644 --- a/docs/Makefile.am +++ b/docs/Makefile.am @@ -4,8 +4,24 @@ CMAKE_DIST = CMakeLists.txt EXTRA_DIST = \ $(CMAKE_DIST) \ conf.py \ + api/asdf/core/datatype.h.rst \ + api/asdf/core/ndarray.h.rst \ + api/asdf/error.h.rst \ + api/asdf/emitter.h.rst \ + api/asdf/error.h.rst \ + api/asdf/extension.h.rst \ + api/asdf/file.h.rst \ + api/asdf/value.h.rst \ + api/asdf/yaml.h.rst \ environment.yml \ index.rst \ + links.rst \ + usage/examples.rst \ + usage/extensions.rst \ + usage/opening.rst \ + usage/overview.rst \ + usage/values.rst \ + usage/writing.rst \ _static/css/globalnav.css \ _static/images/favicon.ico \ _static/images/logo-dark-mode.png \ diff --git a/docs/conf.py b/docs/conf.py index 877eae08..055d53eb 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -2,6 +2,9 @@ from datetime import datetime from pathlib import Path +from docutils.parsers.rst import directives +from sphinx.directives.patches import Code + # -- Project information ------------------------------------------------------ def read_config_h() -> tuple[str, str, str]: @@ -169,5 +172,24 @@ def read_config_h() -> tuple[str, str, str]: latex_logo = "_static/images/logo-light-mode.png" +# -- Doc-example test directive options ---------------------------------------- +# The tests/scripts/extract_doc_examples.py script extracts ``.. code:: c`` +# blocks from the documentation and compiles/runs them as part of the test +# suite. A block is marked for extraction with the ``:test:`` option (whose +# value is the test name) and may declare an input file with ``:fixture:``. +# +# These options are meaningful only to the extraction script; here we simply +# extend the ``code`` directive to accept (and otherwise ignore) them so that +# the documentation still builds without "unknown option" errors. +# +# TODO: Make this more extensible; maybe spin out to a separate plugin +# Sphinx could use an extension for compilable source code doctests like this... +class TestableCode(Code): + option_spec = dict(Code.option_spec) + option_spec["test"] = directives.unchanged + option_spec["fixture"] = directives.unchanged + + def setup(app): app.add_css_file("css/globalnav.css") + app.add_directive("code", TestableCode, override=True) diff --git a/docs/index.rst b/docs/index.rst index 9a0801f8..8d3ba672 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -19,6 +19,7 @@ Usage usage/values usage/writing usage/extensions + usage/examples API documentation diff --git a/docs/usage/examples.rst b/docs/usage/examples.rst new file mode 100644 index 00000000..6104265c --- /dev/null +++ b/docs/usage/examples.rst @@ -0,0 +1,180 @@ +.. _examples: + +Additional usage examples +========================= + +Miscellaneous usage examples collected from the libasdf documentation and tests, as well as from +the community. Submissions welcome! + +The following more complete example demonstrates how to read different metadata out of +the ASDF tree, as well as extract block data. Inline comments provide further explanation: + +.. code:: c + :test: test-read-metadata-ndarray + :fixture: cube.asdf + + #include + #include + #include + + int main(int argc, char **argv) { + if (argc < 2) { + fprintf(stderr, "Usage: %s filename\n", argv[0]); + return 1; + } + const char *filename = argv[1]; + + // The mode string "r" is required and is the only currently-supported mode + asdf_file_t *file = asdf_open(filename, "r"); + + if (file == NULL) { + fprintf(stderr, "error opening the ASDF file\n"); + return 1; + } + + // The simplest way to read metadata from the file is with the + // `asdf_get_*` family of functions + // They all return a value by pointer argument and return an + // `asdf_value_error_t` + // For example you can read a string from the metadata like: + + const char *software = NULL; + // Returns a 0-terminated string into *software. + asdf_value_err_t err = asdf_get_string0(file, "asdf_library/author", &software); + + if (err == ASDF_VALUE_OK) { + printf("Software: %s\n", software); + } + + // Other errors could be e.g. ASDF_VALUE_ERR_NOT_FOUND if the key doesn't + // exist, or ASDF_VALUE_ERR_TYPE_MISMATCH if it's not a string. + + // There are also extensions registered for some (not all yet) of the + // core schemas. Objects defined by extension schemas (identified by + // their YAML tags) also have corresponding asdf_get_ functions: + asdf_meta_t *meta = NULL; + + // This reads the top-level core/asdf-1.0.0 schema + err = asdf_get_meta(file, "/", &meta); + if (err == ASDF_VALUE_OK) { + if (meta->history.entries && meta->history.entries[0]) { + // This is a NULL-terminated array of asdf_history_entry_t* + printf("First history entry: %s\n", meta->history.entries[0]->description); + } else { + printf("File does not contain any history entries\n"); + } + } + + // Functions like `asdf_get_meta` that return into a double-pointer to a + // struct allocate memory for that structure automatically. + // The all have a corresponding `asdf__destroy` function. + // The plan is to track these on the file object (issue #34) to make + // memory management easier and cleaner, but for now you have to free + // them manually when you're done with them. This is good practice in any + // case. + asdf_meta_destroy(meta); + + // Find the first ndarray in the file, if any + asdf_value_t *root = asdf_get_value(file, ""); + asdf_value_t *value = asdf_value_find(root, asdf_value_is_ndarray); + + if (!value) { + fprintf(stderr, "no ndarray found in the file\n"); + return 1; + } + + // Generic values can be *cast* to a specific type with the + // `asdf_value_as_` API, for example to cast to an ndarray and check that + // the cast succeeded: + asdf_ndarray_t *ndarray = NULL; + err = asdf_value_as_ndarray(value, &ndarray); + if (err != ASDF_VALUE_OK) { + fprintf(stderr, "error reading ndarray metadata: %d\n", err); + return 1; + } + + printf("Using ndarray at: %s\n", asdf_value_path(value)); + printf("Number of data dimensions: %d\n", ndarray->ndim); + + // The generic value wrappers are no longer needed and should be freed. + asdf_value_destroy(value); + asdf_value_destroy(root); + + // Get just a raw pointer to the ndarray data block (if uncompressed). + // Optionally returns the size in bytes as well + size_t size = 0; + const void *data = asdf_ndarray_data(ndarray, &size); + + if (data == NULL) { + fprintf(stderr, "error reading ndarray data\n"); + return 1; + } + + // The asdf_ndarray_read_tile_ functions copy a rectangular cutout of + // the array into a buffer, converting datatype and endianness as needed. + // If you don't pass your own buffer one is allocated for you; either way + // you are responsible for freeing it. + + // origin and shape must have one entry per array dimension, so we size + // them to the array we found. Here we take a cutout of up to 5 elements + // along each axis, clamped to the array's actual size. + uint64_t *origin = calloc(ndarray->ndim, sizeof(uint64_t)); + uint64_t *shape = calloc(ndarray->ndim, sizeof(uint64_t)); + + if (origin == NULL || shape == NULL) { + fprintf(stderr, "out of memory\n"); + return 1; + } + + uint64_t tile_nelem = 1; + for (uint32_t dim = 0; dim < ndarray->ndim; dim++) { + shape[dim] = ndarray->shape[dim] < 5 ? ndarray->shape[dim] : 5; + tile_nelem *= shape[dim]; + } + + // Read the cutout, converting to double regardless of the source type + double *tile = NULL; + asdf_ndarray_err_t array_err = asdf_ndarray_read_tile_ndim( + ndarray, + origin, + shape, + ASDF_DATATYPE_FLOAT64, + (void **)&tile + ); + + free(origin); + free(shape); + + if (array_err != ASDF_NDARRAY_OK) { + fprintf(stderr, "error reading ndarray: %d\n", array_err); + return 1; + } + + printf("Value at center of cutout: %g\n", tile[tile_nelem / 2]); + + free(tile); + asdf_ndarray_destroy(ndarray); + asdf_close(file); + return 0; + } + + +With libasdf installed on your system you can compile and run this test like: + +.. code:: sh + + $ gcc asdf-test.c -o asdf-test -lasdf + $ ./asdf-test tests/fixtures/cube.asdf + +Which outputs:: + + Software: The ASDF Developers + First history entry: A very small data cube for testing + Using ndarray at: /cube + Number of data dimensions: 3 + Value at center of cutout: 222 + +In this case the test is run on the file +`cube.asdf `__ +in this repository's test fixtures, though the program should work on any ASDF +file containing a non-empty ndarray. diff --git a/docs/usage/writing.rst b/docs/usage/writing.rst index 8697475f..76b186e6 100644 --- a/docs/usage/writing.rst +++ b/docs/usage/writing.rst @@ -170,7 +170,8 @@ the *cubes* of the original values (i.e. ``v^3 mod 256``), and writes the result to an in-memory buffer. .. code:: c - :name: test-write-cube + :test: test-write-cube + :fixture: cube.asdf #include #include diff --git a/include/asdf/core/time.h b/include/asdf/core/time.h index 9f9eae76..293acffd 100644 --- a/include/asdf/core/time.h +++ b/include/asdf/core/time.h @@ -73,6 +73,7 @@ typedef struct { ASDF_DECLARE_EXTENSION(time, asdf_time_t); ASDF_EXPORT int asdf_time_parse(asdf_time_t *time); +ASDF_EXPORT const char *asdf_time_format_string(asdf_time_format_t format); ASDF_END_DECLS diff --git a/src/compat/endian.h b/src/compat/endian.h index 3e844944..a8746626 100644 --- a/src/compat/endian.h +++ b/src/compat/endian.h @@ -5,12 +5,23 @@ #include "config.h" #endif -/* Choose the right endian header */ +/* Include every endian header that exists, NOT just the first one. This must + * match the set of headers used by AX_CHECK_ENDIAN_DECL + * (m4/ax_check_endian_decl.m4), otherwise the HAVE_DECL_* results can disagree + * with what is actually visible here. On macOS, for example, both + * and *may* exist depending on the SDK + * version. but the be64toh/htobe* family is declared only in . + * + * Thank you Apple for respecting the time and sanity of developers who wish + * to support your platforms. + */ #if defined(HAVE_ENDIAN_H) #include -#elif defined(HAVE_MACHINE_ENDIAN_H) +#endif +#if defined(HAVE_MACHINE_ENDIAN_H) #include -#elif defined(HAVE_SYS_ENDIAN_H) +#endif +#if defined(HAVE_SYS_ENDIAN_H) #include #endif diff --git a/src/core/time.c b/src/core/time.c index a16f663a..5a15c0ea 100644 --- a/src/core/time.c +++ b/src/core/time.c @@ -483,6 +483,16 @@ static const char *const asdf_time_scale_names[] = { }; +const char *asdf_time_format_string(asdf_time_format_t format) { + const size_t nformats = sizeof(asdf_time_format_names) / sizeof(asdf_time_format_names[0]); + + if (format < 0 || format > nformats) + return NULL; + + return asdf_time_format_names[format]; +} + + /* Helpers for format detection and range validation */ static bool asdf_time_format_parse(const char *name, asdf_time_format_t *out) { diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 56568738..377a395f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -171,22 +171,42 @@ if(ENABLE_TESTING_DOCS) set(DOC_EXAMPLES_DIR ${CMAKE_CURRENT_BINARY_DIR}/doc_examples) # For now explicitly list files containing extractable doc examples; the # only one currently is README.rst - set(DOC_FILES ${CMAKE_SOURCE_DIR}/README.rst) + set(DOC_FILES + ${CMAKE_SOURCE_DIR}/README.rst + ${CMAKE_SOURCE_DIR}/docs/usage/examples.rst + ${CMAKE_SOURCE_DIR}/docs/usage/writing.rst + ) file(MAKE_DIRECTORY ${DOC_EXAMPLES_DIR}) set(EXTRACT_DOC_EXAMPLES ${CMAKE_CURRENT_SOURCE_DIR}/scripts/extract_doc_examples.py) set(TEST_DOC_EXAMPLES ${DOC_EXAMPLES_DIR}/test-doc-examples.sh) - # Just list the names of the doctests that should be built + # List the doctests that should be built, one per line. Each line is + # either "name" (run with no argument) or "name|arg" where arg is the + # resolved fixture file passed to the program (see extract_doc_examples.py). execute_process( COMMAND ${Python3_EXECUTABLE} ${EXTRACT_DOC_EXAMPLES} - --list-tests ${DOC_FILES} + --list-tests + --fixtures-dir ${CMAKE_CURRENT_SOURCE_DIR}/fixtures + --out-dir ${DOC_EXAMPLES_DIR} + ${DOC_FILES} OUTPUT_VARIABLE DOC_TEST_LIST OUTPUT_STRIP_TRAILING_WHITESPACE ) - string(REGEX REPLACE "\n" ";" DOC_TESTS "${DOC_TEST_LIST}") + string(REGEX REPLACE "\n" ";" DOC_TEST_LINES "${DOC_TEST_LIST}") + set(DOC_TESTS "") set(DOC_TEST_SOURCES "") - foreach (doctest IN LISTS DOC_TESTS) + foreach (line IN LISTS DOC_TEST_LINES) + # Split "name,arg" into name and fixture argument (arg may be absent) + string(REPLACE "," ";" parts "${line}") + list(GET parts 0 doctest) + list(LENGTH parts nparts) + if (nparts GREATER 1) + list(GET parts 1 DOC_TEST_ARG_${doctest}) + else() + set(DOC_TEST_ARG_${doctest} "") + endif() + list(APPEND DOC_TESTS ${doctest}) list(APPEND DOC_TEST_SOURCES ${DOC_EXAMPLES_DIR}/${doctest}.c) endforeach() @@ -217,12 +237,16 @@ if(ENABLE_TESTING_DOCS) target_include_directories(${doctest} PRIVATE ${CMAKE_SOURCE_DIR}/include) target_link_libraries(${doctest} PRIVATE libasdf) add_dependencies(${doctest} extract-doc-examples) - # All the tests are currently hard-coded to work with this specific - # test file; may change that later if we add any more of these - # doctests - add_test( - NAME ${doctest} - COMMAND ${doctest} ${CMAKE_CURRENT_BINARY_DIR}/fixtures/cube.asdf - ) + # Each doctest declares its own input file via the :fixture: option in + # the documentation; tests with no fixture run with no argument. + if (DOC_TEST_ARG_${doctest}) + add_test(NAME ${doctest} + COMMAND ${doctest} ${DOC_TEST_ARG_${doctest}}) + else() + add_test(NAME ${doctest} COMMAND ${doctest}) + endif() + # Prepend the build dir libasdf to the library search path + set_property(TEST ${doctest} + PROPERTY ENVIRONMENT_MODIFICATION "${runtime}") endforeach() endif() diff --git a/tests/Makefile.am b/tests/Makefile.am index f4a7a4a1..c6b0795e 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -287,6 +287,7 @@ DOC_EXAMPLES_DIR = $(builddir)/doc_examples # Documentation files containing example programs DOC_EXAMPLE_FILES = \ $(top_srcdir)/README.rst \ + $(top_srcdir)/docs/usage/examples.rst \ $(top_srcdir)/docs/usage/writing.rst DOC_EXAMPLE_DEPS = \ @@ -299,8 +300,7 @@ TESTS += $(DOC_EXAMPLES_DIR)/test-doc-examples.sh $(DOC_EXAMPLES_DIR)/test-doc-examples.sh: $(DOC_EXAMPLE_DEPS) $(AM_V_at)$(PYTHON3) $(srcdir)/scripts/extract_doc_examples.py \ - $(top_srcdir)/README.rst \ - $(top_srcdir)/docs/usage/writing.rst \ + $(DOC_EXAMPLE_FILES) \ --out-dir $(DOC_EXAMPLES_DIR) \ --fixtures-dir $(srcdir)/fixtures $(QUIET_OUT) $(AM_V_at)for f in $(DOC_EXAMPLES_DIR)/*.c; do \ @@ -376,6 +376,7 @@ EXTRA_DIST += \ fixtures/scalars.asdf \ fixtures/scalars-out.asdf \ fixtures/tagged-scalars.asdf \ + fixtures/test-write-file.asdf \ fixtures/tiles.asdf \ fixtures/time.asdf \ fixtures/trivial-extension.asdf \ diff --git a/tests/fixtures/cube.asdf b/tests/fixtures/cube.asdf index 88b8b926..a95ad397 100644 Binary files a/tests/fixtures/cube.asdf and b/tests/fixtures/cube.asdf differ diff --git a/tests/fixtures/test-write-file.asdf b/tests/fixtures/test-write-file.asdf new file mode 100644 index 00000000..0a3a18f0 Binary files /dev/null and b/tests/fixtures/test-write-file.asdf differ diff --git a/tests/scripts/extract_doc_examples.py b/tests/scripts/extract_doc_examples.py index a61bc871..4c9c92a6 100755 --- a/tests/scripts/extract_doc_examples.py +++ b/tests/scripts/extract_doc_examples.py @@ -2,14 +2,29 @@ """ Extract all C code blocks from ReST files and write them to test files. -Code blocks to test are always introduced by the ``.. code:: c`` block -directive. Blocks to be extracted must include the ``:name:`` option with -a name beginning with ``test-``. -""" +Code blocks to test are introduced by the ``.. code:: c`` block directive and +marked for extraction with the ``:test:`` option, whose value is the name of +the test (conventionally beginning with ``test-``):: + + .. code:: c + :test: test-open-close-file + :fixture: cube.asdf + + #include + ... -# TODO: Clean this up a bit; when I first wrote it it was simple enough to be -# a flat Python script but might be clearer if it were better organized into -# subroutines +The optional ``:fixture:`` option names the file passed to the compiled +program as its first argument. It may take one of two forms: + +* ``:fixture: `` -- an existing fixture file (resolved relative to + ``--fixtures-dir``) passed to the program as an *input* file. +* ``:fixture: temp`` or ``:fixture: temp:`` -- a throwaway *output* + path in the build directory (resolved relative to ``--out-dir``). Use this + for examples that write a file rather than read one. When no ```` is + given the file is named ``.asdf``. + +A block with no ``:fixture:`` is run with no arguments. +""" import argparse import os.path as pth @@ -17,95 +32,160 @@ import stat from pathlib import Path -parser = argparse.ArgumentParser(description='Extract C code blocks from ReST') -parser.add_argument( - 'rst_files', nargs='+', type=Path, - help='ReST files from which to extract code examples' -) -parser.add_argument( - '--list-tests', action='store_true', - help='Just list the test names without extracting them yet' -) -parser.add_argument( - '--out-dir', type=Path, default=Path('doc_examples'), - help='Directory to write extracted .c files' -) -# NOTE: Curently all the tests are designed to just work on a specific file, -# fixtures/cube.asdf; could change this later if need be. -parser.add_argument( - '--fixtures-dir', type=Path, default=Path('fixtures'), - help='Directory containing test files on which the programs are run') -args = parser.parse_args() - -if not args.list_tests: - args.out_dir.mkdir(parents=True, exist_ok=True) - -# Match a .. code:: c block, optionally with :name: +# Match a .. code:: c block with its options and indented body code_block_re = re.compile( r'^\s*\.\. code:: c\s*\n' - r'((?:\s*:\w+:.*\n)*)' # optional directives like :name: - r'((?:\s{3,}.*\n)+)', # indented code block + r'((?:\s*:[\w-]+:.*\n)*)' # options like :test: / :fixture: + r'((?:\s{3,}.*\n)+)', # indented code block re.MULTILINE ) -test_names = [] +test_option_re = re.compile(r':test:\s*(\S+)') +fixture_option_re = re.compile(r':fixture:\s*(\S+)') -for rst_file in args.rst_files: - if not rst_file.is_file(): - parser.exit(1, f'file not found: {rst_file}\n') - content = rst_file.read_text() +def resolve_fixture(fixture, test_name, fixtures_dir, out_dir): + """ + Resolve a ``:fixture:`` option value to the path passed to the program. - for match in code_block_re.finditer(content): - directives, code_lines = match.groups() + Returns ``None`` when the test takes no argument. + """ + if fixture is None: + return None - # Look for :name: directive starting with test_ - name_match = re.search(r':name:\s*(test-\S+)', directives) - if not name_match: - continue + if fixture == 'temp' or fixture.startswith('temp:'): + name = fixture[len('temp:'):] if ':' in fixture else f'{test_name}.asdf' + return out_dir / name - test_name = name_match.group(1) + return fixtures_dir / fixture - if args.list_tests: - print(test_name) - continue - filename = f'{test_name}.c' +def iter_doc_tests(rst_files): + """ + Yield ``(rst_file, lineno, test_name, fixture, code)`` for each marked + block found in ``rst_files``. + """ + for rst_file in rst_files: + if not rst_file.is_file(): + raise FileNotFoundError(rst_file) - # Remove leading indentation (3 or more spaces) - code = '\n'.join( - line[3:] if line.startswith(' ') else line - for line in code_lines.splitlines() - ) + content = rst_file.read_text() - lineno = content.count('\n', 0, match.start()) + 2 - out_file = args.out_dir / filename - rst_file_rel = pth.relpath(rst_file.resolve(), - start=args.out_dir.resolve()) + for match in code_block_re.finditer(content): + options, code_lines = match.groups() - with out_file.open('w') as fobj: - fobj.write(f'// Auto-generated test extracted from ' - f'{rst_file_rel}:{lineno}\n') - fobj.write(code) + name_match = test_option_re.search(options) + if not name_match: + continue + + test_name = name_match.group(1) + + fixture_match = fixture_option_re.search(options) + fixture = fixture_match.group(1) if fixture_match else None + + # Remove leading indentation (3 or more spaces) + code = '\n'.join( + line[3:] if line.startswith(' ') else line + for line in code_lines.splitlines() + ) + + lineno = content.count('\n', 0, match.start()) + 2 + yield rst_file, lineno, test_name, fixture, code - print(f'Wrote {out_file}') - test_names.append(test_name) -if not args.list_tests: - # Generate a script to run the tests - shell_file = args.out_dir / 'test-doc-examples.sh' - fixture_file = args.fixtures_dir / 'cube.asdf' # hard-coded for now +def write_test_source(out_dir, rst_file, lineno, test_name, code): + """Write the extracted code to ``/.c``.""" + out_file = out_dir / f'{test_name}.c' + rst_file_rel = pth.relpath(rst_file.resolve(), start=out_dir.resolve()) + + with out_file.open('w') as fobj: + fobj.write(f'// Auto-generated test extracted from ' + f'{rst_file_rel}:{lineno}\n') + fobj.write(code) + + return out_file + + +def write_runner_script(out_dir, tests, prog): + """ + Write the ``test-doc-examples.sh`` driver that runs every extracted test. + + ``tests`` is a list of ``(test_name, argv)`` pairs where ``argv`` is the + resolved fixture path (or ``None``). + """ + shell_file = out_dir / 'test-doc-examples.sh' with open(shell_file, 'w') as fobj: fobj.write('#!/bin/sh\n') - fobj.write(f'# Auto-generated by {parser.prog}; ' - f'do not edit or commit\n\n') + fobj.write(f'# Auto-generated by {prog}; do not edit or commit\n\n') - for test_name in test_names: - fobj.write(f'echo "Running {test_name}"\n') - fobj.write(f'{args.out_dir / test_name} {fixture_file} ' - f'> {args.out_dir / test_name}.log\n') + # Run every example, capturing both stdout and stderr (e.g. sanitizer + # reports) to its log, and fail the test if any example exits non-zero. + fobj.write('status=0\n\n') + + for test_name, argv in tests: + exe = out_dir / test_name + arg = f' {argv}' if argv is not None else '' + fobj.write(f'echo -n "Running {test_name}... "\n') + fobj.write(f'{exe}{arg} > {exe}.log 2>&1 || status=1\n') + fobj.write('test $status -eq 0 && echo "passed" || echo "failed"\n') + + fobj.write('\nexit $status\n') - print(f'Wrote {shell_file}') shell_file.chmod(shell_file.stat().st_mode | stat.S_IXUSR) + return shell_file + + +def main(): + parser = argparse.ArgumentParser( + description='Extract C code blocks from ReST') + parser.add_argument( + 'rst_files', nargs='+', type=Path, + help='ReST files from which to extract code examples') + parser.add_argument( + '--list-tests', action='store_true', + help='Just list the test names (and resolved fixture arg) without ' + 'extracting them') + parser.add_argument( + '--out-dir', type=Path, default=Path('doc_examples'), + help='Directory to write extracted .c files') + parser.add_argument( + '--fixtures-dir', type=Path, default=Path('fixtures'), + help='Directory containing fixture files passed to the programs') + args = parser.parse_args() + + try: + doc_tests = list(iter_doc_tests(args.rst_files)) + except FileNotFoundError as exc: + parser.exit(1, f'file not found: {exc}\n') + + if args.list_tests: + # One line per test: ``name`` or ``name,arg`` e.g. so the CMake build + # can add each test with its own fixture argument. + for _, _, test_name, fixture, _ in doc_tests: + argv = resolve_fixture( + fixture, test_name, args.fixtures_dir, args.out_dir) + if argv is None: + print(test_name) + else: + print(f'{test_name},{argv}') + return + + args.out_dir.mkdir(parents=True, exist_ok=True) + + tests = [] + for rst_file, lineno, test_name, fixture, code in doc_tests: + out_file = write_test_source( + args.out_dir, rst_file, lineno, test_name, code) + print(f'Wrote {out_file}') + argv = resolve_fixture( + fixture, test_name, args.fixtures_dir, args.out_dir) + tests.append((test_name, argv)) + + shell_file = write_runner_script(args.out_dir, tests, parser.prog) + print(f'Wrote {shell_file}') + + +if __name__ == '__main__': + main()