diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..ff2cf63 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# git must not convert line endings in our reference test file +# https://github.com/ludocode/mpack/issues/64 +*.debug text eol=lf diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml new file mode 100644 index 0000000..962aa1a --- /dev/null +++ b/.github/workflows/unit.yml @@ -0,0 +1,165 @@ +name: Unit Tests + +on: [push, pull_request] + +jobs: + + + macos-clang-amalgamated: + name: "macOS Clang (amalgamated)" + runs-on: macos-latest + env: + AMALGAMATED: 1 + steps: + - uses: actions/checkout@v2 + + - name: Install Dependencies + run: brew install ninja + + - name: Run ci-unix + run: test/unit/ci-unix.sh + + + + ubuntu-scan-build: + name: "Clang Static Analysis (source)" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install -y ninja-build clang-tools + + - name: Run scan-build + run: tools/scan-build.sh + + + + ubuntu-tcc: + name: "TinyCC (source)" + runs-on: ubuntu-latest + env: + CC: tcc + CXX: /bin/false + steps: + - uses: actions/checkout@v2 + + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install -y ninja-build tcc + + - name: Run ci-unix + run: test/unit/ci-unix.sh + + + + ubuntu-gcc-9: + name: "Ubuntu 20.04 GCC 9 (source)" + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + + - name: Install Ninja + run: | + sudo apt-get update + sudo apt-get install -y ninja-build + + - name: Install lcov + run: | + curl -LO https://github.com/linux-test-project/lcov/releases/download/v1.15/lcov-1.15.tar.gz + tar xzf lcov-1.15.tar.gz + cd lcov-1.15 + sudo make install + + - name: Run ci-unix + run: test/unit/ci-unix.sh + + - name: Collect coverage + run: tools/coverage.sh + + - name: Submit coverage to Coveralls + id: coverage + uses: coverallsapp/github-action@1.1.3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Print Coveralls submission result + run: echo ${{ steps.coverage.outputs['coveralls-api-result'] }} + + + + ubuntu-gcc-7: + name: "Ubuntu 20.04 GCC 7 (amalgamated)" + runs-on: ubuntu-20.04 + env: + CC: gcc-7 + CXX: g++-7 + AMALGAMATED: 1 + steps: + - uses: actions/checkout@v2 + + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install -y ninja-build gcc-7 + + - name: Run ci-unix + run: test/unit/ci-unix.sh + + + + ubuntu-gcc-5: + name: "Ubuntu 18.04 GCC 5 (amalgamated)" + runs-on: ubuntu-18.04 + env: + CC: gcc-5 + CXX: g++-5 + AMALGAMATED: 1 + steps: + - uses: actions/checkout@v2 + + - name: Install Dependencies + # https://askubuntu.com/a/1087368 + # note for when 18.04 is EOL: https://askubuntu.com/a/1313032 + run: | + sudo apt-get update + sudo apt-get install -y ninja-build gcc-5 + + - name: Run ci-unix + run: test/unit/ci-unix.sh + + + + windows-cl-2019-x64: + name: "Windows VS2019 amd64 (source)" + runs-on: windows-latest + env: + COMPILER: cl-2019-x64 + steps: + - uses: actions/checkout@v2 + + - name: Run ci-windows (Windows) + shell: cmd + run: test\\unit\\ci-windows.bat + + + + windows-cl-2015-x86: + name: "Windows VS2015 x86 (amalgamated)" + runs-on: windows-latest + env: + COMPILER: cl-2015-x86 + AMALGAMATED: 1 + steps: + - uses: actions/checkout@v2 + + - name: Amalgamate + shell: bash + run: tools\\amalgamate.sh + + - name: Run ci-windows (Windows) + shell: cmd + run: test\\unit\\ci-windows.bat diff --git a/.gitignore b/.gitignore index a81b19d..186c516 100644 --- a/.gitignore +++ b/.gitignore @@ -1,32 +1,20 @@ # build files -/build -*.o +/.build/ +test/.build/ +coverage/ *.swp *.gcov -/.sconsign.dblite -/.sconf_temp/ -/config.log mpack-test-file mpack-test-blank-file /mpack-test-dir/ /analysis/ /docs/html/ - -# visual studio project -*.suo -*.opensdf -*.sdf -*.vcxproj.user -/projects/vs/Debug/ -/projects/vs/Release/ - -# xcode project -xcuserdata/ -*.xcworkspace/ -.DS_Store +/vgcore.* +/.ninja_deps +/.ninja_log # other junk .directory /tags - +/examples/sax-example diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0fdfbe3..0000000 --- a/.travis.yml +++ /dev/null @@ -1,18 +0,0 @@ -language: c - -addons: - apt: - packages: - - clang - - valgrind - -script: tools/ci.sh - -compiler: - - clang - - gcc - - scan-build - -env: - - AMALGAMATED=1 - - STANDARD=1 diff --git a/AUTHORS.md b/AUTHORS.md new file mode 100644 index 0000000..1d8e267 --- /dev/null +++ b/AUTHORS.md @@ -0,0 +1,16 @@ +| Author | GitHub Profile | +| :------------------------------ | :----------------------------------------- | +| Nicholas Fraser | https://github.com/ludocode | +| Jerry Jacobs | https://github.com/xor-gate | +| Rik van der Heijden | https://github.com/rikvdh | +| Chris Heijdens | https://github.com/chris-heijdens | +| Jean-Louis Fuchs | https://github.com/ganwell | +| Christopher Field | https://github.com/volks73 | +| 喜欢兰花山丘 | https://github.com/wangzhione | +| Vasily Postnicov | https://github.com/shamazmazum | +| Tim Gates | https://github.com/timgates42 | +| Dirkson | https://github.com/dirkson | +| Ethan Li | https://github.com/ethanjli | +| Danny Povolotski | https://github.com/israelidanny | +| Weston Schmidt | https://github.com/schmidtw | +| Nikita Danilov | https://github.com/nikitadanilov | diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f7adb06 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,199 @@ +MPack v1.1.1 +------------ + +Bug Fixes: + +- Fixed a crash that could occur when a builder element was aligned exactly at the end of a page. (See #94) + +- Fixed a crash when encountering an I/O error during builder resolution without an error handler callback. (See #98) + +- Fixed an error destroying a writer while a builder is open. (See #88) + +- Fixed an issue with incorrect NULL checks for 0-length buffers. (See #97) + +- Fixed a string formatting issue on platforms where `int` is not 32 bits. (See #103) + +- Fixed some documentation errors. (See #93, #102) + +- Cleaned up some references to old unit test buildsystem. (See #95) + +MPack v1.1 +---------- + +New Features: + +- Maps and arrays can now be built dynamically without specifying their size up front. See `mpack_build_map()` and `mpack_build_array()`. + +New Platforms: + +- Compiling as gnu89 is now supported. (See #68, #69) + +- Compiling in the Linux kernel is now possible using a [standalone configuration file](https://github.com/ludocode/mpack-linux-kernel). (See #80) + +- Compiling for AVR (e.g. Arduino) and other microcontrollers is now supported. MPack now compiles cleanly on platforms with 16-bit `int` and `size_t`. (See #74, #79) + +- `float` and/or `double` can now be disabled individually on platforms with limited floating point support. If `float` is supported but not `double`, MessagePack doubles can be converted to `float`. (See #74, #79) + +- MPack now builds cleanly under /W4 with Visual Studio 2015, 2017 and 2019 build tools. + +Bug Fixes and Other Changes: + +- An `mpack-defaults.h` sample configuration is no longer provided. + +- Replaced SCons unit test buildsystem and XCode/VS projects with Python+Ninja. + +- Fixed an issue where write overloads could be erroneously defined in C++ without `MPACK_WRITER` (#66). + +- Fixed some minor portability issues. + +MPack v1.0 +---------- + +A number of breaking API changes have been made for the 1.0 release. Please take note of these changes when upgrading. + +Breaking Changes: + +- The Node API now separates tree initialization from parsing. After calling one of the `mpack_tree_init()` functions, you must explicitly call `mpack_tree_parse()` before accessing any nodes. + +- The configuration file `mpack-config.h` is now optional, and requires `MPACK_HAS_CONFIG` in order to be included. This means you must define `MPACK_HAS_CONFIG` when upgrading or your config file will be ignored! + +- Extension types are now disabled by default. You must define `MPACK_EXTENSIONS` to use them. + +- `mpack_tag_t` is now considered an opaque type to prevent future breakage when changing its layout. Compatibility is maintained for this release, but this may change in future releases. + +New Features: + +- The Node API can now parse multiple messages from a data source. `mpack_tree_parse()` can be called repeatedly to parse each message. + +- The Node API can now parse messages indefinitely from a continuous stream. A tree can be initialized with `mpack_tree_init_stream()` to receive a callback for more data. + +- The Node API can now parse messages incrementally from a non-blocking stream. Call `mpack_tree_try_parse()` with a non-blocking read function to start and resume parsing. It will return true when a complete message has become available. + +- The stdio helpers now allow reading from a `FILE*`. `_init_file()` functions have been renamed to `_init_filename()`. (The old names will continue to work for a few more versions.) + +- The Node API now returns a node of "missing" type instead of "nil" type for optional map lookups. This allows the caller to tell the difference between a key having value nil and a missing key. + +- The writer now supports a v4 compatibility mode. Call `mpack_writer_set_version(writer, mpack_version_v4);` to encode without using the `raw8`, `bin` and `ext` types. (This requires `MPACK_COMPATIBILITY`.) + +- The timestamp type has been implemented. A timestamp is a signed number of nanoseconds since the Unix epoch (1970-01-01T00:00:00Z). (This requires `MPACK_EXTENSIONS`.) + +Bug Fixes and Other Changes: + +- Fixed an allocation bug when closing a growable writer without having written anything (#58). + +- The reader's skip function is no longer ignored under `MPACK_OPTIMIZE_FOR_SIZE`. + +MPack v0.8.2 +------------ + +Changes: + +- Fixed incorrect element tracking in `mpack_write_tag()` +- Added type-generic writer functions `mpack_write()` and `mpack_write_kv()` +- Added `mpack_write_object_bytes()` to insert pre-encoded MessagePack into a larger message +- Enabled strings in all builds by default +- Fixed unit test errors under `-ffast-math` +- Fixed some compiler warnings + +MPack v0.8.1 +------------ + +Changes: + +- Fixed some compiler warnings +- Added various performance improvements +- Improved documentation + +MPack v0.8 +---------- + +Changes: + +- Added `mpack_peek_tag()` +- Added reader helper functions to [expect re-ordered map keys](http://ludocode.github.io/mpack/md_docs_expect.html) +- [Improved documentation](http://ludocode.github.io/mpack/) and added [Pages](http://ludocode.github.io/mpack/pages.html) +- Made node key lookups check for duplicate keys +- Added various UTF-8 checking functions for reader and nodes +- Added support for compiling as C in recent versions of Visual Studio +- Removed `mpack_expect_str_alloc()` and `mpack_expect_utf8_alloc()` +- Fixed miscellaneous bugs and improved performance + +MPack v0.7.1 +------------ + +Changes: + +- Removed `mpack_reader_destroy_cancel()` and `mpack_writer_destroy_cancel()`. You must now flag an error (such as `mpack_error_data`) in order to cancel reading. +- Added many code size optimizations. `MPACK_OPTIMIZE_FOR_SIZE` is no longer experimental. +- Improved and reorganized [Writer documentation](http://ludocode.github.io/mpack/group__writer.html) +- Made writer flag `mpack_error_too_big` instead of `mpack_error_io` if writing too much data without a flush callback +- Added optional `skip` callback and optimized `mpack_discard()` +- Fixed various compiler and code analysis warnings +- Optimized speed and memory usage + +MPack v0.7 +---------- + +Changes: + +- Fixed various bugs in UTF-8 checking, error handler callbacks, out-of-memory and I/O errors, debug print functions and more +- Added many missing Tag and Expect functions such as `mpack_tag_ext()`, `mpack_expect_int_range()` and `mpack_expect_utf8()` +- Added extensive unit tests + +MPack v0.6 +---------- + +Changes: + +- `setjmp`/`longjmp` support has been replaced by error callbacks. You can safely `longjmp` or throw C++ exceptions out of error callbacks. Be aware of local variable invalidation rules regarding `setjmp` if you use it. See the [documentation for `mpack_reader_error_t`](http://ludocode.github.io/mpack/mpack-reader_8h.html) and issue #19 for more details. +- All `inline` functions in the MPack API are no longer `static`. A single non-`inline` definition of each `inline` function is emitted, so they behave like normal functions with external linkage. +- Configuration options can now be pre-defined before including `mpack-config.h`, so you can customize MPack by defining these in your build system rather than editing the configuration file. + +MPack v0.5.1 +------------ + +Changes: + +- Fixed compile errors in debug print function +- Fixed C++11 warnings + +MPack v0.5 +---------- + +Changes: + +- `mpack_node_t` is now a handle, so it should be passed by value, not by pointer. Porting to the new version should be as simple as replacing `mpack_node_t*` with `mpack_node_t` in your code. +- Various other minor API changes have been made. +- Major performance improvements were made across all aspects of MPack. + +MPack v0.4 +---------- + +Changes + +- Added `mpack_writer_init_growable()` to write to a growable buffer +- Converted tree parser to support node pool and pages. The Node API no longer requires an allocator. +- Added Xcode unit test project, included projects in release package +- Fixed various bugs + +MPack v0.3 +---------- + +Changes: + +- Changed default config and test suite to use `DEBUG` and `_DEBUG` (instead of `NDEBUG`) +- Added Visual Studio project for running unit tests +- Fixed various bugs + +MPack v0.2 +---------- + +Changes: + +- Added teardown callbacks to reader, writer and tree +- Simplified API for working with files (`mpack_file_tree_t` is now internal) + +MPack v0.1 +---------- + +Initial release. diff --git a/LICENSE b/LICENSE index 7f2bc05..9516a88 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright (c) 2015 Nicholas Fraser +Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 9d2fde2..b0c80ce 100644 --- a/README.md +++ b/README.md @@ -1,43 +1,33 @@ ## Introduction -MPack is a C implementation of an encoder and decoder for the [MessagePack](http://msgpack.org/) serialization format. It is intended to be: +MPack is a C implementation of an encoder and decoder for the [MessagePack](http://msgpack.org/) serialization format. It is: * Simple and easy to use * Secure against untrusted data * Lightweight, suitable for embedded * [Extensively documented](http://ludocode.github.io/mpack/) - * Extremely fast + * [Extremely fast](https://github.com/ludocode/schemaless-benchmarks#speed---desktop-pc) -The core of MPack contains a buffered reader and writer with a custom callback to fill or flush the buffer. Helper functions can be enabled to read values of expected type, to work with files, to allocate strings automatically, to check UTF-8 encoding, and more. The MPack featureset can be configured at compile-time to set which features, components and debug checks are compiled, and what dependencies are available. +The core of MPack contains a buffered reader and writer, and a tree-style parser that decodes into a tree of dynamically typed nodes. Helper functions can be enabled to read values of expected type, to work with files, to grow buffers or allocate strings automatically, to check UTF-8 encoding, and more. -The MPack code is small enough to be embedded directly into your codebase. The easiest way to use it is to download the [amalgamation package](https://github.com/ludocode/mpack/releases) and insert the source files directly into your project. Copy `mpack.h` and `mpack.c` into to your codebase, and copy `mpack-config.h.sample` as `mpack-config.h`. You can use the defaults or edit it if you'd like to customize the MPack featureset. +The MPack code is small enough to be embedded directly into your codebase. Simply download the [amalgamation package](https://github.com/ludocode/mpack/releases) and add `mpack.h` and `mpack.c` to your project. -## Build Status - -MPack is beta software under development. +MPack supports all modern compilers, all desktop and smartphone OSes, WebAssembly, [inside the Linux kernel](https://github.com/ludocode/mpack-linux-kernel), and even 8-bit microcontrollers such as Arduino. The MPack featureset can be customized at compile-time to set which features, components and debug checks are compiled, and what dependencies are available. -[travis-home]: https://travis-ci.org/ -[travis-badge]: https://travis-ci.org/ludocode/mpack.svg?branch=develop -[travis-mpack]: https://travis-ci.org/ludocode/mpack/branches -[appveyor-home]: https://ci.appveyor.com/ -[appveyor-badge]: https://ci.appveyor.com/api/projects/status/tux06aefpqq83k30/branch/develop?svg=true -[appveyor-mpack]: https://ci.appveyor.com/project/ludocode/mpack/branch/develop -[coveralls-home]: https://coveralls.io/ -[coveralls-badge]: https://coveralls.io/repos/ludocode/mpack/badge.svg?branch=develop&service=github -[coveralls-mpack]: https://coveralls.io/github/ludocode/mpack?branch=develop +## Build Status -| [Travis-CI][travis-home] | [AppVeyor][appveyor-home] | [Coveralls.io][coveralls-home] | -| :-------: | :----------: | :----------: | -| [![Build Status][travis-badge]][travis-mpack] | [![Build status][appveyor-badge]][appveyor-mpack] | [![Coverage Status][coveralls-badge]][coveralls-mpack] | +[![Unit Tests](https://github.com/ludocode/mpack/workflows/Unit%20Tests/badge.svg)](https://github.com/ludocode/mpack/actions?query=workflow%3A%22Unit+Tests%22) +[![Coverage](https://coveralls.io/repos/ludocode/mpack/badge.svg?branch=develop&service=github)](https://coveralls.io/github/ludocode/mpack?branch=develop) -## The Node Reader API +## The Node API The Node API parses a chunk of MessagePack data into an immutable tree of dynamically-typed nodes. A series of helper functions can be used to extract data of specific types from each node. ```C // parse a file into a node tree mpack_tree_t tree; -mpack_tree_init_file(&tree, "homepage-example.mp", 0); +mpack_tree_init_filename(&tree, "homepage-example.mp", 0); +mpack_tree_parse(&tree); mpack_node_t root = mpack_tree_root(&tree); // extract the example data on the msgpack homepage @@ -45,7 +35,7 @@ bool compact = mpack_node_bool(mpack_node_map_cstr(root, "compact")); int schema = mpack_node_i32(mpack_node_map_cstr(root, "schema")); // clean up and check for errors -if (mpack_tree_destroy(tree) != mpack_ok) { +if (mpack_tree_destroy(&tree) != mpack_ok) { fprintf(stderr, "An error occurred decoding the data!\n"); return; } @@ -53,9 +43,11 @@ if (mpack_tree_destroy(tree) != mpack_ok) { Note that no additional error handling is needed in the above code. If the file is missing or corrupt, if map keys are missing or if nodes are not in the expected types, special "nil" nodes and false/zero values are returned and the tree is placed in an error state. An error check is only needed before using the data. +The above example allocates nodes automatically. A fixed node pool can be provided to the parser instead in memory-constrained environments. For maximum performance and minimal memory usage, the [Expect API](docs/expect.md) can be used to parse data of a predefined schema. + ## The Write API -The MPack Write API encodes structured data to MessagePack. +The Write API encodes structured data to MessagePack. ```C // encode to memory buffer @@ -65,12 +57,12 @@ mpack_writer_t writer; mpack_writer_init_growable(&writer, &data, &size); // write the example on the msgpack homepage -mpack_start_map(&writer, 2); +mpack_build_map(&writer); mpack_write_cstr(&writer, "compact"); mpack_write_bool(&writer, true); mpack_write_cstr(&writer, "schema"); mpack_write_uint(&writer, 0); -mpack_finish_map(&writer); +mpack_complete_map(&writer); // finish writing if (mpack_writer_destroy(&writer) != mpack_ok) { @@ -83,11 +75,11 @@ do_something_with_data(data, size); free(data); ``` -In the above example, we encode to a growable memory buffer. The writer can instead write to a pre-allocated or stack-allocated buffer, avoiding the need for memory allocation. The writer can also be provided with a flush function (such as a file or socket write function) to call when the buffer is full or when writing is done. +In the above example, we encode to a growable memory buffer. The writer can instead write to a pre-allocated or stack-allocated buffer (with up-front sizes for compound types), avoiding the need for memory allocation. The writer can also be provided with a flush function (such as a file or socket write function) to call when the buffer is full or when writing is done. -If any error occurs, the writer is placed in an error state. The writer will flag an error if too much data is written, if the wrong number of elements are written, if the data could not be flushed, etc. No additional error handling is needed in the above code; any subsequent writes are ignored when the writer is in an error state, so you don't need to check every write for errors. +If any error occurs, the writer is placed in an error state. The writer will flag an error if too much data is written, if the wrong number of elements are written, if an allocation failure occurs, if the data could not be flushed, etc. No additional error handling is needed in the above code; any subsequent writes are ignored when the writer is in an error state, so you don't need to check every write for errors. -Note in particular that in debug mode, the `mpack_finish_map()` call above ensures that two key/value pairs were actually written as claimed, something that other MessagePack C/C++ libraries may not do. +The above example uses `mpack_build_map()` to automatically determine the number of key-value pairs contained. If you know up-front the number of elements needed, you can pass it to `mpack_start_map()` instead. In that case the corresponding `mpack_finish_map()` will assert in debug mode that the expected number of elements were actually written, which is something that other MessagePack C/C++ libraries may not do. ## Comparison With Other Parsers @@ -96,22 +88,23 @@ MPack is rich in features while maintaining very high performance and a small co [mpack]: https://github.com/ludocode/mpack [msgpack-c]: https://github.com/msgpack/msgpack-c [cmp]: https://github.com/camgunz/cmp - -| | [MPack][mpack]
(v0.8) | [msgpack-c][msgpack-c]
(v1.3.0) | [CMP][cmp]
(v14) | -|:------------------------------------|:---:|:---:|:---:| -| No libc requirement | ✓ | | ✓ | -| Growable memory writer | ✓ | ✓ | | -| File I/O helpers | ✓ | ✓ | | -| Tree parser | ✓ | ✓ | | -| Propagating errors | ✓ | | ✓ | -| Compound size tracking | ✓ | | | -| Incremental parser | ✓ | | ✓ | -| Incremental range/match helpers | ✓ | | | -| Tree stream parser | | ✓ | | -| UTF-8 verification | ✓ | | | +[cwpack]: https://github.com/clwi/CWPack + +| | [MPack][mpack]
(v1.1) | [msgpack-c][msgpack-c]
(v3.3.0) | [CMP][cmp]
(v19) | [CWPack][cwpack]
(v1.3.1) | +|:------------------------------------|:---:|:---:|:---:|:---:| +| No libc requirement | ✓ | | ✓ | ✓ | +| Growable memory writer | ✓ | ✓ | | ✓\* | +| File I/O helpers | ✓ | ✓ | | ✓\* | +| Stateful error handling | ✓ | | ✓ | | +| Incremental parser | ✓ | | ✓ | ✓ | +| Tree stream parser | ✓ | ✓ | | | +| Compound size tracking | ✓ | | | | +| Automatic compound size | ✓ | | | | A larger feature comparison table is available [here](docs/features.md) which includes descriptions of the various entries in the table. +[This benchmarking suite](https://github.com/ludocode/schemaless-benchmarks) compares the performance of MPack to other implementations of schemaless serialization formats. MPack outperforms all JSON and MessagePack libraries (except [CWPack][cwpack]), and in some tests MPack is several times faster than [RapidJSON](https://github.com/miloyip/rapidjson) for equivalent data. + ## Why Not Just Use JSON? Conceptually, MessagePack stores data similarly to JSON: they are both composed of simple values such as numbers and strings, stored hierarchically in maps and arrays. So why not just use JSON instead? The main reason is that JSON is designed to be human-readable, so it is not as efficient as a binary serialization format: @@ -124,16 +117,12 @@ Conceptually, MessagePack stores data similarly to JSON: they are both composed - Binary data is not supported by JSON at all. Small binary blobs such as icons and thumbnails need to be Base64 encoded or passed out-of-band. -The above issues greatly increase the complexity of the decoder. Full-featured JSON decoders are quite large, and minimal decoders tend to leave out such features as string unescaping and float parsing, instead leaving these up to the user or platform. This can lead to hard-to-find platform-specific and locale-specific bugs. This also significantly decreases performance, making JSON unattractive for use in applications such as mobile games. +The above issues greatly increase the complexity of the decoder. Full-featured JSON decoders are quite large, and minimal decoders tend to leave out such features as string unescaping and float parsing, instead leaving these up to the user or platform. This can lead to hard-to-find platform-specific and locale-specific bugs, as well as a greater potential for security vulnerabilites. This also significantly decreases performance, making JSON unattractive for use in applications such as mobile games. While the space inefficiencies of JSON can be partially mitigated through minification and compression, the performance inefficiencies cannot. More importantly, if you are minifying and compressing the data, then why use a human-readable format in the first place? -## Running the Unit Tests +## Testing MPack The MPack build process does not build MPack into a library; it is used to build and run the unit tests. You do not need to build MPack or the unit testing suite to use MPack. -On Linux, the test suite uses SCons and requires Valgrind, and can be run in the repository or in the amalgamation package. Run `scons` to build and run the test suite in full debug configuration. - -On Windows, there is a Visual Studio solution, and on OS X, there is an Xcode project for building and running the test suite. - -You can also build and run the test suite in all supported configurations, which is what the continuous integration server will build and run. If you are on 64-bit, you will need support for cross-compiling to 32-bit, and running 32-bit binaries with 64-bit Valgrind. On Ubuntu, you'll need `libc6-dbg:i386`. On Arch you'll need `gcc-multilib` or `lib32-clang`, and `valgrind-multilib`. Use `scons all=1 -j16` (or some appropriate thread count) to build and run all tests. +See [test/README.md](test/README.md) for information on how to test MPack. diff --git a/SConscript b/SConscript deleted file mode 100644 index 9024306..0000000 --- a/SConscript +++ /dev/null @@ -1,35 +0,0 @@ -import platform - -Import('env', 'CPPFLAGS', 'LINKFLAGS') - -# we add the C/C++ specific flags here. we can't use CCFLAGS/CXXFLAGS -# because as far as SCons is concerned, they are all C files; we're -# passing -x c++ to force the language. -if "c++" in CPPFLAGS: - CPPFLAGS += ["-Wmissing-declarations"] - LINKFLAGS += ["-lstdc++"] -else: - CPPFLAGS += ["-Wmissing-prototypes", "-Wc++-compat"] - -srcs = env.Object(env.Glob('src/mpack/*.c') + env.Glob('test/*.c'), - CPPFLAGS=env['CPPFLAGS'] + CPPFLAGS) - -prog = env.Program("mpack-test", srcs, - LINKFLAGS=env['LINKFLAGS'] + LINKFLAGS) - -# only some architectures are supported by valgrind. we don't check for it -# though because we want to force mpack developers to install and use it if -# it's available. - -if platform.machine() in ["i386", "x86_64"]: - valgrind = "valgrind --leak-check=full --error-exitcode=1 " - # travis version of valgrind is too old, doesn't support leak kinds - if "TRAVIS" not in env["ENV"]: - valgrind = valgrind + "--show-leak-kinds=all --errors-for-leak-kinds=all " - valgrind = valgrind + "--suppressions=tools/valgrind-suppressions " -else: - valgrind = "" - -env.Default(env.AlwaysBuild(env.Alias("test", - [prog], - valgrind + Dir('.').path + "/mpack-test"))) diff --git a/SConstruct b/SConstruct deleted file mode 100644 index 1cb6ede..0000000 --- a/SConstruct +++ /dev/null @@ -1,171 +0,0 @@ -import platform, os - -def CheckFlags(context, cppflags, linkflags = [], message = None): - if message == None: - message = " ".join(cppflags + ((cppflags != linkflags) and linkflags or [])) - context.Message("Checking for " + message + " support... ") - - env.Prepend(CPPFLAGS = cppflags, LINKFLAGS = linkflags) - result = context.TryLink("int main(void){return 0;}\n", '.c') - env.Replace(CPPFLAGS = env["CPPFLAGS"][len(cppflags):], LINKFLAGS = env["LINKFLAGS"][len(linkflags):]) - - context.Result(result) - return result - - -# Common environment setup - -env = Environment() -conf = Configure(env, custom_tests = {'CheckFlags': CheckFlags}) - -for x in os.environ.keys(): - if x in ["CC", "CXX"]: - env[x] = os.environ[x] - if x in ["PATH", "TRAVIS", "TERM"] or x.startswith("CLANG_") or x.startswith("CCC_"): - env["ENV"][x] = os.environ[x] - -env.Append(CPPFLAGS = [ - "-Wall", "-Wextra", "-Werror", - "-Wconversion", "-Wno-sign-conversion", "-Wundef", "-Wshadow", - "-Isrc", "-Itest", - "-DMPACK_SCONS=1", - "-g", - ]) -env.Append(LINKFLAGS = [ - "-g", - ]) - -if conf.CheckFlags(["-Wmissing-variable-declarations"]): - env.Append(CPPFLAGS = ["-Wmissing-variable-declarations"]) -if conf.CheckFlags(["-Wstrict-aliasing=1"]): - env.Append(CPPFLAGS = ["-Wstrict-aliasing=1"]) # use level 1 for maximum false positives - -# Additional warning flags are passed in SConscript based on the language (C/C++) - -if 'TRAVIS' not in env["ENV"]: - # Travis-CI currently uses Clang 3.4 which does not support this option, - # and it also appears to be incompatible with other GCC options on Travis-CI - env.Append(CPPFLAGS = ["-Wno-float-conversion"]) - - -# Optional flags used in various builds - -allfeatures = [ - "-DMPACK_READER=1", - "-DMPACK_WRITER=1", - "-DMPACK_EXPECT=1", - "-DMPACK_NODE=1", -] -noioconfigs = [ - "-DMPACK_STDLIB=1", - "-DMPACK_MALLOC=test_malloc", - "-DMPACK_FREE=test_free", -] -allconfigs = noioconfigs + ["-DMPACK_STDIO=1"] - -hasOg = conf.CheckFlags(["-Og"]) -if hasOg: - debugflags = ["-DDEBUG", "-Og"] -else: - debugflags = ["-DDEBUG", "-O0"] -releaseflags = ["-O2"] -cflags = ["-std=c99"] - -gcovflags = [] -if ARGUMENTS.get('gcov'): - gcovflags = ["-DMPACK_GCOV=1", "--coverage"] - -ltoflags = ["-O3", "-flto", "-fuse-linker-plugin", "-fno-fat-lto-objects"] - -if conf.CheckFlags(["-Wstrict-aliasing=3"]): - ltoflags.append("-Wstrict-aliasing=3") -elif conf.CheckFlags(["-Wstrict-aliasing=2"]): - ltoflags.append("-Wstrict-aliasing=2") - - -# -lstdc++ is added in SConscript -cxxflags = ["-x", "c++"] - - -# Functions to add a variant build. One variant build will build and run the -# entire library and test suite in a given configuration. - -def AddBuild(variant_dir, cppflags, linkflags = []): - env.SConscript("SConscript", - variant_dir="build/" + variant_dir, - src="../..", - exports={ - 'env': env, - 'CPPFLAGS': cppflags, - 'LINKFLAGS': linkflags - }, - duplicate=0) - -def AddBuilds(variant_dir, cppflags, linkflags = []): - AddBuild("debug-" + variant_dir, debugflags + cppflags, debugflags + linkflags) - if ARGUMENTS.get('all'): - AddBuild("release-" + variant_dir, releaseflags + cppflags, releaseflags + linkflags) - - -# The default build, everything in debug. This is the build used -# for code coverage measurement and static analysis. -AddBuild("debug", allfeatures + allconfigs + debugflags + cflags + gcovflags, gcovflags) - - -# Run "scons more=1" to run a handful of builds that are likely -# to reveal configuration errors. -if ARGUMENTS.get('more') or ARGUMENTS.get('all'): - AddBuild("release", allfeatures + allconfigs + releaseflags + cflags) - AddBuilds("embed", allfeatures + cflags + ["-DMPACK_NO_BUILTINS=1"]) - AddBuilds("noio", allfeatures + noioconfigs + cflags) - AddBuild("debug-size", ["-DMPACK_OPTIMIZE_FOR_SIZE=1"] + debugflags + allfeatures + allconfigs + cflags) - - -# Run "scons all=1" to run all builds. This is what the CI runs. -if ARGUMENTS.get('all'): - - # various release builds - AddBuild("release-unopt", allfeatures + allconfigs + cflags + ["-O0"]) - AddBuild("release-fastmath", allfeatures + allconfigs + releaseflags + cflags + ["-ffast-math"]) - if conf.CheckFlags(ltoflags, ltoflags, "-flto"): - AddBuild("release-lto", allfeatures + allconfigs + ltoflags + cflags, ltoflags) - AddBuild("release-size", ["-Os"] + allfeatures + allconfigs + cflags) - - # feature subsets with default configuration - AddBuilds("empty", allconfigs + cflags) - AddBuilds("writer", ["-DMPACK_WRITER=1"] + allconfigs + cflags) - AddBuilds("reader", ["-DMPACK_READER=1"] + allconfigs + cflags) - AddBuilds("expect", ["-DMPACK_READER=1", "-DMPACK_EXPECT=1"] + allconfigs + cflags) - AddBuilds("node", ["-DMPACK_NODE=1"] + allconfigs + cflags) - - # no i/o - AddBuilds("noio-writer", ["-DMPACK_WRITER=1"] + noioconfigs + cflags) - AddBuilds("noio-reader", ["-DMPACK_READER=1"] + noioconfigs + cflags) - AddBuilds("noio-expect", ["-DMPACK_READER=1", "-DMPACK_EXPECT=1"] + noioconfigs + cflags) - AddBuilds("noio-node", ["-DMPACK_NODE=1"] + noioconfigs + cflags) - - # embedded builds without libc (using builtins) - AddBuilds("embed-writer", ["-DMPACK_WRITER=1"] + cflags) - AddBuilds("embed-reader", ["-DMPACK_READER=1"] + cflags) - AddBuilds("embed-expect", ["-DMPACK_READER=1", "-DMPACK_EXPECT=1"] + cflags) - AddBuilds("embed-node", ["-DMPACK_NODE=1"] + cflags) - - # miscellaneous test builds - AddBuilds("notrack", ["-DMPACK_NO_TRACKING=1"] + allfeatures + allconfigs + cflags) - AddBuilds("realloc", allfeatures + allconfigs + debugflags + cflags + ["-DMPACK_REALLOC=test_realloc"]) - if hasOg: - AddBuild("debug-O0", allfeatures + allconfigs + ["-DDEBUG", "-O0"] + cflags) - - # other language standards (C11, various C++ versions) - if conf.CheckFlags(["-std=c11"]): - AddBuilds("c11", allfeatures + allconfigs + ["-std=c11"]) - AddBuilds("cxx", allfeatures + allconfigs + cxxflags + ["-std=c++98"]) - if conf.CheckFlags(cxxflags + ["-std=c++11"], [], "-std=c++11"): - AddBuilds("cxx11", allfeatures + allconfigs + cxxflags + ["-std=c++11"]) - if conf.CheckFlags(cxxflags + ["-std=c++14"], [], "-std=c++14"): - AddBuilds("cxx14", allfeatures + allconfigs + cxxflags + ["-std=c++14"]) - - # 32-bit build - if conf.CheckFlags(["-m32"], ["-m32"]): - AddBuilds("32", allfeatures + allconfigs + cflags + ["-m32"], ["-m32"]) - AddBuilds("cxx32", allfeatures + allconfigs + cxxflags + ["-std=c++98", "-m32"], ["-m32"]) diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 2cd2e04..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: 0.0.{build} - -configuration: - - Debug - - Release - -platform: - - Win32 - - x64 - -build: - project: projects\vs\mpack.sln - parallel: true - -test_script: -- cd projects\vs && %PLATFORM%\%CONFIGURATION%\mpack.exe diff --git a/docs/doxyfile b/docs/doxyfile index 0a89b1e..f7c79f3 100644 --- a/docs/doxyfile +++ b/docs/doxyfile @@ -1,4 +1,6 @@ -# Doxyfile 1.8.3.1 +# Doxyfile for MPack +# +# The documentation must be generated with tools/gendocs.sh PROJECT_NAME = "MPack" PROJECT_BRIEF = "A C encoding/decoding library for the MessagePack serialization format." @@ -6,51 +8,52 @@ PROJECT_BRIEF = "A C encoding/decoding library for the MessagePack serialization PROJECT_NUMBER = develop INPUT = \ - README.md \ + .build/docs/README.temp.md \ + docs/reader.md \ docs/expect.md \ + docs/node.md \ docs/features.md \ docs/protocol.md \ + src/mpack/mpack-platform.h \ src/mpack/mpack-common.h \ - src/mpack/mpack-reader.h \ src/mpack/mpack-writer.h \ + src/mpack/mpack-reader.h \ src/mpack/mpack-expect.h \ src/mpack/mpack-node.h \ - src/mpack/mpack.h + src/mpack/mpack.h \ LAYOUT_FILE = docs/doxygen-layout.xml -USE_MDFILE_AS_MAINPAGE = README.md -HTML_OUTPUT = docs/html +USE_MDFILE_AS_MAINPAGE = README.temp.md +HTML_OUTPUT = .build/docs/html GENERATE_LATEX = no -STRIP_FROM_PATH = . ./src +STRIP_FROM_PATH = . src .build/docs HTML_EXTRA_STYLESHEET = docs/doxygen-mpack-css.css PREDEFINED = \ + DEBUG=1 \ + \ inline= \ MPACK_INLINE= \ + MPACK_SILENCE_WARNINGS_BEGIN= \ + MPACK_SILENCE_WARNINGS_END= \ + MPACK_EXTERN_C_BEGIN= \ + MPACK_EXTERN_C_END= \ \ - MPACK_READER=1 \ - MPACK_WRITER=1 \ - MPACK_EXPECT=1 \ - MPACK_NODE=1 \ + MPACK_HAS_GENERIC=1 \ \ - MPACK_STDLIB=1 \ - MPACK_STDIO=1 \ - MPACK_SETJMP=1 \ - MPACK_MALLOC=malloc \ - MPACK_FREE=free \ + MPACK_COMPATIBILITY=1 \ + MPACK_EXTENSIONS=1 \ \ - MPACK_DEBUG=1 \ - MPACK_READ_TRACKING=1 \ - MPACK_WRITE_TRACKING=1 \ - \ - MPACK_HEADER_START= \ - MPACK_HEADER_END= \ + MPACK_DOXYGEN=1 \ MARKDOWN_SUPPORT = YES JAVADOC_AUTOBRIEF = YES ALWAYS_DETAILED_SEC = YES SORT_BRIEF_DOCS = YES +INCLUDE_PATH = .build/docs src + +SEARCH_INCLUDES = YES MACRO_EXPANSION = YES OPTIMIZE_OUTPUT_FOR_C = YES INLINE_SIMPLE_STRUCTS = YES @@ -68,4 +71,3 @@ WARN_IF_DOC_ERROR = YES #WARN_NO_PARAMDOC = YES QUIET = YES - diff --git a/docs/doxygen-layout.xml b/docs/doxygen-layout.xml index a0691cb..15d2416 100644 --- a/docs/doxygen-layout.xml +++ b/docs/doxygen-layout.xml @@ -11,7 +11,7 @@ - + @@ -34,7 +34,6 @@ - diff --git a/docs/doxygen-mpack-css.css b/docs/doxygen-mpack-css.css index 9a3bbf9..143720b 100644 --- a/docs/doxygen-mpack-css.css +++ b/docs/doxygen-mpack-css.css @@ -2,8 +2,11 @@ table.doxtable th { background-color: #e0e7ff; color: #000; } -.textblock code { - background-color: #f3f3f3; +.textblock code, +.contents p code, .contents p em, +.contents dd code, .contents dd em { + padding: 1pt 2pt 1pt 2pt; + background-color: #f5f5f5; } .textblock li { margin: 3pt 0 3pt 0; diff --git a/docs/expect.md b/docs/expect.md index 9edab65..5e9ea09 100644 --- a/docs/expect.md +++ b/docs/expect.md @@ -1,9 +1,10 @@ - # Using the Expect API The Expect API is used to imperatively parse data of a fixed (hardcoded) schema. It is most useful when parsing very large MessagePack files, parsing in memory-constrained environments, or generating parsing code from a schema. The API is similar to [CMP](https://github.com/camgunz/cmp), but has many helper functions especially for map keys and expected value ranges. Some of these will be covered below. -If you are not writing code for an embedded device or generating parsing code from a schema, you should not follow this guide. You should most likely be using the Node API instead. +Check out the [Reader API](docs/reader.md) guide first for information on setting up a reader and reading strings. + +If you are not writing code for an embedded device or generating parsing code from a schema, you should not follow this guide. You should most likely be using the [Node API](docs/node.md) instead. ## A simple example @@ -52,38 +53,38 @@ You can use [msgpack-tools](https://github.com/ludocode/msgpack-tools) with the int main(void) { - // initialize a reader from a file + // Initialize a reader from a file mpack_reader_t reader; mpack_reader_init_file(&reader, "example.mp"); - // the top-level array must have exactly three elements + // The top-level array must have exactly three elements mpack_expect_array_match(&reader, 3); - // the first two elements are short strings + // The first two elements are short strings char first[128]; char second[128]; mpack_expect_utf8_cstr(&reader, first, sizeof(first)); mpack_expect_utf8_cstr(&reader, second, sizeof(second)); - // next we have an array of up to ten ints + // Next we have an array of up to ten ints int32_t numbers[10]; size_t count = mpack_expect_array_max(&reader, sizeof(numbers) / sizeof(numbers[0])); for (size_t i = 0; i < count; ++i) numbers[i] = mpack_expect_i32(&reader); mpack_done_array(&reader); - // done reading the top-level array + // Done reading the top-level array mpack_done_array(&reader); - // clean up and handle errors + // Clean up and handle errors mpack_error_t error = mpack_reader_destroy(&reader); if (error != mpack_ok) { fprintf(stderr, "Error %i occurred reading data!\n", (int)error); return EXIT_FAILURE; } - // we now know the data was parsed correctly and can safely - // be used. the strings are null-terminated and valid UTF-8, + // We now know the data was parsed correctly and can safely + // be used. The strings are null-terminated and valid UTF-8, // the array contained at most ten elements, and the numbers // are all within the range of an int32_t. printf("%s\n", first); @@ -158,7 +159,7 @@ bool compact = false; bool has_schema = false; int schema = -1; -for (size_t i = mpack_expect_map_max(&reader, 2); i > 0; --i) { +for (size_t i = mpack_expect_map_max(&reader, 100); i > 0 && mpack_reader_error(&reader) == mpack_ok; --i) { char key[20]; mpack_expect_cstr(&reader, key, sizeof(key)); @@ -186,9 +187,7 @@ if (!has_compact) mpack_reader_flag_error(&reader, mpack_error_data); ``` -Note above the importance of using `mpack_expect_map_max()` rather than `mpack_expect_map()`. Without the maximum range, an attacker could craft a messaging declaring an array of a billion elements, forcing this code into an infinite loop. Alternatively you could check the reader for errors in each iteration of the loop. - -In order to simplify this code, MPack includes an Expect function called `mpack_expect_key_cstr()` to switch on string keys. This function should be passed an array of key strings and an array of bool flags storing whether each key was found. It will parse the key and check for duplicate keys, and returns the index of the found key (or the key count if it is unrecognized or if an error occurs.) You would use it with an `enum` and a `switch`, like this: +This is obviously way too verbose. In order to simplify this code, MPack includes an Expect function called `mpack_expect_key_cstr()` to switch on string keys. This function should be passed an array of key strings and an array of bool flags storing whether each key was found. It will find the key in the given string array, check for duplicate keys, and return the index of the found key (or the key count if it is unrecognized or if an error occurs.) You would use it with an `enum` and a `switch`, like this: ```C enum key_names {KEY_COMPACT, KEY_SCHEMA, KEY_COUNT}; @@ -198,8 +197,9 @@ bool found[KEY_COUNT] = {0}; bool compact = false; int schema = -1; -for (size_t i = mpack_expect_map_max(&reader, KEY_COUNT); i > 0; --i) { - switch (mpack_expect_key_cstr(&reader, keys, found, key_count)) { +size_t i = mpack_expect_map_max(&reader, 100); // critical check! +for (; i > 0 && mpack_reader_error(&reader) == mpack_ok; --i) { // critical check! + switch (mpack_expect_key_cstr(&reader, keys, found, KEY_COUNT)) { case KEY_COMPACT: compact = mpack_expect_bool(&reader); break; case KEY_SCHEMA: schema = mpack_expect_int(&reader); break; default: mpack_discard(&reader); break; @@ -211,8 +211,10 @@ if (!found[KEY_COMPACT]) mpack_reader_flag_error(&reader, mpack_error_data); ``` -In both of the above examples, the call to `mpack_discard(&reader);` skips over the value for unrecognized keys, allowing the data to be extensible and providing forwards-compatibility. If you want to forbid unrecognized keys, you can flag an error (e.g. `mpack_reader_flag_error(&reader, mpack_error_data);`) instead of discarding the value. +In the above examples, the call to `mpack_discard(&reader);` skips over the value for unrecognized keys, allowing the format to be extensible and providing forwards-compatibility. If you want to forbid unrecognized keys, you can flag an error (e.g. `mpack_reader_flag_error(&reader, mpack_error_data);`) instead of discarding the value. + +WARNING: See above the importance of using a reasonable limit on `mpack_expect_map_max()`, and of checking for errors in each iteration of the loop. If we were to leave these out, an attacker could craft a message declaring an array of a billion elements, forcing this code into a very long loop. We specify a size of 100 here as an arbitrary limit that leaves enough space for the schema to grow in the future. If you forbid unrecognized keys, you could specify the key count as the limit. Unlike JSON, MessagePack supports any type as a map key, so the enum integer values can themselves be used as keys. This reduces message size at some expense of debuggability (losing some of the value of a schemaless format.) There is a simpler function `mpack_expect_key_uint()` which can be used to switch on small non-negative enum values directly. -On the surface this doesn't appear much shorter than the previous code, but it becomes much nicer when you have many possible keys in a map. (Of course if at all possible you should consider using the Node API which is much less error-prone and will handle all of this for you. It can be used with a fixed node pool even without an allocator or libc.) +On the surface this doesn't appear much shorter than the previous code, but it becomes much nicer when you have many possible keys in a map. Of course if at all possible you should consider using the [Node API](docs/node.md) which is much less error-prone and will handle all of this for you. diff --git a/docs/features.md b/docs/features.md index 245ba46..43bc3e1 100644 --- a/docs/features.md +++ b/docs/features.md @@ -1,4 +1,3 @@ - # Feature Comparisons This document compares the features of various C libraries for encoding and decoding MessagePack. (It attempts to be neutral, but you may find it biased towards MPack since it is part of this project.) @@ -10,36 +9,41 @@ Feedback is welcome! Please let me know if any entries in the table are incorrec [mpack]: https://github.com/ludocode/mpack [msgpack-c]: https://github.com/msgpack/msgpack-c [cmp]: https://github.com/camgunz/cmp - -| | [MPack][mpack]
(v0.8) | [msgpack-c][msgpack-c]
(v1.3.0) | [CMP][cmp]
(v14) | -|:------------------------------------|:---:|:---:|:---:| -| No libc requirement | ✓ | | ✓ | -| No allocator requirement | ✓ | | ✓ | -| Growable memory writer | ✓ | ✓ | | -| File I/O helpers | ✓ | ✓ | | -| Tree parser | ✓ | ✓ | | -| Propagating errors | ✓ | | ✓ | -| Descriptive error information | | | | -| Compound size tracking | ✓ | | | -| Automatic compound size | | | | -| Incremental parser | ✓ | | ✓ | -| Incremental type helpers | ✓ | | ✓ | -| Incremental range/match helpers | ✓ | | | -| Asynchronous incremental parser | | | | -| Peek next element | ✓ | | | -| Tree stream parser | | ✓ | | -| Asynchronous tree stream parser | | ✓ | | -| Support for new (2.0) spec | ✓ | ✓ | ✓ | -| Compatible with older (1.0) spec | | ✓ | ✓ | -| UTF-8 verification | ✓ | | | - -Most of the features above are optional when supported and can be configured in all libraries. In particular, UTF-8 verification is optional with MPack; compound size tracking is optional and disabled in release by default with MPack; and 1.0 (v4) spec compatibility is optional with CMP (v5/2.0 is the recommended and default usage.) +[cwpack]: https://github.com/clwi/CWPack + +| | [MPack][mpack]
(v1.1) | [msgpack-c][msgpack-c]
(v3.3.0) | [CMP][cmp]
(v19) | [CWPack][cwpack]
(v1.3.1) | +|:------------------------------------|:---:|:---:|:---:|:---:| +| No libc requirement | ✓ | | ✓ | ✓ | +| No allocator requirement | ✓ | | ✓ | ✓ | +| Growable memory writer | ✓ | ✓ | | ✓ | +| File I/O helpers | ✓ | ✓ | | ✓ | +| Tree parser | ✓ | ✓ | | | +| Stateful error handling | ✓ | | ✓ | ✓ | +| Descriptive error information | | | | | +| Compound size tracking | ✓ | | | | +| Automatic compound size | ✓ | | | | +| Incremental parser | ✓ | | ✓ | ✓ | +| Typed read helpers | ✓ | | ✓ | | +| Range/match read helpers | ✓ | | | | +| Asynchronous incremental parser | | | | | +| Peek next element | ✓ | | | | +| Tree stream parser | ✓ | ✓ | | | +| Asynchronous tree stream parser | ✓ | ✓ | | | +| Support for new (2.0) spec | ✓ | ✓ | ✓ | ✓ | +| Compatible with older (1.0) spec | ✓ | ✓ | ✓ | ✓ | +| UTF-8 verification | ✓ | | | | +| Type-generic write helpers | ✓ | ✓ | | | +| Timestamps | ✓ | ✓ | | | + +Most of the features above are optional when supported and can be configured in all libraries. In particular, UTF-8 verification is optional with MPack; compound size tracking is optional and disabled in release by default with MPack; and 1.0 (v4) spec compatibility is optional in all libraries (v5/2.0 is the recommended and default usage.) ## Glossary *Tree parsing* means parsing a MessagePack object into a DOM-style tree of dynamically-typed elements supporting random access. -*Propagating errors* means a parse error or type error on one element places the whole parser, encoder or tree in an error state. This means you can check for errors only at certain key points rather than at every interaction with the library, and you get a final error state indicating whether any error occurred at any point during parsing or encoding. +*Incremental parsing* means being able to parse one basic MessagePack element at a time (either imperatively or with a SAX-style callback) with no per-element memory usage. + +*Stateful error handling* means a parse error or type error on one element places the whole parser, encoder or tree in an error state. This means you can check for errors only at certain key points rather than at every interaction with the library, and you get a final error state indicating whether any error occurred at any point during parsing or encoding. *Descriptive error information* means being able to get additional information when an error occurs, such as the tree position and byte position in the message where the error occurred. @@ -47,11 +51,9 @@ Most of the features above are optional when supported and can be configured in *Automatic compound size* means not having to specify the number of elements or bytes in an element up-front, instead determining it automatically when the element is closed. -*Incremental parsing* means being able to parse one basic MessagePack element at a time (either imperatively or with a SAX-style callback) with no per-element memory usage. - -*Incremental type helpers* means helper functions for an incremental parser that can check the expected type of an element. +*Typed read helpers* means helper functions for a parser that can check the expected type of an element and return its value in that type. For example `cmp_read_int()` in CMP or `mpack_expect_u32()` in MPack. -*Incremental range/match helpers* means helper functions for an incremental parser that can check not only the expected type of an element, but also enforce an allowed range or exact match on a given value. +*Range/match read helpers* means helper functions for a parser that can check not only the expected type of an element, but also enforce an allowed range or exact match on a given value. *Peeking the next element* means being able to view the next element during incremental parsing without popping it from the data buffer. @@ -61,3 +63,4 @@ Most of the features above are optional when supported and can be configured in *Compatible with older (1.0) spec* means the ability to produce messages compatible with parsers that only understand the old v4/1.0 version of MessagePack. A backwards-compatible encoder must at a minimum support writing an old-style raw without "str8", since there was no "raw8" type in old MessagePack. +*Type-generic write helpers* means a generic write function or macro that can serialize based on the static type of the argument, in at least one of C11 or C++. (The reference [msgpack-c][msgpack-c] currently supports this only in C++ mode. MPack supports this both in C++ with templates and in C11 with `_Generic`.) diff --git a/docs/node.md b/docs/node.md new file mode 100644 index 0000000..29d7cbd --- /dev/null +++ b/docs/node.md @@ -0,0 +1,76 @@ +# Using the Node API + +The Node API is used to parse MessagePack data into a tree in memory, providing DOM-style random access to its contents. + +The Node API can parse from a chunk of data in memory, or it can pull data from a file or stream. When the Node API uses a file or stream, it collects the data for a single message into one contiguous buffer in memory as it parses. The nodes for strings and binary data simply point to the data in the buffer, minimizing copies of the data. In this sense, a tree is just an index over a contiguous MessagePack message in memory. + +## The Basics + +A tree is initialized with one of the `init()` functions, and optionally configured (e.g. with `mpack_set_limits()`.) A complete message is then parsed with `mpack_tree_parse()`. + +The data can then be accessed in random order. For example: + +```C +bool parse_messagepack(const char* data, size_t count) { + mpack_tree_t tree; + mpack_tree_init_data(&tree, data, count); + mpack_tree_parse(&tree); + + mpack_node_t root = mpack_tree_root(&tree); + do_something_with_node(root); + + return mpack_tree_destroy(&tree) == mpack_ok; +} +``` + +As with the Expect API, the Node API contains helper functions for extracting data of expected types from the tree, and for stepping into maps and arrays. These can be nested together to quickly unpack a message. + +For example, to parse the data on the [msgpack.org](https://msgpack.org) homepage from a node: + +```C +void parse_msgpack_homepage(mpack_node_t node, bool* compact, int* schema) { + *compact = mpack_node_bool(mpack_node_map_cstr(root, "compact")); + *schema = mpack_node_int(mpack_node_map_cstr(root, "schema")); +} +``` + +If any error occurs, these functions always return safe values and flag an error on the tree. You do not have to check the tree for errors in between each step; you only need an error check before using the data. + +## Continuous Streams + +The Node API can parse messages indefinitely from a continuous stream. This can be used for inter-process or network communications. See [msgpack-rpc](https://github.com/msgpack-rpc/msgpack-rpc) for an example networking protocol. + +Here's a minimal example that wraps a tree parser around a BSD socket. We've defined a `stream_t` object to contain our file descriptor and other stream state, and we use it as the tree's context. + +```C +#define MAX_SIZE (1024*1024) +#define MAX_NODES 1024 + +typedef struct stream_t { + int fd; +} stream_t; + +static size_t read_stream(mpack_tree_t* tree, char* buffer, size_t count) { + stream_t* stream = mpack_tree_context(tree); + ssize_t step = read(stream->fd, buffer, count); + if (step <= 0) + mpack_tree_flag_error(tree, mpack_error_io); +} + +void parse_stream(stream_t* stream) { + mpack_tree_t tree; + mpack_tree_init_stream(&tree, &read_stream, stream, MAX_SIZE, MAX_NODES); + + while (true) { + mpack_tree_parse(&tree); + if (mpack_tree_error(&tree) != mpack_ok) + break; + + received_message(mpack_tree_root(&tree)); + } +} +``` + +The function `received_message()` will be called with each new message received from the peer. + +The Node API contains many more features, including non-blocking parsing, optional map lookups and more. This document is a work in progress. See the Node API reference for more information. diff --git a/docs/protocol.md b/docs/protocol.md index 335f8f9..d54474d 100644 --- a/docs/protocol.md +++ b/docs/protocol.md @@ -1,9 +1,8 @@ - # Protocol Clarifications -The MessagePack specification contains overlap between different types, allowing the same data to be encoded in many different representations. For example there are overlong sequences, signed/unsigned overlap for non-negative integers, different floating-point widths, raw/str/bin types, and more. +The MessagePack specification contains overlap between different types, allowing the same data to be encoded in many different representations. For example there are overlong sequences, signed/unsigned overlap for non-negative integers, different floating-point widths, raw/str/bin/ext types, and more. -MessagePack also does not specify how types should be interpreted, such as whether maps are ordered, whether strings can be treated as binary data, whether integers can be treated as real numbers, and so on. +MessagePack also does not specify how types should be interpreted, such as whether maps are ordered, whether strings can be treated as binary data, whether integers can be treated as real numbers, and so on. Some of these are explicitly left up to the implementation, such as whether unrecognized extensions should be rejected or treated as opaque data. MPack currently implements the [v5/2.0 MessagePack specification](https://github.com/msgpack/msgpack/blob/0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md?). This document describes MPack's implementation. @@ -25,15 +24,15 @@ As of this writing, all C and C++ libraries seem to write data in the shortest r ## Integer Signedness -MessagePack allows non-negative integers to be encoded both as signed and unsigned, and the specification does not specify how a library should serialize them. For example the number 231 in the shortest representation could be encoded as either `CC E7` (unsigned) or `D0 E7` (signed). +MessagePack allows non-negative integers to be encoded both as signed and unsigned, and the specification does not specify how a library should serialize them. For example the number 100 in the shortest representation could be encoded as either `CC 64` (unsigned) or `D0 64` (signed). -- When encoding, MPack writes all non-negative integers in the shortest unsigned integer type, regardless of the signedness of the input type. +- When encoding, MPack writes all non-negative integers in the shortest unsigned integer type, regardless of the signedness of the input type. (The signedness of the input type is discarded.) - When decoding as a dynamic tag or node, MPack returns the signedness of the serialized type. (This means you always need to handle both `mpack_type_int` and `mpack_type_uint`, regardless of whether you want a signed or unsigned integer, and regardless of whether you want a negative or non-negative integer.) -- When expecting an integer with the Expect API, MPack will automatically convert between signedness without loss of data. (For example if you call `mpack_expect_uint()`, MPack will allow both signed and unsigned data, and will flag an error if the type is signed with a negative value. Likewise if you call `mpack_expect_i8()`, MPack will allow both signed and unsigned data, and will flag an error for values above `INT8_MAX`.) The expect integer functions expect an integer of any size or signedness, and are only checking that it falls within the range of the expected type. +- When retrieving an integer with the Expect or Node APIs, MPack will automatically convert between signedness without loss of data. (For example if you call `mpack_expect_uint()` or `mpack_node_uint()`, MPack will allow both signed and unsigned data, and will flag an error if the type is signed with a negative value. Likewise if you call `mpack_expect_i8()` or `mpack_node_i8()`, MPack will allow both signed and unsigned data, and will flag an error for values below `INT8_MIN` or above `INT8_MAX`.) The expect and node integer functions allow an integer of any size or signedness, and are only checking that it falls within the range of the requested type. -A library could technically preserve the signedness of variables by writing any signed variable as int8/int16/int32/int64 or a negative fixint even if the value is non-negative. This does not seem to be the intent of the specification. For example there are no positive signed fixint values, so encoding the `int` with value 1 would take two bytes (`D0 01`) to preserve signedness. This is why MPack discards signedness. Moreover, many languages do not support unsigned integers (even popular statically typed languages such as Java), so there would be portability concerns in trying to preserve it. +A library could technically preserve the signedness of variables by writing any signed variable as int8/int16/int32/int64 or a negative fixint even if the value is non-negative. This does not seem to be the intent of the specification. For example there are no positive signed fixint values, so encoding the signed int with value 1 would take two bytes (`D0 01`) to preserve signedness. This is why MPack discards signedness. As of this writing, all C and C++ libraries supporting the modern MessagePack specification appear to discard signedness and write all non-negative ints as unsigned (including the reference implementation.) @@ -83,10 +82,20 @@ Despite the allowance for duplicate keys, MPack recommends against providing mul ## v4 Compatibility -The MessagePack [v4/1.0 spec](https://github.com/msgpack/msgpack/blob/acbcdf6b2a5a62666987c041124a10c69124be0d/spec-old.md?) did not distinguish between strings and binary data. It only provided the "raw" type in widths of fixraw, raw16 and raw32, which was used for both. The [v5/2.0 spec](https://github.com/msgpack/msgpack/blob/0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md?) on the other hand renames the raw type to str, adds the bin type to represent binary data, and adds an 8-bit width for strings. This means that even when binary data is not used, the new specification is not backwards compatible with the old one that expects raw to contain strings, because a modern encoder will use the str8 type. The new specification also adds an ext type to distinguish between arbitrary binary blobs and MessagePack extensions. +The MessagePack [v4/1.0 spec](https://github.com/msgpack/msgpack/blob/acbcdf6b2a5a62666987c041124a10c69124be0d/spec-old.md?) did not distinguish between strings and binary data. It only provided the "raw" type in widths of fixraw, raw16 and raw32, which was used for both. The [v5/2.0 spec](https://github.com/msgpack/msgpack/blob/0b8f5ac67cdd130f4d4d4fe6afb839b989fdb86a/spec.md?) on the other hand renames the raw type to str, adds the bin type to represent binary data, and adds an 8-bit width for strings called str8. This means that even when binary data is not used, the new specification is not backwards compatible with the old one that expects raw to contain strings, because a modern encoder will use the str8 type. The new specification also adds an ext type to distinguish between arbitrary binary blobs and MessagePack extensions. -- MPack always encodes with the str8 type for strings when possible. This means that MessagePack encoded with MPack is not backwards compatible with decoders that only understand the raw types from the old specification. This matches the behaviour of other C/C++ libraries that support the modern spec, including the reference implementation. +- MPack by default always encodes with the str8 type for strings where possible. This means that MessagePack encoded with MPack is by default not backwards compatible with decoders that only understand the raw types from the old specification. This matches the behaviour of other C/C++ libraries that support the modern spec, including the reference implementation. - Since MPack allows overlong sequences, it does not require that the str8 type be used, so data encoded with an old-style encoder will be parsed correctly by MPack (with raw types parsed as strings.) -However, other libraries typically also include functions to write an old-style raw in order to create backwards-compatible data, such as `msgpack_pack_v4raw()` in the reference implementation. MPack does not. There hasn't been any demand to create backwards-compatible data so far. If there were, I would be more likely to implement an option on a writer to always generate an old-style raw for both str and bin (and to flag an error if ext is used.) This is not implemented yet, and will hopefully never be implemented if libraries for the old specification can be phased out. If you need this feature, please let me know. +MPack supports a compatibility mode if interoperability is required with applications or data that do not support the new (v5) MessagePack spec. To use it, you must define `MPACK_COMPATIBILITY`, and then call `mpack_writer_set_version()` with the value `mpack_version_v4`. A writer in this mode will never use the str8 type, and will output the old fixraw, raw16 and raw32 types when writing both strings and bins. + +Note that there is no mode to configure readers to interpret the old style raws as either bins or strings. They will only be interpreted as strings. + +## Extension Types + +The MessagePack specification defines support for extension types. These are like bin types, but with an additional marker to define the semantic type, giving a clue to parsers about how to interpret the contained bits. + +This author considers them redundant with the bin type. Adding semantic information about binary data does nothing to improve interoperability. It instead only increases the complexity of decoders and fragments support for MessagePack. In my opinion the MessagePack spec should be forever frozen at v5 without extension types, similar (in theory) to JSON. See the discussion in [msgpack/msgpack!206](https://github.com/msgpack/msgpack/issues/206#issuecomment-386066548) for more. + +MPack discourages the use of extension types. In its default configuration, MPack will flag `mpack_error_unsupported` when encountering them, and functions to encode them are preproc'd out. Support for extension types can be enabled by defining `MPACK_EXTENSIONS`. diff --git a/docs/reader.md b/docs/reader.md new file mode 100644 index 0000000..f2e20a5 --- /dev/null +++ b/docs/reader.md @@ -0,0 +1,310 @@ +# Using the Reader API + +The Reader API is used to parse MessagePack incrementally with no per-element memory usage. Elements can be read one-by-one, and the content of strings and binary blobs can be read in chunks. + +Reading incrementally is much more difficult and verbose than using the [Node API](docs/node.md). If you are not constrained in memory or performance, you should use the Node API. + +## The Basics + +A reader is first initialized against a data source. This can be a chunk of data in memory, or it can be a file or stream. A reader that uses a file or stream will read data in chunks into an internal buffer. This allows parsing very large messages efficiently with minimal memory usage. + +Once initialized, the fundamental operation of a reader is to read a tag. A tag is a value struct of type `mpack_tag_t` that contains the value of a single element, or the metadata for a compound element. + +Here's a minimal example that parses the first element out of a chunk of MessagePack data. + +```C +bool parse_first_element(const char* data, size_t length) { + mpack_reader_t reader; + mpack_reader_init_data(&reader, data, length); + + mpack_tag_t tag = mpack_read_tag(&reader); + do_something_with_tag(&tag); + + return mpack_reader_destroy(&reader) == mpack_ok; +}; +``` + +The struct `mpack_tag_t` contains the element type, accessible with `mpack_tag_type()`. Depending on the type, you can access the value with tag accessor functions such as `mpack_tag_bool_value()` or `mpack_tag_array_count()`. + +Of course, parsing single values isn't terribly useful. You'll need to know how to parse strings, maps and arrays. + +## Compound Types + +Compound types are either containers (map, array) or data chunks (strings, binary blobs). For any compound type, the tag only contains the compound type's size. You still need to read the contained data, and then you need to let the reader know that you're done reading the element (so it can check that you did it correctly.) + +### Containers + +To parse a container, you must read as many additional elements as are specified by the tag's count. Note that a map tag specifies the number of key value pairs it contains, so you must actually read double the tag's count. You must then call `mpack_done_array()` or `mpack_done_map()` so that the reader can verify that you read the correct number of elements. + +Here's an example of a function that reads a tag from a reader, then reads all of the contained elements recursively: + +```C +void parse_element(mpack_reader_t* reader, int depth) { + if (depth >= 32) { // critical check! + mpack_reader_flag_error(reader, mpack_error_too_big); + return; + } + + mpack_tag_t tag = mpack_read_tag(&reader); + if (mpack_reader_error(reader) != mpack_ok) + return; + + do_something_with_tag(&tag); + + if (mpack_tag_type(&tag) == mpack_type_array) { + for (uint32_t i = mpack_tag_array_count(&tag); i > 0; --i) { + parse_element(reader, depth + 1); + if (mpack_reader_error(reader) != mpack_ok) // critical check! + break; + } + mpack_done_array(reader); + } + + if (mpack_tag_type(&tag) == mpack_type_map) { + for (uint32_t i = mpack_tag_map_count(&tag); i > 0; --i) { + parse_element(reader, depth + 1); + parse_element(reader, depth + 1); + if (mpack_reader_error(reader) != mpack_ok) // critical check! + break; + } + mpack_done_map(reader); + } +} +``` + +WARNING: It is critical that we check for errors during each iteration of our array and map loops. If we skip these checks, malicious data could claim to contain billions of elements, throwing us into a very long loop. These checks allow us to break out as soon as we run out of data. Since this function is recursive, we've also added a depth limit to prevent bad data from causing a stack overflow. + +(The [Node API](docs/node.md) is not vulnerable to such problems. In fact, it can break out of such bad data even sooner. The Node API will automatically flag an error as soon as it realizes that there aren't enough bytes left in the message to fill the remaining elements of all open compound types. If possible, consider using the Node API.) + +Unfortunately, the above code has a problem: it does not handle strings or binary blobs. If it encounters them, it will fail because it's not reading their contents. + +### Compound Data Chunks + +The data for chunks such as strings and binary blobs must be read separately. Bytes can be read across multiple read calls, as long the total number of bytes you read matches the number of bytes contained in the element. And as with containers, you must call `mpack_done_str()` or `mpack_done_bin()` so that the reader can verify that you read the correct total number of bytes. + +We can add support for strings to the previous example with the below code. In this example, we read the string incrementally in 128 byte chunks to keep memory usage at an absolute minimum: + +```C + if (mpack_tag_type(&tag) == mpack_type_str) { + uint32_t length = mpack_tag_str_length(&tag); + char buffer[128]; + + while (length > 0) { + size_t step = (length < sizeof(buffer)) ? length : sizeof(buffer); + mpack_read_bytes(reader, buffer, sizeof(buffer); + if (mpack_reader_error(reader) != mpack_ok) // critical check! + break; + + do_something_with_partial_string(buffer, step); + } + + mpack_done_str(reader); + } +``` + +As above, we need a safety check to ensure that bad data cannot get us stuck in a loop. + +Code for `mpack_type_bin` is identical except that the type and function names contain `bin` instead of `str`. + +### In-Place strings + +The MPack reader supports accessing the data contained in compound data types directly out of the reader's buffer. This can provide zero-copy access to strings and binary blobs, provided they fit within the buffer. + +To access string or bin data in-place, use `mpack_read_bytes_inplace()` (or `mpack_read_utf8_inplace()` to also check a string's UTF-8 encoding.) This provides a pointer directly into the reader's buffer, moving data around as necessary to make it fit. The previous code could be replaced with: + +```C + if (mpack_tag_type(&tag) == mpack_type_str) { + uint32_t length = mpack_tag_str_length(&tag); + const char* data = mpack_read_bytes_inplace(reader, length); + if (mpack_reader_error(reader) != mpack_ok) + return; + do_something_with_string(data, length); + mpack_done_str(reader); + } +``` + +There's an important caveat here though: this will flag an error if the string does not fit in the buffer. If you are decoding a chunk of MessagePack data in memory (without a fill function), then this is not a problem, as it would simply mean that the message was truncated. But if you are decoding from a file or stream, you need to account for the fact that the string may not fit in the buffer. + +To work around this, you can provide two paths for reading data depending on the size of the string. MPack provides a helper `mpack_should_read_bytes_inplace()` to tell you if it's a good idea to read in-place. Here's another example that uses zero-copy strings where possible, and falls back to an allocation otherwise: + +```C + if (mpack_tag_type(&tag) == mpack_type_str) { + uint32_t length = mpack_tag_str_length(&tag); + if (length >= 16 * 1024) { // critical check! limit length to avoid a huge allocation + mpack_reader_flag_error(reader, mpack_error_too_big); + return; + } + + if (mpack_should_read_bytes_inplace(reader, length)) { + const char* data = mpack_read_bytes_inplace(reader, length); + if (mpack_reader_error(reader) != mpack_ok) + return; + do_something_with_string(data, length); + + } else { + char* data = malloc(length); + mpack_read_bytes(reader, data, length); + if (mpack_reader_error(reader) != mpack_ok) { + free(data); + return; + } + do_something_with_string(data, length); + free(data); + } + + mpack_done_str(reader); + } +``` + +## Expected Types + +When parsing data of expected types from a dynamic stream, you will likely want to hardcode type checks before each access. For example: + +```C +// get a bool +mpack_tag_t tag = mpack_read_tag(reader); +if (mpack_tag_type(&tag) != mpack_type_bool) { + perror("not a bool!"); + return false; +} +bool value = mpack_tag_bool_value(&tag); +``` + +MPack provides helper functions that perform these checks in the [Expect API](docs/expect.md). The above code for example could be replaced by a single call to `mpack_expect_bool()`. If the value is not a bool, it will flag an error and return `false`. This is the incremental reader analogue to `mpack_write_bool()`. + +These helpers are strongly recommended because they will perform range checks and lossless conversions where needed. For example, suppose you want to read an `int`. A positive integer could be stored in a tag as two different types (`mpack_type_int` or `mpack_type_uint`), and the value of that integer could be outside the range of an `int`. The functions `mpack_expect_int()` supports both types and performs the appropriate range checks. It will convert losslessly as needed and flag an error if the value doesn't fit. + +Note that when using the expect functions, you still need to read the contents of compound types and call the corresponding done function. A call to an expect function only replaces a call to `mpack_read_tag()`. For example: + +```C +uint32_t length = mpack_expect_str(reader); +const char* data = mpack_read_bytes_inplace(reader, length); +if (mpack_reader_error(reader) == mpack_ok) + do_something_with_string(data, length); +mpack_done_str(reader); +``` + +If you want to always check that types match what you expect, head on over to the [Expect API](docs/expect.md) and use these type checking functions. But if you are still interested in handling the dynamic runtime type of elements, read on. + +## Event-Based Parser + +In this example, we'll implement a SAX-style event-based parser for blobs of MessagePack using the Reader API. This is a good example of non-trivial MPack usage that handles the dynamic type of incrementally parsed MessagePack data. It's easy to convert any incremental parser to an event-based parser, as we'll soon see. + +First let's define a header file with a list of callbacks for our parser, and our single function to launch the parser: + +```C +typedef struct sax_callbacks_t { + void (*nil_element)(void* context); + void (*bool_element)(void* context, int64_t value); + void (*int_element)(void* context, int64_t value); + void (*uint_element)(void* context, uint64_t value); + void (*string_element)(void* context, const char* data, uint32_t length); + + void (*start_map)(void* context, uint32_t pair_count); + void (*start_array)(void* context, uint32_t element_count); + void (*finish_map)(void* context); + void (*finish_array)(void* context); +} sax_callbacks_t; + +/** + * Parse a blob of MessagePack data, calling the appropriate callback for each + * element encountered. + * + * @return true if successful, false if any error occurs. + */ +bool parse_messagepack(const char* data, size_t length, + const sax_callbacks_t* callbacks, void* context); +``` + +Next we'll define our `parse_messagepack()` function in a corresponding source file to set our our reader. This will wrap another function called `parse_element()`. + +```C +static void parse_element(mpack_reader_t* reader, int depth, + const sax_callbacks_t* callbacks, void* context); + +bool parse_messagepack(const char* data, size_t length, + const sax_callbacks_t* callbacks, void* context) +{ + mpack_reader_t reader; + mpack_reader_init_data(&reader, data, length); + parse_element(&reader, 0, callbacks, context); + return mpack_ok == mpack_reader_destroy(&reader); +} +``` + +We'll make `parse_element()` recursive to keep things simple. This makes it extremely straightforward to implement. We just parse a tag, switch on the type, and call the appropriate callback function. + +Note that we can access all strings in-place because the source data is a single chunk of contiguous memory. + +```C +static void parse_element(mpack_reader_t* reader, int depth, + const sax_callbacks_t* callbacks, void* context) +{ + if (depth >= 32) { // critical check! + mpack_reader_flag_error(reader, mpack_error_too_big); + return; + } + + mpack_tag_t tag = mpack_read_tag(reader); + if (mpack_reader_error(reader) != mpack_ok) + return; + + switch (mpack_tag_type(&tag)) { + case mpack_type_nil: + callbacks->nil_element(context); + break; + case mpack_type_bool: + callbacks->bool_element(context, mpack_tag_bool_value(&tag)); + break; + case mpack_type_int: + callbacks->int_element(context, mpack_tag_int_value(&tag)); + break; + case mpack_type_uint: + callbacks->uint_element(context, mpack_tag_uint_value(&tag)); + break; + + case mpack_type_str: { + uint32_t length = mpack_tag_str_length(&tag); + const char* data = mpack_read_bytes_inplace(reader, length); + callbacks->string_element(context, data, length); + mpack_done_str(reader); + break; + } + + case mpack_type_array: { + uint32_t count = mpack_tag_array_count(&tag); + callbacks->start_array(context, count); + while (count-- > 0) { + parse_element(reader, depth + 1, callbacks, context); + if (mpack_reader_error(reader) != mpack_ok) // critical check! + break; + } + callbacks->finish_array(context); + mpack_done_array(reader); + break; + } + + case mpack_type_map: { + uint32_t count = mpack_tag_map_count(&tag); + callbacks->start_map(context, count); + while (count-- > 0) { + parse_element(reader, depth + 1, callbacks, context); + parse_element(reader, depth + 1, callbacks, context); + if (mpack_reader_error(reader) != mpack_ok) // critical check! + break; + } + callbacks->finish_map(context); + mpack_done_map(reader); + break; + } + + default: + mpack_reader_flag_error(reader, mpack_error_unsupported); + break; + } +} +``` + +As above, the error checks within loops are critical to keep the parser safe against untrusted data. + +This is all that is needed to convert MPack into an event-based parser. diff --git a/examples/Makefile b/examples/Makefile new file mode 100644 index 0000000..b08b8f1 --- /dev/null +++ b/examples/Makefile @@ -0,0 +1,9 @@ +.PHONY: run-sax-example +run-sax-example: sax-example + ./sax-example ../test/messagepack/data-1-2.mp + +sax-example: Makefile sax-example.c sax-example.h $(shell echo ../src/*) + cc -g -Wall -Werror -DMPACK_EXTENSIONS=1 sax-example.c ../src/mpack/*.c -I../src -o sax-example + +clean: + rm -f sax-example diff --git a/examples/sax-example.c b/examples/sax-example.c new file mode 100644 index 0000000..c3ba99a --- /dev/null +++ b/examples/sax-example.c @@ -0,0 +1,192 @@ +#include "sax-example.h" + +static void parse_element(mpack_reader_t* reader, int depth, + const sax_callbacks_t* callbacks, void* context); + +bool parse_messagepack(const char* data, size_t length, + const sax_callbacks_t* callbacks, void* context) +{ + mpack_reader_t reader; + mpack_reader_init_data(&reader, data, length); + parse_element(&reader, 0, callbacks, context); + return mpack_ok == mpack_reader_destroy(&reader); +} + +static void parse_element(mpack_reader_t* reader, int depth, + const sax_callbacks_t* callbacks, void* context) +{ + if (depth >= 32) { // critical check! + mpack_reader_flag_error(reader, mpack_error_too_big); + return; + } + + mpack_tag_t tag = mpack_read_tag(reader); + if (mpack_reader_error(reader) != mpack_ok) + return; + + switch (mpack_tag_type(&tag)) { + case mpack_type_nil: + callbacks->nil_element(context, depth); + break; + case mpack_type_bool: + callbacks->bool_element(context, depth, mpack_tag_bool_value(&tag)); + break; + case mpack_type_int: + callbacks->int_element(context, depth, mpack_tag_int_value(&tag)); + break; + case mpack_type_uint: + callbacks->uint_element(context, depth, mpack_tag_uint_value(&tag)); + break; + + case mpack_type_str: { + uint32_t length = mpack_tag_str_length(&tag); + const char* data = mpack_read_bytes_inplace(reader, length); + callbacks->string_element(context, depth, data, length); + mpack_done_str(reader); + break; + } + + case mpack_type_bin: { + uint32_t length = mpack_tag_bin_length(&tag); + const char* data = mpack_read_bytes_inplace(reader, length); + callbacks->bin_element(context, depth, data, length); + mpack_done_bin(reader); + break; + } + + case mpack_type_array: { + uint32_t count = mpack_tag_array_count(&tag); + callbacks->start_array(context, depth, count); + while (count-- > 0) { + parse_element(reader, depth + 1, callbacks, context); + if (mpack_reader_error(reader) != mpack_ok) // critical check! + break; + } + callbacks->finish_array(context, depth); + mpack_done_array(reader); + break; + } + + case mpack_type_map: { + uint32_t count = mpack_tag_map_count(&tag); + callbacks->start_map(context, depth, count); + while (count-- > 0) { + parse_element(reader, depth + 1, callbacks, context); + parse_element(reader, depth + 1, callbacks, context); + if (mpack_reader_error(reader) != mpack_ok) // critical check! + break; + } + callbacks->finish_map(context, depth); + mpack_done_map(reader); + break; + } + + default: + fprintf(stderr, "Error: type %s not implemented by this example SAX parser.\n", + mpack_type_to_string(mpack_tag_type(&tag))); + exit(1); + } +} + +#define SAX_EXAMPLE_TEST 1 +#if SAX_EXAMPLE_TEST +#include +#include +#include +#include +#include +#include + +static void indent(int depth) { + const char ruler[] = " "; + printf("%*.*s", depth * 4, depth * 4, ruler); +} + +static void nil_element(void* context, int depth) { + indent(depth); + printf("nil\n"); +} + +static void bool_element(void* context, int depth, int64_t value) { + indent(depth); + printf("bool: %s\n", value ? "true" : "false"); +} + +static void int_element(void* context, int depth, int64_t value) { + indent(depth); + printf("int: %" PRIi64 "\n", value); +} + +static void uint_element(void* context, int depth, uint64_t value) { + indent(depth); + printf("uint: %" PRIu64 "\n", value); +} + +static void string_element(void* context, int depth, + const char* data, uint32_t length) { + indent(depth); + printf("string: \""); + fwrite(data, 1, length, stdout); + printf("\"\n"); +} + +static void bin_element(void* context, int depth, + const char* data, uint32_t length) { + uint32_t i; + indent(depth); + printf("bin: \""); + for (i = 0; i < length; ++i) { + printf("%2.2x", data[i] & 0xff); + } + printf("\"\n"); +} + +static void start_map(void* context, int depth, uint32_t pair_count) { + indent(depth); + printf("starting map of %u key-value pairs\n", pair_count); +} + +static void start_array(void* context, int depth, uint32_t element_count) { + indent(depth); + printf("starting array of %u key-value pairs\n", element_count); +} + +static void finish_map(void* context, int depth) { + indent(depth); + printf("finishing map\n"); +} + +static void finish_array(void* context, int depth) { + indent(depth); + printf("finishing array\n"); +} + +static sax_callbacks_t callbacks = { + nil_element, + bool_element, + int_element, + uint_element, + string_element, + bin_element, + start_map, + start_array, + finish_map, + finish_array, +}; + + +int main(int argc, char** argv) { + if (argc != 2) { + fprintf(stderr, "First argument must be path to MessagePack file."); + exit(1); + } + struct stat stat; + int fd = open(argv[1], O_RDONLY); + fstat(fd, &stat); + const char* p = (const char*) mmap(0, stat.st_size, PROT_READ, MAP_SHARED, fd, 0); + bool ok = parse_messagepack(p, stat.st_size, &callbacks, NULL); + if (!ok) + fprintf(stderr, "Parse failed!\n"); + return ok ? EXIT_SUCCESS : EXIT_FAILURE; +} +#endif diff --git a/examples/sax-example.h b/examples/sax-example.h new file mode 100644 index 0000000..574a57c --- /dev/null +++ b/examples/sax-example.h @@ -0,0 +1,26 @@ +#include "mpack/mpack.h" + +typedef struct sax_callbacks_t { + void (*nil_element)(void* context, int depth); + void (*bool_element)(void* context, int depth, int64_t value); + void (*int_element)(void* context, int depth, int64_t value); + void (*uint_element)(void* context, int depth, uint64_t value); + void (*string_element)(void* context, int depth, + const char* data, uint32_t length); + void (*bin_element)(void* context, int depth, + const char* data, uint32_t length); + + void (*start_map)(void* context, int depth, uint32_t pair_count); + void (*start_array)(void* context, int depth, uint32_t element_count); + void (*finish_map)(void* context, int depth); + void (*finish_array)(void* context, int depth); +} sax_callbacks_t; + +/** + * Parse a blob of MessagePack data, calling the appropriate callback for each + * element encountered. + * + * @return true if successful, false if any error occurs. + */ +bool parse_messagepack(const char* data, size_t length, + const sax_callbacks_t* callbacks, void* context); diff --git a/projects/vs/mpack.sln b/projects/vs/mpack.sln deleted file mode 100755 index 03c403c..0000000 --- a/projects/vs/mpack.sln +++ /dev/null @@ -1,28 +0,0 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2013 -VisualStudioVersion = 12.0.31101.0 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "mpack", "mpack.vcxproj", "{43A2C901-A389-4C88-9FA6-1A7FF2BDF9BF}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Debug|x64 = Debug|x64 - Release|Win32 = Release|Win32 - Release|x64 = Release|x64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {43A2C901-A389-4C88-9FA6-1A7FF2BDF9BF}.Debug|Win32.ActiveCfg = Debug|Win32 - {43A2C901-A389-4C88-9FA6-1A7FF2BDF9BF}.Debug|Win32.Build.0 = Debug|Win32 - {43A2C901-A389-4C88-9FA6-1A7FF2BDF9BF}.Debug|x64.ActiveCfg = Debug|x64 - {43A2C901-A389-4C88-9FA6-1A7FF2BDF9BF}.Debug|x64.Build.0 = Debug|x64 - {43A2C901-A389-4C88-9FA6-1A7FF2BDF9BF}.Release|Win32.ActiveCfg = Release|Win32 - {43A2C901-A389-4C88-9FA6-1A7FF2BDF9BF}.Release|Win32.Build.0 = Release|Win32 - {43A2C901-A389-4C88-9FA6-1A7FF2BDF9BF}.Release|x64.ActiveCfg = Release|x64 - {43A2C901-A389-4C88-9FA6-1A7FF2BDF9BF}.Release|x64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection -EndGlobal diff --git a/projects/vs/mpack.vcxproj b/projects/vs/mpack.vcxproj deleted file mode 100755 index a93f0cb..0000000 --- a/projects/vs/mpack.vcxproj +++ /dev/null @@ -1,179 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - Release - Win32 - - - Release - x64 - - - - {43A2C901-A389-4C88-9FA6-1A7FF2BDF9BF} - Win32Proj - - - - Application - true - v120 - - - Application - true - v120 - - - Application - false - v120 - - - Application - false - v120 - - - - - - - - - - - - - - - - - - - true - $(SolutionDir)$(Platform)\$(Configuration)\ - $(Platform)\$(Configuration)\ - - - true - - - true - $(SolutionDir)$(Platform)\$(Configuration)\ - $(Platform)\$(Configuration)\ - - - true - - - - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDebugDLL - Level3 - ProgramDatabase - Disabled - ..\..\src;..\..\test;%(AdditionalIncludeDirectories) - Default - - - true - Console - - - - - WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDebugDLL - Level3 - ProgramDatabase - Disabled - ..\..\src;..\..\test;%(AdditionalIncludeDirectories) - Default - - - true - Console - - - - - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDLL - Level3 - ProgramDatabase - ..\..\src;..\..\test;%(AdditionalIncludeDirectories) - Default - - - true - Console - true - true - - - - - WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) - MultiThreadedDLL - Level3 - ProgramDatabase - ..\..\src;..\..\test;%(AdditionalIncludeDirectories) - Default - - - true - Console - true - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/projects/vs/mpack.vcxproj.filters b/projects/vs/mpack.vcxproj.filters deleted file mode 100755 index 397e03d..0000000 --- a/projects/vs/mpack.vcxproj.filters +++ /dev/null @@ -1,113 +0,0 @@ - - - - - {4FC737F1-C7A5-4376-A066-2A32D752A2FF} - cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx - - - {93995380-89BD-4b04-88EB-625FBE52EBFB} - h;hh;hpp;hxx;hm;inl;inc;xsd - - - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - Source Files - - - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - Header Files - - - \ No newline at end of file diff --git a/projects/xcode/MPack.xcodeproj/project.pbxproj b/projects/xcode/MPack.xcodeproj/project.pbxproj deleted file mode 100644 index 41382f2..0000000 --- a/projects/xcode/MPack.xcodeproj/project.pbxproj +++ /dev/null @@ -1,343 +0,0 @@ -// !$*UTF8*$! -{ - archiveVersion = 1; - classes = { - }; - objectVersion = 46; - objects = { - -/* Begin PBXBuildFile section */ - 012F29D31AD4524700346AC7 /* mpack-common.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29B11AD4524700346AC7 /* mpack-common.c */; }; - 012F29D41AD4524700346AC7 /* mpack-expect.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29B31AD4524700346AC7 /* mpack-expect.c */; }; - 012F29D61AD4524700346AC7 /* mpack-node.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29B71AD4524700346AC7 /* mpack-node.c */; }; - 012F29D71AD4524700346AC7 /* mpack-platform.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29B91AD4524700346AC7 /* mpack-platform.c */; }; - 012F29D81AD4524700346AC7 /* mpack-reader.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29BB1AD4524700346AC7 /* mpack-reader.c */; }; - 012F29D91AD4524700346AC7 /* mpack-writer.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29BD1AD4524700346AC7 /* mpack-writer.c */; }; - 012F29DA1AD4524700346AC7 /* test.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29C21AD4524700346AC7 /* test.c */; }; - 012F29DB1AD4524700346AC7 /* test-buffer.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29C41AD4524700346AC7 /* test-buffer.c */; }; - 012F29DC1AD4524700346AC7 /* test-file.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29C61AD4524700346AC7 /* test-file.c */; }; - 012F29DD1AD4524700346AC7 /* test-system.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29C81AD4524700346AC7 /* test-system.c */; }; - 012F29DE1AD4524700346AC7 /* test-node.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29CA1AD4524700346AC7 /* test-node.c */; }; - 012F29DF1AD4524700346AC7 /* test-expect.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29CC1AD4524700346AC7 /* test-expect.c */; }; - 012F29E01AD4524700346AC7 /* test-common.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29CE1AD4524700346AC7 /* test-common.c */; }; - 012F29E11AD4524700346AC7 /* test-write.c in Sources */ = {isa = PBXBuildFile; fileRef = 012F29D01AD4524700346AC7 /* test-write.c */; }; - 014246B61BE5426200347D5E /* test-reader.c in Sources */ = {isa = PBXBuildFile; fileRef = 014246B41BE5426200347D5E /* test-reader.c */; }; -/* End PBXBuildFile section */ - -/* Begin PBXCopyFilesBuildPhase section */ - 012C2DA71AD450E00034C7BE /* CopyFiles */ = { - isa = PBXCopyFilesBuildPhase; - buildActionMask = 2147483647; - dstPath = /usr/share/man/man1/; - dstSubfolderSpec = 0; - files = ( - ); - runOnlyForDeploymentPostprocessing = 1; - }; -/* End PBXCopyFilesBuildPhase section */ - -/* Begin PBXFileReference section */ - 012C2DA91AD450E10034C7BE /* MPack */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = MPack; sourceTree = BUILT_PRODUCTS_DIR; }; - 012F29B11AD4524700346AC7 /* mpack-common.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "mpack-common.c"; sourceTree = ""; }; - 012F29B21AD4524700346AC7 /* mpack-common.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "mpack-common.h"; sourceTree = ""; }; - 012F29B31AD4524700346AC7 /* mpack-expect.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "mpack-expect.c"; sourceTree = ""; }; - 012F29B41AD4524700346AC7 /* mpack-expect.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "mpack-expect.h"; sourceTree = ""; }; - 012F29B71AD4524700346AC7 /* mpack-node.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "mpack-node.c"; sourceTree = ""; }; - 012F29B81AD4524700346AC7 /* mpack-node.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "mpack-node.h"; sourceTree = ""; }; - 012F29B91AD4524700346AC7 /* mpack-platform.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "mpack-platform.c"; sourceTree = ""; }; - 012F29BA1AD4524700346AC7 /* mpack-platform.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "mpack-platform.h"; sourceTree = ""; }; - 012F29BB1AD4524700346AC7 /* mpack-reader.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "mpack-reader.c"; sourceTree = ""; }; - 012F29BC1AD4524700346AC7 /* mpack-reader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "mpack-reader.h"; sourceTree = ""; }; - 012F29BD1AD4524700346AC7 /* mpack-writer.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "mpack-writer.c"; sourceTree = ""; }; - 012F29BE1AD4524700346AC7 /* mpack-writer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "mpack-writer.h"; sourceTree = ""; }; - 012F29BF1AD4524700346AC7 /* mpack.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = mpack.h; sourceTree = ""; }; - 012F29C01AD4524700346AC7 /* mpack-config.h.sample */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = "mpack-config.h.sample"; sourceTree = ""; }; - 012F29C21AD4524700346AC7 /* test.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = test.c; sourceTree = ""; }; - 012F29C31AD4524700346AC7 /* mpack-config.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "mpack-config.h"; sourceTree = ""; }; - 012F29C41AD4524700346AC7 /* test-buffer.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "test-buffer.c"; sourceTree = ""; }; - 012F29C51AD4524700346AC7 /* test-buffer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "test-buffer.h"; sourceTree = ""; }; - 012F29C61AD4524700346AC7 /* test-file.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "test-file.c"; sourceTree = ""; }; - 012F29C71AD4524700346AC7 /* test-file.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "test-file.h"; sourceTree = ""; }; - 012F29C81AD4524700346AC7 /* test-system.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "test-system.c"; sourceTree = ""; }; - 012F29C91AD4524700346AC7 /* test-system.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "test-system.h"; sourceTree = ""; }; - 012F29CA1AD4524700346AC7 /* test-node.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "test-node.c"; sourceTree = ""; }; - 012F29CB1AD4524700346AC7 /* test-node.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "test-node.h"; sourceTree = ""; }; - 012F29CC1AD4524700346AC7 /* test-expect.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "test-expect.c"; sourceTree = ""; }; - 012F29CD1AD4524700346AC7 /* test-expect.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "test-expect.h"; sourceTree = ""; }; - 012F29CE1AD4524700346AC7 /* test-common.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "test-common.c"; sourceTree = ""; }; - 012F29CF1AD4524700346AC7 /* test-common.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "test-common.h"; sourceTree = ""; }; - 012F29D01AD4524700346AC7 /* test-write.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "test-write.c"; sourceTree = ""; }; - 012F29D11AD4524700346AC7 /* test-write.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "test-write.h"; sourceTree = ""; }; - 012F29D21AD4524700346AC7 /* test.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = test.h; sourceTree = ""; }; - 014246B41BE5426200347D5E /* test-reader.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = "test-reader.c"; sourceTree = ""; }; - 014246B51BE5426200347D5E /* test-reader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "test-reader.h"; sourceTree = ""; }; -/* End PBXFileReference section */ - -/* Begin PBXFrameworksBuildPhase section */ - 012C2DA61AD450E00034C7BE /* Frameworks */ = { - isa = PBXFrameworksBuildPhase; - buildActionMask = 2147483647; - files = ( - ); - runOnlyForDeploymentPostprocessing = 0; - }; -/* End PBXFrameworksBuildPhase section */ - -/* Begin PBXGroup section */ - 012C2DA01AD450E00034C7BE = { - isa = PBXGroup; - children = ( - 012F29AF1AD4524700346AC7 /* src */, - 012F29C11AD4524700346AC7 /* test */, - 012C2DAA1AD450E10034C7BE /* Products */, - ); - sourceTree = ""; - }; - 012C2DAA1AD450E10034C7BE /* Products */ = { - isa = PBXGroup; - children = ( - 012C2DA91AD450E10034C7BE /* MPack */, - ); - name = Products; - sourceTree = ""; - }; - 012F29AF1AD4524700346AC7 /* src */ = { - isa = PBXGroup; - children = ( - 012F29B01AD4524700346AC7 /* mpack */, - 012F29C01AD4524700346AC7 /* mpack-config.h.sample */, - ); - name = src; - path = ../../src; - sourceTree = ""; - }; - 012F29B01AD4524700346AC7 /* mpack */ = { - isa = PBXGroup; - children = ( - 012F29B11AD4524700346AC7 /* mpack-common.c */, - 012F29B21AD4524700346AC7 /* mpack-common.h */, - 012F29B31AD4524700346AC7 /* mpack-expect.c */, - 012F29B41AD4524700346AC7 /* mpack-expect.h */, - 012F29B71AD4524700346AC7 /* mpack-node.c */, - 012F29B81AD4524700346AC7 /* mpack-node.h */, - 012F29B91AD4524700346AC7 /* mpack-platform.c */, - 012F29BA1AD4524700346AC7 /* mpack-platform.h */, - 012F29BB1AD4524700346AC7 /* mpack-reader.c */, - 012F29BC1AD4524700346AC7 /* mpack-reader.h */, - 012F29BD1AD4524700346AC7 /* mpack-writer.c */, - 012F29BE1AD4524700346AC7 /* mpack-writer.h */, - 012F29BF1AD4524700346AC7 /* mpack.h */, - ); - path = mpack; - sourceTree = ""; - }; - 012F29C11AD4524700346AC7 /* test */ = { - isa = PBXGroup; - children = ( - 012F29C31AD4524700346AC7 /* mpack-config.h */, - 012F29C41AD4524700346AC7 /* test-buffer.c */, - 012F29C51AD4524700346AC7 /* test-buffer.h */, - 012F29CE1AD4524700346AC7 /* test-common.c */, - 012F29CF1AD4524700346AC7 /* test-common.h */, - 012F29CC1AD4524700346AC7 /* test-expect.c */, - 012F29CD1AD4524700346AC7 /* test-expect.h */, - 012F29C61AD4524700346AC7 /* test-file.c */, - 012F29C71AD4524700346AC7 /* test-file.h */, - 012F29CA1AD4524700346AC7 /* test-node.c */, - 012F29CB1AD4524700346AC7 /* test-node.h */, - 014246B41BE5426200347D5E /* test-reader.c */, - 014246B51BE5426200347D5E /* test-reader.h */, - 012F29C81AD4524700346AC7 /* test-system.c */, - 012F29C91AD4524700346AC7 /* test-system.h */, - 012F29D01AD4524700346AC7 /* test-write.c */, - 012F29D11AD4524700346AC7 /* test-write.h */, - 012F29C21AD4524700346AC7 /* test.c */, - 012F29D21AD4524700346AC7 /* test.h */, - ); - name = test; - path = ../../test; - sourceTree = ""; - }; -/* End PBXGroup section */ - -/* Begin PBXNativeTarget section */ - 012C2DA81AD450E00034C7BE /* MPack */ = { - isa = PBXNativeTarget; - buildConfigurationList = 012C2DB21AD450E10034C7BE /* Build configuration list for PBXNativeTarget "MPack" */; - buildPhases = ( - 012C2DA51AD450E00034C7BE /* Sources */, - 012C2DA61AD450E00034C7BE /* Frameworks */, - 012C2DA71AD450E00034C7BE /* CopyFiles */, - ); - buildRules = ( - ); - dependencies = ( - ); - name = MPack; - productName = MPack; - productReference = 012C2DA91AD450E10034C7BE /* MPack */; - productType = "com.apple.product-type.tool"; - }; -/* End PBXNativeTarget section */ - -/* Begin PBXProject section */ - 012C2DA11AD450E00034C7BE /* Project object */ = { - isa = PBXProject; - attributes = { - LastUpgradeCheck = 0510; - ORGANIZATIONNAME = ludocode; - }; - buildConfigurationList = 012C2DA41AD450E00034C7BE /* Build configuration list for PBXProject "MPack" */; - compatibilityVersion = "Xcode 3.2"; - developmentRegion = English; - hasScannedForEncodings = 0; - knownRegions = ( - en, - ); - mainGroup = 012C2DA01AD450E00034C7BE; - productRefGroup = 012C2DAA1AD450E10034C7BE /* Products */; - projectDirPath = ""; - projectRoot = ""; - targets = ( - 012C2DA81AD450E00034C7BE /* MPack */, - ); - }; -/* End PBXProject section */ - -/* Begin PBXSourcesBuildPhase section */ - 012C2DA51AD450E00034C7BE /* Sources */ = { - isa = PBXSourcesBuildPhase; - buildActionMask = 2147483647; - files = ( - 012F29D61AD4524700346AC7 /* mpack-node.c in Sources */, - 012F29DB1AD4524700346AC7 /* test-buffer.c in Sources */, - 012F29D31AD4524700346AC7 /* mpack-common.c in Sources */, - 012F29D41AD4524700346AC7 /* mpack-expect.c in Sources */, - 012F29D81AD4524700346AC7 /* mpack-reader.c in Sources */, - 012F29DA1AD4524700346AC7 /* test.c in Sources */, - 012F29DD1AD4524700346AC7 /* test-system.c in Sources */, - 012F29DE1AD4524700346AC7 /* test-node.c in Sources */, - 014246B61BE5426200347D5E /* test-reader.c in Sources */, - 012F29E01AD4524700346AC7 /* test-common.c in Sources */, - 012F29DF1AD4524700346AC7 /* test-expect.c in Sources */, - 012F29D91AD4524700346AC7 /* mpack-writer.c in Sources */, - 012F29D71AD4524700346AC7 /* mpack-platform.c in Sources */, - 012F29E11AD4524700346AC7 /* test-write.c in Sources */, - 012F29DC1AD4524700346AC7 /* test-file.c in Sources */, - ); - runOnlyForDeploymentPostprocessing = 0; - }; -/* End PBXSourcesBuildPhase section */ - -/* Begin XCBuildConfiguration section */ - 012C2DB01AD450E10034C7BE /* Debug */ = { - isa = XCBuildConfiguration; - buildSettings = { - ALWAYS_SEARCH_USER_PATHS = NO; - CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; - CLANG_CXX_LIBRARY = "libc++"; - CLANG_ENABLE_MODULES = YES; - CLANG_ENABLE_OBJC_ARC = YES; - CLANG_WARN_BOOL_CONVERSION = YES; - CLANG_WARN_CONSTANT_CONVERSION = YES; - CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; - CLANG_WARN_EMPTY_BODY = YES; - CLANG_WARN_ENUM_CONVERSION = YES; - CLANG_WARN_INT_CONVERSION = YES; - CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; - CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; - COPY_PHASE_STRIP = NO; - GCC_C_LANGUAGE_STANDARD = gnu99; - GCC_DYNAMIC_NO_PIC = NO; - GCC_ENABLE_OBJC_EXCEPTIONS = YES; - GCC_OPTIMIZATION_LEVEL = 0; - GCC_PREPROCESSOR_DEFINITIONS = ( - "DEBUG=1", - "$(inherited)", - ); - GCC_SYMBOLS_PRIVATE_EXTERN = NO; - GCC_WARN_64_TO_32_BIT_CONVERSION = YES; - GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; - GCC_WARN_UNDECLARED_SELECTOR = YES; - GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; - GCC_WARN_UNUSED_FUNCTION = YES; - GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 10.9; - ONLY_ACTIVE_ARCH = YES; - SDKROOT = macosx; - }; - name = Debug; - }; - 012C2DB11AD450E10034C7BE /* Release */ = { - isa = XCBuildConfiguration; - buildSettings = { - ALWAYS_SEARCH_USER_PATHS = NO; - CLANG_CXX_LANGUAGE_STANDARD = "gnu++0x"; - CLANG_CXX_LIBRARY = "libc++"; - CLANG_ENABLE_MODULES = YES; - CLANG_ENABLE_OBJC_ARC = YES; - CLANG_WARN_BOOL_CONVERSION = YES; - CLANG_WARN_CONSTANT_CONVERSION = YES; - CLANG_WARN_DIRECT_OBJC_ISA_USAGE = YES_ERROR; - CLANG_WARN_EMPTY_BODY = YES; - CLANG_WARN_ENUM_CONVERSION = YES; - CLANG_WARN_INT_CONVERSION = YES; - CLANG_WARN_OBJC_ROOT_CLASS = YES_ERROR; - CLANG_WARN__DUPLICATE_METHOD_MATCH = YES; - COPY_PHASE_STRIP = YES; - DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; - ENABLE_NS_ASSERTIONS = NO; - GCC_C_LANGUAGE_STANDARD = gnu99; - GCC_ENABLE_OBJC_EXCEPTIONS = YES; - GCC_WARN_64_TO_32_BIT_CONVERSION = YES; - GCC_WARN_ABOUT_RETURN_TYPE = YES_ERROR; - GCC_WARN_UNDECLARED_SELECTOR = YES; - GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; - GCC_WARN_UNUSED_FUNCTION = YES; - GCC_WARN_UNUSED_VARIABLE = YES; - MACOSX_DEPLOYMENT_TARGET = 10.9; - SDKROOT = macosx; - }; - name = Release; - }; - 012C2DB31AD450E10034C7BE /* Debug */ = { - isa = XCBuildConfiguration; - buildSettings = { - PRODUCT_NAME = "$(TARGET_NAME)"; - USER_HEADER_SEARCH_PATHS = "$(SRCROOT)/../../test $(SRCROOT)/../../src"; - }; - name = Debug; - }; - 012C2DB41AD450E10034C7BE /* Release */ = { - isa = XCBuildConfiguration; - buildSettings = { - PRODUCT_NAME = "$(TARGET_NAME)"; - USER_HEADER_SEARCH_PATHS = "$(SRCROOT)/../../test $(SRCROOT)/../../src"; - }; - name = Release; - }; -/* End XCBuildConfiguration section */ - -/* Begin XCConfigurationList section */ - 012C2DA41AD450E00034C7BE /* Build configuration list for PBXProject "MPack" */ = { - isa = XCConfigurationList; - buildConfigurations = ( - 012C2DB01AD450E10034C7BE /* Debug */, - 012C2DB11AD450E10034C7BE /* Release */, - ); - defaultConfigurationIsVisible = 0; - defaultConfigurationName = Release; - }; - 012C2DB21AD450E10034C7BE /* Build configuration list for PBXNativeTarget "MPack" */ = { - isa = XCConfigurationList; - buildConfigurations = ( - 012C2DB31AD450E10034C7BE /* Debug */, - 012C2DB41AD450E10034C7BE /* Release */, - ); - defaultConfigurationIsVisible = 0; - defaultConfigurationName = Release; - }; -/* End XCConfigurationList section */ - }; - rootObject = 012C2DA11AD450E00034C7BE /* Project object */; -} diff --git a/projects/xcode/MPack.xcodeproj/xcshareddata/xcschemes/MPack.xcscheme b/projects/xcode/MPack.xcodeproj/xcshareddata/xcschemes/MPack.xcscheme deleted file mode 100644 index e6fa1b0..0000000 --- a/projects/xcode/MPack.xcodeproj/xcshareddata/xcschemes/MPack.xcscheme +++ /dev/null @@ -1,87 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/mpack-config.h.sample b/src/mpack-config.h.sample deleted file mode 100644 index bb55388..0000000 --- a/src/mpack-config.h.sample +++ /dev/null @@ -1,254 +0,0 @@ - -/** - * This is a sample MPack configuration file. Copy it to mpack-config.h somewhere - * in your project's include tree and, optionally, edit it to suit your setup. - * - * In most cases you can leave this file with the default config. - * - * You can also override the default configuration by pre-defining options to 0 or 1. - */ - -#ifndef MPACK_CONFIG_H -#define MPACK_CONFIG_H 1 - - -/* - * Features - */ - -/** Enables compilation of the base Tag Reader. */ -#ifndef MPACK_READER -#define MPACK_READER 1 -#endif - -/** Enables compilation of the static Expect API. */ -#ifndef MPACK_EXPECT -#define MPACK_EXPECT 1 -#endif - -/** Enables compilation of the dynamic Node API. */ -#ifndef MPACK_NODE -#define MPACK_NODE 1 -#endif - -/** Enables compilation of the Writer. */ -#ifndef MPACK_WRITER -#define MPACK_WRITER 1 -#endif - - -/* - * Dependencies - */ - -/** - * Enables the use of C stdlib. This allows the library to use malloc - * for debugging and in allocation helpers. - */ -#ifndef MPACK_STDLIB -#define MPACK_STDLIB 1 -#endif - -/** - * Enables the use of C stdio. This adds helpers for easily - * reading/writing C files and makes debugging easier. - */ -#ifndef MPACK_STDIO -#define MPACK_STDIO 1 -#endif - - -/* - * System Functions - */ - -/** - * \def MPACK_MALLOC - * - * Defines the memory allocation function used by mpack. This is used by - * helpers for automatically allocating data the correct size, and for - * debugging functions. If this macro is undefined, the allocation helpers - * will not be compiled. - * - * The default is malloc() if MPACK_STDLIB is enabled. - */ -/** - * \def MPACK_FREE - * - * Defines the memory free function used by mpack. This is used by helpers - * for automatically allocating data the correct size. If this macro is - * undefined, the allocation helpers will not be compiled. - * - * The default is free() if MPACK_MALLOC has not been customized and - * MPACK_STDLIB is enabled. - */ -/** - * \def MPACK_REALLOC - * - * Defines the realloc function used by mpack. It is used by growable - * buffers to resize more efficiently. - * - * The default is realloc() if MPACK_MALLOC has not been customized and - * MPACK_STDLIB is enabled. - * - * This is optional, even when MPACK_MALLOC is used. If MPACK_MALLOC is - * set and MPACK_REALLOC is not, MPACK_MALLOC is used with a simple copy - * to grow buffers. - */ -#if defined(MPACK_STDLIB) && !defined(MPACK_MALLOC) -#define MPACK_MALLOC malloc -#define MPACK_REALLOC realloc -#define MPACK_FREE free -#endif - - -/* - * Debugging options - */ - -/** - * \def MPACK_DEBUG - * - * Enables debug features. You may want to wrap this around your - * own debug preprocs. By default, they are enabled if DEBUG or _DEBUG - * are defined. - * - * Note that MPACK_DEBUG cannot be defined differently for different - * source files because it affects layout of structs defined in header - * files. Your entire project must be compiled with the same value of - * MPACK_DEBUG. (This is why NDEBUG is not used.) - */ -#if !defined(MPACK_DEBUG) && (defined(DEBUG) || defined(_DEBUG)) -#define MPACK_DEBUG 1 -#endif - -/** - * Set this to 1 to implement a custom mpack_assert_fail() function. This - * function must not return, and must have the following signature: - * - * void mpack_assert_fail(const char* message) - * - * Asserts are only used when MPACK_DEBUG is enabled, and can be triggered - * by bugs in mpack or bugs due to incorrect usage of mpack. - */ -#ifndef MPACK_CUSTOM_ASSERT -#define MPACK_CUSTOM_ASSERT 0 -#endif - -/** - * \def MPACK_READ_TRACKING - * - * Enables compound type size tracking for readers. This ensures that the - * correct number of elements or bytes are read from a compound type. - * - * This is enabled by default in debug builds (provided a malloc() is - * available.) - */ -#if !defined(MPACK_READ_TRACKING) && \ - defined(MPACK_DEBUG) && MPACK_DEBUG && \ - defined(MPACK_READER) && MPACK_READER && \ - defined(MPACK_MALLOC) -#define MPACK_READ_TRACKING 1 -#endif - -/** - * \def MPACK_WRITE_TRACKING - * - * Enables compound type size tracking for writers. This ensures that the - * correct number of elements or bytes are written in a compound type. - * - * Note that without write tracking enabled, it is possible for buggy code - * to emit invalid MessagePack without flagging an error by writing the wrong - * number of elements or bytes in a compound type. With tracking enabled, - * MPACK will catch such errors and break on the offending line of code. - * - * This is enabled by default in debug builds (provided a malloc() is - * available.) - */ -#if !defined(MPACK_WRITE_TRACKING) && \ - defined(MPACK_DEBUG) && MPACK_DEBUG && \ - defined(MPACK_WRITER) && MPACK_WRITER && \ - defined(MPACK_MALLOC) -#define MPACK_WRITE_TRACKING 1 -#endif - - -/* - * Miscellaneous - */ - -/** - * Whether to optimize for size or speed. - * - * Optimizing for size simplifies some parsing and encoding algorithms - * at the expense of speed, and saves a few kilobytes of space in the - * resulting executable. - * - * This automatically detects -Os with GCC/Clang. Unfortunately there - * doesn't seem to be a macro defined for /Os under MSVC. - */ -#ifndef MPACK_OPTIMIZE_FOR_SIZE -#ifdef __OPTIMIZE_SIZE__ -#define MPACK_OPTIMIZE_FOR_SIZE 1 -#else -#define MPACK_OPTIMIZE_FOR_SIZE 0 -#endif -#endif - -/** - * Stack space in bytes to use when initializing a reader or writer - * with a stack-allocated buffer. - */ -#ifndef MPACK_STACK_SIZE -#define MPACK_STACK_SIZE 4096 -#endif - -/** - * Buffer size to use for allocated buffers (such as for a file writer.) - * - * Starting with a single page and growing as needed seems to - * provide the best performance with minimal memory waste. - * Increasing this does not improve performance even when writing - * huge messages. - */ -#ifndef MPACK_BUFFER_SIZE -#define MPACK_BUFFER_SIZE 4096 -#endif - -/** - * Size of an allocated node page in bytes. - * - * The children for a given compound element must be contiguous, so - * larger pages than this may be allocated as needed. (Safety checks - * exist to prevent malicious data from causing too large allocations.) - * - * Nodes are 16 bytes on most common architectures (32-bit and 64-bit.) - * - * Using as many nodes fit in one memory page seems to provide the - * best performance, and has very little waste when parsing small - * messages. - */ -#ifndef MPACK_NODE_PAGE_SIZE -#define MPACK_NODE_PAGE_SIZE 4096 -#endif - -/** - * The initial depth for the node parser. When MPACK_MALLOC is available, - * the node parser has no practical depth limit, and it is not recursive - * so there is no risk of overflowing the call stack. - */ -#ifndef MPACK_NODE_INITIAL_DEPTH -#define MPACK_NODE_INITIAL_DEPTH 8 -#endif - -/** - * The maximum depth for the node parser if MPACK_MALLOC is not available. - * The parsing stack is placed on the call stack. - */ -#ifndef MPACK_NODE_MAX_DEPTH_WITHOUT_MALLOC -#define MPACK_NODE_MAX_DEPTH_WITHOUT_MALLOC 32 -#endif - - -#endif - diff --git a/src/mpack/mpack-common.c b/src/mpack/mpack-common.c index 1c4f9fe..fb76a43 100644 --- a/src/mpack/mpack-common.c +++ b/src/mpack/mpack-common.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -23,24 +23,23 @@ #include "mpack-common.h" -#if MPACK_DEBUG && MPACK_STDIO -#include -#endif +MPACK_SILENCE_WARNINGS_BEGIN const char* mpack_error_to_string(mpack_error_t error) { - #if MPACK_DEBUG + #if MPACK_STRINGS switch (error) { #define MPACK_ERROR_STRING_CASE(e) case e: return #e MPACK_ERROR_STRING_CASE(mpack_ok); MPACK_ERROR_STRING_CASE(mpack_error_io); MPACK_ERROR_STRING_CASE(mpack_error_invalid); + MPACK_ERROR_STRING_CASE(mpack_error_unsupported); MPACK_ERROR_STRING_CASE(mpack_error_type); MPACK_ERROR_STRING_CASE(mpack_error_too_big); MPACK_ERROR_STRING_CASE(mpack_error_memory); MPACK_ERROR_STRING_CASE(mpack_error_bug); MPACK_ERROR_STRING_CASE(mpack_error_data); + MPACK_ERROR_STRING_CASE(mpack_error_eof); #undef MPACK_ERROR_STRING_CASE - default: break; } mpack_assert(0, "unrecognized error %i", (int)error); return "(unknown mpack_error_t)"; @@ -51,9 +50,10 @@ const char* mpack_error_to_string(mpack_error_t error) { } const char* mpack_type_to_string(mpack_type_t type) { - #if MPACK_DEBUG + #if MPACK_STRINGS switch (type) { #define MPACK_TYPE_STRING_CASE(e) case e: return #e + MPACK_TYPE_STRING_CASE(mpack_type_missing); MPACK_TYPE_STRING_CASE(mpack_type_nil); MPACK_TYPE_STRING_CASE(mpack_type_bool); MPACK_TYPE_STRING_CASE(mpack_type_float); @@ -62,11 +62,12 @@ const char* mpack_type_to_string(mpack_type_t type) { MPACK_TYPE_STRING_CASE(mpack_type_uint); MPACK_TYPE_STRING_CASE(mpack_type_str); MPACK_TYPE_STRING_CASE(mpack_type_bin); - MPACK_TYPE_STRING_CASE(mpack_type_ext); MPACK_TYPE_STRING_CASE(mpack_type_array); MPACK_TYPE_STRING_CASE(mpack_type_map); + #if MPACK_EXTENSIONS + MPACK_TYPE_STRING_CASE(mpack_type_ext); + #endif #undef MPACK_TYPE_STRING_CASE - default: break; } mpack_assert(0, "unrecognized type %i", (int)type); return "(unknown mpack_type_t)"; @@ -81,17 +82,18 @@ int mpack_tag_cmp(mpack_tag_t left, mpack_tag_t right) { // positive numbers may be stored as int; convert to uint if (left.type == mpack_type_int && left.v.i >= 0) { left.type = mpack_type_uint; - left.v.u = left.v.i; + left.v.u = (uint64_t)left.v.i; } if (right.type == mpack_type_int && right.v.i >= 0) { right.type = mpack_type_uint; - right.v.u = right.v.i; + right.v.u = (uint64_t)right.v.i; } if (left.type != right.type) - return (int)left.type - (int)right.type; + return ((int)left.type < (int)right.type) ? -1 : 1; switch (left.type) { + case mpack_type_missing: // fallthrough case mpack_type_nil: return 0; @@ -120,6 +122,7 @@ int mpack_tag_cmp(mpack_tag_t left, mpack_tag_t right) { return 0; return (left.v.l < right.v.l) ? -1 : 1; + #if MPACK_EXTENSIONS case mpack_type_ext: if (left.exttype == right.exttype) { if (left.v.l == right.v.l) @@ -127,6 +130,7 @@ int mpack_tag_cmp(mpack_tag_t left, mpack_tag_t right) { return (left.v.l < right.v.l) ? -1 : 1; } return (int)left.exttype - (int)right.exttype; + #endif // floats should not normally be compared for equality. we compare // with memcmp() to silence compiler warnings, but this will return @@ -143,15 +147,213 @@ int mpack_tag_cmp(mpack_tag_t left, mpack_tag_t right) { return mpack_memcmp(&left.v.f, &right.v.f, sizeof(left.v.f)); case mpack_type_double: return mpack_memcmp(&left.v.d, &right.v.d, sizeof(left.v.d)); - - default: - break; } - + mpack_assert(0, "unrecognized type %i", (int)left.type); return false; } +#if MPACK_DEBUG && MPACK_STDIO +static char mpack_hex_char(uint8_t hex_value) { + // Older compilers (e.g. GCC 4.4.7) promote the result of this ternary to + // int and warn under -Wconversion, so we have to cast it back to char. + return (char)((hex_value < 10) ? (char)('0' + hex_value) : (char)('a' + (hex_value - 10))); +} + +static void mpack_tag_debug_complete_bin_ext(mpack_tag_t tag, size_t string_length, char* buffer, size_t buffer_size, + const char* prefix, size_t prefix_size) +{ + // If at any point in this function we run out of space in the buffer, we + // bail out. The outer tag print wrapper will make sure we have a + // null-terminator. + + if (string_length == 0 || string_length >= buffer_size) + return; + buffer += string_length; + buffer_size -= string_length; + + size_t total = mpack_tag_bytes(&tag); + if (total == 0) { + strncpy(buffer, ">", buffer_size); + return; + } + + strncpy(buffer, ": ", buffer_size); + if (buffer_size < 2) + return; + buffer += 2; + buffer_size -= 2; + + size_t hex_bytes = 0; + size_t i; + for (i = 0; i < MPACK_PRINT_BYTE_COUNT && i < prefix_size && buffer_size > 2; ++i) { + uint8_t byte = (uint8_t)prefix[i]; + buffer[0] = mpack_hex_char((uint8_t)(byte >> 4)); + buffer[1] = mpack_hex_char((uint8_t)(byte & 0xfu)); + buffer += 2; + buffer_size -= 2; + ++hex_bytes; + } + + if (buffer_size != 0) + mpack_snprintf(buffer, buffer_size, "%s>", (total > hex_bytes) ? "..." : ""); +} + +static void mpack_tag_debug_pseudo_json_bin(mpack_tag_t tag, char* buffer, size_t buffer_size, + const char* prefix, size_t prefix_size) +{ + mpack_assert(mpack_tag_type(&tag) == mpack_type_bin); + size_t length = (size_t)mpack_snprintf(buffer, buffer_size, ""); + return; + case mpack_type_nil: + mpack_snprintf(buffer, buffer_size, "null"); + return; + case mpack_type_bool: + mpack_snprintf(buffer, buffer_size, tag.v.b ? "true" : "false"); + return; + case mpack_type_int: + mpack_snprintf(buffer, buffer_size, "%" PRIi64, tag.v.i); + return; + case mpack_type_uint: + mpack_snprintf(buffer, buffer_size, "%" PRIu64, tag.v.u); + return; + case mpack_type_float: + #if MPACK_FLOAT + mpack_snprintf(buffer, buffer_size, "%f", tag.v.f); + #else + mpack_snprintf(buffer, buffer_size, ""); + #endif + return; + case mpack_type_double: + #if MPACK_DOUBLE + mpack_snprintf(buffer, buffer_size, "%f", tag.v.d); + #else + mpack_snprintf(buffer, buffer_size, ""); + #endif + return; + + case mpack_type_str: + mpack_snprintf(buffer, buffer_size, "", tag.v.l); + return; + case mpack_type_bin: + mpack_tag_debug_pseudo_json_bin(tag, buffer, buffer_size, prefix, prefix_size); + return; + #if MPACK_EXTENSIONS + case mpack_type_ext: + mpack_tag_debug_pseudo_json_ext(tag, buffer, buffer_size, prefix, prefix_size); + return; + #endif + + case mpack_type_array: + mpack_snprintf(buffer, buffer_size, "", tag.v.n); + return; + case mpack_type_map: + mpack_snprintf(buffer, buffer_size, "", tag.v.n); + return; + } + + mpack_snprintf(buffer, buffer_size, ""); +} + +void mpack_tag_debug_pseudo_json(mpack_tag_t tag, char* buffer, size_t buffer_size, + const char* prefix, size_t prefix_size) +{ + mpack_assert(buffer_size > 0, "buffer size cannot be zero!"); + buffer[0] = 0; + + mpack_tag_debug_pseudo_json_impl(tag, buffer, buffer_size, prefix, prefix_size); + + // We always null-terminate the buffer manually just in case the snprintf() + // function doesn't null-terminate when the string doesn't fit. + buffer[buffer_size - 1] = 0; +} + +static void mpack_tag_debug_describe_impl(mpack_tag_t tag, char* buffer, size_t buffer_size) { + switch (tag.type) { + case mpack_type_missing: + mpack_snprintf(buffer, buffer_size, "missing"); + return; + case mpack_type_nil: + mpack_snprintf(buffer, buffer_size, "nil"); + return; + case mpack_type_bool: + mpack_snprintf(buffer, buffer_size, tag.v.b ? "true" : "false"); + return; + case mpack_type_int: + mpack_snprintf(buffer, buffer_size, "int %" PRIi64, tag.v.i); + return; + case mpack_type_uint: + mpack_snprintf(buffer, buffer_size, "uint %" PRIu64, tag.v.u); + return; + case mpack_type_float: + #if MPACK_FLOAT + mpack_snprintf(buffer, buffer_size, "float %f", tag.v.f); + #else + mpack_snprintf(buffer, buffer_size, "float"); + #endif + return; + case mpack_type_double: + #if MPACK_DOUBLE + mpack_snprintf(buffer, buffer_size, "double %f", tag.v.d); + #else + mpack_snprintf(buffer, buffer_size, "double"); + #endif + return; + case mpack_type_str: + mpack_snprintf(buffer, buffer_size, "str of %" PRIu32 " bytes", tag.v.l); + return; + case mpack_type_bin: + mpack_snprintf(buffer, buffer_size, "bin of %" PRIu32 " bytes", tag.v.l); + return; + #if MPACK_EXTENSIONS + case mpack_type_ext: + mpack_snprintf(buffer, buffer_size, "ext of type %i, %" PRIu32 " bytes", + mpack_tag_ext_exttype(&tag), mpack_tag_ext_length(&tag)); + return; + #endif + case mpack_type_array: + mpack_snprintf(buffer, buffer_size, "array of %" PRIu32 " elements", tag.v.n); + return; + case mpack_type_map: + mpack_snprintf(buffer, buffer_size, "map of %" PRIu32 " key-value pairs", tag.v.n); + return; + } + + mpack_snprintf(buffer, buffer_size, "unknown!"); +} + +void mpack_tag_debug_describe(mpack_tag_t tag, char* buffer, size_t buffer_size) { + mpack_assert(buffer_size > 0, "buffer size cannot be zero!"); + buffer[0] = 0; + + mpack_tag_debug_describe_impl(tag, buffer, buffer_size); + + // We always null-terminate the buffer manually just in case the snprintf() + // function doesn't null-terminate when the string doesn't fit. + buffer[buffer_size - 1] = 0; +} +#endif + #if MPACK_READ_TRACKING || MPACK_WRITE_TRACKING @@ -187,14 +389,10 @@ mpack_error_t mpack_track_grow(mpack_track_t* track) { return mpack_ok; } -mpack_error_t mpack_track_push(mpack_track_t* track, mpack_type_t type, uint64_t count) { +mpack_error_t mpack_track_push(mpack_track_t* track, mpack_type_t type, uint32_t count) { mpack_assert(track->elements, "null track elements!"); mpack_log("track pushing %s count %i\n", mpack_type_to_string(type), (int)count); - // maps have twice the number of elements (key/value pairs) - if (type == mpack_type_map) - count *= 2; - // grow if needed if (track->count == track->capacity) { mpack_error_t error = mpack_track_grow(track); @@ -205,11 +403,34 @@ mpack_error_t mpack_track_push(mpack_track_t* track, mpack_type_t type, uint64_t // insert new track track->elements[track->count].type = type; track->elements[track->count].left = count; + track->elements[track->count].builder = false; + track->elements[track->count].key_needs_value = false; ++track->count; return mpack_ok; } -mpack_error_t mpack_track_pop(mpack_track_t* track, mpack_type_t type) { +// TODO dedupe this +mpack_error_t mpack_track_push_builder(mpack_track_t* track, mpack_type_t type) { + mpack_assert(track->elements, "null track elements!"); + mpack_log("track pushing %s builder\n", mpack_type_to_string(type)); + + // grow if needed + if (track->count == track->capacity) { + mpack_error_t error = mpack_track_grow(track); + if (error != mpack_ok) + return error; + } + + // insert new track + track->elements[track->count].type = type; + track->elements[track->count].left = 0; + track->elements[track->count].builder = true; + track->elements[track->count].key_needs_value = false; + ++track->count; + return mpack_ok; +} + +static mpack_error_t mpack_track_pop_impl(mpack_track_t* track, mpack_type_t type, bool builder) { mpack_assert(track->elements, "null track elements!"); mpack_log("track popping %s\n", mpack_type_to_string(type)); @@ -226,17 +447,39 @@ mpack_error_t mpack_track_pop(mpack_track_t* track, mpack_type_t type) { return mpack_error_bug; } + if (element->key_needs_value) { + mpack_assert(type == mpack_type_map, "key_needs_value can only be true for maps!"); + mpack_break("attempting to close a %s but an odd number of elements were written", + mpack_type_to_string(type)); + return mpack_error_bug; + } + if (element->left != 0) { - mpack_break("attempting to close a %s but there are %" PRIu64 " %s left", + mpack_break("attempting to close a %s but there are %i %s left", mpack_type_to_string(type), element->left, (type == mpack_type_map || type == mpack_type_array) ? "elements" : "bytes"); return mpack_error_bug; } + if (element->builder != builder) { + mpack_break("attempting to pop a %sbuilder but the open element is %sa builder", + builder ? "" : "non-", + element->builder ? "" : "not "); + return mpack_error_bug; + } + --track->count; return mpack_ok; } +mpack_error_t mpack_track_pop(mpack_track_t* track, mpack_type_t type) { + return mpack_track_pop_impl(track, type, false); +} + +mpack_error_t mpack_track_pop_builder(mpack_track_t* track, mpack_type_t type) { + return mpack_track_pop_impl(track, type, true); +} + mpack_error_t mpack_track_peek_element(mpack_track_t* track, bool read) { MPACK_UNUSED(read); mpack_assert(track->elements, "null track elements!"); @@ -253,7 +496,7 @@ mpack_error_t mpack_track_peek_element(mpack_track_t* track, bool read) { return mpack_error_bug; } - if (element->left == 0) { + if (!element->builder && element->left == 0 && !element->key_needs_value) { mpack_break("too many elements %s for %s", read ? "read" : "written", mpack_type_to_string(element->type)); return mpack_error_bug; @@ -264,15 +507,34 @@ mpack_error_t mpack_track_peek_element(mpack_track_t* track, bool read) { mpack_error_t mpack_track_element(mpack_track_t* track, bool read) { mpack_error_t error = mpack_track_peek_element(track, read); - if (track->count > 0 && error == mpack_ok) - --track->elements[track->count - 1].left; - return error; + if (track->count == 0 || error != mpack_ok) + return error; + + mpack_track_element_t* element = &track->elements[track->count - 1]; + + if (element->type == mpack_type_map) { + if (!element->key_needs_value) { + element->key_needs_value = true; + return mpack_ok; // don't decrement + } + element->key_needs_value = false; + } + + if (!element->builder) + --element->left; + return mpack_ok; } -mpack_error_t mpack_track_bytes(mpack_track_t* track, bool read, uint64_t count) { +mpack_error_t mpack_track_bytes(mpack_track_t* track, bool read, size_t count) { MPACK_UNUSED(read); mpack_assert(track->elements, "null track elements!"); + if (count > MPACK_UINT32_MAX) { + mpack_break("%s more bytes than could possibly fit in a str/bin/ext!", + read ? "reading" : "writing"); + return mpack_error_bug; + } + if (track->count == 0) { mpack_break("bytes cannot be %s with no open bin, str or ext", read ? "read" : "written"); return mpack_error_bug; @@ -292,11 +554,11 @@ mpack_error_t mpack_track_bytes(mpack_track_t* track, bool read, uint64_t count) return mpack_error_bug; } - element->left -= count; + element->left -= (uint32_t)count; return mpack_ok; } -mpack_error_t mpack_track_str_bytes_all(mpack_track_t* track, bool read, uint64_t count) { +mpack_error_t mpack_track_str_bytes_all(mpack_track_t* track, bool read, size_t count) { mpack_error_t error = mpack_track_bytes(track, read, count); if (error != mpack_ok) return error; @@ -435,9 +697,55 @@ bool mpack_utf8_check_no_null(const char* str, size_t bytes) { } bool mpack_str_check_no_null(const char* str, size_t bytes) { - for (size_t i = 0; i < bytes; ++i) + size_t i; + for (i = 0; i < bytes; ++i) if (str[i] == '\0') return false; return true; } +#if MPACK_DEBUG && MPACK_STDIO +void mpack_print_append(mpack_print_t* print, const char* data, size_t count) { + + // copy whatever fits into the buffer + size_t copy = print->size - print->count; + if (copy > count) + copy = count; + mpack_memcpy(print->buffer + print->count, data, copy); + print->count += copy; + data += copy; + count -= copy; + + // if we don't need to flush or can't flush there's nothing else to do + if (count == 0 || print->callback == NULL) + return; + + // flush the buffer + print->callback(print->context, print->buffer, print->count); + + if (count > print->size / 2) { + // flush the rest of the data + print->count = 0; + print->callback(print->context, data, count); + } else { + // copy the rest of the data into the buffer + mpack_memcpy(print->buffer, data, count); + print->count = count; + } + +} + +void mpack_print_flush(mpack_print_t* print) { + if (print->count > 0 && print->callback != NULL) { + print->callback(print->context, print->buffer, print->count); + print->count = 0; + } +} + +void mpack_print_file_callback(void* context, const char* data, size_t count) { + FILE* file = (FILE*)context; + fwrite(data, 1, count, file); +} +#endif + +MPACK_SILENCE_WARNINGS_END diff --git a/src/mpack/mpack-common.h b/src/mpack/mpack-common.h index f5beb66..3e77e07 100644 --- a/src/mpack/mpack-common.h +++ b/src/mpack/mpack-common.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -30,14 +30,28 @@ #include "mpack-platform.h" -MPACK_HEADER_START +#ifndef MPACK_PRINT_BYTE_COUNT +#define MPACK_PRINT_BYTE_COUNT 12 +#endif + +MPACK_SILENCE_WARNINGS_BEGIN +MPACK_EXTERN_C_BEGIN +/** + * @defgroup common Tags and Common Elements + * + * Contains types, constants and functions shared by both the encoding + * and decoding portions of MPack. + * + * @{ + */ + /* Version information */ -#define MPACK_VERSION_MAJOR 0 /**< The major version number of MPack. */ -#define MPACK_VERSION_MINOR 8 /**< The minor version number of MPack. */ +#define MPACK_VERSION_MAJOR 1 /**< The major version number of MPack. */ +#define MPACK_VERSION_MINOR 1 /**< The minor version number of MPack. */ #define MPACK_VERSION_PATCH 1 /**< The patch version number of MPack. */ /** A number containing the version number of MPack for comparison purposes. */ @@ -85,23 +99,57 @@ MPACK_HEADER_START #define MPACK_LIBRARY_STRING "MPack " MPACK_VERSION_STRING #endif +/** @cond */ /** * @def MPACK_MAXIMUM_TAG_SIZE * - * The maximum size of a tag in bytes, as of the "new" MessagePack spec. + * The maximum encoded size of a tag in bytes. */ #define MPACK_MAXIMUM_TAG_SIZE 9 +/** @endcond */ + +#if MPACK_EXTENSIONS +/** + * @def MPACK_TIMESTAMP_NANOSECONDS_MAX + * + * The maximum value of nanoseconds for a timestamp. + * + * @note This requires @ref MPACK_EXTENSIONS. + */ +#define MPACK_TIMESTAMP_NANOSECONDS_MAX 999999999 +#endif +#if MPACK_COMPATIBILITY /** - * @defgroup common Common Elements + * Versions of the MessagePack format. * - * Contains types and functions shared by both the encoding and decoding - * portions of MPack. + * A reader, writer, or tree can be configured to serialize in an older + * version of the MessagePack spec. This is necessary to interface with + * older MessagePack libraries that do not support new MessagePack features. * - * @{ + * @note This requires @ref MPACK_COMPATIBILITY. */ +typedef enum mpack_version_t { + + /** + * Version 1.0/v4, supporting only the @c raw type without @c str8. + */ + mpack_version_v4 = 4, + + /** + * Version 2.0/v5, supporting the @c str8, @c bin and @c ext types. + */ + mpack_version_v5 = 5, + + /** + * The most recent supported version of MessagePack. This is the default. + */ + mpack_version_current = mpack_version_v5, + +} mpack_version_t; +#endif /** * Error states for MPack objects. @@ -114,11 +162,13 @@ typedef enum mpack_error_t { mpack_ok = 0, /**< No error. */ mpack_error_io = 2, /**< The reader or writer failed to fill or flush, or some other file or socket error occurred. */ mpack_error_invalid, /**< The data read is not valid MessagePack. */ + mpack_error_unsupported, /**< The data read is not supported by this configuration of MPack. (See @ref MPACK_EXTENSIONS.) */ mpack_error_type, /**< The type or value range did not match what was expected by the caller. */ mpack_error_too_big, /**< A read or write was bigger than the maximum size allowed for that operation. */ mpack_error_memory, /**< An allocation failure occurred. */ mpack_error_bug, /**< The MPack API was used incorrectly. (This will always assert in debug mode.) */ mpack_error_data, /**< The contained data is not valid. */ + mpack_error_eof, /**< The reader failed to read because of file or socket EOF */ } mpack_error_t; /** @@ -129,19 +179,32 @@ const char* mpack_error_to_string(mpack_error_t error); /** * Defines the type of a MessagePack tag. + * + * Note that extension types, both user defined and built-in, are represented + * in tags as @ref mpack_type_ext. The value for an extension type is stored + * separately. */ typedef enum mpack_type_t { - mpack_type_nil = 1, /**< A null value. */ - mpack_type_bool, /**< A boolean (true or false.) */ - mpack_type_float, /**< A 32-bit IEEE 754 floating point number. */ - mpack_type_double, /**< A 64-bit IEEE 754 floating point number. */ - mpack_type_int, /**< A 64-bit signed integer. */ - mpack_type_uint, /**< A 64-bit unsigned integer. */ - mpack_type_str, /**< A string. */ - mpack_type_bin, /**< A chunk of binary data. */ - mpack_type_ext, /**< A typed MessagePack extension object containing a chunk of binary data. */ - mpack_type_array, /**< An array of MessagePack objects. */ - mpack_type_map, /**< An ordered map of key/value pairs of MessagePack objects. */ + mpack_type_missing = 0, /**< Special type indicating a missing optional value. */ + mpack_type_nil, /**< A null value. */ + mpack_type_bool, /**< A boolean (true or false.) */ + mpack_type_int, /**< A 64-bit signed integer. */ + mpack_type_uint, /**< A 64-bit unsigned integer. */ + mpack_type_float, /**< A 32-bit IEEE 754 floating point number. */ + mpack_type_double, /**< A 64-bit IEEE 754 floating point number. */ + mpack_type_str, /**< A string. */ + mpack_type_bin, /**< A chunk of binary data. */ + mpack_type_array, /**< An array of MessagePack objects. */ + mpack_type_map, /**< An ordered map of key/value pairs of MessagePack objects. */ + + #if MPACK_EXTENSIONS + /** + * A typed MessagePack extension object containing a chunk of binary data. + * + * @note This requires @ref MPACK_EXTENSIONS. + */ + mpack_type_ext, + #endif } mpack_type_t; /** @@ -150,151 +213,441 @@ typedef enum mpack_type_t { */ const char* mpack_type_to_string(mpack_type_t type); +#if MPACK_EXTENSIONS /** - * An MPack tag is a MessagePack object header. It is a variant type representing - * any kind of object, and includes the value of that object when it is not a - * compound type (i.e. boolean, integer, float.) + * A timestamp. * - * If the type is compound (str, bin, ext, array or map), the embedded data is - * stored separately. + * @note This requires @ref MPACK_EXTENSIONS. */ -typedef struct mpack_tag_t { - mpack_type_t type; /**< The type of value. */ +typedef struct mpack_timestamp_t { + int64_t seconds; /*< The number of seconds (signed) since 1970-01-01T00:00:00Z. */ + uint32_t nanoseconds; /*< The number of additional nanoseconds, between 0 and 999,999,999. */ +} mpack_timestamp_t; +#endif - int8_t exttype; /**< The extension type if the type is @ref mpack_type_ext. */ +/** + * An MPack tag is a MessagePack object header. It is a variant type + * representing any kind of object, and includes the length of compound types + * (e.g. map, array, string) or the value of non-compound types (e.g. boolean, + * integer, float.) + * + * If the type is compound (str, bin, ext, array or map), the contained + * elements or bytes are stored separately. + * + * This structure is opaque; its fields should not be accessed outside + * of MPack. + */ +typedef struct mpack_tag_t mpack_tag_t; - /** The value for non-compound types. */ - union - { - bool b; /**< The value if the type is bool. */ - float f; /**< The value if the type is float. */ - double d; /**< The value if the type is double. */ - int64_t i; /**< The value if the type is signed int. */ - uint64_t u; /**< The value if the type is unsigned int. */ - uint32_t l; /**< The number of bytes if the type is str, bin or ext. */ +/* Hide internals from documentation */ +/** @cond */ +struct mpack_tag_t { + mpack_type_t type; /*< The type of value. */ + + #if MPACK_EXTENSIONS + int8_t exttype; /*< The extension type if the type is @ref mpack_type_ext. */ + #endif - /** The element count if the type is an array, or the number of + /* The value for non-compound types. */ + union { + uint64_t u; /*< The value if the type is unsigned int. */ + int64_t i; /*< The value if the type is signed int. */ + bool b; /*< The value if the type is bool. */ + + #if MPACK_FLOAT + float f; /*< The value if the type is float. */ + #else + uint32_t f; /*< The raw value if the type is float. */ + #endif + + #if MPACK_DOUBLE + double d; /*< The value if the type is double. */ + #else + uint64_t d; /*< The raw value if the type is double. */ + #endif + + /* The number of bytes if the type is str, bin or ext. */ + uint32_t l; + + /* The element count if the type is an array, or the number of key/value pairs if the type is map. */ uint32_t n; } v; -} mpack_tag_t; +}; +/** @endcond */ + +/** + * @name Tag Generators + * @{ + */ + +/** + * @def MPACK_TAG_ZERO + * + * An @ref mpack_tag_t initializer that zeroes the given tag. + * + * @warning This does not make the tag nil! The tag's type is invalid when + * initialized this way. Use @ref mpack_tag_make_nil() to generate a nil tag. + */ +#if MPACK_EXTENSIONS +#define MPACK_TAG_ZERO {(mpack_type_t)0, 0, {0}} +#else +#define MPACK_TAG_ZERO {(mpack_type_t)0, {0}} +#endif /** Generates a nil tag. */ -MPACK_INLINE mpack_tag_t mpack_tag_nil(void) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_nil(void) { + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_nil; return ret; } /** Generates a bool tag. */ -MPACK_INLINE mpack_tag_t mpack_tag_bool(bool value) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_bool(bool value) { + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_bool; ret.v.b = value; return ret; } /** Generates a bool tag with value true. */ -MPACK_INLINE mpack_tag_t mpack_tag_true(void) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_true(void) { + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_bool; ret.v.b = true; return ret; } /** Generates a bool tag with value false. */ -MPACK_INLINE mpack_tag_t mpack_tag_false(void) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_false(void) { + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_bool; ret.v.b = false; return ret; } /** Generates a signed int tag. */ -MPACK_INLINE mpack_tag_t mpack_tag_int(int64_t value) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_int(int64_t value) { + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_int; ret.v.i = value; return ret; } /** Generates an unsigned int tag. */ -MPACK_INLINE mpack_tag_t mpack_tag_uint(uint64_t value) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_uint(uint64_t value) { + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_uint; ret.v.u = value; return ret; } +#if MPACK_FLOAT /** Generates a float tag. */ -MPACK_INLINE mpack_tag_t mpack_tag_float(float value) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_float(float value) +#else +/** Generates a float tag from a raw uint32_t. */ +MPACK_INLINE mpack_tag_t mpack_tag_make_raw_float(uint32_t value) +#endif +{ + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_float; ret.v.f = value; return ret; } +#if MPACK_DOUBLE /** Generates a double tag. */ -MPACK_INLINE mpack_tag_t mpack_tag_double(double value) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_double(double value) +#else +/** Generates a double tag from a raw uint64_t. */ +MPACK_INLINE mpack_tag_t mpack_tag_make_raw_double(uint64_t value) +#endif +{ + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_double; ret.v.d = value; return ret; } /** Generates an array tag. */ -MPACK_INLINE mpack_tag_t mpack_tag_array(int32_t count) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_array(uint32_t count) { + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_array; ret.v.n = count; return ret; } /** Generates a map tag. */ -MPACK_INLINE mpack_tag_t mpack_tag_map(int32_t count) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_map(uint32_t count) { + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_map; ret.v.n = count; return ret; } /** Generates a str tag. */ -MPACK_INLINE mpack_tag_t mpack_tag_str(int32_t length) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_str(uint32_t length) { + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_str; ret.v.l = length; return ret; } /** Generates a bin tag. */ -MPACK_INLINE mpack_tag_t mpack_tag_bin(int32_t length) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +MPACK_INLINE mpack_tag_t mpack_tag_make_bin(uint32_t length) { + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_bin; ret.v.l = length; return ret; } -/** Generates an ext tag. */ -MPACK_INLINE mpack_tag_t mpack_tag_ext(int8_t exttype, int32_t length) { - mpack_tag_t ret; - mpack_memset(&ret, 0, sizeof(ret)); +#if MPACK_EXTENSIONS +/** + * Generates an ext tag. + * + * @note This requires @ref MPACK_EXTENSIONS. + */ +MPACK_INLINE mpack_tag_t mpack_tag_make_ext(int8_t exttype, uint32_t length) { + mpack_tag_t ret = MPACK_TAG_ZERO; ret.type = mpack_type_ext; ret.exttype = exttype; ret.v.l = length; return ret; } +#endif + +/** + * @} + */ + +/** + * @name Tag Querying Functions + * @{ + */ + +/** + * Gets the type of a tag. + */ +MPACK_INLINE mpack_type_t mpack_tag_type(mpack_tag_t* tag) { + return tag->type; +} + +/** + * Gets the boolean value of a bool-type tag. The tag must be of type @ref + * mpack_type_bool. + * + * This asserts that the type in the tag is @ref mpack_type_bool. (No check is + * performed if MPACK_DEBUG is not set.) + */ +MPACK_INLINE bool mpack_tag_bool_value(mpack_tag_t* tag) { + mpack_assert(tag->type == mpack_type_bool, "tag is not a bool!"); + return tag->v.b; +} + +/** + * Gets the signed integer value of an int-type tag. + * + * This asserts that the type in the tag is @ref mpack_type_int. (No check is + * performed if MPACK_DEBUG is not set.) + * + * @warning This does not convert between signed and unsigned tags! A positive + * integer may be stored in a tag as either @ref mpack_type_int or @ref + * mpack_type_uint. You must check the type first; this can only be used if the + * type is @ref mpack_type_int. + * + * @see mpack_type_int + */ +MPACK_INLINE int64_t mpack_tag_int_value(mpack_tag_t* tag) { + mpack_assert(tag->type == mpack_type_int, "tag is not an int!"); + return tag->v.i; +} + +/** + * Gets the unsigned integer value of a uint-type tag. + * + * This asserts that the type in the tag is @ref mpack_type_uint. (No check is + * performed if MPACK_DEBUG is not set.) + * + * @warning This does not convert between signed and unsigned tags! A positive + * integer may be stored in a tag as either @ref mpack_type_int or @ref + * mpack_type_uint. You must check the type first; this can only be used if the + * type is @ref mpack_type_uint. + * + * @see mpack_type_uint + */ +MPACK_INLINE uint64_t mpack_tag_uint_value(mpack_tag_t* tag) { + mpack_assert(tag->type == mpack_type_uint, "tag is not a uint!"); + return tag->v.u; +} + +/** + * Gets the float value of a float-type tag. + * + * This asserts that the type in the tag is @ref mpack_type_float. (No check is + * performed if MPACK_DEBUG is not set.) + * + * @warning This does not convert between float and double tags! This can only + * be used if the type is @ref mpack_type_float. + * + * @see mpack_type_float + */ +MPACK_INLINE +#if MPACK_FLOAT +float mpack_tag_float_value(mpack_tag_t* tag) +#else +uint32_t mpack_tag_raw_float_value(mpack_tag_t* tag) +#endif +{ + mpack_assert(tag->type == mpack_type_float, "tag is not a float!"); + return tag->v.f; +} + +/** + * Gets the double value of a double-type tag. + * + * This asserts that the type in the tag is @ref mpack_type_double. (No check + * is performed if MPACK_DEBUG is not set.) + * + * @warning This does not convert between float and double tags! This can only + * be used if the type is @ref mpack_type_double. + * + * @see mpack_type_double + */ +MPACK_INLINE +#if MPACK_DOUBLE +double mpack_tag_double_value(mpack_tag_t* tag) +#else +uint64_t mpack_tag_raw_double_value(mpack_tag_t* tag) +#endif +{ + mpack_assert(tag->type == mpack_type_double, "tag is not a double!"); + return tag->v.d; +} + +/** + * Gets the number of elements in an array tag. + * + * This asserts that the type in the tag is @ref mpack_type_array. (No check is + * performed if MPACK_DEBUG is not set.) + * + * @see mpack_type_array + */ +MPACK_INLINE uint32_t mpack_tag_array_count(mpack_tag_t* tag) { + mpack_assert(tag->type == mpack_type_array, "tag is not an array!"); + return tag->v.n; +} + +/** + * Gets the number of key-value pairs in a map tag. + * + * This asserts that the type in the tag is @ref mpack_type_map. (No check is + * performed if MPACK_DEBUG is not set.) + * + * @see mpack_type_map + */ +MPACK_INLINE uint32_t mpack_tag_map_count(mpack_tag_t* tag) { + mpack_assert(tag->type == mpack_type_map, "tag is not a map!"); + return tag->v.n; +} + +/** + * Gets the length in bytes of a str-type tag. + * + * This asserts that the type in the tag is @ref mpack_type_str. (No check is + * performed if MPACK_DEBUG is not set.) + * + * @see mpack_type_str + */ +MPACK_INLINE uint32_t mpack_tag_str_length(mpack_tag_t* tag) { + mpack_assert(tag->type == mpack_type_str, "tag is not a str!"); + return tag->v.l; +} + +/** + * Gets the length in bytes of a bin-type tag. + * + * This asserts that the type in the tag is @ref mpack_type_bin. (No check is + * performed if MPACK_DEBUG is not set.) + * + * @see mpack_type_bin + */ +MPACK_INLINE uint32_t mpack_tag_bin_length(mpack_tag_t* tag) { + mpack_assert(tag->type == mpack_type_bin, "tag is not a bin!"); + return tag->v.l; +} + +#if MPACK_EXTENSIONS +/** + * Gets the length in bytes of an ext-type tag. + * + * This asserts that the type in the tag is @ref mpack_type_ext. (No check is + * performed if MPACK_DEBUG is not set.) + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @see mpack_type_ext + */ +MPACK_INLINE uint32_t mpack_tag_ext_length(mpack_tag_t* tag) { + mpack_assert(tag->type == mpack_type_ext, "tag is not an ext!"); + return tag->v.l; +} + +/** + * Gets the extension type (exttype) of an ext-type tag. + * + * This asserts that the type in the tag is @ref mpack_type_ext. (No check is + * performed if MPACK_DEBUG is not set.) + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @see mpack_type_ext + */ +MPACK_INLINE int8_t mpack_tag_ext_exttype(mpack_tag_t* tag) { + mpack_assert(tag->type == mpack_type_ext, "tag is not an ext!"); + return tag->exttype; +} +#endif + +/** + * Gets the length in bytes of a str-, bin- or ext-type tag. + * + * This asserts that the type in the tag is @ref mpack_type_str, @ref + * mpack_type_bin or @ref mpack_type_ext. (No check is performed if MPACK_DEBUG + * is not set.) + * + * @see mpack_type_str + * @see mpack_type_bin + * @see mpack_type_ext + */ +MPACK_INLINE uint32_t mpack_tag_bytes(mpack_tag_t* tag) { + #if MPACK_EXTENSIONS + mpack_assert(tag->type == mpack_type_str || tag->type == mpack_type_bin + || tag->type == mpack_type_ext, "tag is not a str, bin or ext!"); + #else + mpack_assert(tag->type == mpack_type_str || tag->type == mpack_type_bin, + "tag is not a str or bin!"); + #endif + return tag->v.l; +} + +/** + * @} + */ + +/** + * @name Other tag functions + * @{ + */ + +#if MPACK_EXTENSIONS +/** + * The extension type for a timestamp. + * + * @note This requires @ref MPACK_EXTENSIONS. + */ +#define MPACK_EXTTYPE_TIMESTAMP ((int8_t)(-1)) +#endif /** * Compares two tags with an arbitrary fixed ordering. Returns 0 if the tags are @@ -322,7 +675,7 @@ int mpack_tag_cmp(mpack_tag_t left, mpack_tag_t right); * is equal to the value 1 stored in a 64-bit unsigned integer field. * * The "extension type" of an extension object is considered part of the value - * and much match exactly. + * and must match exactly. * * \warning Floating point numbers are compared bit-for-bit, not using the language's * operator==. This means that NaNs with matching representation will compare equal. @@ -332,25 +685,163 @@ MPACK_INLINE bool mpack_tag_equal(mpack_tag_t left, mpack_tag_t right) { return mpack_tag_cmp(left, right) == 0; } +#if MPACK_DEBUG && MPACK_STDIO +/** + * Generates a json-like debug description of the given tag into the given buffer. + * + * This is only available in debug mode, and only if stdio is available (since + * it uses snprintf().) It's strictly for debugging purposes. + * + * The prefix is used to print the first few hexadecimal bytes of a bin or ext + * type. Pass NULL if not a bin or ext. + */ +void mpack_tag_debug_pseudo_json(mpack_tag_t tag, char* buffer, size_t buffer_size, + const char* prefix, size_t prefix_size); + +/** + * Generates a debug string description of the given tag into the given buffer. + * + * This is only available in debug mode, and only if stdio is available (since + * it uses snprintf().) It's strictly for debugging purposes. + */ +void mpack_tag_debug_describe(mpack_tag_t tag, char* buffer, size_t buffer_size); + +/** @cond */ + +/* + * A callback function for printing pseudo-JSON for debugging purposes. + * + * @see mpack_node_print_callback + */ +typedef void (*mpack_print_callback_t)(void* context, const char* data, size_t count); + +// helpers for printing debug output +// i feel a bit like i'm re-implementing a buffered writer again... +typedef struct mpack_print_t { + char* buffer; + size_t size; + size_t count; + mpack_print_callback_t callback; + void* context; +} mpack_print_t; + +void mpack_print_append(mpack_print_t* print, const char* data, size_t count); + +MPACK_INLINE void mpack_print_append_cstr(mpack_print_t* print, const char* cstr) { + mpack_print_append(print, cstr, mpack_strlen(cstr)); +} + +void mpack_print_flush(mpack_print_t* print); + +void mpack_print_file_callback(void* context, const char* data, size_t count); + +/** @endcond */ + +#endif + /** * @} */ +/** + * @name Deprecated Tag Generators + * @{ + */ + +/* + * "make" has been added to their names to disambiguate them from the + * value-fetching functions (e.g. mpack_tag_make_bool() vs + * mpack_tag_bool_value().) + * + * The length and count for all compound types was the wrong sign (int32_t + * instead of uint32_t.) These preserve the old behaviour; the new "make" + * functions have the correct sign. + */ +/** \deprecated Renamed to mpack_tag_make_nil(). */ +MPACK_INLINE mpack_tag_t mpack_tag_nil(void) { + return mpack_tag_make_nil(); +} + +/** \deprecated Renamed to mpack_tag_make_bool(). */ +MPACK_INLINE mpack_tag_t mpack_tag_bool(bool value) { + return mpack_tag_make_bool(value); +} + +/** \deprecated Renamed to mpack_tag_make_true(). */ +MPACK_INLINE mpack_tag_t mpack_tag_true(void) { + return mpack_tag_make_true(); +} + +/** \deprecated Renamed to mpack_tag_make_false(). */ +MPACK_INLINE mpack_tag_t mpack_tag_false(void) { + return mpack_tag_make_false(); +} + +/** \deprecated Renamed to mpack_tag_make_int(). */ +MPACK_INLINE mpack_tag_t mpack_tag_int(int64_t value) { + return mpack_tag_make_int(value); +} + +/** \deprecated Renamed to mpack_tag_make_uint(). */ +MPACK_INLINE mpack_tag_t mpack_tag_uint(uint64_t value) { + return mpack_tag_make_uint(value); +} + +#if MPACK_FLOAT +/** \deprecated Renamed to mpack_tag_make_float(). */ +MPACK_INLINE mpack_tag_t mpack_tag_float(float value) { + return mpack_tag_make_float(value); +} +#endif + +#if MPACK_DOUBLE +/** \deprecated Renamed to mpack_tag_make_double(). */ +MPACK_INLINE mpack_tag_t mpack_tag_double(double value) { + return mpack_tag_make_double(value); +} +#endif + +/** \deprecated Renamed to mpack_tag_make_array(). */ +MPACK_INLINE mpack_tag_t mpack_tag_array(int32_t count) { + return mpack_tag_make_array((uint32_t)count); +} + +/** \deprecated Renamed to mpack_tag_make_map(). */ +MPACK_INLINE mpack_tag_t mpack_tag_map(int32_t count) { + return mpack_tag_make_map((uint32_t)count); +} + +/** \deprecated Renamed to mpack_tag_make_str(). */ +MPACK_INLINE mpack_tag_t mpack_tag_str(int32_t length) { + return mpack_tag_make_str((uint32_t)length); +} + +/** \deprecated Renamed to mpack_tag_make_bin(). */ +MPACK_INLINE mpack_tag_t mpack_tag_bin(int32_t length) { + return mpack_tag_make_bin((uint32_t)length); +} + +#if MPACK_EXTENSIONS +/** \deprecated Renamed to mpack_tag_make_ext(). */ +MPACK_INLINE mpack_tag_t mpack_tag_ext(int8_t exttype, int32_t length) { + return mpack_tag_make_ext(exttype, (uint32_t)length); +} +#endif + +/** + * @} + */ /** @cond */ /* * Helpers to perform unaligned network-endian loads and stores - * at arbitrary addresses. + * at arbitrary addresses. Byte-swapping builtins are used if they + * are available and if they improve performance. * * These will remain available in the public API so feel free to * use them for other purposes, but they are undocumented. - * - * The bswap builtins are used when needed and available. With - * GCC 5.2 they appear to give better performance and smaller - * code size on little-endian ARM while compiling to the same - * assembly as the bit-shifting code on x86_64. */ MPACK_INLINE uint8_t mpack_load_u8(const char* p) { @@ -453,8 +944,10 @@ MPACK_INLINE void mpack_store_i16(char* p, int16_t val) {mpack_store_u16(p, (uin MPACK_INLINE void mpack_store_i32(char* p, int32_t val) {mpack_store_u32(p, (uint32_t)val);} MPACK_INLINE void mpack_store_i64(char* p, int64_t val) {mpack_store_u64(p, (uint64_t)val);} +#if MPACK_FLOAT MPACK_INLINE float mpack_load_float(const char* p) { MPACK_CHECK_FLOAT_ORDER(); + MPACK_STATIC_ASSERT(sizeof(float) == sizeof(uint32_t), "float is wrong size??"); union { float f; uint32_t u; @@ -462,9 +955,12 @@ MPACK_INLINE float mpack_load_float(const char* p) { v.u = mpack_load_u32(p); return v.f; } +#endif +#if MPACK_DOUBLE MPACK_INLINE double mpack_load_double(const char* p) { MPACK_CHECK_FLOAT_ORDER(); + MPACK_STATIC_ASSERT(sizeof(double) == sizeof(uint64_t), "double is wrong size??"); union { double d; uint64_t u; @@ -472,7 +968,9 @@ MPACK_INLINE double mpack_load_double(const char* p) { v.u = mpack_load_u64(p); return v.d; } +#endif +#if MPACK_FLOAT MPACK_INLINE void mpack_store_float(char* p, float value) { MPACK_CHECK_FLOAT_ORDER(); union { @@ -482,7 +980,9 @@ MPACK_INLINE void mpack_store_float(char* p, float value) { v.f = value; mpack_store_u32(p, v.u); } +#endif +#if MPACK_DOUBLE MPACK_INLINE void mpack_store_double(char* p, double value) { MPACK_CHECK_FLOAT_ORDER(); union { @@ -492,6 +992,72 @@ MPACK_INLINE void mpack_store_double(char* p, double value) { v.d = value; mpack_store_u64(p, v.u); } +#endif + +#if MPACK_FLOAT && !MPACK_DOUBLE +/** + * Performs a manual shortening conversion on the raw 64-bit representation of + * a double. This is useful for parsing doubles on platforms that only support + * floats (such as AVR.) + * + * The significand is truncated rather than rounded and subnormal numbers are + * set to 0 so this may not be quite as accurate as a real double-to-float + * conversion. + */ +MPACK_INLINE float mpack_shorten_raw_double_to_float(uint64_t d) { + MPACK_CHECK_FLOAT_ORDER(); + union { + float f; + uint32_t u; + } v; + + // float has 1 bit sign, 8 bits exponent, 23 bits significand + // double has 1 bit sign, 11 bits exponent, 52 bits significand + + uint64_t d_sign = (uint64_t)(d >> 63); + uint64_t d_exponent = (uint32_t)(d >> 52) & ((1 << 11) - 1); + uint64_t d_significand = d & (((uint64_t)1 << 52) - 1); + + uint32_t f_sign = (uint32_t)d_sign; + uint32_t f_exponent; + uint32_t f_significand; + + if (MPACK_UNLIKELY(d_exponent == ((1 << 11) - 1))) { + // infinity or NAN. shift down to preserve the top bit since it + // indicates signaling NAN, but also set the low bit if any bits were + // set (that way we can't shift NAN to infinity.) + f_exponent = ((1 << 8) - 1); + f_significand = (uint32_t)(d_significand >> 29) | (d_significand ? 1 : 0); + + } else { + int fix_bias = (int)d_exponent - ((1 << 10) - 1) + ((1 << 7) - 1); + if (MPACK_UNLIKELY(fix_bias <= 0)) { + // we don't currently handle subnormal numbers. just set it to zero. + f_exponent = 0; + f_significand = 0; + } else if (MPACK_UNLIKELY(fix_bias > 0xff)) { + // exponent is too large; saturate to infinity + f_exponent = 0xff; + f_significand = 0; + } else { + // a normal number that fits in a float. this is the usual case. + f_exponent = (uint32_t)fix_bias; + f_significand = (uint32_t)(d_significand >> 29); + } + } + + #if 0 + printf("\n===============\n"); + for (size_t i = 0; i < 64; ++i) + printf("%i%s",(int)((d>>(63-i))&1),((i%8)==7)?" ":""); + printf("\n%lu %lu %lu\n", d_sign, d_exponent, d_significand); + printf("%u %u %u\n", f_sign, f_exponent, f_significand); + #endif + + v.u = (f_sign << 31) | (f_exponent << 23) | f_significand; + return v.f; +} +#endif /** @endcond */ @@ -534,6 +1100,11 @@ MPACK_INLINE void mpack_store_double(char* p, double value) { #define MPACK_TAG_SIZE_EXT16 4 #define MPACK_TAG_SIZE_EXT32 6 +// size in bytes for complete ext types +#define MPACK_EXT_SIZE_TIMESTAMP4 (MPACK_TAG_SIZE_FIXEXT4 + 4) +#define MPACK_EXT_SIZE_TIMESTAMP8 (MPACK_TAG_SIZE_FIXEXT8 + 8) +#define MPACK_EXT_SIZE_TIMESTAMP12 (MPACK_TAG_SIZE_EXT8 + 12) + /** @endcond */ @@ -545,7 +1116,16 @@ MPACK_INLINE void mpack_store_double(char* p, double value) { typedef struct mpack_track_element_t { mpack_type_t type; - uint64_t left; // we need 64-bit because (2 * INT32_MAX) elements can be stored in a map + uint32_t left; + + // indicates that a value still needs to be read/written for an already + // read/written key. left is not decremented until both key and value are + // read/written. + bool key_needs_value; + + // tracks whether the map/array being written is using a builder. if true, + // the number of elements is automatic, and left is 0. + bool builder; } mpack_track_element_t; typedef struct mpack_track_t { @@ -557,12 +1137,14 @@ typedef struct mpack_track_t { #if MPACK_INTERNAL mpack_error_t mpack_track_init(mpack_track_t* track); mpack_error_t mpack_track_grow(mpack_track_t* track); -mpack_error_t mpack_track_push(mpack_track_t* track, mpack_type_t type, uint64_t count); +mpack_error_t mpack_track_push(mpack_track_t* track, mpack_type_t type, uint32_t count); +mpack_error_t mpack_track_push_builder(mpack_track_t* track, mpack_type_t type); mpack_error_t mpack_track_pop(mpack_track_t* track, mpack_type_t type); +mpack_error_t mpack_track_pop_builder(mpack_track_t* track, mpack_type_t type); mpack_error_t mpack_track_element(mpack_track_t* track, bool read); mpack_error_t mpack_track_peek_element(mpack_track_t* track, bool read); -mpack_error_t mpack_track_bytes(mpack_track_t* track, bool read, uint64_t count); -mpack_error_t mpack_track_str_bytes_all(mpack_track_t* track, bool read, uint64_t count); +mpack_error_t mpack_track_bytes(mpack_track_t* track, bool read, size_t count); +mpack_error_t mpack_track_str_bytes_all(mpack_track_t* track, bool read, size_t count); mpack_error_t mpack_track_check_empty(mpack_track_t* track); mpack_error_t mpack_track_destroy(mpack_track_t* track, bool cancel); #endif @@ -601,7 +1183,12 @@ bool mpack_str_check_no_null(const char* str, size_t bytes); -MPACK_HEADER_END +/** + * @} + */ + +MPACK_EXTERN_C_END +MPACK_SILENCE_WARNINGS_END #endif diff --git a/src/mpack/mpack-expect.c b/src/mpack/mpack-expect.c index 9391f2e..81576d1 100644 --- a/src/mpack/mpack-expect.c +++ b/src/mpack/mpack-expect.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -23,6 +23,8 @@ #include "mpack-expect.h" +MPACK_SILENCE_WARNINGS_BEGIN + #if MPACK_EXPECT @@ -34,9 +36,8 @@ MPACK_STATIC_INLINE uint8_t mpack_expect_native_u8(mpack_reader_t* reader) { uint8_t type; if (!mpack_reader_ensure(reader, sizeof(type))) return 0; - type = mpack_load_u8(reader->buffer + reader->pos); - reader->pos += sizeof(type); - reader->left -= sizeof(type); + type = mpack_load_u8(reader->data); + reader->data += sizeof(type); return type; } @@ -47,9 +48,8 @@ MPACK_STATIC_INLINE uint16_t mpack_expect_native_u16(mpack_reader_t* reader) { uint16_t type; if (!mpack_reader_ensure(reader, sizeof(type))) return 0; - type = mpack_load_u16(reader->buffer + reader->pos); - reader->pos += sizeof(type); - reader->left -= sizeof(type); + type = mpack_load_u16(reader->data); + reader->data += sizeof(type); return type; } @@ -59,9 +59,8 @@ MPACK_STATIC_INLINE uint32_t mpack_expect_native_u32(mpack_reader_t* reader) { uint32_t type; if (!mpack_reader_ensure(reader, sizeof(type))) return 0; - type = mpack_load_u32(reader->buffer + reader->pos); - reader->pos += sizeof(type); - reader->left -= sizeof(type); + type = mpack_load_u32(reader->data); + reader->data += sizeof(type); return type; } #endif @@ -77,10 +76,10 @@ MPACK_STATIC_INLINE uint8_t mpack_expect_type_byte(mpack_reader_t* reader) { uint8_t mpack_expect_u8(mpack_reader_t* reader) { mpack_tag_t var = mpack_read_tag(reader); if (var.type == mpack_type_uint) { - if (var.v.u <= UINT8_MAX) + if (var.v.u <= MPACK_UINT8_MAX) return (uint8_t)var.v.u; } else if (var.type == mpack_type_int) { - if (var.v.i >= 0 && var.v.i <= UINT8_MAX) + if (var.v.i >= 0 && var.v.i <= MPACK_UINT8_MAX) return (uint8_t)var.v.i; } mpack_reader_flag_error(reader, mpack_error_type); @@ -90,10 +89,10 @@ uint8_t mpack_expect_u8(mpack_reader_t* reader) { uint16_t mpack_expect_u16(mpack_reader_t* reader) { mpack_tag_t var = mpack_read_tag(reader); if (var.type == mpack_type_uint) { - if (var.v.u <= UINT16_MAX) + if (var.v.u <= MPACK_UINT16_MAX) return (uint16_t)var.v.u; } else if (var.type == mpack_type_int) { - if (var.v.i >= 0 && var.v.i <= UINT16_MAX) + if (var.v.i >= 0 && var.v.i <= MPACK_UINT16_MAX) return (uint16_t)var.v.i; } mpack_reader_flag_error(reader, mpack_error_type); @@ -103,10 +102,10 @@ uint16_t mpack_expect_u16(mpack_reader_t* reader) { uint32_t mpack_expect_u32(mpack_reader_t* reader) { mpack_tag_t var = mpack_read_tag(reader); if (var.type == mpack_type_uint) { - if (var.v.u <= UINT32_MAX) + if (var.v.u <= MPACK_UINT32_MAX) return (uint32_t)var.v.u; } else if (var.type == mpack_type_int) { - if (var.v.i >= 0 && var.v.i <= UINT32_MAX) + if (var.v.i >= 0 && var.v.i <= MPACK_UINT32_MAX) return (uint32_t)var.v.i; } mpack_reader_flag_error(reader, mpack_error_type); @@ -128,10 +127,10 @@ uint64_t mpack_expect_u64(mpack_reader_t* reader) { int8_t mpack_expect_i8(mpack_reader_t* reader) { mpack_tag_t var = mpack_read_tag(reader); if (var.type == mpack_type_uint) { - if (var.v.u <= INT8_MAX) + if (var.v.u <= MPACK_INT8_MAX) return (int8_t)var.v.u; } else if (var.type == mpack_type_int) { - if (var.v.i >= INT8_MIN && var.v.i <= INT8_MAX) + if (var.v.i >= MPACK_INT8_MIN && var.v.i <= MPACK_INT8_MAX) return (int8_t)var.v.i; } mpack_reader_flag_error(reader, mpack_error_type); @@ -141,10 +140,10 @@ int8_t mpack_expect_i8(mpack_reader_t* reader) { int16_t mpack_expect_i16(mpack_reader_t* reader) { mpack_tag_t var = mpack_read_tag(reader); if (var.type == mpack_type_uint) { - if (var.v.u <= INT16_MAX) + if (var.v.u <= MPACK_INT16_MAX) return (int16_t)var.v.u; } else if (var.type == mpack_type_int) { - if (var.v.i >= INT16_MIN && var.v.i <= INT16_MAX) + if (var.v.i >= MPACK_INT16_MIN && var.v.i <= MPACK_INT16_MAX) return (int16_t)var.v.i; } mpack_reader_flag_error(reader, mpack_error_type); @@ -154,10 +153,10 @@ int16_t mpack_expect_i16(mpack_reader_t* reader) { int32_t mpack_expect_i32(mpack_reader_t* reader) { mpack_tag_t var = mpack_read_tag(reader); if (var.type == mpack_type_uint) { - if (var.v.u <= INT32_MAX) + if (var.v.u <= MPACK_INT32_MAX) return (int32_t)var.v.u; } else if (var.type == mpack_type_int) { - if (var.v.i >= INT32_MIN && var.v.i <= INT32_MAX) + if (var.v.i >= MPACK_INT32_MIN && var.v.i <= MPACK_INT32_MAX) return (int32_t)var.v.i; } mpack_reader_flag_error(reader, mpack_error_type); @@ -167,7 +166,7 @@ int32_t mpack_expect_i32(mpack_reader_t* reader) { int64_t mpack_expect_i64(mpack_reader_t* reader) { mpack_tag_t var = mpack_read_tag(reader); if (var.type == mpack_type_uint) { - if (var.v.u <= INT64_MAX) + if (var.v.u <= MPACK_INT64_MAX) return (int64_t)var.v.u; } else if (var.type == mpack_type_int) { return var.v.i; @@ -176,20 +175,30 @@ int64_t mpack_expect_i64(mpack_reader_t* reader) { return 0; } +#if MPACK_FLOAT float mpack_expect_float(mpack_reader_t* reader) { mpack_tag_t var = mpack_read_tag(reader); if (var.type == mpack_type_uint) return (float)var.v.u; - else if (var.type == mpack_type_int) + if (var.type == mpack_type_int) return (float)var.v.i; - else if (var.type == mpack_type_float) + if (var.type == mpack_type_float) return var.v.f; - else if (var.type == mpack_type_double) + + if (var.type == mpack_type_double) { + #if MPACK_DOUBLE return (float)var.v.d; + #else + return mpack_shorten_raw_double_to_float(var.v.d); + #endif + } + mpack_reader_flag_error(reader, mpack_error_type); return 0.0f; } +#endif +#if MPACK_DOUBLE double mpack_expect_double(mpack_reader_t* reader) { mpack_tag_t var = mpack_read_tag(reader); if (var.type == mpack_type_uint) @@ -203,7 +212,9 @@ double mpack_expect_double(mpack_reader_t* reader) { mpack_reader_flag_error(reader, mpack_error_type); return 0.0; } +#endif +#if MPACK_FLOAT float mpack_expect_float_strict(mpack_reader_t* reader) { mpack_tag_t var = mpack_read_tag(reader); if (var.type == mpack_type_float) @@ -211,7 +222,9 @@ float mpack_expect_float_strict(mpack_reader_t* reader) { mpack_reader_flag_error(reader, mpack_error_type); return 0.0f; } +#endif +#if MPACK_DOUBLE double mpack_expect_double_strict(mpack_reader_t* reader) { mpack_tag_t var = mpack_read_tag(reader); if (var.type == mpack_type_float) @@ -221,6 +234,27 @@ double mpack_expect_double_strict(mpack_reader_t* reader) { mpack_reader_flag_error(reader, mpack_error_type); return 0.0; } +#endif + +#if !MPACK_FLOAT +uint32_t mpack_expect_raw_float(mpack_reader_t* reader) { + mpack_tag_t var = mpack_read_tag(reader); + if (var.type == mpack_type_float) + return var.v.f; + mpack_reader_flag_error(reader, mpack_error_type); + return 0; +} +#endif + +#if !MPACK_DOUBLE +uint64_t mpack_expect_raw_double(mpack_reader_t* reader) { + mpack_tag_t var = mpack_read_tag(reader); + if (var.type == mpack_type_double) + return var.v.d; + mpack_reader_flag_error(reader, mpack_error_type); + return 0; +} +#endif // Ranged Number Functions @@ -259,8 +293,12 @@ int16_t mpack_expect_i16_range(mpack_reader_t* reader, int16_t min_value, int16_ int32_t mpack_expect_i32_range(mpack_reader_t* reader, int32_t min_value, int32_t max_value) {MPACK_EXPECT_RANGE_IMPL(i32, int32_t)} int64_t mpack_expect_i64_range(mpack_reader_t* reader, int64_t min_value, int64_t max_value) {MPACK_EXPECT_RANGE_IMPL(i64, int64_t)} +#if MPACK_FLOAT float mpack_expect_float_range(mpack_reader_t* reader, float min_value, float max_value) {MPACK_EXPECT_RANGE_IMPL(float, float)} +#endif +#if MPACK_DOUBLE double mpack_expect_double_range(mpack_reader_t* reader, double min_value, double max_value) {MPACK_EXPECT_RANGE_IMPL(double, double)} +#endif uint32_t mpack_expect_map_range(mpack_reader_t* reader, uint32_t min_value, uint32_t max_value) {MPACK_EXPECT_RANGE_IMPL(map, uint32_t)} uint32_t mpack_expect_array_range(mpack_reader_t* reader, uint32_t min_value, uint32_t max_value) {MPACK_EXPECT_RANGE_IMPL(array, uint32_t)} @@ -303,6 +341,28 @@ void mpack_expect_false(mpack_reader_t* reader) { mpack_reader_flag_error(reader, mpack_error_type); } +#if MPACK_EXTENSIONS +mpack_timestamp_t mpack_expect_timestamp(mpack_reader_t* reader) { + mpack_timestamp_t zero = {0, 0}; + + mpack_tag_t tag = mpack_read_tag(reader); + if (tag.type != mpack_type_ext) { + mpack_reader_flag_error(reader, mpack_error_type); + return zero; + } + if (mpack_tag_ext_exttype(&tag) != MPACK_EXTTYPE_TIMESTAMP) { + mpack_reader_flag_error(reader, mpack_error_type); + return zero; + } + + return mpack_read_timestamp(reader, mpack_tag_ext_length(&tag)); +} + +int64_t mpack_expect_timestamp_truncate(mpack_reader_t* reader) { + return mpack_expect_timestamp(reader).seconds; +} +#endif + // Compound Types @@ -517,6 +577,46 @@ size_t mpack_expect_bin_buf(mpack_reader_t* reader, char* buf, size_t bufsize) { return binsize; } +void mpack_expect_bin_size_buf(mpack_reader_t* reader, char* buf, uint32_t size) { + mpack_assert(buf != NULL, "buf cannot be NULL"); + mpack_expect_bin_size(reader, size); + mpack_read_bytes(reader, buf, size); + mpack_done_bin(reader); +} + +#if MPACK_EXTENSIONS +uint32_t mpack_expect_ext(mpack_reader_t* reader, int8_t* type) { + mpack_tag_t var = mpack_read_tag(reader); + if (var.type == mpack_type_ext) { + *type = mpack_tag_ext_exttype(&var); + return mpack_tag_ext_length(&var); + } + *type = 0; + mpack_reader_flag_error(reader, mpack_error_type); + return 0; +} + +size_t mpack_expect_ext_buf(mpack_reader_t* reader, int8_t* type, char* buf, size_t bufsize) { + mpack_assert(buf != NULL, "buf cannot be NULL"); + + size_t extsize = mpack_expect_ext(reader, type); + if (mpack_reader_error(reader)) + return 0; + if (extsize > bufsize) { + *type = 0; + mpack_reader_flag_error(reader, mpack_error_too_big); + return 0; + } + mpack_read_bytes(reader, buf, extsize); + if (mpack_reader_error(reader)) { + *type = 0; + return 0; + } + mpack_done_ext(reader); + return extsize; +} +#endif + void mpack_expect_cstr(mpack_reader_t* reader, char* buf, size_t bufsize) { uint32_t length = mpack_expect_str(reader); mpack_read_cstr(reader, buf, bufsize, length); @@ -541,8 +641,13 @@ static char* mpack_expect_cstr_alloc_unchecked(mpack_reader_t* reader, size_t ma return NULL; } - if (maxsize > UINT32_MAX) - maxsize = UINT32_MAX; + if (SIZE_MAX < MPACK_UINT32_MAX) { + if (maxsize > SIZE_MAX) + maxsize = SIZE_MAX; + } else { + if (maxsize > (size_t)MPACK_UINT32_MAX) + maxsize = (size_t)MPACK_UINT32_MAX; + } size_t length = mpack_expect_str_max(reader, (uint32_t)maxsize - 1); char* str = mpack_read_bytes_alloc_impl(reader, length, true); @@ -584,12 +689,12 @@ void mpack_expect_str_match(mpack_reader_t* reader, const char* str, size_t len) mpack_assert(str != NULL, "str cannot be NULL"); // expect a str the correct length - if (len > UINT32_MAX) + if (len > MPACK_UINT32_MAX) mpack_reader_flag_error(reader, mpack_error_type); mpack_expect_str_length(reader, (uint32_t)len); if (mpack_reader_error(reader)) return; - mpack_reader_track_bytes(reader, len); + mpack_reader_track_bytes(reader, (uint32_t)len); // check each byte one by one (matched strings are likely to be very small) for (; len > 0; --len) { @@ -613,10 +718,18 @@ char* mpack_expect_bin_alloc(mpack_reader_t* reader, size_t maxsize, size_t* siz mpack_assert(size != NULL, "size cannot be NULL"); *size = 0; - if (maxsize > UINT32_MAX) - maxsize = UINT32_MAX; + if (SIZE_MAX < MPACK_UINT32_MAX) { + if (maxsize > SIZE_MAX) + maxsize = SIZE_MAX; + } else { + if (maxsize > (size_t)MPACK_UINT32_MAX) + maxsize = (size_t)MPACK_UINT32_MAX; + } size_t length = mpack_expect_bin_max(reader, (uint32_t)maxsize); + if (mpack_reader_error(reader)) + return NULL; + char* data = mpack_read_bytes_alloc(reader, length); mpack_done_bin(reader); @@ -626,6 +739,91 @@ char* mpack_expect_bin_alloc(mpack_reader_t* reader, size_t maxsize, size_t* siz } #endif +#if MPACK_EXTENSIONS && defined(MPACK_MALLOC) +char* mpack_expect_ext_alloc(mpack_reader_t* reader, int8_t* type, size_t maxsize, size_t* size) { + mpack_assert(size != NULL, "size cannot be NULL"); + *size = 0; + + if (SIZE_MAX < MPACK_UINT32_MAX) { + if (maxsize > SIZE_MAX) + maxsize = SIZE_MAX; + } else { + if (maxsize > (size_t)MPACK_UINT32_MAX) + maxsize = (size_t)MPACK_UINT32_MAX; + } + + size_t length = mpack_expect_ext_max(reader, type, (uint32_t)maxsize); + if (mpack_reader_error(reader)) + return NULL; + + char* data = mpack_read_bytes_alloc(reader, length); + mpack_done_ext(reader); + + if (data) { + *size = length; + } else { + *type = 0; + } + return data; +} +#endif + +size_t mpack_expect_enum(mpack_reader_t* reader, const char* strings[], size_t count) { + + // read the string in-place + size_t keylen = mpack_expect_str(reader); + const char* key = mpack_read_bytes_inplace(reader, keylen); + mpack_done_str(reader); + if (mpack_reader_error(reader) != mpack_ok) + return count; + + // find what key it matches + size_t i; + for (i = 0; i < count; ++i) { + const char* other = strings[i]; + size_t otherlen = mpack_strlen(other); + if (keylen == otherlen && mpack_memcmp(key, other, keylen) == 0) + return i; + } + + // no matches + mpack_reader_flag_error(reader, mpack_error_type); + return count; +} + +size_t mpack_expect_enum_optional(mpack_reader_t* reader, const char* strings[], size_t count) { + if (mpack_reader_error(reader) != mpack_ok) + return count; + + mpack_assert(count != 0, "count cannot be zero; no strings are valid!"); + mpack_assert(strings != NULL, "strings cannot be NULL"); + + // the key is only recognized if it is a string + if (mpack_peek_tag(reader).type != mpack_type_str) { + mpack_discard(reader); + return count; + } + + // read the string in-place + size_t keylen = mpack_expect_str(reader); + const char* key = mpack_read_bytes_inplace(reader, keylen); + mpack_done_str(reader); + if (mpack_reader_error(reader) != mpack_ok) + return count; + + // find what key it matches + size_t i; + for (i = 0; i < count; ++i) { + const char* other = strings[i]; + size_t otherlen = mpack_strlen(other); + if (keylen == otherlen && mpack_memcmp(key, other, keylen) == 0) + return i; + } + + // no matches + return count; +} + size_t mpack_expect_key_uint(mpack_reader_t* reader, bool found[], size_t count) { if (mpack_reader_error(reader) != mpack_ok) return count; @@ -663,44 +861,14 @@ size_t mpack_expect_key_uint(mpack_reader_t* reader, bool found[], size_t count) } size_t mpack_expect_key_cstr(mpack_reader_t* reader, const char* keys[], bool found[], size_t count) { - if (mpack_reader_error(reader) != mpack_ok) - return count; - - if (count == 0) { - mpack_break("count cannot be zero; no keys are valid!"); - mpack_reader_flag_error(reader, mpack_error_bug); - return count; - } - mpack_assert(keys != NULL, "keys cannot be NULL"); - mpack_assert(found != NULL, "found cannot be NULL"); - - // the key is only recognized if it is a string - if (mpack_peek_tag(reader).type != mpack_type_str) { - mpack_discard(reader); - return count; - } - - // read the string in-place - size_t keylen = mpack_expect_str(reader); - const char* key = mpack_read_bytes_inplace(reader, keylen); - if (mpack_reader_error(reader) != mpack_ok) - return count; - mpack_done_str(reader); - - // find what key it matches - size_t i = 0; - for (; i < count; ++i) { - const char* other = keys[i]; - size_t otherlen = mpack_strlen(other); - if (keylen == otherlen && mpack_memcmp(key, other, keylen) == 0) - break; - } + size_t i = mpack_expect_enum_optional(reader, keys, count); // unrecognized keys are fine, we just return count if (i == count) return count; // check if this key is a duplicate + mpack_assert(found != NULL, "found cannot be NULL"); if (found[i]) { mpack_reader_flag_error(reader, mpack_error_invalid); return count; @@ -712,3 +880,4 @@ size_t mpack_expect_key_cstr(mpack_reader_t* reader, const char* keys[], bool fo #endif +MPACK_SILENCE_WARNINGS_END diff --git a/src/mpack/mpack-expect.h b/src/mpack/mpack-expect.h index b9dff1a..02a1abc 100644 --- a/src/mpack/mpack-expect.h +++ b/src/mpack/mpack-expect.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -30,7 +30,8 @@ #include "mpack-reader.h" -MPACK_HEADER_START +MPACK_SILENCE_WARNINGS_BEGIN +MPACK_EXTERN_C_BEGIN #if MPACK_EXPECT @@ -44,6 +45,10 @@ MPACK_HEADER_START * The MPack Expect API allows you to easily read MessagePack data when you * expect it to follow a predefined schema. * + * @note If you are not writing code for an embedded device (or otherwise do + * not need maximum performance with minimal memory usage), you should not use + * this. You probably want to use the @link node Node API@endlink instead. + * * See @ref docs/expect.md for examples. * * The main purpose of the Expect API is convenience, so the API is lax. It @@ -141,28 +146,33 @@ int32_t mpack_expect_i32(mpack_reader_t* reader); */ int64_t mpack_expect_i64(mpack_reader_t* reader); +#if MPACK_FLOAT /** * Reads a number, returning the value as a float. The underlying value can be an * integer, float or double; the value is converted to a float. * - * Note that reading a double or a large integer with this function can incur a + * @note Reading a double or a large integer with this function can incur a * loss of precision. * * @throws mpack_error_type if the underlying value is not a float, double or integer. */ float mpack_expect_float(mpack_reader_t* reader); +#endif +#if MPACK_DOUBLE /** * Reads a number, returning the value as a double. The underlying value can be an * integer, float or double; the value is converted to a double. * - * Note that reading a very large integer with this function can incur a + * @note Reading a very large integer with this function can incur a * loss of precision. * * @throws mpack_error_type if the underlying value is not a float, double or integer. */ double mpack_expect_double(mpack_reader_t* reader); +#endif +#if MPACK_FLOAT /** * Reads a float. The underlying value must be a float, not a double or an integer. * This ensures no loss of precision can occur. @@ -170,7 +180,9 @@ double mpack_expect_double(mpack_reader_t* reader); * @throws mpack_error_type if the underlying value is not a float. */ float mpack_expect_float_strict(mpack_reader_t* reader); +#endif +#if MPACK_DOUBLE /** * Reads a double. The underlying value must be a float or double, not an integer. * This ensures no loss of precision can occur. @@ -178,6 +190,27 @@ float mpack_expect_float_strict(mpack_reader_t* reader); * @throws mpack_error_type if the underlying value is not a float or double. */ double mpack_expect_double_strict(mpack_reader_t* reader); +#endif + +#if !MPACK_FLOAT +/** + * Reads a float as a raw uint32_t. The underlying value must be a float, not a + * double or an integer. + * + * @throws mpack_error_type if the underlying value is not a float. + */ +uint32_t mpack_expect_raw_float(mpack_reader_t* reader); +#endif + +#if !MPACK_DOUBLE +/** + * Reads a double as a raw uint64_t. The underlying value must be a double, not a + * float or an integer. + * + * @throws mpack_error_type if the underlying value is not a double. + */ +uint64_t mpack_expect_raw_double(mpack_reader_t* reader); +#endif /** * @} @@ -245,7 +278,7 @@ MPACK_INLINE unsigned int mpack_expect_uint_range(mpack_reader_t* reader, unsign } /** - * Reads an 8-bit unsigned integer, ensuring that it is at most max_value. + * Reads an 8-bit unsigned integer, ensuring that it is at most @a max_value. * * The underlying type may be an integer type of any size and signedness, * as long as the value can be represented in an 8-bit unsigned int. @@ -257,7 +290,7 @@ MPACK_INLINE uint8_t mpack_expect_u8_max(mpack_reader_t* reader, uint8_t max_val } /** - * Reads a 16-bit unsigned integer, ensuring that it is at most max_value. + * Reads a 16-bit unsigned integer, ensuring that it is at most @a max_value. * * The underlying type may be an integer type of any size and signedness, * as long as the value can be represented in a 16-bit unsigned int. @@ -269,7 +302,7 @@ MPACK_INLINE uint16_t mpack_expect_u16_max(mpack_reader_t* reader, uint16_t max_ } /** - * Reads a 32-bit unsigned integer, ensuring that it is at most max_value. + * Reads a 32-bit unsigned integer, ensuring that it is at most @a max_value. * * The underlying type may be an integer type of any size and signedness, * as long as the value can be represented in a 32-bit unsigned int. @@ -281,7 +314,7 @@ MPACK_INLINE uint32_t mpack_expect_u32_max(mpack_reader_t* reader, uint32_t max_ } /** - * Reads a 64-bit unsigned integer, ensuring that it is at most max_value. + * Reads a 64-bit unsigned integer, ensuring that it is at most @a max_value. * * The underlying type may be an integer type of any size and signedness, * as long as the value can be represented in a 64-bit unsigned int. @@ -293,7 +326,7 @@ MPACK_INLINE uint64_t mpack_expect_u64_max(mpack_reader_t* reader, uint64_t max_ } /** - * Reads an unsigned integer, ensuring that it is at most max_value. + * Reads an unsigned integer, ensuring that it is at most @a max_value. * * The underlying type may be an integer type of any size and signedness, * as long as the value can be represented in an unsigned int. @@ -362,7 +395,7 @@ MPACK_INLINE int mpack_expect_int_range(mpack_reader_t* reader, int min_value, i /** * Reads an 8-bit signed integer, ensuring that it is at least zero and at - * most max_value. + * most @a max_value. * * The underlying type may be an integer type of any size and signedness, * as long as the value can be represented in an 8-bit signed int. @@ -375,7 +408,7 @@ MPACK_INLINE int8_t mpack_expect_i8_max(mpack_reader_t* reader, int8_t max_value /** * Reads a 16-bit signed integer, ensuring that it is at least zero and at - * most max_value. + * most @a max_value. * * The underlying type may be an integer type of any size and signedness, * as long as the value can be represented in a 16-bit signed int. @@ -388,7 +421,7 @@ MPACK_INLINE int16_t mpack_expect_i16_max(mpack_reader_t* reader, int16_t max_va /** * Reads a 32-bit signed integer, ensuring that it is at least zero and at - * most max_value. + * most @a max_value. * * The underlying type may be an integer type of any size and signedness, * as long as the value can be represented in a 32-bit signed int. @@ -401,7 +434,7 @@ MPACK_INLINE int32_t mpack_expect_i32_max(mpack_reader_t* reader, int32_t max_va /** * Reads a 64-bit signed integer, ensuring that it is at least zero and at - * most max_value. + * most @a max_value. * * The underlying type may be an integer type of any size and signedness, * as long as the value can be represented in a 64-bit signed int. @@ -413,7 +446,7 @@ MPACK_INLINE int64_t mpack_expect_i64_max(mpack_reader_t* reader, int64_t max_va } /** - * Reads an int, ensuring that it is at least zero and at most max_value. + * Reads an int, ensuring that it is at least zero and at most @a max_value. * * The underlying type may be an integer type of any size and signedness, * as long as the value can be represented in a signed int. @@ -424,29 +457,33 @@ MPACK_INLINE int mpack_expect_int_max(mpack_reader_t* reader, int max_value) { return mpack_expect_int_range(reader, 0, max_value); } +#if MPACK_FLOAT /** * Reads a number, ensuring that it falls within the given range and returning * the value as a float. The underlying value can be an integer, float or * double; the value is converted to a float. * - * Note that reading a double or a large integer with this function can incur a + * @note Reading a double or a large integer with this function can incur a * loss of precision. * * @throws mpack_error_type if the underlying value is not a float, double or integer. */ float mpack_expect_float_range(mpack_reader_t* reader, float min_value, float max_value); +#endif +#if MPACK_DOUBLE /** * Reads a number, ensuring that it falls within the given range and returning * the value as a double. The underlying value can be an integer, float or * double; the value is converted to a double. * - * Note that reading a very large integer with this function can incur a + * @note Reading a very large integer with this function can incur a * loss of precision. * * @throws mpack_error_type if the underlying value is not a float, double or integer. */ double mpack_expect_double_range(mpack_reader_t* reader, double min_value, double max_value); +#endif /** * @} @@ -476,7 +513,7 @@ MPACK_INLINE unsigned int mpack_expect_uint(mpack_reader_t* reader) { return (unsigned int)mpack_expect_u32(reader); // Otherwise we wrap the max function to ensure it fits. - return (unsigned int)mpack_expect_u64_max(reader, UINT_MAX); + return (unsigned int)mpack_expect_u64_max(reader, MPACK_UINT_MAX); } @@ -495,7 +532,7 @@ MPACK_INLINE int mpack_expect_int(mpack_reader_t* reader) { return (int)mpack_expect_i32(reader); // Otherwise we wrap the range function to ensure it fits. - return (int)mpack_expect_i64_range(reader, INT_MIN, INT_MAX); + return (int)mpack_expect_i64_range(reader, MPACK_INT_MIN, MPACK_INT_MAX); } @@ -526,32 +563,61 @@ void mpack_expect_uint_match(mpack_reader_t* reader, uint64_t value); */ void mpack_expect_int_match(mpack_reader_t* reader, int64_t value); +/** + * @} + */ + /** * @name Other Basic Types * @{ */ /** - * Reads a nil. + * Reads a nil, raising @ref mpack_error_type if the value is not nil. */ void mpack_expect_nil(mpack_reader_t* reader); /** - * Reads a bool. Note that integers will raise mpack_error_type; the value - * must be strictly a bool. + * Reads a boolean. + * + * @note Integers will raise mpack_error_type; the value must be strictly a boolean. */ bool mpack_expect_bool(mpack_reader_t* reader); /** - * Reads a bool, raising a mpack_error_type if it is not true. + * Reads a boolean, raising @ref mpack_error_type if its value is not @c true. */ void mpack_expect_true(mpack_reader_t* reader); /** - * Reads a bool, raising a mpack_error_type if it is not false. + * Reads a boolean, raising @ref mpack_error_type if its value is not @c false. */ void mpack_expect_false(mpack_reader_t* reader); +/** + * @} + */ + +/** + * @name Extension Functions + * @{ + */ + +#if MPACK_EXTENSIONS +/** + * Reads a timestamp. + * + * @note This requires @ref MPACK_EXTENSIONS. + */ +mpack_timestamp_t mpack_expect_timestamp(mpack_reader_t* reader); + +/** + * Reads a timestamp in seconds, truncating the nanoseconds (if any). + * + * @note This requires @ref MPACK_EXTENSIONS. + */ +int64_t mpack_expect_timestamp_truncate(mpack_reader_t* reader); +#endif /** * @} @@ -569,10 +635,19 @@ void mpack_expect_false(mpack_reader_t* reader); * alternating between keys and values. @ref mpack_done_map() must be called * once all elements have been read. * - * Note that maps in JSON are unordered, so it is recommended not to expect + * @note Maps in JSON are unordered, so it is recommended not to expect * a specific ordering for your map values in case your data is converted * to/from JSON. * + * @warning This call is dangerous! It does not have a size limit, and it + * does not have any way of checking whether there is enough data in the + * message (since the data could be coming from a stream.) When looping + * through the map's contents, you must check for errors on each iteration + * of the loop. Otherwise an attacker could craft a message declaring a map + * of a billion elements which would throw your parsing code into an + * infinite loop! You should strongly consider using mpack_expect_map_max() + * with a safe maximum size instead. + * * @throws mpack_error_type if the value is not a map. */ uint32_t mpack_expect_map(mpack_reader_t* reader); @@ -585,7 +660,7 @@ uint32_t mpack_expect_map(mpack_reader_t* reader); * alternating between keys and values. @ref mpack_done_map() must be called * once all elements have been read. * - * Note that maps in JSON are unordered, so it is recommended not to expect + * @note Maps in JSON are unordered, so it is recommended not to expect * a specific ordering for your map values in case your data is converted * to/from JSON. * @@ -597,14 +672,14 @@ uint32_t mpack_expect_map(mpack_reader_t* reader); uint32_t mpack_expect_map_range(mpack_reader_t* reader, uint32_t min_count, uint32_t max_count); /** - * Reads the start of a map with a number of elements at most max_count, + * Reads the start of a map with a number of elements at most @a max_count, * returning its element count. * * A number of values follow equal to twice the element count of the map, * alternating between keys and values. @ref mpack_done_map() must be called * once all elements have been read. * - * Note that maps in JSON are unordered, so it is recommended not to expect + * @note Maps in JSON are unordered, so it is recommended not to expect * a specific ordering for your map values in case your data is converted * to/from JSON. * @@ -624,7 +699,7 @@ MPACK_INLINE uint32_t mpack_expect_map_max(mpack_reader_t* reader, uint32_t max_ * alternating between keys and values. @ref mpack_done_map() must be called * once all elements have been read. * - * Note that maps in JSON are unordered, so it is recommended not to expect + * @note Maps in JSON are unordered, so it is recommended not to expect * a specific ordering for your map values in case your data is converted * to/from JSON. * @@ -641,11 +716,21 @@ void mpack_expect_map_match(mpack_reader_t* reader, uint32_t count); * of the map, alternating between keys and values. @ref mpack_done_map() should * also be called once all elements have been read (only if a map was read.) * - * Note that maps in JSON are unordered, so it is recommended not to expect + * @note Maps in JSON are unordered, so it is recommended not to expect * a specific ordering for your map values in case your data is converted * to/from JSON. * - * @returns true if a map was read successfully; false if nil was read or an error occured. + * @warning This call is dangerous! It does not have a size limit, and it + * does not have any way of checking whether there is enough data in the + * message (since the data could be coming from a stream.) When looping + * through the map's contents, you must check for errors on each iteration + * of the loop. Otherwise an attacker could craft a message declaring a map + * of a billion elements which would throw your parsing code into an + * infinite loop! You should strongly consider using mpack_expect_map_max_or_nil() + * with a safe maximum size instead. + * + * @returns @c true if a map was read successfully; @c false if nil was read + * or an error occurred. * @throws mpack_error_type if the value is not a nil or map. */ bool mpack_expect_map_or_nil(mpack_reader_t* reader, uint32_t* count); @@ -659,11 +744,13 @@ bool mpack_expect_map_or_nil(mpack_reader_t* reader, uint32_t* count); * of the map, alternating between keys and values. @ref mpack_done_map() should * anlso be called once all elements have been read (only if a map was read.) * - * Note that maps in JSON are unordered, so it is recommended not to expect + * @note Maps in JSON are unordered, so it is recommended not to expect * a specific ordering for your map values in case your data is converted - * to/from JSON. + * to/from JSON. Consider using mpack_expect_key_cstr() or mpack_expect_key_uint() + * to switch on the key; see @ref docs/expect.md for examples. * - * @returns true if a map was read successfully; false if nil was read or an error occured. + * @returns @c true if a map was read successfully; @c false if nil was read + * or an error occurred. * @throws mpack_error_type if the value is not a nil or map. */ bool mpack_expect_map_max_or_nil(mpack_reader_t* reader, uint32_t max_count, uint32_t* count); @@ -673,6 +760,15 @@ bool mpack_expect_map_max_or_nil(mpack_reader_t* reader, uint32_t max_count, uin * * A number of values follow equal to the element count of the array. * @ref mpack_done_array() must be called once all elements have been read. + * + * @warning This call is dangerous! It does not have a size limit, and it + * does not have any way of checking whether there is enough data in the + * message (since the data could be coming from a stream.) When looping + * through the array's contents, you must check for errors on each iteration + * of the loop. Otherwise an attacker could craft a message declaring an array + * of a billion elements which would throw your parsing code into an + * infinite loop! You should strongly consider using mpack_expect_array_max() + * with a safe maximum size instead. */ uint32_t mpack_expect_array(mpack_reader_t* reader); @@ -691,7 +787,7 @@ uint32_t mpack_expect_array(mpack_reader_t* reader); uint32_t mpack_expect_array_range(mpack_reader_t* reader, uint32_t min_count, uint32_t max_count); /** - * Reads the start of an array with a number of elements at most max_count, + * Reads the start of an array with a number of elements at most @a max_count, * returning its element count. * * A number of values follow equal to the element count of the array. @@ -725,7 +821,17 @@ void mpack_expect_array_match(mpack_reader_t* reader, uint32_t count); * of the array. @ref mpack_done_array() should also be called once all elements * have been read (only if an array was read.) * - * @returns true if an array was read successfully; false if nil was read or an error occured. + * @warning This call is dangerous! It does not have a size limit, and it + * does not have any way of checking whether there is enough data in the + * message (since the data could be coming from a stream.) When looping + * through the array's contents, you must check for errors on each iteration + * of the loop. Otherwise an attacker could craft a message declaring an array + * of a billion elements which would throw your parsing code into an + * infinite loop! You should strongly consider using mpack_expect_array_max_or_nil() + * with a safe maximum size instead. + * + * @returns @c true if an array was read successfully; @c false if nil was read + * or an error occurred. * @throws mpack_error_type if the value is not a nil or array. */ bool mpack_expect_array_or_nil(mpack_reader_t* reader, uint32_t* count); @@ -739,7 +845,8 @@ bool mpack_expect_array_or_nil(mpack_reader_t* reader, uint32_t* count); * of the array. @ref mpack_done_array() should also be called once all elements * have been read (only if an array was read.) * - * @returns true if an array was read successfully; false if nil was read or an error occured. + * @returns @c true if an array was read successfully; @c false if nil was read + * or an error occurred. * @throws mpack_error_type if the value is not a nil or array. */ bool mpack_expect_array_max_or_nil(mpack_reader_t* reader, uint32_t max_count, uint32_t* count); @@ -932,8 +1039,8 @@ void mpack_expect_utf8_cstr(mpack_reader_t* reader, char* buf, size_t size); * The allocated string must be freed with MPACK_FREE() (or simply free() * if MPack's allocator hasn't been customized.) * - * @throws mpack_error_too_big if the string plus null-terminator is larger than the given maxsize. - * @throws mpack_error_invalid if the value is not a string or contains a null byte. + * @throws mpack_error_too_big If the string plus null-terminator is larger than the given maxsize. + * @throws mpack_error_type If the value is not a string or contains a null byte. */ char* mpack_expect_cstr_alloc(mpack_reader_t* reader, size_t maxsize); @@ -954,9 +1061,9 @@ char* mpack_expect_cstr_alloc(mpack_reader_t* reader, size_t maxsize); * if MPack's allocator hasn't been customized.) * if you want a null-terminator. * - * @throws mpack_error_too_big if the string plus null-terminator is larger + * @throws mpack_error_too_big If the string plus null-terminator is larger * than the given maxsize. - * @throws mpack_error_invalid if the value is not a string or contains + * @throws mpack_error_type If the value is not a string or contains * invalid UTF-8 or a null byte. */ char* mpack_expect_utf8_cstr_alloc(mpack_reader_t* reader, size_t maxsize); @@ -969,12 +1076,9 @@ char* mpack_expect_utf8_cstr_alloc(mpack_reader_t* reader, size_t maxsize); * Remember that maps are unordered in JSON. Don't use this for map keys * unless the map has only a single key! */ -MPACK_INLINE void mpack_expect_cstr_match(mpack_reader_t* reader, const char* str) { - if (mpack_strlen(str) > UINT32_MAX) { - mpack_reader_flag_error(reader, mpack_error_type); - return; - } - mpack_expect_str_match(reader, str, mpack_strlen(str)); +MPACK_INLINE void mpack_expect_cstr_match(mpack_reader_t* reader, const char* cstr) { + mpack_assert(cstr != NULL, "cstr pointer is NULL"); + mpack_expect_str_match(reader, cstr, mpack_strlen(cstr)); } /** @@ -982,7 +1086,7 @@ MPACK_INLINE void mpack_expect_cstr_match(mpack_reader_t* reader, const char* st */ /** - * @name Binary Data / Extension Functions + * @name Binary Data * @{ */ @@ -1025,8 +1129,8 @@ MPACK_INLINE uint32_t mpack_expect_bin_max(mpack_reader_t* reader, uint32_t maxs * or mpack_read_bytes_inplace(). @ref mpack_done_bin() must be called * once all bytes have been read. * - * mpack_error_type is raised if the value is not a binary blob or if its - * length does not match. + * @throws mpack_error_type if the value is not a binary blob or if its size + * does not match. */ MPACK_INLINE void mpack_expect_bin_size(mpack_reader_t* reader, uint32_t count) { if (mpack_expect_bin(reader) != count) @@ -1042,11 +1146,178 @@ MPACK_INLINE void mpack_expect_bin_size(mpack_reader_t* reader, uint32_t count) */ size_t mpack_expect_bin_buf(mpack_reader_t* reader, char* buf, size_t size); +/** + * Reads a binary blob with the exact given size into the given buffer. + * + * For compatibility, this will accept if the underlying type is string or + * binary (since in MessagePack 1.0, strings and binary data were combined + * under the "raw" type which became string in 1.1.) + * + * @throws mpack_error_type if the value is not a binary blob or if its size + * does not match. + */ +void mpack_expect_bin_size_buf(mpack_reader_t* reader, char* buf, uint32_t size); + /** * Reads a binary blob with the given total maximum size, allocating storage for it. */ char* mpack_expect_bin_alloc(mpack_reader_t* reader, size_t maxsize, size_t* size); +/** + * @} + */ + +/** + * @name Extension Functions + * @{ + */ + +#if MPACK_EXTENSIONS +/** + * Reads the start of an extension blob, returning its size in bytes and + * placing the type into @p type. + * + * The bytes follow and must be read separately with mpack_read_bytes() + * or mpack_read_bytes_inplace(). @ref mpack_done_ext() must be called + * once all bytes have been read. + * + * @p type will be a user-defined type in the range [0,127] or a reserved type + * in the range [-128,-2]. + * + * mpack_error_type is raised if the value is not an extension blob. The @p + * type value is zero if an error occurs. + * + * @note This cannot be used to match a timestamp. @ref mpack_error_type will + * be flagged if the value is a timestamp. Use mpack_expect_timestamp() or + * mpack_expect_timestamp_truncate() instead. + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @warning Be careful when using reserved types. They may no longer be ext + * types in the future, and previously valid data containing reserved types may + * become invalid in the future. + */ +uint32_t mpack_expect_ext(mpack_reader_t* reader, int8_t* type); + +/** + * Reads the start of an extension blob, raising an error if its length is not + * at most the given number of bytes and placing the type into @p type. + * + * The bytes follow and must be read separately with mpack_read_bytes() + * or mpack_read_bytes_inplace(). @ref mpack_done_ext() must be called + * once all bytes have been read. + * + * mpack_error_type is raised if the value is not an extension blob or if its + * length does not match. The @p type value is zero if an error is raised. + * + * @p type will be a user-defined type in the range [0,127] or a reserved type + * in the range [-128,-2]. + * + * @note This cannot be used to match a timestamp. @ref mpack_error_type will + * be flagged if the value is a timestamp. Use mpack_expect_timestamp() or + * mpack_expect_timestamp_truncate() instead. + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @warning Be careful when using reserved types. They may no longer be ext + * types in the future, and previously valid data containing reserved types may + * become invalid in the future. + * + * @see mpack_expect_ext() + */ +MPACK_INLINE uint32_t mpack_expect_ext_max(mpack_reader_t* reader, int8_t* type, uint32_t maxsize) { + uint32_t length = mpack_expect_ext(reader, type); + if (length > maxsize) { + mpack_reader_flag_error(reader, mpack_error_type); + return 0; + } + return length; +} + +/** + * Reads the start of an extension blob, raising an error if its length is not + * exactly the given number of bytes and placing the type into @p type. + * + * The bytes follow and must be read separately with mpack_read_bytes() + * or mpack_read_bytes_inplace(). @ref mpack_done_ext() must be called + * once all bytes have been read. + * + * mpack_error_type is raised if the value is not an extension blob or if its + * length does not match. The @p type value is zero if an error is raised. + * + * @p type will be a user-defined type in the range [0,127] or a reserved type + * in the range [-128,-2]. + * + * @note This cannot be used to match a timestamp. @ref mpack_error_type will + * be flagged if the value is a timestamp. Use mpack_expect_timestamp() or + * mpack_expect_timestamp_truncate() instead. + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @warning Be careful when using reserved types. They may no longer be ext + * types in the future, and previously valid data containing reserved types may + * become invalid in the future. + * + * @see mpack_expect_ext() + */ +MPACK_INLINE void mpack_expect_ext_size(mpack_reader_t* reader, int8_t* type, uint32_t count) { + if (mpack_expect_ext(reader, type) != count) { + *type = 0; + mpack_reader_flag_error(reader, mpack_error_type); + } +} + +/** + * Reads an extension blob into the given buffer, returning its size in bytes + * and placing the type into @p type. + * + * mpack_error_type is raised if the value is not an extension blob or if its + * length does not match. The @p type value is zero if an error is raised. + * + * @p type will be a user-defined type in the range [0,127] or a reserved type + * in the range [-128,-2]. + * + * @note This cannot be used to match a timestamp. @ref mpack_error_type will + * be flagged if the value is a timestamp. Use mpack_expect_timestamp() or + * mpack_expect_timestamp_truncate() instead. + * + * @warning Be careful when using reserved types. They may no longer be ext + * types in the future, and previously valid data containing reserved types may + * become invalid in the future. + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @see mpack_expect_ext() + */ +size_t mpack_expect_ext_buf(mpack_reader_t* reader, int8_t* type, char* buf, size_t size); +#endif + +#if MPACK_EXTENSIONS && defined(MPACK_MALLOC) +/** + * Reads an extension blob with the given total maximum size, allocating + * storage for it, and placing the type into @p type. + * + * mpack_error_type is raised if the value is not an extension blob or if its + * length does not match. The @p type value is zero if an error is raised. + * + * @p type will be a user-defined type in the range [0,127] or a reserved type + * in the range [-128,-2]. + * + * @note This cannot be used to match a timestamp. @ref mpack_error_type will + * be flagged if the value is a timestamp. Use mpack_expect_timestamp() or + * mpack_expect_timestamp_truncate() instead. + * + * @warning Be careful when using reserved types. They may no longer be ext + * types in the future, and previously valid data containing reserved types may + * become invalid in the future. + * + * @note This requires @ref MPACK_EXTENSIONS and @ref MPACK_MALLOC. + * + * @see mpack_expect_ext() + */ +char* mpack_expect_ext_alloc(mpack_reader_t* reader, int8_t* type, size_t maxsize, size_t* size); +#endif + /** * @} */ @@ -1075,6 +1346,75 @@ char* mpack_expect_bin_alloc(mpack_reader_t* reader, size_t maxsize, size_t* siz */ void mpack_expect_tag(mpack_reader_t* reader, mpack_tag_t tag); +/** + * Expects a string matching one of the strings in the given array, + * returning its array index. + * + * If the value does not match any of the given strings, + * @ref mpack_error_type is flagged. Use mpack_expect_enum_optional() + * if you want to allow other values than the given strings. + * + * If any error occurs or the reader is in an error state, @a count + * is returned. + * + * This can be used to quickly parse a string into an enum when the + * enum values range from 0 to @a count-1. If the last value in the + * enum is a special "count" value, it can be passed as the count, + * and the return value can be cast directly to the enum type. + * + * @code{.c} + * typedef enum { APPLE , BANANA , ORANGE , COUNT} fruit_t; + * const char* fruits[] = {"apple", "banana", "orange"}; + * + * fruit_t fruit = (fruit_t)mpack_expect_enum(reader, fruits, COUNT); + * @endcode + * + * See @ref docs/expect.md for more examples. + * + * The maximum string length is the size of the buffer (strings are read in-place.) + * + * @param reader The reader + * @param strings An array of expected strings of length count + * @param count The number of strings + * @return The index of the matched string, or @a count in case of error + */ +size_t mpack_expect_enum(mpack_reader_t* reader, const char* strings[], size_t count); + +/** + * Expects a string matching one of the strings in the given array + * returning its array index, or @a count if no strings match. + * + * If the value is not a string, or it does not match any of the + * given strings, @a count is returned and no error is flagged. + * + * If any error occurs or the reader is in an error state, @a count + * is returned. + * + * This can be used to quickly parse a string into an enum when the + * enum values range from 0 to @a count-1. If the last value in the + * enum is a special "count" value, it can be passed as the count, + * and the return value can be cast directly to the enum type. + * + * @code{.c} + * typedef enum { APPLE , BANANA , ORANGE , COUNT} fruit_t; + * const char* fruits[] = {"apple", "banana", "orange"}; + * + * fruit_t fruit = (fruit_t)mpack_expect_enum_optional(reader, fruits, COUNT); + * @endcode + * + * See @ref docs/expect.md for more examples. + * + * The maximum string length is the size of the buffer (strings are read in-place.) + * + * @param reader The reader + * @param strings An array of expected strings of length count + * @param count The number of strings + * + * @return The index of the matched string, or @a count if it does not + * match or an error occurs + */ +size_t mpack_expect_enum_optional(mpack_reader_t* reader, const char* strings[], size_t count); + /** * Expects an unsigned integer map key between 0 and count-1, marking it * as found in the given bool array and returning it. @@ -1084,14 +1424,14 @@ void mpack_expect_tag(mpack_reader_t* reader, mpack_tag_t tag); * be called in the expression of a switch() statement. See @ref * docs/expect.md for an example. * - * The found array should be cleared before expecting a key. If the flag for - * a given key is already set when found (i.e. the map contains a duplicate - * key), mpack_error_invalid is flagged. + * The found array must be cleared before expecting the first key. If the + * flag for a given key is already set when found (i.e. the map contains a + * duplicate key), mpack_error_invalid is flagged. * - * If the key is count or larger, count is returned and no error is flagged. - * If you want an error on unrecognized keys, flag an error in the default - * case in your switch; otherwise you must call mpack_discard() to discard - * its content. + * If the key is not a non-negative integer, or if the key is @a count or + * larger, @a count is returned and no error is flagged. If you want an error + * on unrecognized keys, flag an error in the default case in your switch; + * otherwise you must call mpack_discard() to discard its content. * * @param reader The reader * @param found An array of bool flags of length count @@ -1111,9 +1451,9 @@ size_t mpack_expect_key_uint(mpack_reader_t* reader, bool found[], size_t count) * array to define the key indices. It should be called in the expression * of a switch() statement. See @ref docs/expect.md for an example. * - * The found array should be cleared before expecting a key. If the flag for - * a given key is already set when found (i.e. the map contains a duplicate - * key), mpack_error_invalid is flagged. + * The found array must be cleared before expecting the first key. If the + * flag for a given key is already set when found (i.e. the map contains a + * duplicate key), mpack_error_invalid is flagged. * * If the key is unrecognized, count is returned and no error is flagged. If * you want an error on unrecognized keys, flag an error in the default case @@ -1141,7 +1481,8 @@ size_t mpack_expect_key_cstr(mpack_reader_t* reader, const char* keys[], #endif -MPACK_HEADER_END +MPACK_EXTERN_C_END +MPACK_SILENCE_WARNINGS_END #endif diff --git a/src/mpack/mpack-node.c b/src/mpack/mpack-node.c index e939af7..8af1562 100644 --- a/src/mpack/mpack-node.c +++ b/src/mpack/mpack-node.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -23,20 +23,52 @@ #include "mpack-node.h" +MPACK_SILENCE_WARNINGS_BEGIN + #if MPACK_NODE +MPACK_STATIC_INLINE const char* mpack_node_data_unchecked(mpack_node_t node) { + mpack_assert(mpack_node_error(node) == mpack_ok, "tree is in an error state!"); + + mpack_type_t type = node.data->type; + MPACK_UNUSED(type); + #if MPACK_EXTENSIONS + mpack_assert(type == mpack_type_str || type == mpack_type_bin || type == mpack_type_ext, + "node of type %i (%s) is not a data type!", type, mpack_type_to_string(type)); + #else + mpack_assert(type == mpack_type_str || type == mpack_type_bin, + "node of type %i (%s) is not a data type!", type, mpack_type_to_string(type)); + #endif + + return node.tree->data + node.data->value.offset; +} + +#if MPACK_EXTENSIONS +MPACK_STATIC_INLINE int8_t mpack_node_exttype_unchecked(mpack_node_t node) { + mpack_assert(mpack_node_error(node) == mpack_ok, "tree is in an error state!"); + + mpack_type_t type = node.data->type; + MPACK_UNUSED(type); + mpack_assert(type == mpack_type_ext, "node of type %i (%s) is not an ext type!", + type, mpack_type_to_string(type)); + + // the exttype of an ext node is stored in the byte preceding the data + return mpack_load_i8(mpack_node_data_unchecked(node) - 1); +} +#endif + /* * Tree Parsing */ +#ifdef MPACK_MALLOC + // fix up the alloc size to make sure it exactly fits the // maximum number of nodes it can contain (the allocator will // waste it back anyway, but we round it down just in case) -#ifdef MPACK_MALLOC - #define MPACK_NODES_PER_PAGE \ ((MPACK_NODE_PAGE_SIZE - sizeof(mpack_tree_page_t)) / sizeof(mpack_node_data_t) + 1) @@ -45,154 +77,185 @@ #endif -typedef struct mpack_level_t { - mpack_node_data_t* child; - size_t left; // children left in level -} mpack_level_t; - -typedef struct mpack_tree_parser_t { - mpack_tree_t* tree; - const char* data; - - // We keep track of the number of "possible nodes" left in the data rather - // than the number of bytes. - // - // When a map or array is parsed, we ensure at least one byte for each child - // exists and subtract them right away. This ensures that if ever a map or - // array declares more elements than could possibly be contained in the data, - // we will error out immediately rather than allocating storage for them. - // - // For example malicious data that repeats 0xDE 0xFF 0xFF would otherwise - // cause us to run out of memory. With this, the parser can only allocate - // as many nodes as there are bytes in the data (plus the paging overhead, - // 12%.) An error will be flagged immediately if and when there isn't enough - // data left to fully read all children of all open compound types on the - // parsing stack. - // - // Once an entire message has been parsed (and there are no nodes left to - // parse whose bytes have been subtracted), this matches the number of left - // over bytes in the data. - size_t possible_nodes_left; - - mpack_node_data_t* nodes; - size_t nodes_left; // nodes left in current page/pool - - size_t level; - size_t depth; - mpack_level_t* stack; - bool stack_owned; -} mpack_tree_parser_t; - -MPACK_STATIC_INLINE uint8_t mpack_tree_u8(mpack_tree_parser_t* parser) { - if (parser->possible_nodes_left < sizeof(uint8_t)) { - mpack_tree_flag_error(parser->tree, mpack_error_invalid); - return 0; +#ifdef MPACK_MALLOC +/* + * Fills the tree until we have at least enough bytes for the current node. + */ +static bool mpack_tree_reserve_fill(mpack_tree_t* tree) { + mpack_assert(tree->parser.state == mpack_tree_parse_state_in_progress); + + size_t bytes = tree->parser.current_node_reserved; + mpack_assert(bytes > tree->parser.possible_nodes_left, + "there are already enough bytes! call mpack_tree_ensure() instead."); + mpack_log("filling to reserve %i bytes\n", (int)bytes); + + // if the necessary bytes would put us over the maximum tree + // size, fail right away. + // TODO: check for overflow? + if (tree->data_length + bytes > tree->max_size) { + mpack_tree_flag_error(tree, mpack_error_too_big); + return false; } - uint8_t val = mpack_load_u8(parser->data); - parser->data += sizeof(uint8_t); - parser->possible_nodes_left -= sizeof(uint8_t); - return val; -} -MPACK_STATIC_INLINE uint16_t mpack_tree_u16(mpack_tree_parser_t* parser) { - if (parser->possible_nodes_left < sizeof(uint16_t)) { - mpack_tree_flag_error(parser->tree, mpack_error_invalid); - return 0; + // we'll need a read function to fetch more data. if there's + // no read function, the data should contain an entire message + // (or messages), so we flag it as invalid. + if (tree->read_fn == NULL) { + mpack_log("tree has no read function!\n"); + mpack_tree_flag_error(tree, mpack_error_invalid); + return false; } - uint16_t val = mpack_load_u16(parser->data); - parser->data += sizeof(uint16_t); - parser->possible_nodes_left -= sizeof(uint16_t); - return val; -} -MPACK_STATIC_INLINE uint32_t mpack_tree_u32(mpack_tree_parser_t* parser) { - if (parser->possible_nodes_left < sizeof(uint32_t)) { - mpack_tree_flag_error(parser->tree, mpack_error_invalid); - return 0; + // expand the buffer if needed + if (tree->data_length + bytes > tree->buffer_capacity) { + + // TODO: check for overflow? + size_t new_capacity = (tree->buffer_capacity == 0) ? MPACK_BUFFER_SIZE : tree->buffer_capacity; + while (new_capacity < tree->data_length + bytes) + new_capacity *= 2; + if (new_capacity > tree->max_size) + new_capacity = tree->max_size; + + mpack_log("expanding buffer from %i to %i\n", (int)tree->buffer_capacity, (int)new_capacity); + + char* new_buffer; + if (tree->buffer == NULL) + new_buffer = (char*)MPACK_MALLOC(new_capacity); + else + new_buffer = (char*)mpack_realloc(tree->buffer, tree->data_length, new_capacity); + + if (new_buffer == NULL) { + mpack_tree_flag_error(tree, mpack_error_memory); + return false; + } + + tree->data = new_buffer; + tree->buffer = new_buffer; + tree->buffer_capacity = new_capacity; } - uint32_t val = mpack_load_u32(parser->data); - parser->data += sizeof(uint32_t); - parser->possible_nodes_left -= sizeof(uint32_t); - return val; + + // request as much data as possible, looping until we have + // all the data we need + do { + size_t read = tree->read_fn(tree, tree->buffer + tree->data_length, tree->buffer_capacity - tree->data_length); + + // If the fill function encounters an error, it should flag an error on + // the tree. + if (mpack_tree_error(tree) != mpack_ok) + return false; + + // We guard against fill functions that return -1 just in case. + if (read == (size_t)(-1)) { + mpack_tree_flag_error(tree, mpack_error_io); + return false; + } + + // If the fill function returns 0, the data is not available yet. We + // return false to stop parsing the current node. + if (read == 0) { + mpack_log("not enough data.\n"); + return false; + } + + mpack_log("read %" PRIu32 " more bytes\n", (uint32_t)read); + tree->data_length += read; + tree->parser.possible_nodes_left += read; + } while (tree->parser.possible_nodes_left < bytes); + + return true; } +#endif -MPACK_STATIC_INLINE uint64_t mpack_tree_u64(mpack_tree_parser_t* parser) { - if (parser->possible_nodes_left < sizeof(uint64_t)) { - mpack_tree_flag_error(parser->tree, mpack_error_invalid); - return 0; +/* + * Ensures there are enough additional bytes in the tree for the current node + * (including reserved bytes for the children of this node, and in addition to + * the reserved bytes for children of previous compound nodes), reading more + * data if needed. + * + * extra_bytes is the number of additional bytes to reserve for the current + * node beyond the type byte (since one byte is already reserved for each node + * by its parent array or map.) + * + * This may reallocate the tree, which means the tree->data pointer may change! + * + * Returns false if not enough bytes could be read. + */ +MPACK_STATIC_INLINE bool mpack_tree_reserve_bytes(mpack_tree_t* tree, size_t extra_bytes) { + mpack_assert(tree->parser.state == mpack_tree_parse_state_in_progress); + + // We guard against overflow here. A compound type could declare more than + // MPACK_UINT32_MAX contents which overflows SIZE_MAX on 32-bit platforms. We + // flag mpack_error_invalid instead of mpack_error_too_big since it's far + // more likely that the message is corrupt than that the data is valid but + // not parseable on this architecture (see test_read_node_possible() in + // test-node.c .) + if ((uint64_t)tree->parser.current_node_reserved + (uint64_t)extra_bytes > SIZE_MAX) { + mpack_tree_flag_error(tree, mpack_error_invalid); + return false; } - uint64_t val = mpack_load_u64(parser->data); - parser->data += sizeof(uint64_t); - parser->possible_nodes_left -= sizeof(uint64_t); - return val; -} -MPACK_STATIC_INLINE int8_t mpack_tree_i8 (mpack_tree_parser_t* parser) {return (int8_t) mpack_tree_u8(parser); } -MPACK_STATIC_INLINE int16_t mpack_tree_i16(mpack_tree_parser_t* parser) {return (int16_t)mpack_tree_u16(parser);} -MPACK_STATIC_INLINE int32_t mpack_tree_i32(mpack_tree_parser_t* parser) {return (int32_t)mpack_tree_u32(parser);} -MPACK_STATIC_INLINE int64_t mpack_tree_i64(mpack_tree_parser_t* parser) {return (int64_t)mpack_tree_u64(parser);} + tree->parser.current_node_reserved += extra_bytes; -MPACK_STATIC_INLINE void mpack_skip_exttype(mpack_tree_parser_t* parser) { - // the exttype is stored right before the data. we - // skip it and get it out of the data when needed. - mpack_tree_i8(parser); -} + // Note that possible_nodes_left already accounts for reserved bytes for + // children of previous compound nodes. So even if there are hundreds of + // bytes left in the buffer, we might need to read anyway. + if (tree->parser.current_node_reserved <= tree->parser.possible_nodes_left) + return true; -MPACK_STATIC_INLINE float mpack_tree_float(mpack_tree_parser_t* parser) { - union { - float f; - uint32_t i; - } u; - u.i = mpack_tree_u32(parser); - return u.f; + #ifdef MPACK_MALLOC + return mpack_tree_reserve_fill(tree); + #else + return false; + #endif } -MPACK_STATIC_INLINE double mpack_tree_double(mpack_tree_parser_t* parser) { - union { - double d; - uint64_t i; - } u; - u.i = mpack_tree_u64(parser); - return u.d; +MPACK_STATIC_INLINE size_t mpack_tree_parser_stack_capacity(mpack_tree_t* tree) { + #ifdef MPACK_MALLOC + return tree->parser.stack_capacity; + #else + return sizeof(tree->parser.stack) / sizeof(tree->parser.stack[0]); + #endif } -static void mpack_tree_push_stack(mpack_tree_parser_t* parser, mpack_node_data_t* first_child, size_t total) { +static bool mpack_tree_push_stack(mpack_tree_t* tree, mpack_node_data_t* first_child, size_t total) { + mpack_tree_parser_t* parser = &tree->parser; + mpack_assert(parser->state == mpack_tree_parse_state_in_progress); // No need to push empty containers if (total == 0) - return; + return true; // Make sure we have enough room in the stack - if (parser->level + 1 == parser->depth) { + if (parser->level + 1 == mpack_tree_parser_stack_capacity(tree)) { #ifdef MPACK_MALLOC - size_t new_depth = parser->depth * 2; - mpack_log("growing stack to depth %i\n", (int)new_depth); + size_t new_capacity = parser->stack_capacity * 2; + mpack_log("growing parse stack to capacity %i\n", (int)new_capacity); // Replace the stack-allocated parsing stack if (!parser->stack_owned) { - mpack_level_t* new_stack = (mpack_level_t*)MPACK_MALLOC(sizeof(mpack_level_t) * new_depth); + mpack_level_t* new_stack = (mpack_level_t*)MPACK_MALLOC(sizeof(mpack_level_t) * new_capacity); if (!new_stack) { - mpack_tree_flag_error(parser->tree, mpack_error_memory); - return; + mpack_tree_flag_error(tree, mpack_error_memory); + return false; } - mpack_memcpy(new_stack, parser->stack, sizeof(mpack_level_t) * parser->depth); + mpack_memcpy(new_stack, parser->stack, sizeof(mpack_level_t) * parser->stack_capacity); parser->stack = new_stack; parser->stack_owned = true; // Realloc the allocated parsing stack } else { mpack_level_t* new_stack = (mpack_level_t*)mpack_realloc(parser->stack, - sizeof(mpack_level_t) * parser->depth, sizeof(mpack_level_t) * new_depth); + sizeof(mpack_level_t) * parser->stack_capacity, sizeof(mpack_level_t) * new_capacity); if (!new_stack) { - mpack_tree_flag_error(parser->tree, mpack_error_memory); - return; + mpack_tree_flag_error(tree, mpack_error_memory); + return false; } parser->stack = new_stack; } - parser->depth = new_depth; + parser->stack_capacity = new_capacity; #else - mpack_tree_flag_error(parser->tree, mpack_error_too_big); - return; + mpack_tree_flag_error(tree, mpack_error_too_big); + return false; #endif } @@ -200,28 +263,36 @@ static void mpack_tree_push_stack(mpack_tree_parser_t* parser, mpack_node_data_t ++parser->level; parser->stack[parser->level].child = first_child; parser->stack[parser->level].left = total; + return true; } -static void mpack_tree_parse_children(mpack_tree_parser_t* parser, mpack_node_data_t* node) { +static bool mpack_tree_parse_children(mpack_tree_t* tree, mpack_node_data_t* node) { + mpack_tree_parser_t* parser = &tree->parser; + mpack_assert(parser->state == mpack_tree_parse_state_in_progress); + mpack_type_t type = node->type; size_t total = node->len; // Calculate total elements to read if (type == mpack_type_map) { - if ((uint64_t)total * 2 > (uint64_t)SIZE_MAX) { - mpack_tree_flag_error(parser->tree, mpack_error_too_big); - return; + if ((uint64_t)total * 2 > SIZE_MAX) { + mpack_tree_flag_error(tree, mpack_error_too_big); + return false; } total *= 2; } + // Make sure we are under our total node limit (TODO can this overflow?) + tree->node_count += total; + if (tree->node_count > tree->max_nodes) { + mpack_tree_flag_error(tree, mpack_error_too_big); + return false; + } + // Each node is at least one byte. Count these bytes now to make // sure there is enough data left. - if (total > parser->possible_nodes_left) { - mpack_tree_flag_error(parser->tree, mpack_error_invalid); - return; - } - parser->possible_nodes_left -= total; + if (!mpack_tree_reserve_bytes(tree, total)) + return false; // If there are enough nodes left in the current page, no need to grow if (total <= parser->nodes_left) { @@ -234,9 +305,9 @@ static void mpack_tree_parse_children(mpack_tree_parser_t* parser, mpack_node_da #ifdef MPACK_MALLOC // We can't grow if we're using a fixed pool (i.e. we didn't start with a page) - if (!parser->tree->next) { - mpack_tree_flag_error(parser->tree, mpack_error_too_big); - return; + if (!tree->next) { + mpack_tree_flag_error(tree, mpack_error_too_big); + return false; } // Otherwise we need to grow, and the node's children need to be contiguous. @@ -251,62 +322,70 @@ static void mpack_tree_parse_children(mpack_tree_parser_t* parser, mpack_node_da mpack_tree_page_t* page; if (total > MPACK_NODES_PER_PAGE || parser->nodes_left > MPACK_NODES_PER_PAGE / 8) { + // TODO: this should check for overflow page = (mpack_tree_page_t*)MPACK_MALLOC( sizeof(mpack_tree_page_t) + sizeof(mpack_node_data_t) * (total - 1)); if (page == NULL) { - mpack_tree_flag_error(parser->tree, mpack_error_memory); - return; + mpack_tree_flag_error(tree, mpack_error_memory); + return false; } mpack_log("allocated seperate page %p for %i children, %i left in page of %i total\n", - page, (int)total, (int)parser->nodes_left, (int)MPACK_NODES_PER_PAGE); + (void*)page, (int)total, (int)parser->nodes_left, (int)MPACK_NODES_PER_PAGE); node->value.children = page->nodes; } else { page = (mpack_tree_page_t*)MPACK_MALLOC(MPACK_PAGE_ALLOC_SIZE); if (page == NULL) { - mpack_tree_flag_error(parser->tree, mpack_error_memory); - return; + mpack_tree_flag_error(tree, mpack_error_memory); + return false; } mpack_log("allocated new page %p for %i children, wasting %i in page of %i total\n", - page, (int)total, (int)parser->nodes_left, (int)MPACK_NODES_PER_PAGE); + (void*)page, (int)total, (int)parser->nodes_left, (int)MPACK_NODES_PER_PAGE); node->value.children = page->nodes; parser->nodes = page->nodes + total; parser->nodes_left = MPACK_NODES_PER_PAGE - total; } - page->next = parser->tree->next; - parser->tree->next = page; + page->next = tree->next; + tree->next = page; #else // We can't grow if we don't have an allocator - mpack_tree_flag_error(parser->tree, mpack_error_too_big); - return; + mpack_tree_flag_error(tree, mpack_error_too_big); + return false; #endif } - mpack_tree_push_stack(parser, node->value.children, total); + return mpack_tree_push_stack(tree, node->value.children, total); } -static void mpack_tree_parse_bytes(mpack_tree_parser_t* parser, mpack_node_data_t* node) { - size_t length = node->len; - if (length > parser->possible_nodes_left) { - mpack_tree_flag_error(parser->tree, mpack_error_invalid); - return; - } - node->value.bytes = parser->data; - parser->data += length; - parser->possible_nodes_left -= length; +static bool mpack_tree_parse_bytes(mpack_tree_t* tree, mpack_node_data_t* node) { + node->value.offset = tree->size + tree->parser.current_node_reserved + 1; + return mpack_tree_reserve_bytes(tree, node->len); +} + +#if MPACK_EXTENSIONS +static bool mpack_tree_parse_ext(mpack_tree_t* tree, mpack_node_data_t* node) { + // reserve space for exttype + tree->parser.current_node_reserved += sizeof(int8_t); + node->type = mpack_type_ext; + return mpack_tree_parse_bytes(tree, node); } +#endif -static void mpack_tree_parse_node(mpack_tree_parser_t* parser, mpack_node_data_t* node) { +static bool mpack_tree_parse_node_contents(mpack_tree_t* tree, mpack_node_data_t* node) { + mpack_assert(tree->parser.state == mpack_tree_parse_state_in_progress); + mpack_assert(node != NULL, "null node?"); // read the type. we've already accounted for this byte in - // possible_nodes_left, so we know it is in bounds and don't - // need to subtract it. - uint8_t type = mpack_load_u8(parser->data); - parser->data += sizeof(uint8_t); + // possible_nodes_left, so we already know it is in bounds, and we don't + // need to reserve it for this node. + mpack_assert(tree->data_length > tree->size); + uint8_t type = mpack_load_u8(tree->data + tree->size); + mpack_log("node type %x\n", type); + tree->parser.current_node_reserved = 0; // as with mpack_read_tag(), the fastest way to parse a node is to switch // on the first byte, and to explicitly list every possible byte. we switch @@ -320,34 +399,31 @@ static void mpack_tree_parse_node(mpack_tree_parser_t* parser, mpack_node_data_t case 0x4: case 0x5: case 0x6: case 0x7: node->type = mpack_type_uint; node->value.u = type; - return; + return true; // negative fixnum case 0xe: case 0xf: node->type = mpack_type_int; node->value.i = (int8_t)type; - return; + return true; // fixmap case 0x8: node->type = mpack_type_map; - node->len = type & ~0xf0; - mpack_tree_parse_children(parser, node); - return; + node->len = (uint32_t)(type & ~0xf0); + return mpack_tree_parse_children(tree, node); // fixarray case 0x9: node->type = mpack_type_array; - node->len = type & ~0xf0; - mpack_tree_parse_children(parser, node); - return; + node->len = (uint32_t)(type & ~0xf0); + return mpack_tree_parse_children(tree, node); // fixstr case 0xa: case 0xb: node->type = mpack_type_str; - node->len = type & ~0xe0; - mpack_tree_parse_bytes(parser, node); - return; + node->len = (uint32_t)(type & ~0xe0); + return mpack_tree_parse_bytes(tree, node); // not one of the common infix types default: @@ -377,7 +453,7 @@ static void mpack_tree_parse_node(mpack_tree_parser_t* parser, mpack_node_data_t case 0x78: case 0x79: case 0x7a: case 0x7b: case 0x7c: case 0x7d: case 0x7e: case 0x7f: node->type = mpack_type_uint; node->value.u = type; - return; + return true; // negative fixnum case 0xe0: case 0xe1: case 0xe2: case 0xe3: case 0xe4: case 0xe5: case 0xe6: case 0xe7: @@ -386,23 +462,21 @@ static void mpack_tree_parse_node(mpack_tree_parser_t* parser, mpack_node_data_t case 0xf8: case 0xf9: case 0xfa: case 0xfb: case 0xfc: case 0xfd: case 0xfe: case 0xff: node->type = mpack_type_int; node->value.i = (int8_t)type; - return; + return true; // fixmap case 0x80: case 0x81: case 0x82: case 0x83: case 0x84: case 0x85: case 0x86: case 0x87: case 0x88: case 0x89: case 0x8a: case 0x8b: case 0x8c: case 0x8d: case 0x8e: case 0x8f: node->type = mpack_type_map; - node->len = type & ~0xf0; - mpack_tree_parse_children(parser, node); - return; + node->len = (uint32_t)(type & ~0xf0); + return mpack_tree_parse_children(tree, node); // fixarray case 0x90: case 0x91: case 0x92: case 0x93: case 0x94: case 0x95: case 0x96: case 0x97: case 0x98: case 0x99: case 0x9a: case 0x9b: case 0x9c: case 0x9d: case 0x9e: case 0x9f: node->type = mpack_type_array; - node->len = type & ~0xf0; - mpack_tree_parse_children(parser, node); - return; + node->len = (uint32_t)(type & ~0xf0); + return mpack_tree_parse_children(tree, node); // fixstr case 0xa0: case 0xa1: case 0xa2: case 0xa3: case 0xa4: case 0xa5: case 0xa6: case 0xa7: @@ -410,245 +484,331 @@ static void mpack_tree_parse_node(mpack_tree_parser_t* parser, mpack_node_data_t case 0xb0: case 0xb1: case 0xb2: case 0xb3: case 0xb4: case 0xb5: case 0xb6: case 0xb7: case 0xb8: case 0xb9: case 0xba: case 0xbb: case 0xbc: case 0xbd: case 0xbe: case 0xbf: node->type = mpack_type_str; - node->len = type & ~0xe0; - mpack_tree_parse_bytes(parser, node); - return; + node->len = (uint32_t)(type & ~0xe0); + return mpack_tree_parse_bytes(tree, node); #endif // nil case 0xc0: node->type = mpack_type_nil; - return; + return true; // bool case 0xc2: case 0xc3: node->type = mpack_type_bool; node->value.b = type & 1; - return; + return true; // bin8 case 0xc4: node->type = mpack_type_bin; - node->len = mpack_tree_u8(parser); - mpack_tree_parse_bytes(parser, node); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(uint8_t))) + return false; + node->len = mpack_load_u8(tree->data + tree->size + 1); + return mpack_tree_parse_bytes(tree, node); // bin16 case 0xc5: node->type = mpack_type_bin; - node->len = mpack_tree_u16(parser); - mpack_tree_parse_bytes(parser, node); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(uint16_t))) + return false; + node->len = mpack_load_u16(tree->data + tree->size + 1); + return mpack_tree_parse_bytes(tree, node); // bin32 case 0xc6: node->type = mpack_type_bin; - node->len = mpack_tree_u32(parser); - mpack_tree_parse_bytes(parser, node); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(uint32_t))) + return false; + node->len = mpack_load_u32(tree->data + tree->size + 1); + return mpack_tree_parse_bytes(tree, node); + #if MPACK_EXTENSIONS // ext8 case 0xc7: - node->type = mpack_type_ext; - node->len = mpack_tree_u8(parser); - mpack_skip_exttype(parser); - mpack_tree_parse_bytes(parser, node); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(uint8_t))) + return false; + node->len = mpack_load_u8(tree->data + tree->size + 1); + return mpack_tree_parse_ext(tree, node); // ext16 case 0xc8: - node->type = mpack_type_ext; - node->len = mpack_tree_u16(parser); - mpack_skip_exttype(parser); - mpack_tree_parse_bytes(parser, node); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(uint16_t))) + return false; + node->len = mpack_load_u16(tree->data + tree->size + 1); + return mpack_tree_parse_ext(tree, node); // ext32 case 0xc9: - node->type = mpack_type_ext; - node->len = mpack_tree_u32(parser); - mpack_skip_exttype(parser); - mpack_tree_parse_bytes(parser, node); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(uint32_t))) + return false; + node->len = mpack_load_u32(tree->data + tree->size + 1); + return mpack_tree_parse_ext(tree, node); + #endif // float case 0xca: + #if MPACK_FLOAT + if (!mpack_tree_reserve_bytes(tree, sizeof(float))) + return false; + node->value.f = mpack_load_float(tree->data + tree->size + 1); + #else + if (!mpack_tree_reserve_bytes(tree, sizeof(uint32_t))) + return false; + node->value.f = mpack_load_u32(tree->data + tree->size + 1); + #endif node->type = mpack_type_float; - node->value.f = mpack_tree_float(parser); - return; + return true; // double case 0xcb: + #if MPACK_DOUBLE + if (!mpack_tree_reserve_bytes(tree, sizeof(double))) + return false; + node->value.d = mpack_load_double(tree->data + tree->size + 1); + #else + if (!mpack_tree_reserve_bytes(tree, sizeof(uint64_t))) + return false; + node->value.d = mpack_load_u64(tree->data + tree->size + 1); + #endif node->type = mpack_type_double; - node->value.d = mpack_tree_double(parser); - return; + return true; // uint8 case 0xcc: node->type = mpack_type_uint; - node->value.u = mpack_tree_u8(parser); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(uint8_t))) + return false; + node->value.u = mpack_load_u8(tree->data + tree->size + 1); + return true; // uint16 case 0xcd: node->type = mpack_type_uint; - node->value.u = mpack_tree_u16(parser); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(uint16_t))) + return false; + node->value.u = mpack_load_u16(tree->data + tree->size + 1); + return true; // uint32 case 0xce: node->type = mpack_type_uint; - node->value.u = mpack_tree_u32(parser); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(uint32_t))) + return false; + node->value.u = mpack_load_u32(tree->data + tree->size + 1); + return true; // uint64 case 0xcf: node->type = mpack_type_uint; - node->value.u = mpack_tree_u64(parser); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(uint64_t))) + return false; + node->value.u = mpack_load_u64(tree->data + tree->size + 1); + return true; // int8 case 0xd0: node->type = mpack_type_int; - node->value.i = mpack_tree_i8(parser); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(int8_t))) + return false; + node->value.i = mpack_load_i8(tree->data + tree->size + 1); + return true; // int16 case 0xd1: node->type = mpack_type_int; - node->value.i = mpack_tree_i16(parser); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(int16_t))) + return false; + node->value.i = mpack_load_i16(tree->data + tree->size + 1); + return true; // int32 case 0xd2: node->type = mpack_type_int; - node->value.i = mpack_tree_i32(parser); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(int32_t))) + return false; + node->value.i = mpack_load_i32(tree->data + tree->size + 1); + return true; // int64 case 0xd3: node->type = mpack_type_int; - node->value.i = mpack_tree_i64(parser); - return; + if (!mpack_tree_reserve_bytes(tree, sizeof(int64_t))) + return false; + node->value.i = mpack_load_i64(tree->data + tree->size + 1); + return true; + #if MPACK_EXTENSIONS // fixext1 case 0xd4: - node->type = mpack_type_ext; node->len = 1; - mpack_skip_exttype(parser); - mpack_tree_parse_bytes(parser, node); - return; + return mpack_tree_parse_ext(tree, node); // fixext2 case 0xd5: - node->type = mpack_type_ext; node->len = 2; - mpack_skip_exttype(parser); - mpack_tree_parse_bytes(parser, node); - return; + return mpack_tree_parse_ext(tree, node); // fixext4 case 0xd6: - node->type = mpack_type_ext; node->len = 4; - mpack_skip_exttype(parser); - mpack_tree_parse_bytes(parser, node); - return; + return mpack_tree_parse_ext(tree, node); // fixext8 case 0xd7: - node->type = mpack_type_ext; node->len = 8; - mpack_skip_exttype(parser); - mpack_tree_parse_bytes(parser, node); - return; + return mpack_tree_parse_ext(tree, node); // fixext16 case 0xd8: - node->type = mpack_type_ext; node->len = 16; - mpack_skip_exttype(parser); - mpack_tree_parse_bytes(parser, node); - return; + return mpack_tree_parse_ext(tree, node); + #endif // str8 case 0xd9: + if (!mpack_tree_reserve_bytes(tree, sizeof(uint8_t))) + return false; + node->len = mpack_load_u8(tree->data + tree->size + 1); node->type = mpack_type_str; - node->len = mpack_tree_u8(parser); - mpack_tree_parse_bytes(parser, node); - return; + return mpack_tree_parse_bytes(tree, node); // str16 case 0xda: + if (!mpack_tree_reserve_bytes(tree, sizeof(uint16_t))) + return false; + node->len = mpack_load_u16(tree->data + tree->size + 1); node->type = mpack_type_str; - node->len = mpack_tree_u16(parser); - mpack_tree_parse_bytes(parser, node); - return; + return mpack_tree_parse_bytes(tree, node); // str32 case 0xdb: + if (!mpack_tree_reserve_bytes(tree, sizeof(uint32_t))) + return false; + node->len = mpack_load_u32(tree->data + tree->size + 1); node->type = mpack_type_str; - node->len = mpack_tree_u32(parser); - mpack_tree_parse_bytes(parser, node); - return; + return mpack_tree_parse_bytes(tree, node); // array16 case 0xdc: + if (!mpack_tree_reserve_bytes(tree, sizeof(uint16_t))) + return false; + node->len = mpack_load_u16(tree->data + tree->size + 1); node->type = mpack_type_array; - node->len = mpack_tree_u16(parser); - mpack_tree_parse_children(parser, node); - return; + return mpack_tree_parse_children(tree, node); // array32 case 0xdd: + if (!mpack_tree_reserve_bytes(tree, sizeof(uint32_t))) + return false; + node->len = mpack_load_u32(tree->data + tree->size + 1); node->type = mpack_type_array; - node->len = mpack_tree_u32(parser); - mpack_tree_parse_children(parser, node); - return; + return mpack_tree_parse_children(tree, node); // map16 case 0xde: + if (!mpack_tree_reserve_bytes(tree, sizeof(uint16_t))) + return false; + node->len = mpack_load_u16(tree->data + tree->size + 1); node->type = mpack_type_map; - node->len = mpack_tree_u16(parser); - mpack_tree_parse_children(parser, node); - return; + return mpack_tree_parse_children(tree, node); // map32 case 0xdf: + if (!mpack_tree_reserve_bytes(tree, sizeof(uint32_t))) + return false; + node->len = mpack_load_u32(tree->data + tree->size + 1); node->type = mpack_type_map; - node->len = mpack_tree_u32(parser); - mpack_tree_parse_children(parser, node); - return; + return mpack_tree_parse_children(tree, node); // reserved case 0xc1: - mpack_tree_flag_error(parser->tree, mpack_error_invalid); - return; + mpack_tree_flag_error(tree, mpack_error_invalid); + return false; + + #if !MPACK_EXTENSIONS + // ext + case 0xc7: // fallthrough + case 0xc8: // fallthrough + case 0xc9: // fallthrough + // fixext + case 0xd4: // fallthrough + case 0xd5: // fallthrough + case 0xd6: // fallthrough + case 0xd7: // fallthrough + case 0xd8: + mpack_tree_flag_error(tree, mpack_error_unsupported); + return false; + #endif #if MPACK_OPTIMIZE_FOR_SIZE // any other bytes should have been handled by the infix switch default: - mpack_assert(0, "unreachable"); - return; + break; #endif } + mpack_assert(0, "unreachable"); + return false; +} + +static bool mpack_tree_parse_node(mpack_tree_t* tree, mpack_node_data_t* node) { + mpack_log("parsing a node at position %i in level %i\n", + (int)tree->size, (int)tree->parser.level); + + if (!mpack_tree_parse_node_contents(tree, node)) { + mpack_log("node parsing returned false\n"); + return false; + } + + tree->parser.possible_nodes_left -= tree->parser.current_node_reserved; + + // The reserve for the current node does not include the initial byte + // previously reserved as part of its parent. + size_t node_size = tree->parser.current_node_reserved + 1; + + // If the parsed type is a map or array, the reserve includes one byte for + // each child. We want to subtract these out of possible_nodes_left, but + // not out of the current size of the tree. + if (node->type == mpack_type_array) + node_size -= node->len; + else if (node->type == mpack_type_map) + node_size -= node->len * 2; + tree->size += node_size; + + mpack_log("parsed a node of type %s of %i bytes and " + "%i additional bytes reserved for children.\n", + mpack_type_to_string(node->type), (int)node_size, + (int)tree->parser.current_node_reserved + 1 - (int)node_size); + + return true; } -static void mpack_tree_parse_elements(mpack_tree_parser_t* parser) { - mpack_log("parsing tree elements\n"); +/* + * We read nodes in a loop instead of recursively for maximum performance. The + * stack holds the amount of children left to read in each level of the tree. + * Parsing can pause and resume when more data becomes available. + */ +static bool mpack_tree_continue_parsing(mpack_tree_t* tree) { + if (mpack_tree_error(tree) != mpack_ok) + return false; + + mpack_tree_parser_t* parser = &tree->parser; + mpack_assert(parser->state == mpack_tree_parse_state_in_progress); + mpack_log("parsing tree elements, %i bytes in buffer\n", (int)tree->data_length); // we loop parsing nodes until the parse stack is empty. we break // by returning out of the function. while (true) { mpack_node_data_t* node = parser->stack[parser->level].child; - --parser->stack[parser->level].left; - ++parser->stack[parser->level].child; - - mpack_tree_parse_node(parser, node); + size_t level = parser->level; + if (!mpack_tree_parse_node(tree, node)) + return false; + --parser->stack[level].left; + ++parser->stack[level].child; - if (mpack_tree_error(parser->tree) != mpack_ok) - break; + mpack_assert(mpack_tree_error(tree) == mpack_ok, + "mpack_tree_parse_node() should have returned false due to error!"); // pop empty stack levels, exiting the outer loop when the stack is empty. // (we could tail-optimize containers by pre-emptively popping empty @@ -658,75 +818,170 @@ static void mpack_tree_parse_elements(mpack_tree_parser_t* parser) { // it needs to be complete.) while (parser->stack[parser->level].left == 0) { if (parser->level == 0) - return; + return true; --parser->level; } } } -static void mpack_tree_parse(mpack_tree_t* tree, const char* data, size_t length, - mpack_node_data_t* initial_nodes, size_t initial_nodes_count) -{ - mpack_log("starting parse\n"); +static void mpack_tree_cleanup(mpack_tree_t* tree) { + MPACK_UNUSED(tree); - if (length == 0) { - mpack_tree_flag_error(tree, mpack_error_invalid); - return; + #ifdef MPACK_MALLOC + if (tree->parser.stack_owned) { + MPACK_FREE(tree->parser.stack); + tree->parser.stack = NULL; + tree->parser.stack_owned = false; } - if (initial_nodes_count == 0) { - mpack_break("initial page has no nodes!"); - mpack_tree_flag_error(tree, mpack_error_bug); - return; + + mpack_tree_page_t* page = tree->next; + while (page != NULL) { + mpack_tree_page_t* next = page->next; + mpack_log("freeing page %p\n", (void*)page); + MPACK_FREE(page); + page = next; } - tree->root = initial_nodes; + tree->next = NULL; + #endif +} - // Setup parser - mpack_tree_parser_t parser; - mpack_memset(&parser, 0, sizeof(parser)); - parser.tree = tree; - parser.data = data; - parser.nodes = initial_nodes + 1; - parser.nodes_left = initial_nodes_count - 1; +static bool mpack_tree_parse_start(mpack_tree_t* tree) { + if (mpack_tree_error(tree) != mpack_ok) + return false; - // We read nodes in a loop instead of recursively for maximum - // performance. The stack holds the amount of children left to - // read in each level of the tree. + mpack_tree_parser_t* parser = &tree->parser; + mpack_assert(parser->state != mpack_tree_parse_state_in_progress, + "previous parsing was not finished!"); - // Even when we have a malloc() function, it's much faster to - // allocate the initial parsing stack on the call stack. We - // replace it with a heap allocation if we need to grow it. - #ifdef MPACK_MALLOC - #define MPACK_NODE_STACK_LOCAL_DEPTH MPACK_NODE_INITIAL_DEPTH - parser.stack_owned = false; - #else - #define MPACK_NODE_STACK_LOCAL_DEPTH MPACK_NODE_MAX_DEPTH_WITHOUT_MALLOC - #endif - mpack_level_t stack_local[MPACK_NODE_STACK_LOCAL_DEPTH]; // no VLAs in VS 2013 - parser.depth = MPACK_NODE_STACK_LOCAL_DEPTH; - parser.stack = stack_local; - parser.possible_nodes_left = length; - #undef MPACK_NODE_STACK_LOCAL_DEPTH - - // configure the root node - --parser.possible_nodes_left; - tree->node_count = 1; - parser.level = 0; - parser.stack[0].child = tree->root; - parser.stack[0].left = 1; + if (parser->state == mpack_tree_parse_state_parsed) + mpack_tree_cleanup(tree); + + mpack_log("starting parse\n"); + tree->parser.state = mpack_tree_parse_state_in_progress; + tree->parser.current_node_reserved = 0; + + // check if we previously parsed a tree + if (tree->size > 0) { + #ifdef MPACK_MALLOC + // if we're buffered, move the remaining data back to the + // start of the buffer + // TODO: This is not ideal performance-wise. We should only move data + // when we need to call the fill function. + // TODO: We could consider shrinking the buffer here, especially if we + // determine that the fill function is providing less than a quarter of + // the buffer size or if messages take up less than a quarter of the + // buffer size. Maybe this should be configurable. + if (tree->buffer != NULL) { + mpack_memmove(tree->buffer, tree->buffer + tree->size, tree->data_length - tree->size); + } + else + #endif + // otherwise advance past the parsed data + { + tree->data += tree->size; + } + tree->data_length -= tree->size; + tree->size = 0; + tree->node_count = 0; + } - mpack_tree_parse_elements(&parser); + // make sure we have at least one byte available before allocating anything + parser->possible_nodes_left = tree->data_length; + if (!mpack_tree_reserve_bytes(tree, sizeof(uint8_t))) { + tree->parser.state = mpack_tree_parse_state_not_started; + return false; + } + mpack_log("parsing tree at %p starting with byte %x\n", tree->data, (uint8_t)tree->data[0]); + parser->possible_nodes_left -= 1; + tree->node_count = 1; #ifdef MPACK_MALLOC - if (parser.stack_owned) - MPACK_FREE(parser.stack); + parser->stack = parser->stack_local; + parser->stack_owned = false; + parser->stack_capacity = sizeof(parser->stack_local) / sizeof(*parser->stack_local); + + if (tree->pool == NULL) { + + // allocate first page + mpack_tree_page_t* page = (mpack_tree_page_t*)MPACK_MALLOC(MPACK_PAGE_ALLOC_SIZE); + mpack_log("allocated initial page %p of size %i count %i\n", + (void*)page, (int)MPACK_PAGE_ALLOC_SIZE, (int)MPACK_NODES_PER_PAGE); + if (page == NULL) { + tree->error = mpack_error_memory; + return false; + } + page->next = NULL; + tree->next = page; + + parser->nodes = page->nodes; + parser->nodes_left = MPACK_NODES_PER_PAGE; + } + else #endif + { + // otherwise use the provided pool + mpack_assert(tree->pool != NULL, "no pool provided?"); + parser->nodes = tree->pool; + parser->nodes_left = tree->pool_count; + } + + tree->root = parser->nodes; + ++parser->nodes; + --parser->nodes_left; + + parser->level = 0; + parser->stack[0].child = tree->root; + parser->stack[0].left = 1; - // now that there are no longer any nodes to read, possible_nodes_left - // is the number of bytes left in the data. - if (mpack_tree_error(tree) == mpack_ok) - tree->size = length - parser.possible_nodes_left; - mpack_log("parsed tree of %i bytes, %i bytes left\n", (int)tree->size, (int)parser.possible_nodes_left); - mpack_log("%i nodes in final page\n", (int)parser.nodes_left); + return true; +} + +void mpack_tree_parse(mpack_tree_t* tree) { + if (mpack_tree_error(tree) != mpack_ok) + return; + + if (tree->parser.state != mpack_tree_parse_state_in_progress) { + if (!mpack_tree_parse_start(tree)) { + mpack_tree_flag_error(tree, (tree->read_fn == NULL) ? + mpack_error_invalid : mpack_error_io); + return; + } + } + + if (!mpack_tree_continue_parsing(tree)) { + if (mpack_tree_error(tree) != mpack_ok) + return; + + // We're parsing synchronously on a blocking fill function. If we + // didn't completely finish parsing the tree, it's an error. + mpack_log("tree parsing incomplete. flagging error.\n"); + mpack_tree_flag_error(tree, (tree->read_fn == NULL) ? + mpack_error_invalid : mpack_error_io); + return; + } + + mpack_assert(mpack_tree_error(tree) == mpack_ok); + mpack_assert(tree->parser.level == 0); + tree->parser.state = mpack_tree_parse_state_parsed; + mpack_log("parsed tree of %i bytes, %i bytes left\n", (int)tree->size, (int)tree->parser.possible_nodes_left); + mpack_log("%i nodes in final page\n", (int)tree->parser.nodes_left); +} + +bool mpack_tree_try_parse(mpack_tree_t* tree) { + if (mpack_tree_error(tree) != mpack_ok) + return false; + + if (tree->parser.state != mpack_tree_parse_state_in_progress) + if (!mpack_tree_parse_start(tree)) + return false; + + if (!mpack_tree_continue_parsing(tree)) + return false; + + mpack_assert(mpack_tree_error(tree) == mpack_ok); + mpack_assert(tree->parser.level == 0); + tree->parser.state = mpack_tree_parse_state_parsed; + return true; } @@ -736,16 +991,32 @@ static void mpack_tree_parse(mpack_tree_t* tree, const char* data, size_t length */ mpack_node_t mpack_tree_root(mpack_tree_t* tree) { - return mpack_node(tree, (mpack_tree_error(tree) != mpack_ok) ? &tree->nil_node : tree->root); + if (mpack_tree_error(tree) != mpack_ok) + return mpack_tree_nil_node(tree); + + // We check that a tree was parsed successfully and assert if not. You must + // call mpack_tree_parse() (or mpack_tree_try_parse() with a success + // result) in order to access the root node. + if (tree->parser.state != mpack_tree_parse_state_parsed) { + mpack_break("Tree has not been parsed! " + "Did you call mpack_tree_parse() or mpack_tree_try_parse()?"); + mpack_tree_flag_error(tree, mpack_error_bug); + return mpack_tree_nil_node(tree); + } + + return mpack_node(tree, tree->root); } static void mpack_tree_init_clear(mpack_tree_t* tree) { mpack_memset(tree, 0, sizeof(*tree)); tree->nil_node.type = mpack_type_nil; + tree->missing_node.type = mpack_type_missing; + tree->max_size = SIZE_MAX; + tree->max_nodes = SIZE_MAX; } #ifdef MPACK_MALLOC -void mpack_tree_init(mpack_tree_t* tree, const char* data, size_t length) { +void mpack_tree_init_data(mpack_tree_t* tree, const char* data, size_t length) { mpack_tree_init_clear(tree); MPACK_STATIC_ASSERT(MPACK_NODE_PAGE_SIZE >= sizeof(mpack_tree_page_t), @@ -754,21 +1025,14 @@ void mpack_tree_init(mpack_tree_t* tree, const char* data, size_t length) { MPACK_STATIC_ASSERT(MPACK_PAGE_ALLOC_SIZE <= MPACK_NODE_PAGE_SIZE, "incorrect page rounding?"); - // allocate first page - mpack_tree_page_t* page = (mpack_tree_page_t*)MPACK_MALLOC(MPACK_PAGE_ALLOC_SIZE); - mpack_log("allocated initial page %p of size %i count %i\n", - page, (int)MPACK_PAGE_ALLOC_SIZE, (int)MPACK_NODES_PER_PAGE); - if (page == NULL) { - tree->error = mpack_error_memory; - return; - } - page->next = NULL; - tree->next = page; + tree->data = data; + tree->data_length = length; + tree->pool = NULL; + tree->pool_count = 0; + tree->next = NULL; mpack_log("===========================\n"); mpack_log("initializing tree with data of size %i\n", (int)length); - - mpack_tree_parse(tree, data, length, page->nodes, MPACK_NODES_PER_PAGE); } #endif @@ -780,10 +1044,20 @@ void mpack_tree_init_pool(mpack_tree_t* tree, const char* data, size_t length, tree->next = NULL; #endif - mpack_log("===========================\n"); - mpack_log("initializing tree with data of size %i and pool of count %i\n", (int)length, (int)node_pool_count); + if (node_pool_count == 0) { + mpack_break("initial page has no nodes!"); + mpack_tree_flag_error(tree, mpack_error_bug); + return; + } - mpack_tree_parse(tree, data, length, node_pool, node_pool_count); + tree->data = data; + tree->data_length = length; + tree->pool = node_pool; + tree->pool_count = node_pool_count; + + mpack_log("===========================\n"); + mpack_log("initializing tree with data of size %i and pool of count %i\n", + (int)length, (int)node_pool_count); } void mpack_tree_init_error(mpack_tree_t* tree, mpack_error_t error) { @@ -794,6 +1068,31 @@ void mpack_tree_init_error(mpack_tree_t* tree, mpack_error_t error) { mpack_log("initializing tree error state %i\n", (int)error); } +#ifdef MPACK_MALLOC +void mpack_tree_init_stream(mpack_tree_t* tree, mpack_tree_read_t read_fn, void* context, + size_t max_message_size, size_t max_message_nodes) { + mpack_tree_init_clear(tree); + + tree->read_fn = read_fn; + tree->context = context; + + mpack_tree_set_limits(tree, max_message_size, max_message_nodes); + tree->max_size = max_message_size; + tree->max_nodes = max_message_nodes; + + mpack_log("===========================\n"); + mpack_log("initializing tree with stream, max size %i max nodes %i\n", + (int)max_message_size, (int)max_message_nodes); +} +#endif + +void mpack_tree_set_limits(mpack_tree_t* tree, size_t max_message_size, size_t max_message_nodes) { + mpack_assert(max_message_size > 0); + mpack_assert(max_message_nodes > 0); + tree->max_size = max_message_size; + tree->max_nodes = max_message_nodes; +} + #if MPACK_STDIO typedef struct mpack_file_tree_t { char* data; @@ -807,14 +1106,7 @@ static void mpack_file_tree_teardown(mpack_tree_t* tree) { MPACK_FREE(file_tree); } -static bool mpack_file_tree_read(mpack_tree_t* tree, mpack_file_tree_t* file_tree, const char* filename, size_t max_size) { - - // open the file - FILE* file = fopen(filename, "rb"); - if (!file) { - mpack_tree_init_error(tree, mpack_error_io); - return false; - } +static bool mpack_file_tree_read(mpack_tree_t* tree, mpack_file_tree_t* file_tree, FILE* file, size_t max_bytes) { // get the file size errno = 0; @@ -828,28 +1120,24 @@ static bool mpack_file_tree_read(mpack_tree_t* tree, mpack_file_tree_t* file_tre // check for errors if (error != 0 || size < 0) { - fclose(file); mpack_tree_init_error(tree, mpack_error_io); return false; } if (size == 0) { - fclose(file); mpack_tree_init_error(tree, mpack_error_invalid); return false; } - // make sure the size is less than max_size + // make sure the size is less than max_bytes // (this mess exists to safely convert between long and size_t regardless of their widths) - if (max_size != 0 && (((uint64_t)LONG_MAX > (uint64_t)SIZE_MAX && size > (long)SIZE_MAX) || (size_t)size > max_size)) { - fclose(file); + if (max_bytes != 0 && (((uint64_t)LONG_MAX > (uint64_t)SIZE_MAX && size > (long)SIZE_MAX) || (size_t)size > max_bytes)) { mpack_tree_init_error(tree, mpack_error_too_big); return false; } // allocate data - file_tree->data = (char*)MPACK_MALLOC(size); + file_tree->data = (char*)MPACK_MALLOC((size_t)size); if (file_tree->data == NULL) { - fclose(file); mpack_tree_init_error(tree, mpack_error_memory); return false; } @@ -859,28 +1147,31 @@ static bool mpack_file_tree_read(mpack_tree_t* tree, mpack_file_tree_t* file_tre while (total < size) { size_t read = fread(file_tree->data + total, 1, (size_t)(size - total), file); if (read <= 0) { - fclose(file); mpack_tree_init_error(tree, mpack_error_io); MPACK_FREE(file_tree->data); return false; } - total += read; + total += (long)read; } - fclose(file); file_tree->size = (size_t)size; return true; } -void mpack_tree_init_file(mpack_tree_t* tree, const char* filename, size_t max_size) { +static bool mpack_tree_file_check_max_bytes(mpack_tree_t* tree, size_t max_bytes) { // the C STDIO family of file functions use long (e.g. ftell) - if (max_size > LONG_MAX) { - mpack_break("max_size of %" PRIu64 " is invalid, maximum is LONG_MAX", (uint64_t)max_size); + if (max_bytes > LONG_MAX) { + mpack_break("max_bytes of %" PRIu64 " is invalid, maximum is LONG_MAX", (uint64_t)max_bytes); mpack_tree_init_error(tree, mpack_error_bug); - return; + return false; } + return true; +} + +static void mpack_tree_init_stdfile_noclose(mpack_tree_t* tree, FILE* stdfile, size_t max_bytes) { + // allocate file tree mpack_file_tree_t* file_tree = (mpack_file_tree_t*) MPACK_MALLOC(sizeof(mpack_file_tree_t)); if (file_tree == NULL) { @@ -889,26 +1180,47 @@ void mpack_tree_init_file(mpack_tree_t* tree, const char* filename, size_t max_s } // read all data - if (!mpack_file_tree_read(tree, file_tree, filename, max_size)) { + if (!mpack_file_tree_read(tree, file_tree, stdfile, max_bytes)) { MPACK_FREE(file_tree); return; } - mpack_tree_init(tree, file_tree->data, file_tree->size); + mpack_tree_init_data(tree, file_tree->data, file_tree->size); mpack_tree_set_context(tree, file_tree); mpack_tree_set_teardown(tree, mpack_file_tree_teardown); } + +void mpack_tree_init_stdfile(mpack_tree_t* tree, FILE* stdfile, size_t max_bytes, bool close_when_done) { + if (!mpack_tree_file_check_max_bytes(tree, max_bytes)) + return; + + mpack_tree_init_stdfile_noclose(tree, stdfile, max_bytes); + + if (close_when_done) + fclose(stdfile); +} + +void mpack_tree_init_filename(mpack_tree_t* tree, const char* filename, size_t max_bytes) { + if (!mpack_tree_file_check_max_bytes(tree, max_bytes)) + return; + + // open the file + FILE* file = fopen(filename, "rb"); + if (!file) { + mpack_tree_init_error(tree, mpack_error_io); + return; + } + + mpack_tree_init_stdfile(tree, file, max_bytes, true); +} #endif mpack_error_t mpack_tree_destroy(mpack_tree_t* tree) { + mpack_tree_cleanup(tree); + #ifdef MPACK_MALLOC - mpack_tree_page_t* page = tree->next; - while (page) { - mpack_tree_page_t* next = page->next; - mpack_log("freeing page %p\n", page); - MPACK_FREE(page); - page = next; - } + if (tree->buffer) + MPACK_FREE(tree->buffer); #endif if (tree->teardown) @@ -919,9 +1231,8 @@ mpack_error_t mpack_tree_destroy(mpack_tree_t* tree) { } void mpack_tree_flag_error(mpack_tree_t* tree, mpack_error_t error) { - mpack_log("tree %p setting error %i: %s\n", tree, (int)error, mpack_error_to_string(error)); - if (tree->error == mpack_ok) { + mpack_log("tree %p setting error %i: %s\n", (void*)tree, (int)error, mpack_error_to_string(error)); tree->error = error; if (tree->error_fn) tree->error_fn(tree, error); @@ -943,11 +1254,16 @@ mpack_tag_t mpack_node_tag(mpack_node_t node) { if (mpack_node_error(node) != mpack_ok) return mpack_tag_nil(); - mpack_tag_t tag; - mpack_memset(&tag, 0, sizeof(tag)); + mpack_tag_t tag = MPACK_TAG_ZERO; tag.type = node.data->type; switch (node.data->type) { + case mpack_type_missing: + // If a node is missing, I don't know if it makes sense to ask for + // a tag for it. We'll return a missing tag to match the missing + // node I guess, but attempting to use the tag for anything (like + // writing it for example) will flag mpack_error_bug. + break; case mpack_type_nil: break; case mpack_type_bool: tag.v.b = node.data->value.b; break; case mpack_type_float: tag.v.f = node.data->value.f; break; @@ -958,11 +1274,12 @@ mpack_tag_t mpack_node_tag(mpack_node_t node) { case mpack_type_str: tag.v.l = node.data->len; break; case mpack_type_bin: tag.v.l = node.data->len; break; + #if MPACK_EXTENSIONS case mpack_type_ext: tag.v.l = node.data->len; - // the exttype of an ext node is stored in the byte preceding the data - tag.exttype = (int8_t)*(node.data->value.bytes - 1); + tag.exttype = mpack_node_exttype_unchecked(node); break; + #endif case mpack_type_array: tag.v.n = node.data->len; break; case mpack_type_map: tag.v.n = node.data->len; break; @@ -974,99 +1291,191 @@ mpack_tag_t mpack_node_tag(mpack_node_t node) { return tag; } -#if MPACK_STDIO -static void mpack_node_print_element(mpack_node_t node, size_t depth, FILE* file) { +#if MPACK_DEBUG && MPACK_STDIO +static void mpack_node_print_element(mpack_node_t node, mpack_print_t* print, size_t depth) { mpack_node_data_t* data = node.data; + size_t i,j; switch (data->type) { - - case mpack_type_nil: - fprintf(file, "null"); - break; - case mpack_type_bool: - fprintf(file, data->value.b ? "true" : "false"); - break; - - case mpack_type_float: - fprintf(file, "%f", data->value.f); - break; - case mpack_type_double: - fprintf(file, "%f", data->value.d); - break; - - case mpack_type_int: - fprintf(file, "%" PRIi64, data->value.i); - break; - case mpack_type_uint: - fprintf(file, "%" PRIu64, data->value.u); - break; - - case mpack_type_bin: - fprintf(file, "", data->len); - break; - - case mpack_type_ext: - fprintf(file, "", - mpack_node_exttype(node), data->len); - break; - case mpack_type_str: { - putc('"', file); - const char* bytes = mpack_node_data(node); - for (size_t i = 0; i < data->len; ++i) { + mpack_print_append_cstr(print, "\""); + const char* bytes = mpack_node_data_unchecked(node); + for (i = 0; i < data->len; ++i) { char c = bytes[i]; switch (c) { - case '\n': fprintf(file, "\\n"); break; - case '\\': fprintf(file, "\\\\"); break; - case '"': fprintf(file, "\\\""); break; - default: putc(c, file); break; + case '\n': mpack_print_append_cstr(print, "\\n"); break; + case '\\': mpack_print_append_cstr(print, "\\\\"); break; + case '"': mpack_print_append_cstr(print, "\\\""); break; + default: mpack_print_append(print, &c, 1); break; } } - putc('"', file); + mpack_print_append_cstr(print, "\""); } break; case mpack_type_array: - fprintf(file, "[\n"); - for (size_t i = 0; i < data->len; ++i) { - for (size_t j = 0; j < depth + 1; ++j) - fprintf(file, " "); - mpack_node_print_element(mpack_node_array_at(node, i), depth + 1, file); + mpack_print_append_cstr(print, "[\n"); + for (i = 0; i < data->len; ++i) { + for (j = 0; j < depth + 1; ++j) + mpack_print_append_cstr(print, " "); + mpack_node_print_element(mpack_node_array_at(node, i), print, depth + 1); if (i != data->len - 1) - putc(',', file); - putc('\n', file); + mpack_print_append_cstr(print, ","); + mpack_print_append_cstr(print, "\n"); } - for (size_t i = 0; i < depth; ++i) - fprintf(file, " "); - putc(']', file); + for (i = 0; i < depth; ++i) + mpack_print_append_cstr(print, " "); + mpack_print_append_cstr(print, "]"); break; case mpack_type_map: - fprintf(file, "{\n"); - for (size_t i = 0; i < data->len; ++i) { - for (size_t j = 0; j < depth + 1; ++j) - fprintf(file, " "); - mpack_node_print_element(mpack_node_map_key_at(node, i), depth + 1, file); - fprintf(file, ": "); - mpack_node_print_element(mpack_node_map_value_at(node, i), depth + 1, file); + mpack_print_append_cstr(print, "{\n"); + for (i = 0; i < data->len; ++i) { + for (j = 0; j < depth + 1; ++j) + mpack_print_append_cstr(print, " "); + mpack_node_print_element(mpack_node_map_key_at(node, i), print, depth + 1); + mpack_print_append_cstr(print, ": "); + mpack_node_print_element(mpack_node_map_value_at(node, i), print, depth + 1); if (i != data->len - 1) - putc(',', file); - putc('\n', file); + mpack_print_append_cstr(print, ","); + mpack_print_append_cstr(print, "\n"); } - for (size_t i = 0; i < depth; ++i) - fprintf(file, " "); - putc('}', file); + for (i = 0; i < depth; ++i) + mpack_print_append_cstr(print, " "); + mpack_print_append_cstr(print, "}"); break; + + default: + { + const char* prefix = NULL; + size_t prefix_length = 0; + if (mpack_node_type(node) == mpack_type_bin + #if MPACK_EXTENSIONS + || mpack_node_type(node) == mpack_type_ext + #endif + ) { + prefix = mpack_node_data(node); + prefix_length = mpack_node_data_len(node); + } + + char buf[256]; + mpack_tag_t tag = mpack_node_tag(node); + mpack_tag_debug_pseudo_json(tag, buf, sizeof(buf), prefix, prefix_length); + mpack_print_append_cstr(print, buf); + } + break; + } +} + +void mpack_node_print_to_buffer(mpack_node_t node, char* buffer, size_t buffer_size) { + if (buffer_size == 0) { + mpack_assert(false, "buffer size is zero!"); + return; } + + mpack_print_t print; + mpack_memset(&print, 0, sizeof(print)); + print.buffer = buffer; + print.size = buffer_size; + mpack_node_print_element(node, &print, 0); + mpack_print_append(&print, "", 1); // null-terminator + mpack_print_flush(&print); + + // we always make sure there's a null-terminator at the end of the buffer + // in case we ran out of space. + print.buffer[print.size - 1] = '\0'; } -void mpack_node_print_file(mpack_node_t node, FILE* file) { +void mpack_node_print_to_callback(mpack_node_t node, mpack_print_callback_t callback, void* context) { + char buffer[1024]; + mpack_print_t print; + mpack_memset(&print, 0, sizeof(print)); + print.buffer = buffer; + print.size = sizeof(buffer); + print.callback = callback; + print.context = context; + mpack_node_print_element(node, &print, 0); + mpack_print_flush(&print); +} + +void mpack_node_print_to_file(mpack_node_t node, FILE* file) { mpack_assert(file != NULL, "file is NULL"); - int depth = 2; - for (int i = 0; i < depth; ++i) - fprintf(file, " "); - mpack_node_print_element(node, depth, file); - putc('\n', file); + + char buffer[1024]; + mpack_print_t print; + mpack_memset(&print, 0, sizeof(print)); + print.buffer = buffer; + print.size = sizeof(buffer); + print.callback = &mpack_print_file_callback; + print.context = file; + + size_t depth = 2; + size_t i; + for (i = 0; i < depth; ++i) + mpack_print_append_cstr(&print, " "); + mpack_node_print_element(node, &print, depth); + mpack_print_append_cstr(&print, "\n"); + mpack_print_flush(&print); +} +#endif + + + +/* + * Node Value Functions + */ + +#if MPACK_EXTENSIONS +mpack_timestamp_t mpack_node_timestamp(mpack_node_t node) { + mpack_timestamp_t timestamp = {0, 0}; + + // we'll let mpack_node_exttype() do most checks + if (mpack_node_exttype(node) != MPACK_EXTTYPE_TIMESTAMP) { + mpack_log("exttype %i\n", mpack_node_exttype(node)); + mpack_node_flag_error(node, mpack_error_type); + return timestamp; + } + + const char* p = mpack_node_data_unchecked(node); + + switch (node.data->len) { + case 4: + timestamp.nanoseconds = 0; + timestamp.seconds = mpack_load_u32(p); + break; + + case 8: { + uint64_t value = mpack_load_u64(p); + timestamp.nanoseconds = (uint32_t)(value >> 34); + timestamp.seconds = value & ((MPACK_UINT64_C(1) << 34) - 1); + break; + } + + case 12: + timestamp.nanoseconds = mpack_load_u32(p); + timestamp.seconds = mpack_load_i64(p + 4); + break; + + default: + mpack_tree_flag_error(node.tree, mpack_error_invalid); + return timestamp; + } + + if (timestamp.nanoseconds > MPACK_TIMESTAMP_NANOSECONDS_MAX) { + mpack_tree_flag_error(node.tree, mpack_error_invalid); + mpack_timestamp_t zero = {0, 0}; + return zero; + } + + return timestamp; +} + +int64_t mpack_node_timestamp_seconds(mpack_node_t node) { + return mpack_node_timestamp(node).seconds; +} + +uint32_t mpack_node_timestamp_nanoseconds(mpack_node_t node) { + return mpack_node_timestamp(node).nanoseconds; } #endif @@ -1080,7 +1489,7 @@ void mpack_node_check_utf8(mpack_node_t node) { if (mpack_node_error(node) != mpack_ok) return; mpack_node_data_t* data = node.data; - if (data->type != mpack_type_str || !mpack_utf8_check(data->value.bytes, data->len)) + if (data->type != mpack_type_str || !mpack_utf8_check(mpack_node_data_unchecked(node), data->len)) mpack_node_flag_error(node, mpack_error_type); } @@ -1088,7 +1497,7 @@ void mpack_node_check_utf8_cstr(mpack_node_t node) { if (mpack_node_error(node) != mpack_ok) return; mpack_node_data_t* data = node.data; - if (data->type != mpack_type_str || !mpack_utf8_check_no_null(data->value.bytes, data->len)) + if (data->type != mpack_type_str || !mpack_utf8_check_no_null(mpack_node_data_unchecked(node), data->len)) mpack_node_flag_error(node, mpack_error_type); } @@ -1099,7 +1508,11 @@ size_t mpack_node_copy_data(mpack_node_t node, char* buffer, size_t bufsize) { mpack_assert(bufsize == 0 || buffer != NULL, "buffer is NULL for maximum of %i bytes", (int)bufsize); mpack_type_t type = node.data->type; - if (type != mpack_type_str && type != mpack_type_bin && type != mpack_type_ext) { + if (type != mpack_type_str && type != mpack_type_bin + #if MPACK_EXTENSIONS + && type != mpack_type_ext + #endif + ) { mpack_node_flag_error(node, mpack_error_type); return 0; } @@ -1109,7 +1522,7 @@ size_t mpack_node_copy_data(mpack_node_t node, char* buffer, size_t bufsize) { return 0; } - mpack_memcpy(buffer, node.data->value.bytes, node.data->len); + mpack_memcpy(buffer, mpack_node_data_unchecked(node), node.data->len); return (size_t)node.data->len; } @@ -1130,12 +1543,12 @@ size_t mpack_node_copy_utf8(mpack_node_t node, char* buffer, size_t bufsize) { return 0; } - if (!mpack_utf8_check(node.data->value.bytes, node.data->len)) { + if (!mpack_utf8_check(mpack_node_data_unchecked(node), node.data->len)) { mpack_node_flag_error(node, mpack_error_type); return 0; } - mpack_memcpy(buffer, node.data->value.bytes, node.data->len); + mpack_memcpy(buffer, mpack_node_data_unchecked(node), node.data->len); return (size_t)node.data->len; } @@ -1163,13 +1576,13 @@ void mpack_node_copy_cstr(mpack_node_t node, char* buffer, size_t bufsize) { return; } - if (!mpack_str_check_no_null(node.data->value.bytes, node.data->len)) { + if (!mpack_str_check_no_null(mpack_node_data_unchecked(node), node.data->len)) { buffer[0] = '\0'; mpack_node_flag_error(node, mpack_error_type); return; } - mpack_memcpy(buffer, node.data->value.bytes, node.data->len); + mpack_memcpy(buffer, mpack_node_data_unchecked(node), node.data->len); buffer[node.data->len] = '\0'; } @@ -1197,13 +1610,13 @@ void mpack_node_copy_utf8_cstr(mpack_node_t node, char* buffer, size_t bufsize) return; } - if (!mpack_utf8_check_no_null(node.data->value.bytes, node.data->len)) { + if (!mpack_utf8_check_no_null(mpack_node_data_unchecked(node), node.data->len)) { buffer[0] = '\0'; mpack_node_flag_error(node, mpack_error_type); return; } - mpack_memcpy(buffer, node.data->value.bytes, node.data->len); + mpack_memcpy(buffer, mpack_node_data_unchecked(node), node.data->len); buffer[node.data->len] = '\0'; } @@ -1214,7 +1627,11 @@ char* mpack_node_data_alloc(mpack_node_t node, size_t maxlen) { // make sure this is a valid data type mpack_type_t type = node.data->type; - if (type != mpack_type_str && type != mpack_type_bin && type != mpack_type_ext) { + if (type != mpack_type_str && type != mpack_type_bin + #if MPACK_EXTENSIONS + && type != mpack_type_ext + #endif + ) { mpack_node_flag_error(node, mpack_error_type); return NULL; } @@ -1230,7 +1647,7 @@ char* mpack_node_data_alloc(mpack_node_t node, size_t maxlen) { return NULL; } - mpack_memcpy(ret, node.data->value.bytes, node.data->len); + mpack_memcpy(ret, mpack_node_data_unchecked(node), node.data->len); return ret; } @@ -1255,7 +1672,7 @@ char* mpack_node_cstr_alloc(mpack_node_t node, size_t maxlen) { return NULL; } - if (!mpack_str_check_no_null(node.data->value.bytes, node.data->len)) { + if (!mpack_str_check_no_null(mpack_node_data_unchecked(node), node.data->len)) { mpack_node_flag_error(node, mpack_error_type); return NULL; } @@ -1266,7 +1683,7 @@ char* mpack_node_cstr_alloc(mpack_node_t node, size_t maxlen) { return NULL; } - mpack_memcpy(ret, node.data->value.bytes, node.data->len); + mpack_memcpy(ret, mpack_node_data_unchecked(node), node.data->len); ret[node.data->len] = '\0'; return ret; } @@ -1292,7 +1709,7 @@ char* mpack_node_utf8_cstr_alloc(mpack_node_t node, size_t maxlen) { return NULL; } - if (!mpack_utf8_check_no_null(node.data->value.bytes, node.data->len)) { + if (!mpack_utf8_check_no_null(mpack_node_data_unchecked(node), node.data->len)) { mpack_node_flag_error(node, mpack_error_type); return NULL; } @@ -1303,7 +1720,7 @@ char* mpack_node_utf8_cstr_alloc(mpack_node_t node, size_t maxlen) { return NULL; } - mpack_memcpy(ret, node.data->value.bytes, node.data->len); + mpack_memcpy(ret, mpack_node_data_unchecked(node), node.data->len); ret[node.data->len] = '\0'; return ret; } @@ -1325,7 +1742,8 @@ static mpack_node_data_t* mpack_node_map_int_impl(mpack_node_t node, int64_t num mpack_node_data_t* found = NULL; - for (size_t i = 0; i < node.data->len; ++i) { + size_t i; + for (i = 0; i < node.data->len; ++i) { mpack_node_data_t* key = mpack_node_child(node, i * 2); if ((key->type == mpack_type_int && key->value.i == num) || @@ -1356,7 +1774,8 @@ static mpack_node_data_t* mpack_node_map_uint_impl(mpack_node_t node, uint64_t n mpack_node_data_t* found = NULL; - for (size_t i = 0; i < node.data->len; ++i) { + size_t i; + for (i = 0; i < node.data->len; ++i) { mpack_node_data_t* key = mpack_node_child(node, i * 2); if ((key->type == mpack_type_uint && key->value.u == num) || @@ -1387,12 +1806,15 @@ static mpack_node_data_t* mpack_node_map_str_impl(mpack_node_t node, const char* return NULL; } + mpack_tree_t* tree = node.tree; mpack_node_data_t* found = NULL; - for (size_t i = 0; i < node.data->len; ++i) { + size_t i; + for (i = 0; i < node.data->len; ++i) { mpack_node_data_t* key = mpack_node_child(node, i * 2); - if (key->type == mpack_type_str && key->len == length && mpack_memcmp(str, key->value.bytes, length) == 0) { + if (key->type == mpack_type_str && key->len == length && + mpack_memcmp(str, mpack_node_data_unchecked(mpack_node(tree, key)), length) == 0) { if (found) { mpack_node_flag_error(node, mpack_error_data); return NULL; @@ -1417,8 +1839,11 @@ static mpack_node_t mpack_node_wrap_lookup(mpack_tree_t* tree, mpack_node_data_t } static mpack_node_t mpack_node_wrap_lookup_optional(mpack_tree_t* tree, mpack_node_data_t* data) { - if (!data) + if (!data) { + if (tree->error == mpack_ok) + return mpack_tree_missing_node(tree); return mpack_tree_nil_node(tree); + } return mpack_node(tree, data); } @@ -1473,6 +1898,507 @@ bool mpack_node_map_contains_cstr(mpack_node_t node, const char* cstr) { return mpack_node_map_contains_str(node, cstr, mpack_strlen(cstr)); } +size_t mpack_node_enum_optional(mpack_node_t node, const char* strings[], size_t count) { + if (mpack_node_error(node) != mpack_ok) + return count; + + // the value is only recognized if it is a string + if (mpack_node_type(node) != mpack_type_str) + return count; + + // fetch the string + const char* key = mpack_node_str(node); + size_t keylen = mpack_node_strlen(node); + mpack_assert(mpack_node_error(node) == mpack_ok, "these should not fail"); + + // find what key it matches + size_t i; + for (i = 0; i < count; ++i) { + const char* other = strings[i]; + size_t otherlen = mpack_strlen(other); + if (keylen == otherlen && mpack_memcmp(key, other, keylen) == 0) + return i; + } + + // no matches + return count; +} + +size_t mpack_node_enum(mpack_node_t node, const char* strings[], size_t count) { + size_t value = mpack_node_enum_optional(node, strings, count); + if (value == count) + mpack_node_flag_error(node, mpack_error_type); + return value; +} + +mpack_type_t mpack_node_type(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return mpack_type_nil; + return node.data->type; +} + +bool mpack_node_is_nil(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) { + // All nodes are treated as nil nodes when we are in error. + return true; + } + return node.data->type == mpack_type_nil; +} + +bool mpack_node_is_missing(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) { + // errors still return nil nodes, not missing nodes. + return false; + } + return node.data->type == mpack_type_missing; +} + +void mpack_node_nil(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return; + if (node.data->type != mpack_type_nil) + mpack_node_flag_error(node, mpack_error_type); +} + +void mpack_node_missing(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return; + if (node.data->type != mpack_type_missing) + mpack_node_flag_error(node, mpack_error_type); +} + +bool mpack_node_bool(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return false; + + if (node.data->type == mpack_type_bool) + return node.data->value.b; + + mpack_node_flag_error(node, mpack_error_type); + return false; +} + +void mpack_node_true(mpack_node_t node) { + if (mpack_node_bool(node) != true) + mpack_node_flag_error(node, mpack_error_type); +} + +void mpack_node_false(mpack_node_t node) { + if (mpack_node_bool(node) != false) + mpack_node_flag_error(node, mpack_error_type); +} + +uint8_t mpack_node_u8(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_uint) { + if (node.data->value.u <= MPACK_UINT8_MAX) + return (uint8_t)node.data->value.u; + } else if (node.data->type == mpack_type_int) { + if (node.data->value.i >= 0 && node.data->value.i <= MPACK_UINT8_MAX) + return (uint8_t)node.data->value.i; + } + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +int8_t mpack_node_i8(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_uint) { + if (node.data->value.u <= MPACK_INT8_MAX) + return (int8_t)node.data->value.u; + } else if (node.data->type == mpack_type_int) { + if (node.data->value.i >= MPACK_INT8_MIN && node.data->value.i <= MPACK_INT8_MAX) + return (int8_t)node.data->value.i; + } + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +uint16_t mpack_node_u16(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_uint) { + if (node.data->value.u <= MPACK_UINT16_MAX) + return (uint16_t)node.data->value.u; + } else if (node.data->type == mpack_type_int) { + if (node.data->value.i >= 0 && node.data->value.i <= MPACK_UINT16_MAX) + return (uint16_t)node.data->value.i; + } + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +int16_t mpack_node_i16(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_uint) { + if (node.data->value.u <= MPACK_INT16_MAX) + return (int16_t)node.data->value.u; + } else if (node.data->type == mpack_type_int) { + if (node.data->value.i >= MPACK_INT16_MIN && node.data->value.i <= MPACK_INT16_MAX) + return (int16_t)node.data->value.i; + } + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +uint32_t mpack_node_u32(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_uint) { + if (node.data->value.u <= MPACK_UINT32_MAX) + return (uint32_t)node.data->value.u; + } else if (node.data->type == mpack_type_int) { + if (node.data->value.i >= 0 && node.data->value.i <= MPACK_UINT32_MAX) + return (uint32_t)node.data->value.i; + } + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +int32_t mpack_node_i32(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_uint) { + if (node.data->value.u <= MPACK_INT32_MAX) + return (int32_t)node.data->value.u; + } else if (node.data->type == mpack_type_int) { + if (node.data->value.i >= MPACK_INT32_MIN && node.data->value.i <= MPACK_INT32_MAX) + return (int32_t)node.data->value.i; + } + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +uint64_t mpack_node_u64(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_uint) { + return node.data->value.u; + } else if (node.data->type == mpack_type_int) { + if (node.data->value.i >= 0) + return (uint64_t)node.data->value.i; + } + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +int64_t mpack_node_i64(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_uint) { + if (node.data->value.u <= (uint64_t)MPACK_INT64_MAX) + return (int64_t)node.data->value.u; + } else if (node.data->type == mpack_type_int) { + return node.data->value.i; + } + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +unsigned int mpack_node_uint(mpack_node_t node) { + + // This should be true at compile-time, so this just wraps the 32-bit function. + if (sizeof(unsigned int) == 4) + return (unsigned int)mpack_node_u32(node); + + // Otherwise we use u64 and check the range. + uint64_t val = mpack_node_u64(node); + if (val <= MPACK_UINT_MAX) + return (unsigned int)val; + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +int mpack_node_int(mpack_node_t node) { + + // This should be true at compile-time, so this just wraps the 32-bit function. + if (sizeof(int) == 4) + return (int)mpack_node_i32(node); + + // Otherwise we use i64 and check the range. + int64_t val = mpack_node_i64(node); + if (val >= MPACK_INT_MIN && val <= MPACK_INT_MAX) + return (int)val; + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +#if MPACK_FLOAT +float mpack_node_float(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0.0f; + + if (node.data->type == mpack_type_uint) + return (float)node.data->value.u; + if (node.data->type == mpack_type_int) + return (float)node.data->value.i; + if (node.data->type == mpack_type_float) + return node.data->value.f; + + if (node.data->type == mpack_type_double) { + #if MPACK_DOUBLE + return (float)node.data->value.d; + #else + return mpack_shorten_raw_double_to_float(node.data->value.d); + #endif + } + + mpack_node_flag_error(node, mpack_error_type); + return 0.0f; +} #endif +#if MPACK_DOUBLE +double mpack_node_double(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0.0; + + if (node.data->type == mpack_type_uint) + return (double)node.data->value.u; + else if (node.data->type == mpack_type_int) + return (double)node.data->value.i; + else if (node.data->type == mpack_type_float) + return (double)node.data->value.f; + else if (node.data->type == mpack_type_double) + return node.data->value.d; + + mpack_node_flag_error(node, mpack_error_type); + return 0.0; +} +#endif + +#if MPACK_FLOAT +float mpack_node_float_strict(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0.0f; + + if (node.data->type == mpack_type_float) + return node.data->value.f; + + mpack_node_flag_error(node, mpack_error_type); + return 0.0f; +} +#endif + +#if MPACK_DOUBLE +double mpack_node_double_strict(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0.0; + + if (node.data->type == mpack_type_float) + return (double)node.data->value.f; + else if (node.data->type == mpack_type_double) + return node.data->value.d; + + mpack_node_flag_error(node, mpack_error_type); + return 0.0; +} +#endif + +#if !MPACK_FLOAT +uint32_t mpack_node_raw_float(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_float) + return node.data->value.f; + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} +#endif + +#if !MPACK_DOUBLE +uint64_t mpack_node_raw_double(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_double) + return node.data->value.d; + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} +#endif + +#if MPACK_EXTENSIONS +int8_t mpack_node_exttype(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_ext) + return mpack_node_exttype_unchecked(node); + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} +#endif + +uint32_t mpack_node_data_len(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + mpack_type_t type = node.data->type; + if (type == mpack_type_str || type == mpack_type_bin + #if MPACK_EXTENSIONS + || type == mpack_type_ext + #endif + ) + return (uint32_t)node.data->len; + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +size_t mpack_node_strlen(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_str) + return (size_t)node.data->len; + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +const char* mpack_node_str(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return NULL; + + mpack_type_t type = node.data->type; + if (type == mpack_type_str) + return mpack_node_data_unchecked(node); + + mpack_node_flag_error(node, mpack_error_type); + return NULL; +} + +const char* mpack_node_data(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return NULL; + + mpack_type_t type = node.data->type; + if (type == mpack_type_str || type == mpack_type_bin + #if MPACK_EXTENSIONS + || type == mpack_type_ext + #endif + ) + return mpack_node_data_unchecked(node); + + mpack_node_flag_error(node, mpack_error_type); + return NULL; +} + +const char* mpack_node_bin_data(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return NULL; + + if (node.data->type == mpack_type_bin) + return mpack_node_data_unchecked(node); + + mpack_node_flag_error(node, mpack_error_type); + return NULL; +} + +size_t mpack_node_bin_size(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type == mpack_type_bin) + return (size_t)node.data->len; + + mpack_node_flag_error(node, mpack_error_type); + return 0; +} + +size_t mpack_node_array_length(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type != mpack_type_array) { + mpack_node_flag_error(node, mpack_error_type); + return 0; + } + + return (size_t)node.data->len; +} + +mpack_node_t mpack_node_array_at(mpack_node_t node, size_t index) { + if (mpack_node_error(node) != mpack_ok) + return mpack_tree_nil_node(node.tree); + + if (node.data->type != mpack_type_array) { + mpack_node_flag_error(node, mpack_error_type); + return mpack_tree_nil_node(node.tree); + } + + if (index >= node.data->len) { + mpack_node_flag_error(node, mpack_error_data); + return mpack_tree_nil_node(node.tree); + } + + return mpack_node(node.tree, mpack_node_child(node, index)); +} + +size_t mpack_node_map_count(mpack_node_t node) { + if (mpack_node_error(node) != mpack_ok) + return 0; + + if (node.data->type != mpack_type_map) { + mpack_node_flag_error(node, mpack_error_type); + return 0; + } + + return node.data->len; +} + +// internal node map lookup +static mpack_node_t mpack_node_map_at(mpack_node_t node, size_t index, size_t offset) { + if (mpack_node_error(node) != mpack_ok) + return mpack_tree_nil_node(node.tree); + + if (node.data->type != mpack_type_map) { + mpack_node_flag_error(node, mpack_error_type); + return mpack_tree_nil_node(node.tree); + } + + if (index >= node.data->len) { + mpack_node_flag_error(node, mpack_error_data); + return mpack_tree_nil_node(node.tree); + } + + return mpack_node(node.tree, mpack_node_child(node, index * 2 + offset)); +} + +mpack_node_t mpack_node_map_key_at(mpack_node_t node, size_t index) { + return mpack_node_map_at(node, index, 0); +} + +mpack_node_t mpack_node_map_value_at(mpack_node_t node, size_t index) { + return mpack_node_map_at(node, index, 1); +} + +#endif +MPACK_SILENCE_WARNINGS_END diff --git a/src/mpack/mpack-node.h b/src/mpack/mpack-node.h index 0f0300a..26046e8 100644 --- a/src/mpack/mpack-node.h +++ b/src/mpack/mpack-node.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -30,27 +30,35 @@ #include "mpack-reader.h" -MPACK_HEADER_START +MPACK_SILENCE_WARNINGS_BEGIN +MPACK_EXTERN_C_BEGIN #if MPACK_NODE /** * @defgroup node Node API * - * The MPack Node API allows you to parse a chunk of MessagePack data - * in-place into a dynamically typed data structure. + * The MPack Node API allows you to parse a chunk of MessagePack into a + * dynamically typed data structure, providing random access to the parsed + * data. + * + * See @ref docs/node.md for examples. * * @{ */ /** - * A handle to node in a parsed MPack tree. Note that mpack_node_t is passed by value. + * A handle to node data in a parsed MPack tree. * * Nodes represent either primitive values or compound types. If a * node is a compound type, it contains a pointer to its child nodes, * or a pointer to its underlying data. * * Nodes are immutable. + * + * @note @ref mpack_node_t is an opaque reference to the node data, not the + * node data itself. (It contains pointers to both the node data and the tree.) + * It is passed by value in the Node API. */ typedef struct mpack_node_t mpack_node_t; @@ -59,14 +67,17 @@ typedef struct mpack_node_t mpack_node_t; * * You only need to use this if you intend to provide your own storage * for nodes instead of letting the tree allocate it. + * + * @ref mpack_node_data_t is 16 bytes on most common architectures (32-bit + * and 64-bit.) */ typedef struct mpack_node_data_t mpack_node_data_t; /** - * An MPack tree parsed from a blob of MessagePack. + * An MPack tree parser to parse a blob or stream of MessagePack. * - * The tree contains a single root node which contains all parsed data. - * The tree and its nodes are immutable. + * When a message is parsed, the tree contains a single root node which + * contains all parsed data. The tree and its nodes are immutable. */ typedef struct mpack_tree_t mpack_tree_t; @@ -96,6 +107,29 @@ typedef struct mpack_tree_t mpack_tree_t; */ typedef void (*mpack_tree_error_t)(mpack_tree_t* tree, mpack_error_t error); +/** + * The MPack tree's read function. It should fill the buffer with as many bytes + * as are immediately available up to the given @c count, returning the number + * of bytes written to the buffer. + * + * In case of error, it should flag an appropriate error on the reader + * (usually @ref mpack_error_io.) + * + * The blocking or non-blocking behaviour of the read should match whether you + * are using mpack_tree_parse() or mpack_tree_try_parse(). + * + * If you are using mpack_tree_parse(), the read should block until at least + * one byte is read. If you return 0, mpack_tree_parse() will raise @ref + * mpack_error_io. + * + * If you are using mpack_tree_try_parse(), the read function can always + * return 0, and must never block waiting for data (otherwise + * mpack_tree_try_parse() would be equivalent to mpack_tree_parse().) + * When you return 0, mpack_tree_try_parse() will return false without flagging + * an error. + */ +typedef size_t (*mpack_tree_read_t)(mpack_tree_t* tree, char* buffer, size_t count); + /** * A teardown function to be called when the tree is destroyed. */ @@ -121,14 +155,25 @@ struct mpack_node_data_t { */ uint32_t len; - union - { + union { bool b; /* The value if the type is bool. */ + + #if MPACK_FLOAT float f; /* The value if the type is float. */ + #else + uint32_t f; /*< The raw value if the type is float. */ + #endif + + #if MPACK_DOUBLE double d; /* The value if the type is double. */ + #else + uint64_t d; /*< The raw value if the type is double. */ + #endif + int64_t i; /* The value if the type is signed int. */ uint64_t u; /* The value if the type is unsigned int. */ - const char* bytes; /* The byte pointer for str, bin and ext */ + size_t offset; /* The byte offset for str, bin and ext */ + mpack_node_data_t* children; /* The children for map or array */ } value; }; @@ -138,19 +183,90 @@ typedef struct mpack_tree_page_t { mpack_node_data_t nodes[1]; // variable size } mpack_tree_page_t; +typedef enum mpack_tree_parse_state_t { + mpack_tree_parse_state_not_started, + mpack_tree_parse_state_in_progress, + mpack_tree_parse_state_parsed, +} mpack_tree_parse_state_t; + +typedef struct mpack_level_t { + mpack_node_data_t* child; + size_t left; // children left in level +} mpack_level_t; + +typedef struct mpack_tree_parser_t { + mpack_tree_parse_state_t state; + + // We keep track of the number of "possible nodes" left in the data rather + // than the number of bytes. + // + // When a map or array is parsed, we ensure at least one byte for each child + // exists and subtract them right away. This ensures that if ever a map or + // array declares more elements than could possibly be contained in the data, + // we will error out immediately rather than allocating storage for them. + // + // For example malicious data that repeats 0xDE 0xFF 0xFF (start of a map + // with 65536 key-value pairs) would otherwise cause us to run out of + // memory. With this, the parser can allocate at most as many nodes as + // there are bytes in the data (plus the paging overhead, 12%.) An error + // will be flagged immediately if and when there isn't enough data left to + // fully read all children of all open compound types on the parsing stack. + // + // Once an entire message has been parsed (and there are no nodes left to + // parse whose bytes have been subtracted), this matches the number of left + // over bytes in the data. + size_t possible_nodes_left; + + mpack_node_data_t* nodes; // next node in current page/pool + size_t nodes_left; // nodes left in current page/pool + + size_t current_node_reserved; + size_t level; + + #ifdef MPACK_MALLOC + // It's much faster to allocate the initial parsing stack inline within the + // parser. We replace it with a heap allocation if we need to grow it. + mpack_level_t* stack; + size_t stack_capacity; + bool stack_owned; + mpack_level_t stack_local[MPACK_NODE_INITIAL_DEPTH]; + #else + // Without malloc(), we have to reserve a parsing stack the maximum allowed + // parsing depth. + mpack_level_t stack[MPACK_NODE_MAX_DEPTH_WITHOUT_MALLOC]; + #endif +} mpack_tree_parser_t; + struct mpack_tree_t { mpack_tree_error_t error_fn; /* Function to call on error */ + mpack_tree_read_t read_fn; /* Function to call to read more data */ mpack_tree_teardown_t teardown; /* Function to teardown the context on destroy */ void* context; /* Context for tree callbacks */ - mpack_node_data_t nil_node; /* a nil node to be returned in case of error */ + mpack_node_data_t nil_node; /* a nil node to be returned in case of error */ + mpack_node_data_t missing_node; /* a missing node to be returned in optional lookups */ mpack_error_t error; - size_t node_count; - size_t size; + #ifdef MPACK_MALLOC + char* buffer; + size_t buffer_capacity; + #endif + + const char* data; + size_t data_length; // length of data (and content of buffer, if used) + size_t size; // size in bytes of tree (usually matches data_length, but not if tree has trailing data) + size_t node_count; // total number of nodes in tree (across all pages) + + size_t max_size; // maximum message size + size_t max_nodes; // maximum nodes in a message + + mpack_tree_parser_t parser; mpack_node_data_t* root; + mpack_node_data_t* pool; // pool, or NULL if no pool provided + size_t pool_count; + #ifdef MPACK_MALLOC mpack_tree_page_t* next; #endif @@ -173,49 +289,99 @@ MPACK_INLINE mpack_node_t mpack_tree_nil_node(mpack_tree_t* tree) { return mpack_node(tree, &tree->nil_node); } +MPACK_INLINE mpack_node_t mpack_tree_missing_node(mpack_tree_t* tree) { + return mpack_node(tree, &tree->missing_node); +} + /** @endcond */ /** - * @name Tree Functions + * @name Tree Initialization * @{ */ #ifdef MPACK_MALLOC /** - * Initializes a tree by parsing the given data buffer. The tree must be destroyed - * with mpack_tree_destroy(), even if parsing fails. + * Initializes a tree parser with the given data. + * + * Configure the tree if desired, then call mpack_tree_parse() to parse it. The + * tree will allocate pages of nodes as needed and will free them when + * destroyed. * - * The tree will allocate pages of nodes as needed, and free them when destroyed. + * The tree must be destroyed with mpack_tree_destroy(). * - * Any string or blob data types reference the original data, so the data + * Any string or blob data types reference the original data, so the given data * pointer must remain valid until after the tree is destroyed. */ -void mpack_tree_init(mpack_tree_t* tree, const char* data, size_t length); +void mpack_tree_init_data(mpack_tree_t* tree, const char* data, size_t length); + +/** + * Deprecated. + * + * \deprecated Renamed to mpack_tree_init_data(). + */ +MPACK_INLINE void mpack_tree_init(mpack_tree_t* tree, const char* data, size_t length) { + mpack_tree_init_data(tree, data, length); +} + +/** + * Initializes a tree parser from an unbounded stream, or a stream of + * unknown length. + * + * The parser can be used to read a single message from a stream of unknown + * length, or multiple messages from an unbounded stream, allowing it to + * be used for RPC communication. Call @ref mpack_tree_parse() to parse + * a message from a blocking stream, or @ref mpack_tree_try_parse() for a + * non-blocking stream. + * + * The stream will use a growable internal buffer to store the most recent + * message, as well as allocated pages of nodes for the parse tree. + * + * Maximum allowances for message size and node count must be specified in this + * function (since the stream is unbounded.) They can be changed later with + * @ref mpack_tree_set_limits(). + * + * @param tree The tree parser + * @param read_fn The read function + * @param context The context for the read function + * @param max_message_size The maximum size of a message in bytes + * @param max_message_nodes The maximum number of nodes per message. See + * @ref mpack_node_data_t for the size of nodes. + * + * @see mpack_tree_read_t + * @see mpack_reader_context() + */ +void mpack_tree_init_stream(mpack_tree_t* tree, mpack_tree_read_t read_fn, void* context, + size_t max_message_size, size_t max_message_nodes); #endif /** - * Initializes a tree by parsing the given data buffer, using the given - * node data pool to store the results. + * Initializes a tree parser with the given data, using the given node data + * pool to store the results. + * + * Configure the tree if desired, then call mpack_tree_parse() to parse it. * - * If the data does not fit in the pool, mpack_error_too_big will be flagged + * If the data does not fit in the pool, @ref mpack_error_too_big will be flagged * on the tree. * * The tree must be destroyed with mpack_tree_destroy(), even if parsing fails. */ -void mpack_tree_init_pool(mpack_tree_t* tree, const char* data, size_t length, mpack_node_data_t* node_pool, size_t node_pool_count); +void mpack_tree_init_pool(mpack_tree_t* tree, const char* data, size_t length, + mpack_node_data_t* node_pool, size_t node_pool_count); /** * Initializes an MPack tree directly into an error state. Use this if you - * are writing a wrapper to mpack_tree_init() which can fail its setup. + * are writing a wrapper to another mpack_tree_init*() function which + * can fail its setup. */ void mpack_tree_init_error(mpack_tree_t* tree, mpack_error_t error); #if MPACK_STDIO /** - * Initializes a tree by reading and parsing the given file. The tree must be - * destroyed with mpack_tree_destroy(), even if parsing fails. + * Initializes a tree to parse the given file. The tree must be destroyed with + * mpack_tree_destroy(), even if parsing fails. * * The file is opened, loaded fully into memory, and closed before this call * returns. @@ -224,12 +390,115 @@ void mpack_tree_init_error(mpack_tree_t* tree, mpack_error_t error); * @param filename The filename passed to fopen() to read the file * @param max_bytes The maximum size of file to load, or 0 for unlimited size. */ -void mpack_tree_init_file(mpack_tree_t* tree, const char* filename, size_t max_bytes); +void mpack_tree_init_filename(mpack_tree_t* tree, const char* filename, size_t max_bytes); + +/** + * Deprecated. + * + * \deprecated Renamed to mpack_tree_init_filename(). + */ +MPACK_INLINE void mpack_tree_init_file(mpack_tree_t* tree, const char* filename, size_t max_bytes) { + mpack_tree_init_filename(tree, filename, max_bytes); +} + +/** + * Initializes a tree to parse the given libc FILE. This can be used to + * read from stdin, or from a file opened separately. + * + * The tree must be destroyed with mpack_tree_destroy(), even if parsing fails. + * + * The FILE is fully loaded fully into memory (and closed if requested) before + * this call returns. + * + * @param tree The tree to initialize. + * @param stdfile The FILE. + * @param max_bytes The maximum size of file to load, or 0 for unlimited size. + * @param close_when_done If true, fclose() will be called on the FILE when it + * is no longer needed. If false, the file will not be closed when + * reading is done. + * + * @warning The tree will read all data in the FILE before parsing it. If this + * is used on stdin, the parser will block until it is closed, even if + * a complete message has been written to it! + */ +void mpack_tree_init_stdfile(mpack_tree_t* tree, FILE* stdfile, size_t max_bytes, bool close_when_done); #endif +/** + * @} + */ + +/** + * @name Tree Functions + * @{ + */ + +/** + * Sets the maximum byte size and maximum number of nodes allowed per message. + * + * The default is SIZE_MAX (no limit) unless @ref mpack_tree_init_stream() is + * called (where maximums are required.) + * + * If a pool of nodes is used, the node limit is the lesser of this limit and + * the pool size. + * + * @param tree The tree parser + * @param max_message_size The maximum size of a message in bytes + * @param max_message_nodes The maximum number of nodes per message. See + * @ref mpack_node_data_t for the size of nodes. + */ +void mpack_tree_set_limits(mpack_tree_t* tree, size_t max_message_size, + size_t max_message_nodes); + +/** + * Parses a MessagePack message into a tree of immutable nodes. + * + * If successful, the root node will be available under @ref mpack_tree_root(). + * If not, an appropriate error will be flagged. + * + * This can be called repeatedly to parse a series of messages from a data + * source. When this is called, all previous nodes from this tree and their + * contents (including the root node) are invalidated. + * + * If this is called with a stream (see @ref mpack_tree_init_stream()), the + * stream must block until data is available. (Otherwise, if this is called on + * a non-blocking stream, parsing will fail with @ref mpack_error_io when the + * fill function returns 0.) + * + * There is no way to recover a tree in an error state. It must be destroyed. + */ +void mpack_tree_parse(mpack_tree_t* tree); + +/** + * Attempts to parse a MessagePack message from a non-blocking stream into a + * tree of immutable nodes. + * + * A non-blocking read function must have been passed to the tree in + * mpack_tree_init_stream(). + * + * If this returns true, a message is available under + * @ref mpack_tree_root(). The tree nodes and data will be valid until + * the next time a parse is started. + * + * If this returns false, no message is available, because either not enough + * data is available yet or an error has occurred. You must check the tree for + * errors whenever this returns false. If there is no error, you should try + * again later when more data is available. (You will want to select()/poll() + * on the underlying socket or use some other asynchronous mechanism to + * determine when it has data.) + * + * There is no way to recover a tree in an error state. It must be destroyed. + * + * @see mpack_tree_init_stream() + */ +bool mpack_tree_try_parse(mpack_tree_t* tree); + /** * Returns the root node of the tree, if the tree is not in an error state. * Returns a nil node otherwise. + * + * @warning You must call mpack_tree_parse() before calling this. If + * @ref mpack_tree_parse() was never called, the tree will assert. */ mpack_node_t mpack_tree_root(mpack_tree_t* tree); @@ -241,9 +510,10 @@ MPACK_INLINE mpack_error_t mpack_tree_error(mpack_tree_t* tree) { } /** - * Returns the number of bytes used in the buffer when the tree was - * parsed. If there is something in the buffer after the MessagePack - * object (such as another object), this can be used to find it. + * Returns the size in bytes of the current parsed message. + * + * If there is something in the buffer after the MessagePack object, this can + * be used to find it. * * This is zero if an error occurred during tree parsing (since the * portion of the data that the first complete object occupies cannot @@ -263,11 +533,23 @@ mpack_error_t mpack_tree_destroy(mpack_tree_t* tree); * * @param tree The MPack tree. * @param context User data to pass to the tree callbacks. + * + * @see mpack_reader_context() */ MPACK_INLINE void mpack_tree_set_context(mpack_tree_t* tree, void* context) { tree->context = context; } +/** + * Returns the custom context for tree callbacks. + * + * @see mpack_tree_set_context + * @see mpack_tree_init_stream + */ +MPACK_INLINE void* mpack_tree_context(mpack_tree_t* tree) { + return tree->context; +} + /** * Sets the error function to call when an error is flagged on the tree. * @@ -310,6 +592,15 @@ MPACK_INLINE void mpack_tree_set_teardown(mpack_tree_t* tree, mpack_tree_teardow */ void mpack_tree_flag_error(mpack_tree_t* tree, mpack_error_t error); +/** + * @} + */ + +/** + * @name Node Core Functions + * @{ + */ + /** * Places the node's tree in the given error state, calling the error callback * if one is set. @@ -322,15 +613,6 @@ void mpack_tree_flag_error(mpack_tree_t* tree, mpack_error_t error); */ void mpack_node_flag_error(mpack_node_t node, mpack_error_t error); -/** - * @} - */ - -/** - * @name Node Core Functions - * @{ - */ - /** * Returns the error state of the node's tree. */ @@ -344,23 +626,63 @@ MPACK_INLINE mpack_error_t mpack_node_error(mpack_node_t node) { */ mpack_tag_t mpack_node_tag(mpack_node_t node); -#if MPACK_STDIO -/** +/** @cond */ + +#if MPACK_DEBUG && MPACK_STDIO +/* + * Converts a node to a pseudo-JSON string for debugging purposes, placing the + * result in the given buffer with a null-terminator. + * + * If the buffer does not have enough space, the result will be truncated (but + * it is guaranteed to be null-terminated.) + * + * This is only available in debug mode, and only if stdio is available (since + * it uses snprintf().) It's strictly for debugging purposes. + */ +void mpack_node_print_to_buffer(mpack_node_t node, char* buffer, size_t buffer_size); + +/* + * Converts a node to pseudo-JSON for debugging purposes, calling the given + * callback as many times as is necessary to output the character data. + * + * No null-terminator or trailing newline will be written. + * + * This is only available in debug mode, and only if stdio is available (since + * it uses snprintf().) It's strictly for debugging purposes. + */ +void mpack_node_print_to_callback(mpack_node_t node, mpack_print_callback_t callback, void* context); + +/* * Converts a node to pseudo-JSON for debugging purposes * and pretty-prints it to the given file. + * + * This is only available in debug mode, and only if stdio is available (since + * it uses snprintf().) It's strictly for debugging purposes. */ -void mpack_node_print_file(mpack_node_t node, FILE* file); +void mpack_node_print_to_file(mpack_node_t node, FILE* file); -#ifndef MPACK_GCOV -/** +/* * Converts a node to pseudo-JSON for debugging purposes * and pretty-prints it to stdout. + * + * This is only available in debug mode, and only if stdio is available (since + * it uses snprintf().) It's strictly for debugging purposes. + */ +MPACK_INLINE void mpack_node_print_to_stdout(mpack_node_t node) { + mpack_node_print_to_file(node, stdout); +} + +/* + * Deprecated. + * + * \deprecated Renamed to mpack_node_print_to_stdout(). */ MPACK_INLINE void mpack_node_print(mpack_node_t node) { - mpack_node_print_file(node, stdout); + mpack_node_print_to_stdout(node); } #endif -#endif + +/** @endcond */ /** * @} @@ -374,212 +696,105 @@ MPACK_INLINE void mpack_node_print(mpack_node_t node) { /** * Returns the type of the node. */ -MPACK_INLINE mpack_type_t mpack_node_type(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return mpack_type_nil; - return node.data->type; -} +mpack_type_t mpack_node_type(mpack_node_t node); /** - * Checks if the given node is of nil type, raising mpack_error_type otherwise. + * Returns true if the given node is a nil node; false otherwise. + * + * To ensure that a node is nil and flag an error otherwise, use + * mpack_node_nil(). */ -MPACK_INLINE void mpack_node_nil(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return; - if (node.data->type != mpack_type_nil) - mpack_node_flag_error(node, mpack_error_type); -} +bool mpack_node_is_nil(mpack_node_t node); /** - * Returns the bool value of the node. If this node is not of the correct - * type, false is returned and mpack_error_type is raised. + * Returns true if the given node handle indicates a missing node; false otherwise. + * + * To ensure that a node is missing and flag an error otherwise, use + * mpack_node_missing(). */ -MPACK_INLINE bool mpack_node_bool(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return false; +bool mpack_node_is_missing(mpack_node_t node); - if (node.data->type == mpack_type_bool) - return node.data->value.b; +/** + * Checks that the given node is of nil type, raising @ref mpack_error_type + * otherwise. + * + * Use mpack_node_is_nil() to return whether the node is nil. + */ +void mpack_node_nil(mpack_node_t node); - mpack_node_flag_error(node, mpack_error_type); - return false; -} +/** + * Checks that the given node indicates a missing node, raising @ref + * mpack_error_type otherwise. + * + * Use mpack_node_is_missing() to return whether the node is missing. + */ +void mpack_node_missing(mpack_node_t node); + +/** + * Returns the bool value of the node. If this node is not of the correct + * type, false is returned and mpack_error_type is raised. + */ +bool mpack_node_bool(mpack_node_t node); /** * Checks if the given node is of bool type with value true, raising * mpack_error_type otherwise. */ -MPACK_INLINE void mpack_node_true(mpack_node_t node) { - if (mpack_node_bool(node) != true) - mpack_node_flag_error(node, mpack_error_type); -} +void mpack_node_true(mpack_node_t node); /** * Checks if the given node is of bool type with value false, raising * mpack_error_type otherwise. */ -MPACK_INLINE void mpack_node_false(mpack_node_t node) { - if (mpack_node_bool(node) != false) - mpack_node_flag_error(node, mpack_error_type); -} +void mpack_node_false(mpack_node_t node); /** * Returns the 8-bit unsigned value of the node. If this node is not - * of a compatible type, mpack_error_type is raised and zero is returned. - */ -MPACK_INLINE uint8_t mpack_node_u8(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - if (node.data->type == mpack_type_uint) { - if (node.data->value.u <= UINT8_MAX) - return (uint8_t)node.data->value.u; - } else if (node.data->type == mpack_type_int) { - if (node.data->value.i >= 0 && node.data->value.i <= UINT8_MAX) - return (uint8_t)node.data->value.i; - } - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} + * of a compatible type, @ref mpack_error_type is raised and zero is returned. + */ +uint8_t mpack_node_u8(mpack_node_t node); /** * Returns the 8-bit signed value of the node. If this node is not - * of a compatible type, mpack_error_type is raised and zero is returned. - */ -MPACK_INLINE int8_t mpack_node_i8(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - if (node.data->type == mpack_type_uint) { - if (node.data->value.u <= INT8_MAX) - return (int8_t)node.data->value.u; - } else if (node.data->type == mpack_type_int) { - if (node.data->value.i >= INT8_MIN && node.data->value.i <= INT8_MAX) - return (int8_t)node.data->value.i; - } - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} + * of a compatible type, @ref mpack_error_type is raised and zero is returned. + */ +int8_t mpack_node_i8(mpack_node_t node); /** * Returns the 16-bit unsigned value of the node. If this node is not - * of a compatible type, mpack_error_type is raised and zero is returned. - */ -MPACK_INLINE uint16_t mpack_node_u16(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - if (node.data->type == mpack_type_uint) { - if (node.data->value.u <= UINT16_MAX) - return (uint16_t)node.data->value.u; - } else if (node.data->type == mpack_type_int) { - if (node.data->value.i >= 0 && node.data->value.i <= UINT16_MAX) - return (uint16_t)node.data->value.i; - } - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} + * of a compatible type, @ref mpack_error_type is raised and zero is returned. + */ +uint16_t mpack_node_u16(mpack_node_t node); /** * Returns the 16-bit signed value of the node. If this node is not - * of a compatible type, mpack_error_type is raised and zero is returned. - */ -MPACK_INLINE int16_t mpack_node_i16(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - if (node.data->type == mpack_type_uint) { - if (node.data->value.u <= INT16_MAX) - return (int16_t)node.data->value.u; - } else if (node.data->type == mpack_type_int) { - if (node.data->value.i >= INT16_MIN && node.data->value.i <= INT16_MAX) - return (int16_t)node.data->value.i; - } - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} + * of a compatible type, @ref mpack_error_type is raised and zero is returned. + */ +int16_t mpack_node_i16(mpack_node_t node); /** * Returns the 32-bit unsigned value of the node. If this node is not - * of a compatible type, mpack_error_type is raised and zero is returned. - */ -MPACK_INLINE uint32_t mpack_node_u32(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - if (node.data->type == mpack_type_uint) { - if (node.data->value.u <= UINT32_MAX) - return (uint32_t)node.data->value.u; - } else if (node.data->type == mpack_type_int) { - if (node.data->value.i >= 0 && node.data->value.i <= UINT32_MAX) - return (uint32_t)node.data->value.i; - } - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} + * of a compatible type, @ref mpack_error_type is raised and zero is returned. + */ +uint32_t mpack_node_u32(mpack_node_t node); /** * Returns the 32-bit signed value of the node. If this node is not - * of a compatible type, mpack_error_type is raised and zero is returned. - */ -MPACK_INLINE int32_t mpack_node_i32(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - if (node.data->type == mpack_type_uint) { - if (node.data->value.u <= INT32_MAX) - return (int32_t)node.data->value.u; - } else if (node.data->type == mpack_type_int) { - if (node.data->value.i >= INT32_MIN && node.data->value.i <= INT32_MAX) - return (int32_t)node.data->value.i; - } - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} + * of a compatible type, @ref mpack_error_type is raised and zero is returned. + */ +int32_t mpack_node_i32(mpack_node_t node); /** * Returns the 64-bit unsigned value of the node. If this node is not - * of a compatible type, mpack_error_type is raised, and zero is returned. + * of a compatible type, @ref mpack_error_type is raised, and zero is returned. */ -MPACK_INLINE uint64_t mpack_node_u64(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - if (node.data->type == mpack_type_uint) { - return node.data->value.u; - } else if (node.data->type == mpack_type_int) { - if (node.data->value.i >= 0) - return (uint64_t)node.data->value.i; - } - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} +uint64_t mpack_node_u64(mpack_node_t node); /** * Returns the 64-bit signed value of the node. If this node is not - * of a compatible type, mpack_error_type is raised and zero is returned. + * of a compatible type, @ref mpack_error_type is raised and zero is returned. */ -MPACK_INLINE int64_t mpack_node_i64(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - if (node.data->type == mpack_type_uint) { - if (node.data->value.u <= (uint64_t)INT64_MAX) - return (int64_t)node.data->value.u; - } else if (node.data->type == mpack_type_int) { - return node.data->value.i; - } - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} +int64_t mpack_node_i64(mpack_node_t node); /** * Returns the unsigned int value of the node. @@ -588,20 +803,7 @@ MPACK_INLINE int64_t mpack_node_i64(mpack_node_t node) { * * @throws mpack_error_type If the node is not an integer type or does not fit in the range of an unsigned int */ -MPACK_INLINE unsigned int mpack_node_uint(mpack_node_t node) { - - // This should be true at compile-time, so this just wraps the 32-bit function. - if (sizeof(unsigned int) == 4) - return (unsigned int)mpack_node_u32(node); - - // Otherwise we use u64 and check the range. - uint64_t val = mpack_node_u64(node); - if (val <= UINT_MAX) - return (unsigned int)val; - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} +unsigned int mpack_node_uint(mpack_node_t node); /** * Returns the int value of the node. @@ -610,115 +812,111 @@ MPACK_INLINE unsigned int mpack_node_uint(mpack_node_t node) { * * @throws mpack_error_type If the node is not an integer type or does not fit in the range of an int */ -MPACK_INLINE int mpack_node_int(mpack_node_t node) { - - // This should be true at compile-time, so this just wraps the 32-bit function. - if (sizeof(int) == 4) - return (int)mpack_node_i32(node); - - // Otherwise we use i64 and check the range. - int64_t val = mpack_node_i64(node); - if (val >= INT_MIN && val <= INT_MAX) - return (int)val; - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} +int mpack_node_int(mpack_node_t node); +#if MPACK_FLOAT /** * Returns the float value of the node. The underlying value can be an * integer, float or double; the value is converted to a float. * - * Note that reading a double or a large integer with this function can incur a + * @note Reading a double or a large integer with this function can incur a * loss of precision. * * @throws mpack_error_type if the underlying value is not a float, double or integer. */ -MPACK_INLINE float mpack_node_float(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0.0f; - - if (node.data->type == mpack_type_uint) - return (float)node.data->value.u; - else if (node.data->type == mpack_type_int) - return (float)node.data->value.i; - else if (node.data->type == mpack_type_float) - return node.data->value.f; - else if (node.data->type == mpack_type_double) - return (float)node.data->value.d; - - mpack_node_flag_error(node, mpack_error_type); - return 0.0f; -} +float mpack_node_float(mpack_node_t node); +#endif +#if MPACK_DOUBLE /** * Returns the double value of the node. The underlying value can be an * integer, float or double; the value is converted to a double. * - * Note that reading a very large integer with this function can incur a + * @note Reading a very large integer with this function can incur a * loss of precision. * * @throws mpack_error_type if the underlying value is not a float, double or integer. */ -MPACK_INLINE double mpack_node_double(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0.0; - - if (node.data->type == mpack_type_uint) - return (double)node.data->value.u; - else if (node.data->type == mpack_type_int) - return (double)node.data->value.i; - else if (node.data->type == mpack_type_float) - return (double)node.data->value.f; - else if (node.data->type == mpack_type_double) - return node.data->value.d; - - mpack_node_flag_error(node, mpack_error_type); - return 0.0; -} +double mpack_node_double(mpack_node_t node); +#endif +#if MPACK_FLOAT /** * Returns the float value of the node. The underlying value must be a float, * not a double or an integer. This ensures no loss of precision can occur. * * @throws mpack_error_type if the underlying value is not a float. */ -MPACK_INLINE float mpack_node_float_strict(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0.0f; - - if (node.data->type == mpack_type_float) - return node.data->value.f; - - mpack_node_flag_error(node, mpack_error_type); - return 0.0f; -} +float mpack_node_float_strict(mpack_node_t node); +#endif +#if MPACK_DOUBLE /** * Returns the double value of the node. The underlying value must be a float * or double, not an integer. This ensures no loss of precision can occur. * * @throws mpack_error_type if the underlying value is not a float or double. */ -MPACK_INLINE double mpack_node_double_strict(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0.0; +double mpack_node_double_strict(mpack_node_t node); +#endif - if (node.data->type == mpack_type_float) - return (double)node.data->value.f; - else if (node.data->type == mpack_type_double) - return node.data->value.d; +#if !MPACK_FLOAT +/** + * Returns the float value of the node as a raw uint32_t. The underlying value + * must be a float, not a double or an integer. + * + * @throws mpack_error_type if the underlying value is not a float. + */ +uint32_t mpack_node_raw_float(mpack_node_t node); +#endif - mpack_node_flag_error(node, mpack_error_type); - return 0.0; -} +#if !MPACK_DOUBLE +/** + * Returns the double value of the node as a raw uint64_t. The underlying value + * must be a double, not a float or an integer. + * + * @throws mpack_error_type if the underlying value is not a float or double. + */ +uint64_t mpack_node_raw_double(mpack_node_t node); +#endif + + +#if MPACK_EXTENSIONS +/** + * Returns a timestamp. + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @throws mpack_error_type if the underlying value is not a timestamp. + */ +mpack_timestamp_t mpack_node_timestamp(mpack_node_t node); + +/** + * Returns a timestamp's (signed) seconds since 1970-01-01T00:00:00Z. + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @throws mpack_error_type if the underlying value is not a timestamp. + */ +int64_t mpack_node_timestamp_seconds(mpack_node_t node); + +/** + * Returns a timestamp's additional nanoseconds. + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @return A nanosecond count between 0 and 999,999,999 inclusive. + * @throws mpack_error_type if the underlying value is not a timestamp. + */ +uint32_t mpack_node_timestamp_nanoseconds(mpack_node_t node); +#endif /** * @} */ /** - * @name Node Data Functions + * @name Node String and Data Functions * @{ */ @@ -762,109 +960,89 @@ void mpack_node_check_utf8(mpack_node_t node); */ void mpack_node_check_utf8_cstr(mpack_node_t node); +#if MPACK_EXTENSIONS /** * Returns the extension type of the given ext node. * * This returns zero if the tree is in an error state. + * + * @note This requires @ref MPACK_EXTENSIONS. */ -MPACK_INLINE int8_t mpack_node_exttype(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - // the exttype of an ext node is stored in the byte preceding the data - if (node.data->type == mpack_type_ext) - return (int8_t)*(node.data->value.bytes - 1); +int8_t mpack_node_exttype(mpack_node_t node); +#endif - mpack_node_flag_error(node, mpack_error_type); - return 0; -} +/** + * Returns the number of bytes in the given bin node. + * + * This returns zero if the tree is in an error state. + * + * If this node is not a bin, @ref mpack_error_type is raised and zero is returned. + */ +size_t mpack_node_bin_size(mpack_node_t node); /** * Returns the length of the given str, bin or ext node. * * This returns zero if the tree is in an error state. + * + * If this node is not a str, bin or ext, @ref mpack_error_type is raised and zero + * is returned. */ -MPACK_INLINE uint32_t mpack_node_data_len(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - mpack_type_t type = node.data->type; - if (type == mpack_type_str || type == mpack_type_bin || type == mpack_type_ext) - return (uint32_t)node.data->len; - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} +uint32_t mpack_node_data_len(mpack_node_t node); /** * Returns the length in bytes of the given string node. This does not * include any null-terminator. * * This returns zero if the tree is in an error state. + * + * If this node is not a str, @ref mpack_error_type is raised and zero is returned. */ -MPACK_INLINE size_t mpack_node_strlen(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - if (node.data->type == mpack_type_str) - return (size_t)node.data->len; - - mpack_node_flag_error(node, mpack_error_type); - return 0; -} +size_t mpack_node_strlen(mpack_node_t node); /** - * Returns a pointer to the data contained by this node, ensuring it is a string. + * Returns a pointer to the data contained by this node, ensuring the node is a + * string. * - * Note that strings are not null-terminated! Use one of the cstr functions + * @warning Strings are not null-terminated! Use one of the cstr functions * to get a null-terminated string. * * The pointer is valid as long as the data backing the tree is valid. * - * If this node is not a string, mpack_error_type is raised and NULL is returned. + * If this node is not a string, @ref mpack_error_type is raised and @c NULL is returned. * * @see mpack_node_copy_cstr() * @see mpack_node_cstr_alloc() * @see mpack_node_utf8_cstr_alloc() */ -MPACK_INLINE const char* mpack_node_str(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return NULL; - - mpack_type_t type = node.data->type; - if (type == mpack_type_str) - return node.data->value.bytes; - - mpack_node_flag_error(node, mpack_error_type); - return NULL; -} +const char* mpack_node_str(mpack_node_t node); /** * Returns a pointer to the data contained by this node. * - * Note that strings are not null-terminated! Use one of the cstr functions + * @note Strings are not null-terminated! Use one of the cstr functions * to get a null-terminated string. * * The pointer is valid as long as the data backing the tree is valid. * - * If this node is not of a str, bin or map, mpack_error_type is raised, and - * NULL is returned. + * If this node is not of a str, bin or ext, @ref mpack_error_type is raised, and + * @c NULL is returned. * * @see mpack_node_copy_cstr() * @see mpack_node_cstr_alloc() * @see mpack_node_utf8_cstr_alloc() */ -MPACK_INLINE const char* mpack_node_data(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return NULL; - - mpack_type_t type = node.data->type; - if (type == mpack_type_str || type == mpack_type_bin || type == mpack_type_ext) - return node.data->value.bytes; +const char* mpack_node_data(mpack_node_t node); - mpack_node_flag_error(node, mpack_error_type); - return NULL; -} +/** + * Returns a pointer to the data contained by this bin node. + * + * The pointer is valid as long as the data backing the tree is valid. + * + * If this node is not a bin, @ref mpack_error_type is raised and @c NULL is + * returned. + */ +const char* mpack_node_bin_data(mpack_node_t node); /** * Copies the bytes contained by this node into the given buffer, returning the @@ -900,8 +1078,8 @@ size_t mpack_node_copy_utf8(mpack_node_t node, char* buffer, size_t bufsize); * Checks that the given node contains a string with no NUL bytes, copies the string * into the given buffer, and adds a null terminator. * - * If this node is not of a string type, mpack_error_type is raised. If the string - * does not fit, mpack_error_data is raised. + * If this node is not of a string type, @ref mpack_error_type is raised. If the string + * does not fit, @ref mpack_error_data is raised. * * If any error occurs, the buffer will contain an empty null-terminated string. * @@ -915,8 +1093,8 @@ void mpack_node_copy_cstr(mpack_node_t node, char* buffer, size_t size); * Checks that the given node contains a valid UTF-8 string with no NUL bytes, * copies the string into the given buffer, and adds a null terminator. * - * If this node is not of a string type, mpack_error_type is raised. If the string - * does not fit, mpack_error_data is raised. + * If this node is not of a string type, @ref mpack_error_type is raised. If the string + * does not fit, @ref mpack_error_data is raised. * * If any error occurs, the buffer will contain an empty null-terminated string. * @@ -928,7 +1106,7 @@ void mpack_node_copy_utf8_cstr(mpack_node_t node, char* buffer, size_t size); #ifdef MPACK_MALLOC /** - * Allocates a new chunk of data using MPACK_MALLOC with the bytes + * Allocates a new chunk of data using MPACK_MALLOC() with the bytes * contained by this node. * * The allocated data must be freed with MPACK_FREE() (or simply free() @@ -947,7 +1125,7 @@ void mpack_node_copy_utf8_cstr(mpack_node_t node, char* buffer, size_t size); char* mpack_node_data_alloc(mpack_node_t node, size_t maxsize); /** - * Allocates a new null-terminated string using MPACK_MALLOC with the string + * Allocates a new null-terminated string using MPACK_MALLOC() with the string * contained by this node. * * The allocated string must be freed with MPACK_FREE() (or simply free() @@ -966,7 +1144,7 @@ char* mpack_node_data_alloc(mpack_node_t node, size_t maxsize); char* mpack_node_cstr_alloc(mpack_node_t node, size_t maxsize); /** - * Allocates a new null-terminated string using MPACK_MALLOC with the UTF-8 + * Allocates a new null-terminated string using MPACK_MALLOC() with the UTF-8 * string contained by this node. * * The allocated string must be freed with MPACK_FREE() (or simply free() @@ -986,6 +1164,65 @@ char* mpack_node_cstr_alloc(mpack_node_t node, size_t maxsize); char* mpack_node_utf8_cstr_alloc(mpack_node_t node, size_t maxsize); #endif +/** + * Searches the given string array for a string matching the given + * node and returns its index. + * + * If the node does not match any of the given strings, + * @ref mpack_error_type is flagged. Use mpack_node_enum_optional() + * if you want to allow values other than the given strings. + * + * If any error occurs or if the tree is in an error state, @a count + * is returned. + * + * This can be used to quickly parse a string into an enum when the + * enum values range from 0 to @a count-1. If the last value in the + * enum is a special "count" value, it can be passed as the count, + * and the return value can be cast directly to the enum type. + * + * @code{.c} + * typedef enum { APPLE , BANANA , ORANGE , COUNT} fruit_t; + * const char* fruits[] = {"apple", "banana", "orange"}; + * + * fruit_t fruit = (fruit_t)mpack_node_enum(node, fruits, COUNT); + * @endcode + * + * @param node The node + * @param strings An array of expected strings of length count + * @param count The number of strings + * @return The index of the matched string, or @a count in case of error + */ +size_t mpack_node_enum(mpack_node_t node, const char* strings[], size_t count); + +/** + * Searches the given string array for a string matching the given node, + * returning its index or @a count if no strings match. + * + * If the value is not a string, or it does not match any of the + * given strings, @a count is returned and no error is flagged. + * + * If any error occurs or if the tree is in an error state, @a count + * is returned. + * + * This can be used to quickly parse a string into an enum when the + * enum values range from 0 to @a count-1. If the last value in the + * enum is a special "count" value, it can be passed as the count, + * and the return value can be cast directly to the enum type. + * + * @code{.c} + * typedef enum { APPLE , BANANA , ORANGE , COUNT} fruit_t; + * const char* fruits[] = {"apple", "banana", "orange"}; + * + * fruit_t fruit = (fruit_t)mpack_node_enum_optional(node, fruits, COUNT); + * @endcode + * + * @param node The node + * @param strings An array of expected strings of length count + * @param count The number of strings + * @return The index of the matched string, or @a count in case of error + */ +size_t mpack_node_enum_optional(mpack_node_t node, const char* strings[], size_t count); + /** * @} */ @@ -999,74 +1236,21 @@ char* mpack_node_utf8_cstr_alloc(mpack_node_t node, size_t maxsize); * Returns the length of the given array node. Raises mpack_error_type * and returns 0 if the given node is not an array. */ -MPACK_INLINE size_t mpack_node_array_length(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - if (node.data->type != mpack_type_array) { - mpack_node_flag_error(node, mpack_error_type); - return 0; - } - - return (size_t)node.data->len; -} +size_t mpack_node_array_length(mpack_node_t node); /** * Returns the node in the given array at the given index. If the node - * is not an array, mpack_error_type is raised and a nil node is returned. - * If the given index is out of bounds, mpack_error_data is raised and + * is not an array, @ref mpack_error_type is raised and a nil node is returned. + * If the given index is out of bounds, @ref mpack_error_data is raised and * a nil node is returned. */ -MPACK_INLINE mpack_node_t mpack_node_array_at(mpack_node_t node, size_t index) { - if (mpack_node_error(node) != mpack_ok) - return mpack_tree_nil_node(node.tree); - - if (node.data->type != mpack_type_array) { - mpack_node_flag_error(node, mpack_error_type); - return mpack_tree_nil_node(node.tree); - } - - if (index >= node.data->len) { - mpack_node_flag_error(node, mpack_error_data); - return mpack_tree_nil_node(node.tree); - } - - return mpack_node(node.tree, mpack_node_child(node, index)); -} +mpack_node_t mpack_node_array_at(mpack_node_t node, size_t index); /** * Returns the number of key/value pairs in the given map node. Raises * mpack_error_type and returns 0 if the given node is not a map. */ -MPACK_INLINE size_t mpack_node_map_count(mpack_node_t node) { - if (mpack_node_error(node) != mpack_ok) - return 0; - - if (node.data->type != mpack_type_map) { - mpack_node_flag_error(node, mpack_error_type); - return 0; - } - - return node.data->len; -} - -// internal node map lookup -MPACK_INLINE mpack_node_t mpack_node_map_at(mpack_node_t node, size_t index, size_t offset) { - if (mpack_node_error(node) != mpack_ok) - return mpack_tree_nil_node(node.tree); - - if (node.data->type != mpack_type_map) { - mpack_node_flag_error(node, mpack_error_type); - return mpack_tree_nil_node(node.tree); - } - - if (index >= node.data->len) { - mpack_node_flag_error(node, mpack_error_data); - return mpack_tree_nil_node(node.tree); - } - - return mpack_node(node.tree, mpack_node_child(node, index * 2 + offset)); -} +size_t mpack_node_map_count(mpack_node_t node); /** * Returns the key node in the given map at the given index. @@ -1076,9 +1260,7 @@ MPACK_INLINE mpack_node_t mpack_node_map_at(mpack_node_t node, size_t index, siz * @throws mpack_error_type if the node is not a map * @throws mpack_error_data if the given index is out of bounds */ -MPACK_INLINE mpack_node_t mpack_node_map_key_at(mpack_node_t node, size_t index) { - return mpack_node_map_at(node, index, 0); -} +mpack_node_t mpack_node_map_key_at(mpack_node_t node, size_t index); /** * Returns the value node in the given map at the given index. @@ -1088,9 +1270,7 @@ MPACK_INLINE mpack_node_t mpack_node_map_key_at(mpack_node_t node, size_t index) * @throws mpack_error_type if the node is not a map * @throws mpack_error_data if the given index is out of bounds */ -MPACK_INLINE mpack_node_t mpack_node_map_value_at(mpack_node_t node, size_t index) { - return mpack_node_map_at(node, index, 1); -} +mpack_node_t mpack_node_map_value_at(mpack_node_t node, size_t index); /** * Returns the value node in the given map for the given integer key. @@ -1109,8 +1289,8 @@ MPACK_INLINE mpack_node_t mpack_node_map_value_at(mpack_node_t node, size_t inde mpack_node_t mpack_node_map_int(mpack_node_t node, int64_t num); /** - * Returns the value node in the given map for the given integer key, or a nil - * node if the map does not contain the given key. + * Returns the value node in the given map for the given integer key, or a + * missing node if the map does not contain the given key. * * The key must be unique. An error is flagged if the node has multiple * entries with the given key. @@ -1118,7 +1298,10 @@ mpack_node_t mpack_node_map_int(mpack_node_t node, int64_t num); * @throws mpack_error_type If the node is not a map * @throws mpack_error_data If the node contains more than one entry with the given key * - * @return The value node for the given key, or a nil node in case of error + * @return The value node for the given key, or a missing node if the key does + * not exist, or a nil node in case of error + * + * @see mpack_node_is_missing() */ mpack_node_t mpack_node_map_int_optional(mpack_node_t node, int64_t num); @@ -1140,7 +1323,7 @@ mpack_node_t mpack_node_map_uint(mpack_node_t node, uint64_t num); /** * Returns the value node in the given map for the given unsigned integer - * key, or a nil node if the map does not contain the given key. + * key, or a missing node if the map does not contain the given key. * * The key must be unique. An error is flagged if the node has multiple * entries with the given key. @@ -1148,7 +1331,10 @@ mpack_node_t mpack_node_map_uint(mpack_node_t node, uint64_t num); * @throws mpack_error_type If the node is not a map * @throws mpack_error_data If the node contains more than one entry with the given key * - * @return The value node for the given key, or a nil node in case of error + * @return The value node for the given key, or a missing node if the key does + * not exist, or a nil node in case of error + * + * @see mpack_node_is_missing() */ mpack_node_t mpack_node_map_uint_optional(mpack_node_t node, uint64_t num); @@ -1169,7 +1355,7 @@ mpack_node_t mpack_node_map_uint_optional(mpack_node_t node, uint64_t num); mpack_node_t mpack_node_map_str(mpack_node_t node, const char* str, size_t length); /** - * Returns the value node in the given map for the given string key, or a nil + * Returns the value node in the given map for the given string key, or a missing * node if the map does not contain the given key. * * The key must be unique. An error is flagged if the node has multiple @@ -1178,7 +1364,10 @@ mpack_node_t mpack_node_map_str(mpack_node_t node, const char* str, size_t lengt * @throws mpack_error_type If the node is not a map * @throws mpack_error_data If the node contains more than one entry with the given key * - * @return The value node for the given key, or a nil node in case of error + * @return The value node for the given key, or a missing node if the key does + * not exist, or a nil node in case of error + * + * @see mpack_node_is_missing() */ mpack_node_t mpack_node_map_str_optional(mpack_node_t node, const char* str, size_t length); @@ -1201,7 +1390,7 @@ mpack_node_t mpack_node_map_cstr(mpack_node_t node, const char* cstr); /** * Returns the value node in the given map for the given null-terminated - * string key, or a nil node if the map does not contain the given key. + * string key, or a missing node if the map does not contain the given key. * * The key must be unique. An error is flagged if the node has multiple * entries with the given key. @@ -1209,7 +1398,10 @@ mpack_node_t mpack_node_map_cstr(mpack_node_t node, const char* cstr); * @throws mpack_error_type If the node is not a map * @throws mpack_error_data If the node contains more than one entry with the given key * - * @return The value node for the given key, or a nil node in case of error + * @return The value node for the given key, or a missing node if the key does + * not exist, or a nil node in case of error + * + * @see mpack_node_is_missing() */ mpack_node_t mpack_node_map_cstr_optional(mpack_node_t node, const char* cstr); @@ -1271,7 +1463,8 @@ bool mpack_node_map_contains_cstr(mpack_node_t node, const char* cstr); #endif -MPACK_HEADER_END +MPACK_EXTERN_C_END +MPACK_SILENCE_WARNINGS_END #endif diff --git a/src/mpack/mpack-platform.c b/src/mpack/mpack-platform.c index c557363..6599e1f 100644 --- a/src/mpack/mpack-platform.c +++ b/src/mpack/mpack-platform.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -29,12 +29,7 @@ #include "mpack-platform.h" #include "mpack.h" - -#if MPACK_DEBUG && MPACK_STDIO -#include -#endif - - +MPACK_SILENCE_WARNINGS_BEGIN #if MPACK_DEBUG @@ -46,7 +41,7 @@ void mpack_assert_fail_format(const char* format, ...) { vsnprintf(buffer, sizeof(buffer), format, args); va_end(args); buffer[sizeof(buffer) - 1] = 0; - mpack_assert_fail(buffer); + mpack_assert_fail_wrapper(buffer); } void mpack_break_hit_format(const char* format, ...) { @@ -67,6 +62,23 @@ void mpack_assert_fail(const char* message) { #if MPACK_STDIO fprintf(stderr, "%s\n", message); #endif +} +#endif + +// We split the assert failure from the wrapper so that a +// custom assert function can return. +void mpack_assert_fail_wrapper(const char* message) { + + #ifdef MPACK_GCOV + // gcov marks even __builtin_unreachable() as an uncovered line. this + // silences it. + (mpack_assert_fail(message), __builtin_unreachable()); + + #else + mpack_assert_fail(message); + + // mpack_assert_fail() is not supposed to return. in case it does, we + // abort. #if !MPACK_NO_BUILTINS #if defined(__GNUC__) || defined(__clang__) @@ -83,8 +95,8 @@ void mpack_assert_fail(const char* message) { #endif MPACK_UNREACHABLE; + #endif } -#endif #if !MPACK_CUSTOM_BREAK @@ -98,7 +110,7 @@ void mpack_assert_fail(const char* message) { #if MPACK_CUSTOM_ASSERT void mpack_break_hit(const char* message) { - mpack_assert_fail(message); + mpack_assert_fail_wrapper(message); } #else void mpack_break_hit(const char* message) { @@ -205,3 +217,5 @@ void* mpack_realloc(void* old_ptr, size_t used_size, size_t new_size) { return new_ptr; } #endif + +MPACK_SILENCE_WARNINGS_END diff --git a/src/mpack/mpack-platform.h b/src/mpack/mpack-platform.h index 6babf5a..0f7465e 100644 --- a/src/mpack/mpack-platform.h +++ b/src/mpack/mpack-platform.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -20,86 +20,909 @@ */ /** - * @file - * - * Abstracts all platform-specific code from MPack. This contains - * implementations of standard C functions when libc is not available, - * as well as wrappers to library functions. + * @file + * + * Abstracts all platform-specific code from MPack and handles configuration + * options. + * + * This verifies the configuration and sets defaults based on the platform, + * contains implementations of standard C functions when libc is not available, + * and provides wrappers to all library functions. + * + * Documentation for configuration options is available here: + * + * https://ludocode.github.io/mpack/group__config.html + */ + +#ifndef MPACK_PLATFORM_H +#define MPACK_PLATFORM_H 1 + + + +/** + * @defgroup config Configuration Options + * + * Defines the MPack configuration options. + * + * Custom configuration of MPack is not usually necessary. In almost all + * cases you can ignore this and use the defaults. + * + * If you do want to configure MPack, you can define the below options as part + * of your build system or project settings. This will override the below + * defaults. + * + * If you'd like to use a file for configuration instead, define + * @ref MPACK_HAS_CONFIG to 1 in your build system or project settings. + * This will cause MPack to include a file you create called @c mpack-config.h + * in which you can define your configuration. This is useful if you need to + * include specific headers (such as a custom allocator) in order to configure + * MPack to use it. + * + * @warning The value of all configuration options must be the same in + * all translation units of your project, as well as in the mpack source + * itself. These configuration options affect the layout of structs, among + * other things, which cannot be different in source files that are linked + * together. + * + * @note MPack does not contain defaults for building inside the Linux kernel. + * There is a + * configuration file for the Linux kernel that can be used instead. + * + * @{ + */ + + + +/* + * Pre-include checks + * + * These need to come before the user's mpack-config.h because they might be + * including headers in it. + */ + +/** @cond */ +#if defined(_MSC_VER) && _MSC_VER < 1800 && !defined(__cplusplus) + #error "In Visual Studio 2012 and earlier, MPack must be compiled as C++. Enable the /Tp compiler flag." +#endif + +#if defined(_WIN32) && MPACK_INTERNAL + #define _CRT_SECURE_NO_WARNINGS 1 +#endif + +#ifndef __STDC_LIMIT_MACROS + #define __STDC_LIMIT_MACROS 1 +#endif +#ifndef __STDC_FORMAT_MACROS + #define __STDC_FORMAT_MACROS 1 +#endif +#ifndef __STDC_CONSTANT_MACROS + #define __STDC_CONSTANT_MACROS 1 +#endif +/** @endcond */ + + + +/** + * @name File Configuration + * @{ + */ + +/** + * @def MPACK_HAS_CONFIG + * + * Causes MPack to include a file you create called @c mpack-config.h . + * + * The file is included before MPack sets any defaults for undefined + * configuration options. You can use it to configure MPack. + * + * This is off by default. + */ +#if defined(MPACK_HAS_CONFIG) + #if MPACK_HAS_CONFIG + #include "mpack-config.h" + #endif +#else + #define MPACK_HAS_CONFIG 0 +#endif + +/** + * @} + */ + +// this needs to come first since some stuff depends on it +/** @cond */ +#ifndef MPACK_NO_BUILTINS + #define MPACK_NO_BUILTINS 0 +#endif +/** @endcond */ + + + +/** + * @name Features + * @{ + */ + +/** + * @def MPACK_READER + * + * Enables compilation of the base Tag Reader. + */ +#ifndef MPACK_READER +#define MPACK_READER 1 +#endif + +/** + * @def MPACK_EXPECT + * + * Enables compilation of the static Expect API. + */ +#ifndef MPACK_EXPECT +#define MPACK_EXPECT 1 +#endif + +/** + * @def MPACK_NODE + * + * Enables compilation of the dynamic Node API. + */ +#ifndef MPACK_NODE +#define MPACK_NODE 1 +#endif + +/** + * @def MPACK_WRITER + * + * Enables compilation of the Writer. + */ +#ifndef MPACK_WRITER +#define MPACK_WRITER 1 +#endif + +/** + * @def MPACK_BUILDER + * + * Enables compilation of the Builder. + * + * The Builder API provides additional functions to the Writer for + * automatically determining the element count of compound elements so you do + * not have to specify them up-front. + * + * This requires a @c malloc(). It is enabled by default if MPACK_WRITER is + * enabled and MPACK_MALLOC is defined. + * + * @see mpack_build_map() + * @see mpack_build_array() + * @see mpack_complete_map() + * @see mpack_complete_array() + */ +// This is defined furthur below after we've resolved whether we have malloc(). + +/** + * @def MPACK_COMPATIBILITY + * + * Enables compatibility features for reading and writing older + * versions of MessagePack. + * + * This is disabled by default. When disabled, the behaviour is equivalent to + * using the default version, @ref mpack_version_current. + * + * Enable this if you need to interoperate with applications or data that do + * not support the new (v5) MessagePack spec. See the section on v4 + * compatibility in @ref docs/protocol.md for more information. + */ +#ifndef MPACK_COMPATIBILITY +#define MPACK_COMPATIBILITY 0 +#endif + +/** + * @def MPACK_EXTENSIONS + * + * Enables the use of extension types. + * + * This is disabled by default. Define it to 1 to enable it. If disabled, + * functions to read and write extensions will not exist, and any occurrence of + * extension types in parsed messages will flag @ref mpack_error_invalid. + * + * MPack discourages the use of extension types. See the section on extension + * types in @ref docs/protocol.md for more information. + */ +#ifndef MPACK_EXTENSIONS +#define MPACK_EXTENSIONS 0 +#endif + +/** + * @} + */ + + + +// workarounds for Doxygen +#if defined(MPACK_DOXYGEN) +#if MPACK_DOXYGEN +// We give these their default values of 0 here even though they are defined to +// 1 in the doxyfile. Doxygen will show this as the value in the docs, even +// though it ignores it when parsing the rest of the source. This is what we +// want, since we want the documentation to show these defaults but still +// generate documentation for the functions they add when they're on. +#define MPACK_COMPATIBILITY 0 +#define MPACK_EXTENSIONS 0 +#endif +#endif + + + +/** + * @name Dependencies + * @{ + */ + +/** + * @def MPACK_CONFORMING + * + * Enables the inclusion of basic C headers to define standard types and + * macros. + * + * This causes MPack to include headers required for conforming implementations + * of C99 even in freestanding, in particular , , + * and . It also includes ; this is + * technically not required for freestanding but MPack needs it to detect + * integer limits. + * + * You can disable this if these headers are unavailable or if they do not + * define the standard types and macros (for example inside the Linux kernel.) + * If this is disabled, MPack will include no headers and will assume a 32-bit + * int. You will probably also want to define @ref MPACK_HAS_CONFIG to 1 and + * include your own headers in the config file. You must provide definitions + * for standard types such as @c size_t, @c bool, @c int32_t and so on. + * + * @see + * cppreference.com documentation on Conformance + */ +#ifndef MPACK_CONFORMING + #define MPACK_CONFORMING 1 +#endif + +/** + * @def MPACK_STDLIB + * + * Enables the use of the C stdlib. + * + * This allows the library to use basic functions like @c memcmp() and @c + * strlen(), as well as @c malloc() for debugging and in allocation helpers. + * + * If this is disabled, allocation helper functions will not be defined, and + * MPack will attempt to detect compiler intrinsics for functions like @c + * memcmp() (assuming @ref MPACK_NO_BUILTINS is not set.) It will fallback to + * its own (slow) implementations if it cannot use builtins. You may want to + * define @ref MPACK_MEMCMP and friends if you disable this. + * + * @see MPACK_MEMCMP + * @see MPACK_MEMCPY + * @see MPACK_MEMMOVE + * @see MPACK_MEMSET + * @see MPACK_STRLEN + * @see MPACK_MALLOC + * @see MPACK_REALLOC + * @see MPACK_FREE + */ +#ifndef MPACK_STDLIB + #if !MPACK_CONFORMING + // If we don't even have a proper we assume we won't have + // malloc() either. + #define MPACK_STDLIB 0 + #else + #define MPACK_STDLIB 1 + #endif +#endif + +/** + * @def MPACK_STDIO + * + * Enables the use of C stdio. This adds helpers for easily + * reading/writing C files and makes debugging easier. + */ +#ifndef MPACK_STDIO + #if !MPACK_STDLIB || defined(__AVR__) + #define MPACK_STDIO 0 + #else + #define MPACK_STDIO 1 + #endif +#endif + +/** + * Whether the 'float' type and floating point operations are supported. + * + * If @ref MPACK_FLOAT is disabled, floats are read and written as @c uint32_t + * instead. This way messages with floats do not result in errors and you can + * still perform manual float parsing yourself. + */ +#ifndef MPACK_FLOAT + #define MPACK_FLOAT 1 +#endif + +/** + * Whether the 'double' type is supported. This requires support for 'float'. + * + * If @ref MPACK_DOUBLE is disabled, doubles are read and written as @c + * uint32_t instead. This way messages with doubles do not result in errors and + * you can still perform manual doubles parsing yourself. + * + * If @ref MPACK_FLOAT is enabled but @ref MPACK_DOUBLE is not, doubles can be + * read as floats using the shortening conversion functions, e.g. @ref + * mpack_expect_float() or @ref mpack_node_float(). + */ +#ifndef MPACK_DOUBLE + #if !MPACK_FLOAT || defined(__AVR__) + // AVR supports only float, not double. + #define MPACK_DOUBLE 0 + #else + #define MPACK_DOUBLE 1 + #endif +#endif + +/** + * @} + */ + + + +/** + * @name Allocation Functions + * @{ + */ + +/** + * @def MPACK_MALLOC + * + * Defines the memory allocation function used by MPack. This is used by + * helpers for automatically allocating data the correct size, and for + * debugging functions. If this macro is undefined, the allocation helpers + * will not be compiled. + * + * Set this to use a custom @c malloc() function. Your function must have the + * signature: + * + * @code + * void* malloc(size_t size); + * @endcode + * + * The default is @c malloc() if @ref MPACK_STDLIB is enabled. + */ +/** + * @def MPACK_FREE + * + * Defines the memory free function used by MPack. This is used by helpers + * for automatically allocating data the correct size. If this macro is + * undefined, the allocation helpers will not be compiled. + * + * Set this to use a custom @c free() function. Your function must have the + * signature: + * + * @code + * void free(void* p); + * @endcode + * + * The default is @c free() if @ref MPACK_MALLOC has not been customized and + * @ref MPACK_STDLIB is enabled. + */ +/** + * @def MPACK_REALLOC + * + * Defines the realloc function used by MPack. It is used by growable + * buffers to resize more efficiently. + * + * The default is @c realloc() if @ref MPACK_MALLOC has not been customized and + * @ref MPACK_STDLIB is enabled. + * + * Set this to use a custom @c realloc() function. Your function must have the + * signature: + * + * @code + * void* realloc(void* p, size_t new_size); + * @endcode + * + * This is optional, even when @ref MPACK_MALLOC is used. If @ref MPACK_MALLOC is + * set and @ref MPACK_REALLOC is not, @ref MPACK_MALLOC is used with a simple copy + * to grow buffers. + */ + +#if defined(MPACK_MALLOC) && !defined(MPACK_FREE) + #error "MPACK_MALLOC requires MPACK_FREE." +#endif +#if !defined(MPACK_MALLOC) && defined(MPACK_FREE) + #error "MPACK_FREE requires MPACK_MALLOC." +#endif + +// These were never configurable in lowercase but we check anyway. +#ifdef mpack_malloc + #error "Define MPACK_MALLOC, not mpack_malloc." +#endif +#ifdef mpack_realloc + #error "Define MPACK_REALLOC, not mpack_realloc." +#endif +#ifdef mpack_free + #error "Define MPACK_FREE, not mpack_free." +#endif + +// We don't use calloc() at all. +#ifdef MPACK_CALLOC + #error "Don't define MPACK_CALLOC. MPack does not use calloc()." +#endif +#ifdef mpack_calloc + #error "Don't define mpack_calloc. MPack does not use calloc()." +#endif + +// Use defaults in stdlib if we have them. Without it we don't use malloc. +#if defined(MPACK_STDLIB) + #if MPACK_STDLIB && !defined(MPACK_MALLOC) + #define MPACK_MALLOC malloc + #define MPACK_REALLOC realloc + #define MPACK_FREE free + #endif +#endif + +/** + * @} + */ + + + +// This needs to be defined after we've decided whether we have malloc(). +#ifndef MPACK_BUILDER + #if defined(MPACK_MALLOC) && MPACK_WRITER + #define MPACK_BUILDER 1 + #else + #define MPACK_BUILDER 0 + #endif +#endif + + + +/** + * @name System Functions + * @{ + */ + +/** + * @def MPACK_MEMCMP + * + * The function MPack will use to provide @c memcmp(). + * + * Set this to use a custom @c memcmp() function. Your function must have the + * signature: + * + * @code + * int memcmp(const void* left, const void* right, size_t count); + * @endcode + */ +/** + * @def MPACK_MEMCPY + * + * The function MPack will use to provide @c memcpy(). + * + * Set this to use a custom @c memcpy() function. Your function must have the + * signature: + * + * @code + * void* memcpy(void* restrict dest, const void* restrict src, size_t count); + * @endcode + */ +/** + * @def MPACK_MEMMOVE + * + * The function MPack will use to provide @c memmove(). + * + * Set this to use a custom @c memmove() function. Your function must have the + * signature: + * + * @code + * void* memmove(void* dest, const void* src, size_t count); + * @endcode + */ +/** + * @def MPACK_MEMSET + * + * The function MPack will use to provide @c memset(). + * + * Set this to use a custom @c memset() function. Your function must have the + * signature: + * + * @code + * void* memset(void* p, int c, size_t count); + * @endcode + */ +/** + * @def MPACK_STRLEN + * + * The function MPack will use to provide @c strlen(). + * + * Set this to use a custom @c strlen() function. Your function must have the + * signature: + * + * @code + * size_t strlen(const char* str); + * @endcode + */ + +// These were briefly configurable in lowercase in an unreleased version. Just +// to make sure no one is doing this, we make sure these aren't already defined. +#ifdef mpack_memcmp + #error "Define MPACK_MEMCMP, not mpack_memcmp." +#endif +#ifdef mpack_memcpy + #error "Define MPACK_MEMCPY, not mpack_memcpy." +#endif +#ifdef mpack_memmove + #error "Define MPACK_MEMMOVE, not mpack_memmove." +#endif +#ifdef mpack_memset + #error "Define MPACK_MEMSET, not mpack_memset." +#endif +#ifdef mpack_strlen + #error "Define MPACK_STRLEN, not mpack_strlen." +#endif + +// If the standard library is available, we prefer to use its functions. +#if MPACK_STDLIB + #ifndef MPACK_MEMCMP + #define MPACK_MEMCMP memcmp + #endif + #ifndef MPACK_MEMCPY + #define MPACK_MEMCPY memcpy + #endif + #ifndef MPACK_MEMMOVE + #define MPACK_MEMMOVE memmove + #endif + #ifndef MPACK_MEMSET + #define MPACK_MEMSET memset + #endif + #ifndef MPACK_STRLEN + #define MPACK_STRLEN strlen + #endif +#endif + +#if !MPACK_NO_BUILTINS + #ifdef __has_builtin + #if !defined(MPACK_MEMCMP) && __has_builtin(__builtin_memcmp) + #define MPACK_MEMCMP __builtin_memcmp + #endif + #if !defined(MPACK_MEMCPY) && __has_builtin(__builtin_memcpy) + #define MPACK_MEMCPY __builtin_memcpy + #endif + #if !defined(MPACK_MEMMOVE) && __has_builtin(__builtin_memmove) + #define MPACK_MEMMOVE __builtin_memmove + #endif + #if !defined(MPACK_MEMSET) && __has_builtin(__builtin_memset) + #define MPACK_MEMSET __builtin_memset + #endif + #if !defined(MPACK_STRLEN) && __has_builtin(__builtin_strlen) + #define MPACK_STRLEN __builtin_strlen + #endif + #elif defined(__GNUC__) + #ifndef MPACK_MEMCMP + #define MPACK_MEMCMP __builtin_memcmp + #endif + #ifndef MPACK_MEMCPY + #define MPACK_MEMCPY __builtin_memcpy + #endif + // There's not always a builtin memmove for GCC. If we can't test for + // it with __has_builtin above, we don't use it. It's been around for + // much longer under clang, but then so has __has_builtin, so we let + // the block above handle it. + #ifndef MPACK_MEMSET + #define MPACK_MEMSET __builtin_memset + #endif + #ifndef MPACK_STRLEN + #define MPACK_STRLEN __builtin_strlen + #endif + #endif +#endif + +/** + * @} + */ + + + +/** + * @name Debugging Options + * @{ + */ + +/** + * @def MPACK_DEBUG + * + * Enables debug features. You may want to wrap this around your + * own debug preprocs. By default, this is enabled if @c DEBUG or @c _DEBUG + * are defined. (@c NDEBUG is not used since it is allowed to have + * different values in different translation units.) + */ +#if !defined(MPACK_DEBUG) + #if defined(DEBUG) || defined(_DEBUG) + #define MPACK_DEBUG 1 + #else + #define MPACK_DEBUG 0 + #endif +#endif + +/** + * @def MPACK_STRINGS + * + * Enables descriptive error and type strings. + * + * This can be turned off (by defining it to 0) to maximize space savings + * on embedded devices. If this is disabled, string functions such as + * mpack_error_to_string() and mpack_type_to_string() return an empty string. + */ +#ifndef MPACK_STRINGS + #ifdef __AVR__ + #define MPACK_STRINGS 0 + #else + #define MPACK_STRINGS 1 + #endif +#endif + +/** + * Set this to 1 to implement a custom @ref mpack_assert_fail() function. + * See the documentation on @ref mpack_assert_fail() for details. + * + * Asserts are only used when @ref MPACK_DEBUG is enabled, and can be + * triggered by bugs in MPack or bugs due to incorrect usage of MPack. + */ +#ifndef MPACK_CUSTOM_ASSERT +#define MPACK_CUSTOM_ASSERT 0 +#endif + +/** + * @def MPACK_READ_TRACKING + * + * Enables compound type size tracking for readers. This ensures that the + * correct number of elements or bytes are read from a compound type. + * + * This is enabled by default in debug builds (provided a @c malloc() is + * available.) + */ +#if !defined(MPACK_READ_TRACKING) + #if MPACK_DEBUG && MPACK_READER && defined(MPACK_MALLOC) + #define MPACK_READ_TRACKING 1 + #else + #define MPACK_READ_TRACKING 0 + #endif +#endif +#if MPACK_READ_TRACKING && !MPACK_READER + #error "MPACK_READ_TRACKING requires MPACK_READER." +#endif + +/** + * @def MPACK_WRITE_TRACKING + * + * Enables compound type size tracking for writers. This ensures that the + * correct number of elements or bytes are written in a compound type. + * + * Note that without write tracking enabled, it is possible for buggy code + * to emit invalid MessagePack without flagging an error by writing the wrong + * number of elements or bytes in a compound type. With tracking enabled, + * MPack will catch such errors and break on the offending line of code. + * + * This is enabled by default in debug builds (provided a @c malloc() is + * available.) + */ +#if !defined(MPACK_WRITE_TRACKING) + #if MPACK_DEBUG && MPACK_WRITER && defined(MPACK_MALLOC) + #define MPACK_WRITE_TRACKING 1 + #else + #define MPACK_WRITE_TRACKING 0 + #endif +#endif +#if MPACK_WRITE_TRACKING && !MPACK_WRITER + #error "MPACK_WRITE_TRACKING requires MPACK_WRITER." +#endif + +/** + * @} + */ + + + + +/** + * @name Miscellaneous Options + * @{ + */ + +/** + * Whether to optimize for size or speed. + * + * Optimizing for size simplifies some parsing and encoding algorithms + * at the expense of speed and saves a few kilobytes of space in the + * resulting executable. + * + * This automatically detects -Os with GCC/Clang. Unfortunately there + * doesn't seem to be a macro defined for /Os under MSVC. + */ +#ifndef MPACK_OPTIMIZE_FOR_SIZE + #ifdef __OPTIMIZE_SIZE__ + #define MPACK_OPTIMIZE_FOR_SIZE 1 + #else + #define MPACK_OPTIMIZE_FOR_SIZE 0 + #endif +#endif + +/** + * Stack space in bytes to use when initializing a reader or writer + * with a stack-allocated buffer. + * + * @warning Make sure you have sufficient stack space. Some libc use relatively + * small stacks even on desktop platforms, e.g. musl. + */ +#ifndef MPACK_STACK_SIZE +#define MPACK_STACK_SIZE 4096 +#endif + +/** + * Buffer size to use for allocated buffers (such as for a file writer.) + * + * Starting with a single page and growing as needed seems to + * provide the best performance with minimal memory waste. + * Increasing this does not improve performance even when writing + * huge messages. + */ +#ifndef MPACK_BUFFER_SIZE +#define MPACK_BUFFER_SIZE 4096 +#endif + +/** + * Minimum size for paged allocations in bytes. + * + * This is the value used by default for MPACK_NODE_PAGE_SIZE and + * MPACK_BUILDER_PAGE_SIZE. + */ +#ifndef MPACK_PAGE_SIZE +#define MPACK_PAGE_SIZE 4096 +#endif + +/** + * Minimum size of an allocated node page in bytes. + * + * The children for a given compound element must be contiguous, so + * larger pages than this may be allocated as needed. (Safety checks + * exist to prevent malicious data from causing too large allocations.) + * + * See @ref mpack_node_data_t for the size of nodes. + * + * Using as many nodes fit in one memory page seems to provide the + * best performance, and has very little waste when parsing small + * messages. + */ +#ifndef MPACK_NODE_PAGE_SIZE +#define MPACK_NODE_PAGE_SIZE MPACK_PAGE_SIZE +#endif + +/** + * Minimum size of an allocated builder page in bytes. + * + * Builder writes are deferred to the allocated builder buffer which is + * composed of a list of buffer pages. This defines the size of those pages. + */ +#ifndef MPACK_BUILDER_PAGE_SIZE +#define MPACK_BUILDER_PAGE_SIZE MPACK_PAGE_SIZE +#endif + +/** + * @def MPACK_BUILDER_INTERNAL_STORAGE + * + * Enables a small amount of internal storage within the writer to avoid some + * allocations when using builders. + * + * This is disabled by default. Enable it to potentially improve performance at + * the expense of a larger writer. + * + * @see MPACK_BUILDER_INTERNAL_STORAGE_SIZE to configure its size. + */ +#ifndef MPACK_BUILDER_INTERNAL_STORAGE +#define MPACK_BUILDER_INTERNAL_STORAGE 0 +#endif + +/** + * Amount of space reserved inside @ref mpack_writer_t for the Builders. This + * can allow small messages to be built with the Builder API without incurring + * an allocation. + * + * Builder metadata is placed in this space in addition to the literal + * MessagePack data. It needs to be big enough to be useful, but not so big as + * to overflow the stack. If more space is needed, pages are allocated. + * + * This is only used if MPACK_BUILDER_INTERNAL_STORAGE is enabled. + * + * @see MPACK_BUILDER_PAGE_SIZE + * @see MPACK_BUILDER_INTERNAL_STORAGE + * + * @warning Writers are typically placed on the stack so make sure you have + * sufficient stack space. Some libc use relatively small stacks even on + * desktop platforms, e.g. musl. + */ +#ifndef MPACK_BUILDER_INTERNAL_STORAGE_SIZE +#define MPACK_BUILDER_INTERNAL_STORAGE_SIZE 256 +#endif + +/** + * The initial depth for the node parser. When MPACK_MALLOC is available, + * the node parser has no practical depth limit, and it is not recursive + * so there is no risk of overflowing the call stack. + */ +#ifndef MPACK_NODE_INITIAL_DEPTH +#define MPACK_NODE_INITIAL_DEPTH 8 +#endif + +/** + * The maximum depth for the node parser if @ref MPACK_MALLOC is not available. */ +#ifndef MPACK_NODE_MAX_DEPTH_WITHOUT_MALLOC +#define MPACK_NODE_MAX_DEPTH_WITHOUT_MALLOC 32 +#endif -#ifndef MPACK_PLATFORM_H -#define MPACK_PLATFORM_H 1 - - - -/* For now, nothing in here should be seen by Doxygen. */ -/** @cond */ +/** + * @def MPACK_NO_BUILTINS + * + * Whether to disable compiler intrinsics and other built-in functions. + * + * If this is enabled, MPack won't use `__attribute__`, `__declspec`, any + * function starting with `__builtin`, or pretty much anything else that isn't + * standard C. + */ +#if defined(MPACK_DOXYGEN) +#if MPACK_DOXYGEN + #define MPACK_NO_BUILTINS 0 +#endif +#endif +/** + * @} + */ -/* Pre-include checks */ -#if defined(_MSC_VER) && _MSC_VER < 1800 && !defined(__cplusplus) -#error "In Visual Studio 2012 and earlier, MPack must be compiled as C++. Enable the /Tp compiler flag." +#if MPACK_DEBUG +/** + * @name Debug Functions + * @{ + */ +/** + * Implement this and define @ref MPACK_CUSTOM_ASSERT to use a custom + * assertion function. + * + * This function should not return. If it does, MPack will @c abort(). + * + * If you use C++, make sure you include @c mpack.h where you define + * this to get the correct linkage (or define it extern "C".) + * + * Asserts are only used when @ref MPACK_DEBUG is enabled, and can be + * triggered by bugs in MPack or bugs due to incorrect usage of MPack. + */ +void mpack_assert_fail(const char* message); +/** + * @} + */ #endif -#if defined(WIN32) && defined(MPACK_INTERNAL) && MPACK_INTERNAL -#define _CRT_SECURE_NO_WARNINGS 1 -#endif +// The rest of this file shouldn't show up in Doxygen docs. +/** @cond */ + -#include "mpack-config.h" /* - * Now that the config is included, we define to 0 any of the configuration - * options and other switches that aren't defined. This supports -Wundef - * without us having to write "#if defined(X) && X" everywhere (and while - * allowing configs to be pre-defined to 0.) + * All remaining pseudo-configuration options that have not yet been set must + * be defined here in order to support -Wundef. + * + * These aren't real configuration options; they are implementation details of + * MPack. */ -#ifndef MPACK_READER -#define MPACK_READER 0 -#endif -#ifndef MPACK_EXPECT -#define MPACK_EXPECT 0 -#endif -#ifndef MPACK_NODE -#define MPACK_NODE 0 -#endif -#ifndef MPACK_WRITER -#define MPACK_WRITER 0 -#endif - -#ifndef MPACK_STDLIB -#define MPACK_STDLIB 0 -#endif -#ifndef MPACK_STDIO -#define MPACK_STDIO 0 -#endif - -#ifndef MPACK_DEBUG -#define MPACK_DEBUG 0 -#endif -#ifndef MPACK_CUSTOM_ASSERT -#define MPACK_CUSTOM_ASSERT 0 -#endif #ifndef MPACK_CUSTOM_BREAK #define MPACK_CUSTOM_BREAK 0 #endif - -#ifndef MPACK_READ_TRACKING -#define MPACK_READ_TRACKING 0 -#endif -#ifndef MPACK_WRITE_TRACKING -#define MPACK_WRITE_TRACKING 0 -#endif -#ifndef MPACK_NO_TRACKING -#define MPACK_NO_TRACKING 0 -#endif -#ifndef MPACK_OPTIMIZE_FOR_SIZE -#define MPACK_OPTIMIZE_FOR_SIZE 0 -#endif - #ifndef MPACK_EMIT_INLINE_DEFS #define MPACK_EMIT_INLINE_DEFS 0 #endif @@ -112,103 +935,219 @@ #ifndef MPACK_INTERNAL #define MPACK_INTERNAL 0 #endif -#ifndef MPACK_NO_BUILTINS -#define MPACK_NO_BUILTINS 0 -#endif /* System headers (based on configuration) */ -#ifndef __STDC_LIMIT_MACROS -#define __STDC_LIMIT_MACROS 1 +#if MPACK_CONFORMING + #include + #include + #include + #include + #include #endif -#ifndef __STDC_FORMAT_MACROS -#define __STDC_FORMAT_MACROS 1 + +#if MPACK_STDLIB + #include + #include #endif -#ifndef __STDC_CONSTANT_MACROS -#define __STDC_CONSTANT_MACROS 1 + +#if MPACK_STDIO + #include + #include + #if MPACK_DEBUG + #include + #endif #endif -#include -#include -#include -#include -#include -#if MPACK_STDLIB -#include -#include + +/* + * Integer Constants and Limits + */ + +#if MPACK_CONFORMING + #define MPACK_INT64_C INT64_C + #define MPACK_UINT64_C UINT64_C + + #define MPACK_INT8_MIN INT8_MIN + #define MPACK_INT16_MIN INT16_MIN + #define MPACK_INT32_MIN INT32_MIN + #define MPACK_INT64_MIN INT64_MIN + #define MPACK_INT_MIN INT_MIN + + #define MPACK_INT8_MAX INT8_MAX + #define MPACK_INT16_MAX INT16_MAX + #define MPACK_INT32_MAX INT32_MAX + #define MPACK_INT64_MAX INT64_MAX + #define MPACK_INT_MAX INT_MAX + + #define MPACK_UINT8_MAX UINT8_MAX + #define MPACK_UINT16_MAX UINT16_MAX + #define MPACK_UINT32_MAX UINT32_MAX + #define MPACK_UINT64_MAX UINT64_MAX + #define MPACK_UINT_MAX UINT_MAX +#else + // For a non-conforming implementation we assume int is 32 bits. + + #define MPACK_INT64_C(x) ((int64_t)(x##LL)) + #define MPACK_UINT64_C(x) ((uint64_t)(x##LLU)) + + #define MPACK_INT8_MIN ((int8_t)(0x80)) + #define MPACK_INT16_MIN ((int16_t)(0x8000)) + #define MPACK_INT32_MIN ((int32_t)(0x80000000)) + #define MPACK_INT64_MIN MPACK_INT64_C(0x8000000000000000) + #define MPACK_INT_MIN MPACK_INT32_MIN + + #define MPACK_INT8_MAX ((int8_t)(0x7f)) + #define MPACK_INT16_MAX ((int16_t)(0x7fff)) + #define MPACK_INT32_MAX ((int32_t)(0x7fffffff)) + #define MPACK_INT64_MAX MPACK_INT64_C(0x7fffffffffffffff) + #define MPACK_INT_MAX MPACK_INT32_MAX + + #define MPACK_UINT8_MAX ((uint8_t)(0xffu)) + #define MPACK_UINT16_MAX ((uint16_t)(0xffffu)) + #define MPACK_UINT32_MAX ((uint32_t)(0xffffffffu)) + #define MPACK_UINT64_MAX MPACK_UINT64_C(0xffffffffffffffff) + #define MPACK_UINT_MAX MPACK_UINT32_MAX #endif -#if MPACK_STDIO -#include -#include + + +/* + * Floating point support + */ + +#if MPACK_DOUBLE && !MPACK_FLOAT + #error "MPACK_DOUBLE requires MPACK_FLOAT." +#endif + +// If we don't have support for float or double, we poison the identifiers to +// make sure we don't define anything related to them. +#if MPACK_INTERNAL && defined(MPACK_UNIT_TESTS) && defined(__GNUC__) + #if !MPACK_FLOAT + #pragma GCC poison float + #endif + #if !MPACK_DOUBLE + #pragma GCC poison double + #endif #endif /* - * Header configuration + * extern C */ #ifdef __cplusplus - #define MPACK_EXTERN_C_START extern "C" { + #define MPACK_EXTERN_C_BEGIN extern "C" { #define MPACK_EXTERN_C_END } #else - #define MPACK_EXTERN_C_START /* nothing */ - #define MPACK_EXTERN_C_END /* nothing */ + #define MPACK_EXTERN_C_BEGIN /*nothing*/ + #define MPACK_EXTERN_C_END /*nothing*/ +#endif + + + +/* + * Warnings + */ + +#if defined(__GNUC__) + // Diagnostic push is not supported before GCC 4.6. + #if defined(__clang__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) + #define MPACK_SILENCE_WARNINGS_PUSH _Pragma ("GCC diagnostic push") + #define MPACK_SILENCE_WARNINGS_POP _Pragma ("GCC diagnostic pop") + #endif +#elif defined(_MSC_VER) + // To support VS2017 and earlier we need to use __pragma and not _Pragma + #define MPACK_SILENCE_WARNINGS_PUSH __pragma(warning(push)) + #define MPACK_SILENCE_WARNINGS_POP __pragma(warning(pop)) +#endif + +#if defined(_MSC_VER) + // These are a bunch of mostly useless warnings emitted under MSVC /W4, + // some as a result of the expansion of macros. + #define MPACK_SILENCE_WARNINGS_MSVC_W4 \ + __pragma(warning(disable:4996)) /* _CRT_SECURE_NO_WARNINGS */ \ + __pragma(warning(disable:4127)) /* comparison is constant */ \ + __pragma(warning(disable:4702)) /* unreachable code */ \ + __pragma(warning(disable:4310)) /* cast truncates constant value */ +#else + #define MPACK_SILENCE_WARNINGS_MSVC_W4 /*nothing*/ #endif -/* GCC versions from 4.6 to before 5.1 warn about defining a C99 - * non-static inline function before declaring it (see issue #20) */ -#ifdef __GNUC__ - #if (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) +/* GCC versions before 5.1 warn about defining a C99 non-static inline function + * before declaring it (see issue #20). */ +#if defined(__GNUC__) && !defined(__clang__) + #if __GNUC__ < 5 || (__GNUC__ == 5 && __GNUC_MINOR__ < 1) #ifdef __cplusplus - #define MPACK_DECLARED_INLINE_WARNING_START \ - _Pragma ("GCC diagnostic push") \ + #define MPACK_SILENCE_WARNINGS_MISSING_PROTOTYPES \ _Pragma ("GCC diagnostic ignored \"-Wmissing-declarations\"") #else - #define MPACK_DECLARED_INLINE_WARNING_START \ - _Pragma ("GCC diagnostic push") \ + #define MPACK_SILENCE_WARNINGS_MISSING_PROTOTYPES \ _Pragma ("GCC diagnostic ignored \"-Wmissing-prototypes\"") #endif - #define MPACK_DECLARED_INLINE_WARNING_END \ - _Pragma ("GCC diagnostic pop") #endif #endif -#ifndef MPACK_DECLARED_INLINE_WARNING_START - #define MPACK_DECLARED_INLINE_WARNING_START /* nothing */ - #define MPACK_DECLARED_INLINE_WARNING_END /* nothing */ +#ifndef MPACK_SILENCE_WARNINGS_MISSING_PROTOTYPES + #define MPACK_SILENCE_WARNINGS_MISSING_PROTOTYPES /*nothing*/ #endif -/* GCC versions before 4.8 warn about shadowing a function with a - * variable that isn't a function or function pointer (like "index") */ -#ifdef __GNUC__ - #if (__GNUC__ < 4) || (__GNUC__ == 4 && __GNUC_MINOR__ < 8) - #define MPACK_WSHADOW_WARNING_START \ - _Pragma ("GCC diagnostic push") \ +/* GCC versions before 4.8 warn about shadowing a function with a variable that + * isn't a function or function pointer (like "index"). */ +#if defined(__GNUC__) && !defined(__clang__) + #if __GNUC__ == 4 && __GNUC_MINOR__ < 8 + #define MPACK_SILENCE_WARNINGS_SHADOW \ _Pragma ("GCC diagnostic ignored \"-Wshadow\"") - #define MPACK_WSHADOW_WARNING_END \ - _Pragma ("GCC diagnostic pop") #endif #endif -#ifndef MPACK_WSHADOW_WARNING_START - #define MPACK_WSHADOW_WARNING_START /* nothing */ - #define MPACK_WSHADOW_WARNING_END /* nothing */ +#ifndef MPACK_SILENCE_WARNINGS_SHADOW + #define MPACK_SILENCE_WARNINGS_SHADOW /*nothing*/ +#endif + +// On platforms with small size_t (e.g. AVR) we get type limits warnings where +// we compare a size_t to e.g. MPACK_UINT32_MAX. +#ifdef __AVR__ + #define MPACK_SILENCE_WARNINGS_TYPE_LIMITS \ + _Pragma ("GCC diagnostic ignored \"-Wtype-limits\"") +#else + #define MPACK_SILENCE_WARNINGS_TYPE_LIMITS /*nothing*/ #endif -#define MPACK_HEADER_START \ - MPACK_EXTERN_C_START \ - MPACK_WSHADOW_WARNING_START \ - MPACK_DECLARED_INLINE_WARNING_START +// MPack uses declarations after statements. This silences warnings about it +// (e.g. when using MPack in a Linux kernel module.) +#if defined(__GNUC__) && !defined(__cplusplus) + #define MPACK_SILENCE_WARNINGS_DECLARATION_AFTER_STATEMENT \ + _Pragma ("GCC diagnostic ignored \"-Wdeclaration-after-statement\"") +#else + #define MPACK_SILENCE_WARNINGS_DECLARATION_AFTER_STATEMENT /*nothing*/ +#endif -#define MPACK_HEADER_END \ - MPACK_DECLARED_INLINE_WARNING_END \ - MPACK_WSHADOW_WARNING_END \ - MPACK_EXTERN_C_END +#ifdef MPACK_SILENCE_WARNINGS_PUSH + // We only silence warnings if push/pop is supported, that way we aren't + // silencing warnings in code that uses MPack. If your compiler doesn't + // support push/pop silencing of warnings, you'll have to turn off + // conflicting warnings manually. + + #define MPACK_SILENCE_WARNINGS_BEGIN \ + MPACK_SILENCE_WARNINGS_PUSH \ + MPACK_SILENCE_WARNINGS_MSVC_W4 \ + MPACK_SILENCE_WARNINGS_MISSING_PROTOTYPES \ + MPACK_SILENCE_WARNINGS_SHADOW \ + MPACK_SILENCE_WARNINGS_TYPE_LIMITS \ + MPACK_SILENCE_WARNINGS_DECLARATION_AFTER_STATEMENT + + #define MPACK_SILENCE_WARNINGS_END \ + MPACK_SILENCE_WARNINGS_POP +#else + #define MPACK_SILENCE_WARNINGS_BEGIN /*nothing*/ + #define MPACK_SILENCE_WARNINGS_END /*nothing*/ +#endif -MPACK_HEADER_START +MPACK_SILENCE_WARNINGS_BEGIN +MPACK_EXTERN_C_BEGIN @@ -219,6 +1158,25 @@ MPACK_HEADER_START #define MPACK_STRINGIFY_IMPL(arg) #arg #define MPACK_STRINGIFY(arg) MPACK_STRINGIFY_IMPL(arg) +// This is a workaround for MSVC's incorrect expansion of __VA_ARGS__. It +// treats __VA_ARGS__ as a single preprocessor token when passed in the +// argument list of another macro unless we use an outer wrapper to expand it +// lexically first. (For safety/consistency we use this in all variadic macros +// that don't ignore the variadic arguments regardless of whether __VA_ARGS__ +// is passed to another macro.) +// https://stackoverflow.com/a/32400131 +#define MPACK_EXPAND(x) x + +// Extracts the first argument of a variadic macro list, where there might only +// be one argument. +#define MPACK_EXTRACT_ARG0_IMPL(first, ...) first +#define MPACK_EXTRACT_ARG0(...) MPACK_EXPAND(MPACK_EXTRACT_ARG0_IMPL( __VA_ARGS__ , ignored)) + +// Stringifies the first argument of a variadic macro list, where there might +// only be one argument. +#define MPACK_STRINGIFY_ARG0_impl(first, ...) #first +#define MPACK_STRINGIFY_ARG0(...) MPACK_EXPAND(MPACK_STRINGIFY_ARG0_impl( __VA_ARGS__ , ignored)) + /* @@ -261,17 +1219,18 @@ MPACK_HEADER_START // translation units. If mpack-platform.c (or the amalgamation) // is compiled as C, its definition will be used, otherwise a // C++ definition will be used, and no other C files will emit - // a defition. + // a definition. #define MPACK_INLINE inline #elif defined(_MSC_VER) - // MSVC 2013 always uses COMDAT linkage, but it doesn't treat - // 'inline' as a keyword in C99 mode. + // MSVC 2013 always uses COMDAT linkage, but it doesn't treat 'inline' as a + // keyword in C99 mode. (This appears to be fixed in a later version of + // MSVC but we don't bother detecting it.) #define MPACK_INLINE __inline #define MPACK_STATIC_INLINE static __inline #elif defined(__GNUC__) && (defined(__GNUC_GNU_INLINE__) || \ - !defined(__GNUC_STDC_INLINE__) && !defined(__GNUC_GNU_INLINE__)) + (!defined(__GNUC_STDC_INLINE__) && !defined(__GNUC_GNU_INLINE__))) // GNU rules #if MPACK_EMIT_INLINE_DEFS #define MPACK_INLINE inline @@ -279,6 +1238,16 @@ MPACK_HEADER_START #define MPACK_INLINE extern inline #endif +#elif defined(__TINYC__) + // tcc ignores the inline keyword, so we have to use static inline. We + // issue a warning to make sure you are aware. You can define the below + // macro to disable the warning. Hopefully this will be fixed soon: + // https://lists.nongnu.org/archive/html/tinycc-devel/2019-06/msg00000.html + #ifndef MPACK_DISABLE_TINYC_INLINE_WARNING + #warning "Single-definition inline is not supported by tcc. All inlines will be static. Define MPACK_DISABLE_TINYC_INLINE_WARNING to disable this warning." + #endif + #define MPACK_INLINE static inline + #else // C99 rules #if MPACK_EMIT_INLINE_DEFS @@ -298,26 +1267,75 @@ MPACK_HEADER_START -/* Some compiler-specific keywords and builtins */ +/* + * Prevent inlining + * + * When a function is only used once, it is almost always inlined + * automatically. This can cause poor instruction cache usage because a + * function that should rarely be called (such as buffer exhaustion handling) + * will get inlined into the middle of a hot code path. + */ #if !MPACK_NO_BUILTINS - #if defined(__GNUC__) || defined(__clang__) - #define MPACK_UNREACHABLE __builtin_unreachable() - #define MPACK_NORETURN(fn) fn __attribute__((noreturn)) + #if defined(_MSC_VER) + #define MPACK_NOINLINE __declspec(noinline) + #elif defined(__GNUC__) || defined(__clang__) + #define MPACK_NOINLINE __attribute__((__noinline__)) + #endif +#endif +#ifndef MPACK_NOINLINE + #define MPACK_NOINLINE /* nothing */ +#endif + + + +/* restrict */ + +// We prefer the builtins even though e.g. MSVC's __restrict may not have +// exactly the same behaviour as the proper C99 restrict keyword because the +// builtins work in C++, so using the same keyword in both C and C++ prevents +// any incompatibilities when using MPack compiled as C in C++ code. +#if !MPACK_NO_BUILTINS + #if defined(__GNUC__) #define MPACK_RESTRICT __restrict__ #elif defined(_MSC_VER) - #define MPACK_UNREACHABLE __assume(0) - #define MPACK_NORETURN(fn) __declspec(noreturn) fn #define MPACK_RESTRICT __restrict #endif #endif #ifndef MPACK_RESTRICT -#ifdef __cplusplus -#define MPACK_RESTRICT /* nothing, unavailable in C++ */ -#else -#define MPACK_RESTRICT restrict /* required in C99 */ + #ifdef __cplusplus + #define MPACK_RESTRICT /* nothing, unavailable in C++ */ + #endif +#endif + +#ifndef MPACK_RESTRICT + #ifdef _MSC_VER + // MSVC 2015 apparently doesn't properly support the restrict keyword + // in C. We're using builtins above which do work on 2015, but when + // MPACK_NO_BUILTINS is enabled we can't use it. + #if _MSC_VER < 1910 + #define MPACK_RESTRICT /*nothing*/ + #endif + #endif +#endif + +#ifndef MPACK_RESTRICT + #define MPACK_RESTRICT restrict /* required in C99 */ #endif + + + +/* Some compiler-specific keywords and builtins */ + +#if !MPACK_NO_BUILTINS + #if defined(__GNUC__) || defined(__clang__) + #define MPACK_UNREACHABLE __builtin_unreachable() + #define MPACK_NORETURN(fn) fn __attribute__((__noreturn__)) + #elif defined(_MSC_VER) + #define MPACK_UNREACHABLE __assume(0) + #define MPACK_NORETURN(fn) __declspec(noreturn) fn + #endif #endif #ifndef MPACK_UNREACHABLE @@ -329,6 +1347,70 @@ MPACK_HEADER_START +/* + * Likely/unlikely + * + * These should only really be used when a branch is taken (or not taken) less + * than 1/1000th of the time. Buffer flush checks when writing very small + * elements are a good example. + */ + +#if !MPACK_NO_BUILTINS + #if defined(__GNUC__) || defined(__clang__) + #ifndef MPACK_LIKELY + #define MPACK_LIKELY(x) __builtin_expect((x),1) + #endif + #ifndef MPACK_UNLIKELY + #define MPACK_UNLIKELY(x) __builtin_expect((x),0) + #endif + #endif +#endif + +#ifndef MPACK_LIKELY + #define MPACK_LIKELY(x) (x) +#endif +#ifndef MPACK_UNLIKELY + #define MPACK_UNLIKELY(x) (x) +#endif + + + +/* alignof */ + +#ifndef MPACK_ALIGNOF + #if defined(__STDC_VERSION__) + #if __STDC_VERSION__ >= 201112L + #define MPACK_ALIGNOF(T) (_Alignof(T)) + #endif + #endif +#endif + +#ifndef MPACK_ALIGNOF + #if defined(__cplusplus) + #if __cplusplus >= 201103L + #define MPACK_ALIGNOF(T) (alignof(T)) + #endif + #endif +#endif + +#ifndef MPACK_ALIGNOF + #if defined(__GNUC__) && !defined(MPACK_NO_BUILTINS) + #if defined(__clang__) || __GNUC__ >= 4 + #define MPACK_ALIGNOF(T) (__alignof__(T)) + #endif + #endif +#endif + +#ifndef MPACK_ALIGNOF + #ifdef _MSC_VER + #define MPACK_ALIGNOF(T) __alignof(T) + #endif +#endif + +// MPACK_ALIGNOF may not exist, in which case a workaround is used. + + + /* Static assert */ #ifndef MPACK_STATIC_ASSERT @@ -356,9 +1438,10 @@ MPACK_HEADER_START #ifndef MPACK_STATIC_ASSERT #if defined(__GNUC__) - #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) + /* Diagnostic push is not supported before GCC 4.6. */ + #if defined(__clang__) || __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 6) #ifndef __cplusplus - #if __GNUC__ >= 5 + #if defined(__clang__) || __GNUC__ >= 5 #define MPACK_IGNORE_PEDANTIC "GCC diagnostic ignored \"-Wpedantic\"" #else #define MPACK_IGNORE_PEDANTIC "GCC diagnostic ignored \"-pedantic\"" @@ -390,6 +1473,64 @@ MPACK_HEADER_START +/* _Generic */ + +#ifndef MPACK_HAS_GENERIC + #if defined(__clang__) && defined(__has_feature) + // With Clang we can test for _Generic support directly + // and ignore C/C++ version + #if __has_feature(c_generic_selections) + #define MPACK_HAS_GENERIC 1 + #else + #define MPACK_HAS_GENERIC 0 + #endif + #endif +#endif + +#ifndef MPACK_HAS_GENERIC + #if defined(__STDC_VERSION__) + #if __STDC_VERSION__ >= 201112L + #if defined(__GNUC__) && !defined(__clang__) + // GCC does not have full C11 support in GCC 4.7 and 4.8 + #if __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9) + #define MPACK_HAS_GENERIC 1 + #endif + #else + // We hope other compilers aren't lying about C11/_Generic support + #define MPACK_HAS_GENERIC 1 + #endif + #endif + #endif +#endif + +#ifndef MPACK_HAS_GENERIC + #define MPACK_HAS_GENERIC 0 +#endif + + + +/* + * Finite Math + * + * -ffinite-math-only, included in -ffast-math, breaks functions that + * that check for non-finite real values such as isnan() and isinf(). + * + * We should use this to trap errors when reading data that contains + * non-finite reals. This isn't currently implemented. + */ + +#ifndef MPACK_FINITE_MATH +#if defined(__FINITE_MATH_ONLY__) && __FINITE_MATH_ONLY__ +#define MPACK_FINITE_MATH 1 +#endif +#endif + +#ifndef MPACK_FINITE_MATH +#define MPACK_FINITE_MATH 0 +#endif + + + /* * Endianness checks * @@ -446,13 +1587,17 @@ MPACK_HEADER_START #endif #endif -#elif defined(_MSC_VER) && defined(_WIN32) && !MPACK_NO_BUILTINS +#elif defined(_MSC_VER) && defined(_WIN32) && MPACK_STDLIB && !MPACK_NO_BUILTINS // On Windows, we assume x86 and x86_64 are always little-endian. // We make no assumptions about ARM even though all current // Windows Phone devices are little-endian in case Microsoft's // compiler is ever used with a big-endian ARM device. + // These are functions in so we depend on MPACK_STDLIB. + // It's not clear if these are actually faster than just doing the + // swap manually; maybe we shouldn't bother with this. + #if defined(_M_IX86) || defined(_M_X64) || defined(_M_AMD64) #define MPACK_NHSWAP16(x) _byteswap_ushort(x) #define MPACK_NHSWAP32(x) _byteswap_ulong(x) @@ -518,104 +1663,131 @@ MPACK_HEADER_START */ #if MPACK_DEBUG - MPACK_NORETURN(void mpack_assert_fail(const char* message)); + MPACK_NORETURN(void mpack_assert_fail_wrapper(const char* message)); #if MPACK_STDIO MPACK_NORETURN(void mpack_assert_fail_format(const char* format, ...)); - #define mpack_assert_fail_at(line, file, expr, ...) \ - mpack_assert_fail_format("mpack assertion failed at " file ":" #line "\n" expr "\n" __VA_ARGS__) + #define mpack_assert_fail_at(line, file, exprstr, format, ...) \ + MPACK_EXPAND(mpack_assert_fail_format("mpack assertion failed at " file ":" #line "\n%s\n" format, exprstr, __VA_ARGS__)) #else - #define mpack_assert_fail_at(line, file, ...) \ - mpack_assert_fail("mpack assertion failed at " file ":" #line ) + #define mpack_assert_fail_at(line, file, exprstr, format, ...) \ + mpack_assert_fail_wrapper("mpack assertion failed at " file ":" #line "\n" exprstr "\n") #endif - #define mpack_assert_fail_pos(line, file, expr, ...) mpack_assert_fail_at(line, file, expr, __VA_ARGS__) - #define mpack_assert(expr, ...) ((!(expr)) ? mpack_assert_fail_pos(__LINE__, __FILE__, #expr, __VA_ARGS__) : (void)0) + #define mpack_assert_fail_pos(line, file, exprstr, expr, ...) \ + MPACK_EXPAND(mpack_assert_fail_at(line, file, exprstr, __VA_ARGS__)) + + // This contains a workaround to the pedantic C99 requirement of having at + // least one argument to a variadic macro. The first argument is the + // boolean expression, the optional second argument (if provided) must be a + // literal format string, and any additional arguments are the format + // argument list. + // + // Unfortunately this means macros are expanded in the expression before it + // gets stringified. I haven't found a workaround to this. + // + // This adds two unused arguments to the format argument list when a + // format string is provided, so this would complicate the use of + // -Wformat and __attribute__((__format__)) on mpack_assert_fail_format() + // if we ever bothered to implement it. + #define mpack_assert(...) \ + MPACK_EXPAND(((!(MPACK_EXTRACT_ARG0(__VA_ARGS__))) ? \ + mpack_assert_fail_pos(__LINE__, __FILE__, MPACK_STRINGIFY_ARG0(__VA_ARGS__) , __VA_ARGS__ , "", NULL) : \ + (void)0)) void mpack_break_hit(const char* message); #if MPACK_STDIO void mpack_break_hit_format(const char* format, ...); #define mpack_break_hit_at(line, file, ...) \ - mpack_break_hit_format("mpack breakpoint hit at " file ":" #line "\n" __VA_ARGS__) + MPACK_EXPAND(mpack_break_hit_format("mpack breakpoint hit at " file ":" #line "\n" __VA_ARGS__)) #else #define mpack_break_hit_at(line, file, ...) \ mpack_break_hit("mpack breakpoint hit at " file ":" #line ) #endif - #define mpack_break_hit_pos(line, file, ...) mpack_break_hit_at(line, file, __VA_ARGS__) - #define mpack_break(...) mpack_break_hit_pos(__LINE__, __FILE__, __VA_ARGS__) + #define mpack_break_hit_pos(line, file, ...) MPACK_EXPAND(mpack_break_hit_at(line, file, __VA_ARGS__)) + #define mpack_break(...) MPACK_EXPAND(mpack_break_hit_pos(__LINE__, __FILE__, __VA_ARGS__)) #else - #define mpack_assert(expr, ...) ((!(expr)) ? MPACK_UNREACHABLE, (void)0 : (void)0) + #define mpack_assert(...) \ + (MPACK_EXPAND((!(MPACK_EXTRACT_ARG0(__VA_ARGS__))) ? \ + (MPACK_UNREACHABLE, (void)0) : \ + (void)0)) #define mpack_break(...) ((void)0) #endif -/* Wrap some needed libc functions */ +// make sure we don't use the stdlib directly during development +#if MPACK_STDLIB && defined(MPACK_UNIT_TESTS) && MPACK_INTERNAL && defined(__GNUC__) + #undef memcmp + #undef memcpy + #undef memmove + #undef memset + #undef strlen + #undef malloc + #undef calloc + #undef realloc + #undef free + #pragma GCC poison memcmp + #pragma GCC poison memcpy + #pragma GCC poison memmove + #pragma GCC poison memset + #pragma GCC poison strlen + #pragma GCC poison malloc + #pragma GCC poison calloc + #pragma GCC poison realloc + #pragma GCC poison free +#endif -#if MPACK_STDLIB - #define mpack_memcmp memcmp - #define mpack_memcpy memcpy - #define mpack_memmove memmove - #define mpack_memset memset - #define mpack_strlen strlen - #if defined(MPACK_UNIT_TESTS) && MPACK_INTERNAL && defined(__GNUC__) - // make sure we don't use the stdlib directly during development - #pragma GCC poison memcmp - #pragma GCC poison memcpy - #pragma GCC poison memmove - #pragma GCC poison memset - #pragma GCC poison strlen - #pragma GCC poison malloc - #pragma GCC poison free - #endif -#elif defined(__GNUC__) && !MPACK_NO_BUILTINS - // there's not always a builtin memmove for GCC, - // and we don't have a way to test for it - #define mpack_memcmp __builtin_memcmp - #define mpack_memcpy __builtin_memcpy - #define mpack_memset __builtin_memset - #define mpack_strlen __builtin_strlen +// If we don't have these stdlib functions, we need to define them ourselves. +// Either way we give them a lowercase name to make the code a bit nicer. -#elif defined(__clang__) && defined(__has_builtin) && !MPACK_NO_BUILTINS - #if __has_builtin(__builtin_memcmp) - #define mpack_memcmp __builtin_memcmp - #endif - #if __has_builtin(__builtin_memcpy) - #define mpack_memcpy __builtin_memcpy - #endif - #if __has_builtin(__builtin_memmove) - #define mpack_memmove __builtin_memmove - #endif - #if __has_builtin(__builtin_memset) - #define mpack_memset __builtin_memset - #endif - #if __has_builtin(__builtin_strlen) - #define mpack_strlen __builtin_strlen - #endif +#ifdef MPACK_MEMCMP + #define mpack_memcmp MPACK_MEMCMP +#else + int mpack_memcmp(const void* s1, const void* s2, size_t n); #endif -#ifndef mpack_memcmp -int mpack_memcmp(const void* s1, const void* s2, size_t n); +#ifdef MPACK_MEMCPY + #define mpack_memcpy MPACK_MEMCPY +#else + void* mpack_memcpy(void* MPACK_RESTRICT s1, const void* MPACK_RESTRICT s2, size_t n); #endif -#ifndef mpack_memcpy -void* mpack_memcpy(void* MPACK_RESTRICT s1, const void* MPACK_RESTRICT s2, size_t n); + +#ifdef MPACK_MEMMOVE + #define mpack_memmove MPACK_MEMMOVE +#else + void* mpack_memmove(void* s1, const void* s2, size_t n); #endif -#ifndef mpack_memmove -void* mpack_memmove(void* s1, const void* s2, size_t n); + +#ifdef MPACK_MEMSET + #define mpack_memset MPACK_MEMSET +#else + void* mpack_memset(void* s, int c, size_t n); #endif -#ifndef mpack_memset -void* mpack_memset(void* s, int c, size_t n); + +#ifdef MPACK_STRLEN + #define mpack_strlen MPACK_STRLEN +#else + size_t mpack_strlen(const char* s); #endif -#ifndef mpack_strlen -size_t mpack_strlen(const char* s); + + + +#if MPACK_STDIO + #if defined(WIN32) + #define mpack_snprintf _snprintf + #else + #define mpack_snprintf snprintf + #endif #endif /* Debug logging */ #if 0 - #define mpack_log(...) printf(__VA_ARGS__); + #include + #define mpack_log(...) (MPACK_EXPAND(printf(__VA_ARGS__)), fflush(stdout)) #else #define mpack_log(...) ((void)0) #endif @@ -623,18 +1795,6 @@ size_t mpack_strlen(const char* s); /* Make sure our configuration makes sense */ -#if defined(MPACK_MALLOC) && !defined(MPACK_FREE) - #error "MPACK_MALLOC requires MPACK_FREE." -#endif -#if !defined(MPACK_MALLOC) && defined(MPACK_FREE) - #error "MPACK_FREE requires MPACK_MALLOC." -#endif -#if MPACK_READ_TRACKING && !defined(MPACK_READER) - #error "MPACK_READ_TRACKING requires MPACK_READER." -#endif -#if MPACK_WRITE_TRACKING && !defined(MPACK_WRITER) - #error "MPACK_WRITE_TRACKING requires MPACK_WRITER." -#endif #ifndef MPACK_MALLOC #if MPACK_STDIO #error "MPACK_STDIO requires preprocessor definitions for MPACK_MALLOC and MPACK_FREE." @@ -663,13 +1823,12 @@ size_t mpack_strlen(const char* s); +/** @endcond */ /** * @} */ -MPACK_HEADER_END - -/** @endcond */ +MPACK_EXTERN_C_END +MPACK_SILENCE_WARNINGS_END #endif - diff --git a/src/mpack/mpack-reader.c b/src/mpack/mpack-reader.c index 07fa58c..dfac4c2 100644 --- a/src/mpack/mpack-reader.c +++ b/src/mpack/mpack-reader.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -23,6 +23,8 @@ #include "mpack-reader.h" +MPACK_SILENCE_WARNINGS_BEGIN + #if MPACK_READER static void mpack_reader_skip_using_fill(mpack_reader_t* reader, size_t count); @@ -33,7 +35,8 @@ void mpack_reader_init(mpack_reader_t* reader, char* buffer, size_t size, size_t mpack_memset(reader, 0, sizeof(*reader)); reader->buffer = buffer; reader->size = size; - reader->left = count; + reader->data = buffer; + reader->end = buffer + count; #if MPACK_READ_TRACKING mpack_reader_flag_if_error(reader, mpack_track_init(&reader->track)); @@ -55,17 +58,8 @@ void mpack_reader_init_data(mpack_reader_t* reader, const char* data, size_t cou mpack_assert(data != NULL, "data is NULL"); mpack_memset(reader, 0, sizeof(*reader)); - reader->left = count; - - // unfortunately we have to cast away the const to store the buffer, - // but we won't be modifying it because there's no fill function. - // the buffer size is left at 0 to ensure no fill function can be - // set or used (see mpack_reader_set_fill().) - #ifdef __cplusplus - reader->buffer = const_cast(data); - #else - reader->buffer = (char*)data; - #endif + reader->data = data; + reader->end = data + count; #if MPACK_READ_TRACKING mpack_reader_flag_if_error(reader, mpack_track_init(&reader->track)); @@ -97,20 +91,18 @@ void mpack_reader_set_fill(mpack_reader_t* reader, mpack_reader_fill_t fill) { void mpack_reader_set_skip(mpack_reader_t* reader, mpack_reader_skip_t skip) { mpack_assert(reader->size != 0, "cannot use skip function without a writeable buffer!"); - #if MPACK_OPTIMIZE_FOR_SIZE - MPACK_UNUSED(reader); - MPACK_UNUSED(skip); - #else reader->skip = skip; - #endif } #if MPACK_STDIO static size_t mpack_file_reader_fill(mpack_reader_t* reader, char* buffer, size_t count) { + if (feof((FILE *)reader->context)) { + mpack_reader_flag_error(reader, mpack_error_eof); + return 0; + } return fread((void*)buffer, 1, count, (FILE*)reader->context); } -#if !MPACK_OPTIMIZE_FOR_SIZE static void mpack_file_reader_skip(mpack_reader_t* reader, size_t count) { if (mpack_reader_error(reader) != mpack_ok) return; @@ -132,48 +124,61 @@ static void mpack_file_reader_skip(mpack_reader_t* reader, size_t count) { // If the stream is not seekable, fall back to the fill function. mpack_reader_skip_using_fill(reader, count); } -#endif static void mpack_file_reader_teardown(mpack_reader_t* reader) { + MPACK_FREE(reader->buffer); + reader->buffer = NULL; + reader->context = NULL; + reader->size = 0; + reader->fill = NULL; + reader->skip = NULL; + reader->teardown = NULL; +} + +static void mpack_file_reader_teardown_close(mpack_reader_t* reader) { FILE* file = (FILE*)reader->context; if (file) { int ret = fclose(file); - reader->context = NULL; if (ret != 0) mpack_reader_flag_error(reader, mpack_error_io); } - MPACK_FREE(reader->buffer); - reader->buffer = NULL; - reader->size = 0; - reader->fill = NULL; + mpack_file_reader_teardown(reader); } -void mpack_reader_init_file(mpack_reader_t* reader, const char* filename) { - mpack_assert(filename != NULL, "filename is NULL"); +void mpack_reader_init_stdfile(mpack_reader_t* reader, FILE* file, bool close_when_done) { + mpack_assert(file != NULL, "file is NULL"); size_t capacity = MPACK_BUFFER_SIZE; char* buffer = (char*)MPACK_MALLOC(capacity); if (buffer == NULL) { mpack_reader_init_error(reader, mpack_error_memory); + if (close_when_done) { + fclose(file); + } return; } + mpack_reader_init(reader, buffer, capacity, 0); + mpack_reader_set_context(reader, file); + mpack_reader_set_fill(reader, mpack_file_reader_fill); + mpack_reader_set_skip(reader, mpack_file_reader_skip); + mpack_reader_set_teardown(reader, close_when_done ? + mpack_file_reader_teardown_close : + mpack_file_reader_teardown); +} + +void mpack_reader_init_filename(mpack_reader_t* reader, const char* filename) { + mpack_assert(filename != NULL, "filename is NULL"); + FILE* file = fopen(filename, "rb"); if (file == NULL) { - MPACK_FREE(buffer); mpack_reader_init_error(reader, mpack_error_io); return; } - mpack_reader_init(reader, buffer, capacity, 0); - mpack_reader_set_context(reader, file); - mpack_reader_set_fill(reader, mpack_file_reader_fill); - #if !MPACK_OPTIMIZE_FOR_SIZE - mpack_reader_set_skip(reader, mpack_file_reader_skip); - #endif - mpack_reader_set_teardown(reader, mpack_file_reader_teardown); + mpack_reader_init_stdfile(reader, file, true); } #endif @@ -201,53 +206,55 @@ size_t mpack_reader_remaining(mpack_reader_t* reader, const char** data) { #endif if (data) - *data = reader->buffer + reader->pos; - return reader->left; + *data = reader->data; + return (size_t)(reader->end - reader->data); } void mpack_reader_flag_error(mpack_reader_t* reader, mpack_error_t error) { - mpack_log("reader %p setting error %i: %s\n", reader, (int)error, mpack_error_to_string(error)); + mpack_log("reader %p setting error %i: %s\n", (void*)reader, (int)error, mpack_error_to_string(error)); if (reader->error == mpack_ok) { reader->error = error; - reader->left = 0; + reader->end = reader->data; if (reader->error_fn) reader->error_fn(reader, error); } } -// A helper to call the reader fill function. This makes sure it's -// implemented and guards against overflow in case it returns -1. -static size_t mpack_fill(mpack_reader_t* reader, char* p, size_t count) { - mpack_assert(reader->fill != NULL, "mpack_fill() called with no fill function?"); - - size_t ret = reader->fill(reader, p, count); - if (ret == ((size_t)(-1))) - return 0; - - return ret; -} +// Loops on the fill function, reading between the minimum and +// maximum number of bytes and flagging an error if it fails. +MPACK_NOINLINE static size_t mpack_fill_range(mpack_reader_t* reader, char* p, size_t min_bytes, size_t max_bytes) { + mpack_assert(reader->fill != NULL, "mpack_fill_range() called with no fill function?"); + mpack_assert(min_bytes > 0, "cannot fill zero bytes!"); + mpack_assert(max_bytes >= min_bytes, "min_bytes %i cannot be larger than max_bytes %i!", + (int)min_bytes, (int)max_bytes); + + size_t count = 0; + while (count < min_bytes) { + size_t read = reader->fill(reader, p + count, max_bytes - count); + + // Reader fill functions can flag an error or return 0 on failure. We + // also guard against functions that return -1 just in case. + if (mpack_reader_error(reader) != mpack_ok) + return 0; + if (read == 0 || read == ((size_t)(-1))) { + mpack_reader_flag_error(reader, mpack_error_io); + return 0; + } -// Fills the buffer when there is already some data in the buffer. The -// existing data is moved to the start of the buffer. -static void mpack_partial_fill(mpack_reader_t* reader) { - mpack_memmove(reader->buffer, reader->buffer + reader->pos, reader->left); - reader->pos = 0; - reader->left += mpack_fill(reader, reader->buffer + reader->left, reader->size - reader->left); + count += read; + } + return count; } -bool mpack_reader_ensure_straddle(mpack_reader_t* reader, size_t count) { +MPACK_NOINLINE bool mpack_reader_ensure_straddle(mpack_reader_t* reader, size_t count) { mpack_assert(count != 0, "cannot ensure zero bytes!"); mpack_assert(reader->error == mpack_ok, "reader cannot be in an error state!"); - if (count <= reader->left) { - mpack_assert(0, - "big ensure requested for %i bytes, but there are %i bytes " - "left in buffer. call mpack_reader_ensure() instead", - (int)count, (int)reader->left); - mpack_reader_flag_error(reader, mpack_error_bug); - return false; - } + mpack_assert(count > (size_t)(reader->end - reader->data), + "straddling ensure requested for %i bytes, but there are %i bytes " + "left in buffer. call mpack_reader_ensure() instead", + (int)count, (int)(reader->end - reader->data)); // we'll need a fill function to get more data. if there's no // fill function, the buffer should contain an entire MessagePack @@ -258,23 +265,33 @@ bool mpack_reader_ensure_straddle(mpack_reader_t* reader, size_t count) { return false; } - mpack_assert(count <= reader->size, "cannot ensure byte count %i larger than buffer size %i", - (int)count, (int)reader->size); - - // re-fill as much as possible - mpack_partial_fill(reader); - - if (reader->left < count) { - mpack_reader_flag_error(reader, mpack_error_io); + // we need enough space in the buffer. if the buffer is not + // big enough, we return mpack_error_too_big (since this is + // for an in-place read larger than the buffer size.) + if (count > reader->size) { + mpack_reader_flag_error(reader, mpack_error_too_big); return false; } + // move the existing data to the start of the buffer + size_t left = (size_t)(reader->end - reader->data); + mpack_memmove(reader->buffer, reader->data, left); + reader->end -= reader->data - reader->buffer; + reader->data = reader->buffer; + + // read at least the necessary number of bytes, accepting up to the + // buffer size + size_t read = mpack_fill_range(reader, reader->buffer + left, + count - left, reader->size - left); + if (mpack_reader_error(reader) != mpack_ok) + return false; + reader->end += read; return true; } // Reads count bytes into p. Used when there are not enough bytes // left in the buffer to satisfy a read. -void mpack_read_native_big(mpack_reader_t* reader, char* p, size_t count) { +MPACK_NOINLINE void mpack_read_native_straddle(mpack_reader_t* reader, char* p, size_t count) { mpack_assert(count == 0 || p != NULL, "data pointer for %i bytes is NULL", (int)count); if (mpack_reader_error(reader) != mpack_ok) { @@ -282,14 +299,15 @@ void mpack_read_native_big(mpack_reader_t* reader, char* p, size_t count) { return; } + size_t left = (size_t)(reader->end - reader->data); mpack_log("big read for %i bytes into %p, %i left in buffer, buffer size %i\n", - (int)count, p, (int)reader->left, (int)reader->size); + (int)count, p, (int)left, (int)reader->size); - if (count <= reader->left) { + if (count <= left) { mpack_assert(0, "big read requested for %i bytes, but there are %i bytes " "left in buffer. call mpack_read_native() instead", - (int)count, (int)reader->left); + (int)count, (int)left); mpack_reader_flag_error(reader, mpack_error_bug); mpack_memset(p, 0, count); return; @@ -317,65 +335,38 @@ void mpack_read_native_big(mpack_reader_t* reader, char* p, size_t count) { } // flush what's left of the buffer - if (reader->left > 0) { - mpack_log("flushing %i bytes remaining in buffer\n", (int)reader->left); - mpack_memcpy(p, reader->buffer + reader->pos, reader->left); - count -= reader->left; - p += reader->left; - reader->pos += reader->left; - reader->left = 0; + if (left > 0) { + mpack_log("flushing %i bytes remaining in buffer\n", (int)left); + mpack_memcpy(p, reader->data, left); + count -= left; + p += left; + reader->data += left; } - // we read only in multiples of the buffer size. read the middle portion, if any - size_t middle = count - (count % reader->size); - if (middle > 0) { - mpack_log("reading %i bytes in middle\n", (int)middle); - if (mpack_fill(reader, p, middle) < middle) { - mpack_reader_flag_error(reader, mpack_error_io); - mpack_memset(p, 0, count); - return; - } - count -= middle; - p += middle; - if (count == 0) + // if the remaining data needed is some small fraction of the + // buffer size, we'll try to fill the buffer as much as possible + // and copy the needed data out. + if (count <= reader->size / MPACK_READER_SMALL_FRACTION_DENOMINATOR) { + size_t read = mpack_fill_range(reader, reader->buffer, count, reader->size); + if (mpack_reader_error(reader) != mpack_ok) return; + mpack_memcpy(p, reader->buffer, count); + reader->data = reader->buffer + count; + reader->end = reader->buffer + read; + + // otherwise we read the remaining data directly into the target. + } else { + mpack_log("reading %i additional bytes\n", (int)count); + mpack_fill_range(reader, p, count, count); } - - // fill the buffer - reader->pos = 0; - reader->left = mpack_fill(reader, reader->buffer, reader->size); - mpack_log("filled %i bytes into buffer\n", (int)reader->left); - if (reader->left < count) { - mpack_reader_flag_error(reader, mpack_error_io); - mpack_memset(p, 0, count); - return; - } - - // serve the remainder - mpack_log("serving %i remaining bytes from %p to %p\n", (int)count, reader->buffer+reader->pos,p); - mpack_memcpy(p, reader->buffer + reader->pos, count); - reader->pos += count; - reader->left -= count; } -void mpack_skip_bytes(mpack_reader_t* reader, size_t count) { - if (mpack_reader_error(reader) != mpack_ok) - return; - mpack_log("skip requested for %i bytes\n", (int)count); - mpack_reader_track_bytes(reader, count); - - // check if we have enough in the buffer already - if (reader->left >= count) { - mpack_log("skipping %i bytes still in buffer\n", (int)count); - reader->left -= count; - reader->pos += count; - return; - } +MPACK_NOINLINE static void mpack_skip_bytes_straddle(mpack_reader_t* reader, size_t count) { // we'll need at least a fill function to skip more data. if there's // no fill function, the buffer should contain an entire MessagePack // object, so we raise mpack_error_invalid instead of mpack_error_io - // on truncated data. (see mpack_read_native_big()) + // on truncated data. (see mpack_read_native_straddle()) if (reader->fill == NULL) { mpack_log("reader has no fill function!\n"); mpack_reader_flag_error(reader, mpack_error_invalid); @@ -383,12 +374,11 @@ void mpack_skip_bytes(mpack_reader_t* reader, size_t count) { } // discard whatever's left in the buffer - mpack_log("discarding %i bytes still in buffer\n", (int)reader->left); - count -= reader->left; - reader->pos += reader->left; - reader->left = 0; + size_t left = (size_t)(reader->end - reader->data); + mpack_log("discarding %i bytes still in buffer\n", (int)left); + count -= left; + reader->data = reader->end; - #if !MPACK_OPTIMIZE_FOR_SIZE // use the skip function if we've got one, and if we're trying // to skip a lot of data. if we only need to skip some tiny // fraction of the buffer size, it's probably better to just @@ -398,21 +388,38 @@ void mpack_skip_bytes(mpack_reader_t* reader, size_t count) { reader->skip(reader, count); return; } - #endif mpack_reader_skip_using_fill(reader, count); } -static void mpack_reader_skip_using_fill(mpack_reader_t* reader, size_t count) { +void mpack_skip_bytes(mpack_reader_t* reader, size_t count) { + if (mpack_reader_error(reader) != mpack_ok) + return; + mpack_log("skip requested for %i bytes\n", (int)count); + + mpack_reader_track_bytes(reader, count); + + // check if we have enough in the buffer already + size_t left = (size_t)(reader->end - reader->data); + if (left >= count) { + mpack_log("skipping %" PRIu32 " bytes still in buffer\n", (uint32_t)count); + reader->data += count; + return; + } + + mpack_skip_bytes_straddle(reader, count); +} + +MPACK_NOINLINE static void mpack_reader_skip_using_fill(mpack_reader_t* reader, size_t count) { mpack_assert(reader->fill != NULL, "missing fill function!"); - mpack_assert(reader->left == 0, "there are bytes left in the buffer!"); + mpack_assert(reader->data == reader->end, "there are bytes left in the buffer!"); mpack_assert(reader->error == mpack_ok, "should not have called this in an error state (%i)", reader->error); mpack_log("skip using fill for %i bytes\n", (int)count); // fill and discard multiples of the buffer size while (count > reader->size) { mpack_log("filling and discarding buffer of %i bytes\n", (int)reader->size); - if (mpack_fill(reader, reader->buffer, reader->size) < reader->size) { + if (mpack_fill_range(reader, reader->buffer, reader->size, reader->size) < reader->size) { mpack_reader_flag_error(reader, mpack_error_io); return; } @@ -420,15 +427,15 @@ static void mpack_reader_skip_using_fill(mpack_reader_t* reader, size_t count) { } // fill the buffer as much as possible - reader->pos = 0; - reader->left = mpack_fill(reader, reader->buffer, reader->size); - if (reader->left < count) { + reader->data = reader->buffer; + size_t read = mpack_fill_range(reader, reader->buffer, count, reader->size); + if (read < count) { mpack_reader_flag_error(reader, mpack_error_io); return; } - mpack_log("filled %i bytes into buffer; discarding %i bytes\n", (int)reader->left, (int)count); - reader->pos += count; - reader->left -= count; + reader->end = reader->data + read; + mpack_log("filled %i bytes into buffer; discarding %i bytes\n", (int)read, (int)count); + reader->data += count; } void mpack_read_bytes(mpack_reader_t* reader, char* p, size_t count) { @@ -533,41 +540,6 @@ char* mpack_read_bytes_alloc_impl(mpack_reader_t* reader, size_t count, bool nul } #endif -// internal inplace reader for when it straddles the end of the buffer -static const char* mpack_read_bytes_inplace_big(mpack_reader_t* reader, size_t count) { - - // we should only arrive here from inplace straddle; this should already be checked - mpack_assert(mpack_reader_error(reader) == mpack_ok, "already in error state? %s", - mpack_error_to_string(mpack_reader_error(reader))); - mpack_assert(reader->left < count, "already enough bytes in buffer: %i left, %i count", (int)reader->left, (int)count); - - // we'll need a fill function to get more data. if there's no - // fill function, the buffer should contain an entire MessagePack - // object, so we raise mpack_error_invalid instead of mpack_error_io - // on truncated data. - if (reader->fill == NULL) { - mpack_reader_flag_error(reader, mpack_error_invalid); - return NULL; - } - - // make sure the buffer is big enough to actually fit the data - if (count > reader->size) { - mpack_reader_flag_error(reader, mpack_error_too_big); - return NULL; - } - - // re-fill as much as possible - mpack_partial_fill(reader); - - if (reader->left < count) { - mpack_reader_flag_error(reader, mpack_error_io); - return NULL; - } - reader->pos += count; - reader->left -= count; - return reader->buffer; -} - // read inplace without tracking (since there are different // tracking modes for different inplace readers) static const char* mpack_read_bytes_inplace_notrack(mpack_reader_t* reader, size_t count) { @@ -575,13 +547,18 @@ static const char* mpack_read_bytes_inplace_notrack(mpack_reader_t* reader, size return NULL; // if we have enough bytes already in the buffer, we can return it directly. - if (reader->left >= count) { - reader->pos += count; - reader->left -= count; - return reader->buffer + reader->pos - count; + if ((size_t)(reader->end - reader->data) >= count) { + const char* bytes = reader->data; + reader->data += count; + return bytes; } - return mpack_read_bytes_inplace_big(reader, count); + if (!mpack_reader_ensure(reader, count)) + return NULL; + + const char* bytes = reader->data; + reader->data += count; + return bytes; } const char* mpack_read_bytes_inplace(mpack_reader_t* reader, size_t count) { @@ -601,14 +578,12 @@ const char* mpack_read_utf8_inplace(mpack_reader_t* reader, size_t count) { return str; } -// Decodes a tag from a byte buffer. The size of the bytes buffer -// must be at least MPACK_MINIMUM_TAG_SIZE. static size_t mpack_parse_tag(mpack_reader_t* reader, mpack_tag_t* tag) { mpack_assert(reader->error == mpack_ok, "reader cannot be in an error state!"); if (!mpack_reader_ensure(reader, 1)) return 0; - uint8_t type = mpack_load_u8(reader->buffer + reader->pos); + uint8_t type = mpack_load_u8(reader->data); // unfortunately, by far the fastest way to parse a tag is to switch // on the first byte, and to explicitly list every possible byte. so for @@ -623,32 +598,27 @@ static size_t mpack_parse_tag(mpack_reader_t* reader, mpack_tag_t* tag) { // positive fixnum case 0x0: case 0x1: case 0x2: case 0x3: case 0x4: case 0x5: case 0x6: case 0x7: - tag->type = mpack_type_uint; - tag->v.u = type; + *tag = mpack_tag_make_uint(type); return 1; // negative fixnum case 0xe: case 0xf: - tag->type = mpack_type_int; - tag->v.i = (int32_t)(int8_t)type; + *tag = mpack_tag_make_int((int8_t)type); return 1; // fixmap case 0x8: - tag->type = mpack_type_map; - tag->v.n = type & ~0xf0; + *tag = mpack_tag_make_map(type & ~0xf0u); return 1; // fixarray case 0x9: - tag->type = mpack_type_array; - tag->v.n = type & ~0xf0; + *tag = mpack_tag_make_array(type & ~0xf0u); return 1; // fixstr case 0xa: case 0xb: - tag->type = mpack_type_str; - tag->v.l = type & ~0xe0; + *tag = mpack_tag_make_str(type & ~0xe0u); return 1; // not one of the common infix types @@ -679,8 +649,7 @@ static size_t mpack_parse_tag(mpack_reader_t* reader, mpack_tag_t* tag) { case 0x68: case 0x69: case 0x6a: case 0x6b: case 0x6c: case 0x6d: case 0x6e: case 0x6f: case 0x70: case 0x71: case 0x72: case 0x73: case 0x74: case 0x75: case 0x76: case 0x77: case 0x78: case 0x79: case 0x7a: case 0x7b: case 0x7c: case 0x7d: case 0x7e: case 0x7f: - tag->type = mpack_type_uint; - tag->v.u = type; + *tag = mpack_tag_make_uint(type); return 1; // negative fixnum @@ -688,22 +657,19 @@ static size_t mpack_parse_tag(mpack_reader_t* reader, mpack_tag_t* tag) { case 0xe8: case 0xe9: case 0xea: case 0xeb: case 0xec: case 0xed: case 0xee: case 0xef: case 0xf0: case 0xf1: case 0xf2: case 0xf3: case 0xf4: case 0xf5: case 0xf6: case 0xf7: case 0xf8: case 0xf9: case 0xfa: case 0xfb: case 0xfc: case 0xfd: case 0xfe: case 0xff: - tag->type = mpack_type_int; - tag->v.i = (int8_t)type; + *tag = mpack_tag_make_int((int8_t)type); return 1; // fixmap case 0x80: case 0x81: case 0x82: case 0x83: case 0x84: case 0x85: case 0x86: case 0x87: case 0x88: case 0x89: case 0x8a: case 0x8b: case 0x8c: case 0x8d: case 0x8e: case 0x8f: - tag->type = mpack_type_map; - tag->v.n = type & ~0xf0; + *tag = mpack_tag_make_map(type & ~0xf0u); return 1; // fixarray case 0x90: case 0x91: case 0x92: case 0x93: case 0x94: case 0x95: case 0x96: case 0x97: case 0x98: case 0x99: case 0x9a: case 0x9b: case 0x9c: case 0x9d: case 0x9e: case 0x9f: - tag->type = mpack_type_array; - tag->v.n = type & ~0xf0; + *tag = mpack_tag_make_array(type & ~0xf0u); return 1; // fixstr @@ -711,268 +677,256 @@ static size_t mpack_parse_tag(mpack_reader_t* reader, mpack_tag_t* tag) { case 0xa8: case 0xa9: case 0xaa: case 0xab: case 0xac: case 0xad: case 0xae: case 0xaf: case 0xb0: case 0xb1: case 0xb2: case 0xb3: case 0xb4: case 0xb5: case 0xb6: case 0xb7: case 0xb8: case 0xb9: case 0xba: case 0xbb: case 0xbc: case 0xbd: case 0xbe: case 0xbf: - tag->type = mpack_type_str; - tag->v.l = type & ~0xe0; + *tag = mpack_tag_make_str(type & ~0xe0u); return 1; #endif // nil case 0xc0: - tag->type = mpack_type_nil; + *tag = mpack_tag_make_nil(); return 1; // bool case 0xc2: case 0xc3: - tag->type = mpack_type_bool; - tag->v.b = type & 1; + *tag = mpack_tag_make_bool((bool)(type & 1)); return 1; // bin8 case 0xc4: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_BIN8)) return 0; - tag->type = mpack_type_bin; - tag->v.l = mpack_load_u8(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_bin(mpack_load_u8(reader->data + 1)); return MPACK_TAG_SIZE_BIN8; // bin16 case 0xc5: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_BIN16)) return 0; - tag->type = mpack_type_bin; - tag->v.l = mpack_load_u16(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_bin(mpack_load_u16(reader->data + 1)); return MPACK_TAG_SIZE_BIN16; // bin32 case 0xc6: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_BIN32)) return 0; - tag->type = mpack_type_bin; - tag->v.l = mpack_load_u32(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_bin(mpack_load_u32(reader->data + 1)); return MPACK_TAG_SIZE_BIN32; + #if MPACK_EXTENSIONS // ext8 case 0xc7: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_EXT8)) return 0; - tag->type = mpack_type_ext; - tag->v.l = mpack_load_u8(reader->buffer + reader->pos + 1); - tag->exttype = mpack_load_i8(reader->buffer + reader->pos + 2); + *tag = mpack_tag_make_ext(mpack_load_i8(reader->data + 2), mpack_load_u8(reader->data + 1)); return MPACK_TAG_SIZE_EXT8; // ext16 case 0xc8: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_EXT16)) return 0; - tag->type = mpack_type_ext; - tag->v.l = mpack_load_u16(reader->buffer + reader->pos + 1); - tag->exttype = mpack_load_i8(reader->buffer + reader->pos + 3); + *tag = mpack_tag_make_ext(mpack_load_i8(reader->data + 3), mpack_load_u16(reader->data + 1)); return MPACK_TAG_SIZE_EXT16; // ext32 case 0xc9: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_EXT32)) return 0; - tag->type = mpack_type_ext; - tag->v.l = mpack_load_u32(reader->buffer + reader->pos + 1); - tag->exttype = mpack_load_i8(reader->buffer + reader->pos + 5); + *tag = mpack_tag_make_ext(mpack_load_i8(reader->data + 5), mpack_load_u32(reader->data + 1)); return MPACK_TAG_SIZE_EXT32; + #endif // float case 0xca: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_FLOAT)) return 0; - tag->type = mpack_type_float; - tag->v.f = mpack_load_float(reader->buffer + reader->pos + 1); + #if MPACK_FLOAT + *tag = mpack_tag_make_float(mpack_load_float(reader->data + 1)); + #else + *tag = mpack_tag_make_raw_float(mpack_load_u32(reader->data + 1)); + #endif return MPACK_TAG_SIZE_FLOAT; // double case 0xcb: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_DOUBLE)) return 0; - tag->type = mpack_type_double; - tag->v.d = mpack_load_double(reader->buffer + reader->pos + 1); + #if MPACK_DOUBLE + *tag = mpack_tag_make_double(mpack_load_double(reader->data + 1)); + #else + *tag = mpack_tag_make_raw_double(mpack_load_u64(reader->data + 1)); + #endif return MPACK_TAG_SIZE_DOUBLE; // uint8 case 0xcc: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_U8)) return 0; - tag->type = mpack_type_uint; - tag->v.u = mpack_load_u8(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_uint(mpack_load_u8(reader->data + 1)); return MPACK_TAG_SIZE_U8; // uint16 case 0xcd: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_U16)) return 0; - tag->type = mpack_type_uint; - tag->v.u = mpack_load_u16(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_uint(mpack_load_u16(reader->data + 1)); return MPACK_TAG_SIZE_U16; // uint32 case 0xce: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_U32)) return 0; - tag->type = mpack_type_uint; - tag->v.u = mpack_load_u32(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_uint(mpack_load_u32(reader->data + 1)); return MPACK_TAG_SIZE_U32; // uint64 case 0xcf: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_U64)) return 0; - tag->type = mpack_type_uint; - tag->v.u = mpack_load_u64(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_uint(mpack_load_u64(reader->data + 1)); return MPACK_TAG_SIZE_U64; // int8 case 0xd0: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_I8)) return 0; - tag->type = mpack_type_int; - tag->v.i = mpack_load_i8(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_int(mpack_load_i8(reader->data + 1)); return MPACK_TAG_SIZE_I8; // int16 case 0xd1: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_I16)) return 0; - tag->type = mpack_type_int; - tag->v.i = mpack_load_i16(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_int(mpack_load_i16(reader->data + 1)); return MPACK_TAG_SIZE_I16; // int32 case 0xd2: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_I32)) return 0; - tag->type = mpack_type_int; - tag->v.i = mpack_load_i32(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_int(mpack_load_i32(reader->data + 1)); return MPACK_TAG_SIZE_I32; // int64 case 0xd3: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_I64)) return 0; - tag->type = mpack_type_int; - tag->v.i = mpack_load_i64(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_int(mpack_load_i64(reader->data + 1)); return MPACK_TAG_SIZE_I64; + #if MPACK_EXTENSIONS // fixext1 case 0xd4: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_FIXEXT1)) return 0; - tag->type = mpack_type_ext; - tag->v.l = 1; - tag->exttype = mpack_load_i8(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_ext(mpack_load_i8(reader->data + 1), 1); return MPACK_TAG_SIZE_FIXEXT1; // fixext2 case 0xd5: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_FIXEXT2)) return 0; - tag->type = mpack_type_ext; - tag->v.l = 2; - tag->exttype = mpack_load_i8(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_ext(mpack_load_i8(reader->data + 1), 2); return MPACK_TAG_SIZE_FIXEXT2; // fixext4 case 0xd6: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_FIXEXT4)) return 0; - tag->type = mpack_type_ext; - tag->v.l = 4; - tag->exttype = mpack_load_i8(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_ext(mpack_load_i8(reader->data + 1), 4); return 2; // fixext8 case 0xd7: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_FIXEXT8)) return 0; - tag->type = mpack_type_ext; - tag->v.l = 8; - tag->exttype = mpack_load_i8(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_ext(mpack_load_i8(reader->data + 1), 8); return MPACK_TAG_SIZE_FIXEXT8; // fixext16 case 0xd8: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_FIXEXT16)) return 0; - tag->type = mpack_type_ext; - tag->v.l = 16; - tag->exttype = mpack_load_i8(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_ext(mpack_load_i8(reader->data + 1), 16); return MPACK_TAG_SIZE_FIXEXT16; + #endif // str8 case 0xd9: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_STR8)) return 0; - tag->type = mpack_type_str; - tag->v.l = mpack_load_u8(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_str(mpack_load_u8(reader->data + 1)); return MPACK_TAG_SIZE_STR8; // str16 case 0xda: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_STR16)) return 0; - tag->type = mpack_type_str; - tag->v.l = mpack_load_u16(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_str(mpack_load_u16(reader->data + 1)); return MPACK_TAG_SIZE_STR16; // str32 case 0xdb: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_STR32)) return 0; - tag->type = mpack_type_str; - tag->v.l = mpack_load_u32(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_str(mpack_load_u32(reader->data + 1)); return MPACK_TAG_SIZE_STR32; // array16 case 0xdc: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_ARRAY16)) return 0; - tag->type = mpack_type_array; - tag->v.n = mpack_load_u16(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_array(mpack_load_u16(reader->data + 1)); return MPACK_TAG_SIZE_ARRAY16; // array32 case 0xdd: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_ARRAY32)) return 0; - tag->type = mpack_type_array; - tag->v.n = mpack_load_u32(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_array(mpack_load_u32(reader->data + 1)); return MPACK_TAG_SIZE_ARRAY32; // map16 case 0xde: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_MAP16)) return 0; - tag->type = mpack_type_map; - tag->v.n = mpack_load_u16(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_map(mpack_load_u16(reader->data + 1)); return MPACK_TAG_SIZE_MAP16; // map32 case 0xdf: if (!mpack_reader_ensure(reader, MPACK_TAG_SIZE_MAP32)) return 0; - tag->type = mpack_type_map; - tag->v.n = mpack_load_u32(reader->buffer + reader->pos + 1); + *tag = mpack_tag_make_map(mpack_load_u32(reader->data + 1)); return MPACK_TAG_SIZE_MAP32; // reserved case 0xc1: - break; + mpack_reader_flag_error(reader, mpack_error_invalid); + return 0; + + #if !MPACK_EXTENSIONS + // ext + case 0xc7: // fallthrough + case 0xc8: // fallthrough + case 0xc9: // fallthrough + // fixext + case 0xd4: // fallthrough + case 0xd5: // fallthrough + case 0xd6: // fallthrough + case 0xd7: // fallthrough + case 0xd8: + mpack_reader_flag_error(reader, mpack_error_unsupported); + return 0; + #endif #if MPACK_OPTIMIZE_FOR_SIZE // any other bytes should have been handled by the infix switch default: - mpack_assert(0, "unreachable"); break; #endif } - // unrecognized type - mpack_reader_flag_error(reader, mpack_error_invalid); + mpack_assert(0, "unreachable"); return 0; } @@ -985,8 +939,7 @@ mpack_tag_t mpack_read_tag(mpack_reader_t* reader) { if (mpack_reader_track_element(reader) != mpack_ok) return mpack_tag_nil(); - mpack_tag_t tag; - mpack_memset(&tag, 0, sizeof(tag)); + mpack_tag_t tag = MPACK_TAG_ZERO; size_t count = mpack_parse_tag(reader, &tag); if (count == 0) return mpack_tag_nil(); @@ -997,12 +950,14 @@ mpack_tag_t mpack_read_tag(mpack_reader_t* reader) { switch (tag.type) { case mpack_type_map: case mpack_type_array: - track_error = mpack_track_push(&reader->track, tag.type, tag.v.l); + track_error = mpack_track_push(&reader->track, tag.type, tag.v.n); break; + #if MPACK_EXTENSIONS + case mpack_type_ext: + #endif case mpack_type_str: case mpack_type_bin: - case mpack_type_ext: - track_error = mpack_track_push(&reader->track, tag.type, tag.v.n); + track_error = mpack_track_push(&reader->track, tag.type, tag.v.l); break; default: break; @@ -1014,11 +969,7 @@ mpack_tag_t mpack_read_tag(mpack_reader_t* reader) { } #endif - // the tag is guaranteed to have been read out of - // the buffer, so we advance past it - reader->pos += count; - reader->left -= count; - + reader->data += count; return tag; } @@ -1031,8 +982,7 @@ mpack_tag_t mpack_peek_tag(mpack_reader_t* reader) { if (mpack_reader_track_peek_element(reader) != mpack_ok) return mpack_tag_nil(); - mpack_tag_t tag; - mpack_memset(&tag, 0, sizeof(tag)); + mpack_tag_t tag = MPACK_TAG_ZERO; if (mpack_parse_tag(reader, &tag) == 0) return mpack_tag_nil(); return tag; @@ -1051,10 +1001,12 @@ void mpack_discard(mpack_reader_t* reader) { mpack_skip_bytes(reader, var.v.l); mpack_done_bin(reader); break; + #if MPACK_EXTENSIONS case mpack_type_ext: mpack_skip_bytes(reader, var.v.l); mpack_done_ext(reader); break; + #endif case mpack_type_array: { for (; var.v.n > 0; --var.v.n) { mpack_discard(reader); @@ -1079,134 +1031,257 @@ void mpack_discard(mpack_reader_t* reader) { } } -#if MPACK_READ_TRACKING -void mpack_done_type(mpack_reader_t* reader, mpack_type_t type) { - if (mpack_reader_error(reader) == mpack_ok) - mpack_reader_flag_if_error(reader, mpack_track_pop(&reader->track, type)); -} -#endif +#if MPACK_EXTENSIONS +mpack_timestamp_t mpack_read_timestamp(mpack_reader_t* reader, size_t size) { + mpack_timestamp_t timestamp = {0, 0}; -#if MPACK_STDIO -static void mpack_print_element(mpack_reader_t* reader, size_t depth, FILE* file) { - mpack_tag_t val = mpack_read_tag(reader); + if (size != 4 && size != 8 && size != 12) { + mpack_reader_flag_error(reader, mpack_error_invalid); + return timestamp; + } + + char buf[12]; + mpack_read_bytes(reader, buf, size); + mpack_done_ext(reader); if (mpack_reader_error(reader) != mpack_ok) - return; - switch (val.type) { + return timestamp; - case mpack_type_nil: - fprintf(file, "null"); - break; - case mpack_type_bool: - fprintf(file, val.v.b ? "true" : "false"); + switch (size) { + case 4: + timestamp.seconds = (int64_t)(uint64_t)mpack_load_u32(buf); break; - case mpack_type_float: - fprintf(file, "%f", val.v.f); - break; - case mpack_type_double: - fprintf(file, "%f", val.v.d); + case 8: { + uint64_t packed = mpack_load_u64(buf); + timestamp.seconds = (int64_t)(packed & ((MPACK_UINT64_C(1) << 34) - 1)); + timestamp.nanoseconds = (uint32_t)(packed >> 34); break; + } - case mpack_type_int: - fprintf(file, "%" PRIi64, val.v.i); - break; - case mpack_type_uint: - fprintf(file, "%" PRIu64, val.v.u); + case 12: + timestamp.nanoseconds = mpack_load_u32(buf); + timestamp.seconds = mpack_load_i64(buf + 4); break; - case mpack_type_bin: - fprintf(file, "", val.v.l); - mpack_skip_bytes(reader, val.v.l); - mpack_done_bin(reader); + default: + mpack_assert(false, "unreachable"); break; + } - case mpack_type_ext: - fprintf(file, "", val.exttype, val.v.l); - mpack_skip_bytes(reader, val.v.l); - mpack_done_ext(reader); - break; + if (timestamp.nanoseconds > MPACK_TIMESTAMP_NANOSECONDS_MAX) { + mpack_reader_flag_error(reader, mpack_error_invalid); + mpack_timestamp_t zero = {0, 0}; + return zero; + } + return timestamp; +} +#endif + +#if MPACK_READ_TRACKING +void mpack_done_type(mpack_reader_t* reader, mpack_type_t type) { + if (mpack_reader_error(reader) == mpack_ok) + mpack_reader_flag_if_error(reader, mpack_track_pop(&reader->track, type)); +} +#endif + +#if MPACK_DEBUG && MPACK_STDIO +static size_t mpack_print_read_prefix(mpack_reader_t* reader, size_t length, char* buffer, size_t buffer_size) { + if (length == 0) + return 0; + + size_t read = (length < buffer_size) ? length : buffer_size; + mpack_read_bytes(reader, buffer, read); + if (mpack_reader_error(reader) != mpack_ok) + return 0; + + mpack_skip_bytes(reader, length - read); + return read; +} + +static void mpack_print_element(mpack_reader_t* reader, mpack_print_t* print, size_t depth) { + mpack_tag_t val = mpack_read_tag(reader); + if (mpack_reader_error(reader) != mpack_ok) + return; + + // We read some bytes from bin and ext so we can print its prefix in hex. + char buffer[MPACK_PRINT_BYTE_COUNT]; + size_t count = 0; + size_t i, j; + + switch (val.type) { case mpack_type_str: - putc('"', file); - for (size_t i = 0; i < val.v.l; ++i) { + mpack_print_append_cstr(print, "\""); + for (i = 0; i < val.v.l; ++i) { char c; mpack_read_bytes(reader, &c, 1); if (mpack_reader_error(reader) != mpack_ok) return; switch (c) { - case '\n': fprintf(file, "\\n"); break; - case '\\': fprintf(file, "\\\\"); break; - case '"': fprintf(file, "\\\""); break; - default: putc(c, file); break; + case '\n': mpack_print_append_cstr(print, "\\n"); break; + case '\\': mpack_print_append_cstr(print, "\\\\"); break; + case '"': mpack_print_append_cstr(print, "\\\""); break; + default: mpack_print_append(print, &c, 1); break; } } - putc('"', file); + mpack_print_append_cstr(print, "\""); mpack_done_str(reader); - break; + return; case mpack_type_array: - fprintf(file, "[\n"); - for (size_t i = 0; i < val.v.n; ++i) { - for (size_t j = 0; j < depth + 1; ++j) - fprintf(file, " "); - mpack_print_element(reader, depth + 1, file); + mpack_print_append_cstr(print, "[\n"); + for (i = 0; i < val.v.n; ++i) { + for (j = 0; j < depth + 1; ++j) + mpack_print_append_cstr(print, " "); + mpack_print_element(reader, print, depth + 1); if (mpack_reader_error(reader) != mpack_ok) return; if (i != val.v.n - 1) - putc(',', file); - putc('\n', file); + mpack_print_append_cstr(print, ","); + mpack_print_append_cstr(print, "\n"); } - for (size_t i = 0; i < depth; ++i) - fprintf(file, " "); - putc(']', file); + for (i = 0; i < depth; ++i) + mpack_print_append_cstr(print, " "); + mpack_print_append_cstr(print, "]"); mpack_done_array(reader); - break; + return; case mpack_type_map: - fprintf(file, "{\n"); - for (size_t i = 0; i < val.v.n; ++i) { - for (size_t j = 0; j < depth + 1; ++j) - fprintf(file, " "); - mpack_print_element(reader, depth + 1, file); + mpack_print_append_cstr(print, "{\n"); + for (i = 0; i < val.v.n; ++i) { + for (j = 0; j < depth + 1; ++j) + mpack_print_append_cstr(print, " "); + mpack_print_element(reader, print, depth + 1); if (mpack_reader_error(reader) != mpack_ok) return; - fprintf(file, ": "); - mpack_print_element(reader, depth + 1, file); + mpack_print_append_cstr(print, ": "); + mpack_print_element(reader, print, depth + 1); if (mpack_reader_error(reader) != mpack_ok) return; if (i != val.v.n - 1) - putc(',', file); - putc('\n', file); + mpack_print_append_cstr(print, ","); + mpack_print_append_cstr(print, "\n"); } - for (size_t i = 0; i < depth; ++i) - fprintf(file, " "); - putc('}', file); + for (i = 0; i < depth; ++i) + mpack_print_append_cstr(print, " "); + mpack_print_append_cstr(print, "}"); mpack_done_map(reader); + return; + + // The above cases return so as not to print a pseudo-json value. The + // below cases break and print pseudo-json. + + case mpack_type_bin: + count = mpack_print_read_prefix(reader, mpack_tag_bin_length(&val), buffer, sizeof(buffer)); + mpack_done_bin(reader); + break; + + #if MPACK_EXTENSIONS + case mpack_type_ext: + count = mpack_print_read_prefix(reader, mpack_tag_ext_length(&val), buffer, sizeof(buffer)); + mpack_done_ext(reader); + break; + #endif + + default: break; } + + char buf[256]; + mpack_tag_debug_pseudo_json(val, buf, sizeof(buf), buffer, count); + mpack_print_append_cstr(print, buf); } -void mpack_print_file(const char* data, size_t len, FILE* file) { - mpack_assert(data != NULL, "data is NULL"); - mpack_assert(file != NULL, "file is NULL"); +static void mpack_print_and_destroy(mpack_reader_t* reader, mpack_print_t* print, size_t depth) { + size_t i; + for (i = 0; i < depth; ++i) + mpack_print_append_cstr(print, " "); + mpack_print_element(reader, print, depth); + + size_t remaining = mpack_reader_remaining(reader, NULL); + + char buf[256]; + if (mpack_reader_destroy(reader) != mpack_ok) { + mpack_snprintf(buf, sizeof(buf), "\n", mpack_error_to_string(mpack_reader_error(reader))); + buf[sizeof(buf) - 1] = '\0'; + mpack_print_append_cstr(print, buf); + } else if (remaining > 0) { + mpack_snprintf(buf, sizeof(buf), "\n<%i extra bytes at end of message>", (int)remaining); + buf[sizeof(buf) - 1] = '\0'; + mpack_print_append_cstr(print, buf); + } +} +static void mpack_print_data(const char* data, size_t len, mpack_print_t* print, size_t depth) { mpack_reader_t reader; mpack_reader_init_data(&reader, data, len); + mpack_print_and_destroy(&reader, print, depth); +} - int depth = 2; - for (int i = 0; i < depth; ++i) - fprintf(file, " "); - mpack_print_element(&reader, depth, file); - putc('\n', file); +void mpack_print_data_to_buffer(const char* data, size_t data_size, char* buffer, size_t buffer_size) { + if (buffer_size == 0) { + mpack_assert(false, "buffer size is zero!"); + return; + } + + mpack_print_t print; + mpack_memset(&print, 0, sizeof(print)); + print.buffer = buffer; + print.size = buffer_size; + mpack_print_data(data, data_size, &print, 0); + mpack_print_append(&print, "", 1); // null-terminator + mpack_print_flush(&print); + + // we always make sure there's a null-terminator at the end of the buffer + // in case we ran out of space. + print.buffer[print.size - 1] = '\0'; +} + +void mpack_print_data_to_callback(const char* data, size_t size, mpack_print_callback_t callback, void* context) { + char buffer[1024]; + mpack_print_t print; + mpack_memset(&print, 0, sizeof(print)); + print.buffer = buffer; + print.size = sizeof(buffer); + print.callback = callback; + print.context = context; + mpack_print_data(data, size, &print, 0); + mpack_print_flush(&print); +} - size_t remaining = mpack_reader_remaining(&reader, NULL); +void mpack_print_data_to_file(const char* data, size_t len, FILE* file) { + mpack_assert(data != NULL, "data is NULL"); + mpack_assert(file != NULL, "file is NULL"); + + char buffer[1024]; + mpack_print_t print; + mpack_memset(&print, 0, sizeof(print)); + print.buffer = buffer; + print.size = sizeof(buffer); + print.callback = &mpack_print_file_callback; + print.context = file; + + mpack_print_data(data, len, &print, 2); + mpack_print_append_cstr(&print, "\n"); + mpack_print_flush(&print); +} + +void mpack_print_stdfile_to_callback(FILE* file, mpack_print_callback_t callback, void* context) { + char buffer[1024]; + mpack_print_t print; + mpack_memset(&print, 0, sizeof(print)); + print.buffer = buffer; + print.size = sizeof(buffer); + print.callback = callback; + print.context = context; - if (mpack_reader_destroy(&reader) != mpack_ok) - fprintf(file, "\n", mpack_error_to_string(mpack_reader_error(&reader))); - else if (remaining > 0) - fprintf(file, "<%i extra bytes at end of mpack>\n", (int)remaining); + mpack_reader_t reader; + mpack_reader_init_stdfile(&reader, file, false); + mpack_print_and_destroy(&reader, &print, 0); + mpack_print_flush(&print); } #endif #endif +MPACK_SILENCE_WARNINGS_END diff --git a/src/mpack/mpack-reader.h b/src/mpack/mpack-reader.h index 96dd01c..092e2ba 100644 --- a/src/mpack/mpack-reader.h +++ b/src/mpack/mpack-reader.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -30,7 +30,8 @@ #include "mpack-common.h" -MPACK_HEADER_START +MPACK_SILENCE_WARNINGS_BEGIN +MPACK_EXTERN_C_BEGIN #if MPACK_READER @@ -38,12 +39,24 @@ MPACK_HEADER_START struct mpack_track_t; #endif +// The denominator to determine whether a read is a small +// fraction of the buffer size. +#define MPACK_READER_SMALL_FRACTION_DENOMINATOR 32 + /** - * @defgroup reader Core Reader API + * @defgroup reader Reader API + * + * The MPack Reader API contains functions for imperatively reading dynamically + * typed data from a MessagePack stream. * - * The MPack Core Reader API contains functions for imperatively reading - * dynamically typed data from a MessagePack stream. This forms the basis - * of the Expect and Node APIs. + * See @ref docs/reader.md for examples. + * + * @note If you are not writing code for an embedded device (or otherwise do + * not need maximum performance with minimal memory usage), you should not use + * this. You probably want to use the @link node Node API@endlink instead. + * + * This forms the basis of the @link expect Expect API@endlink, which can be + * used to interpret the stream of elements in expected types and value ranges. * * @{ */ @@ -71,10 +84,21 @@ struct mpack_track_t; typedef struct mpack_reader_t mpack_reader_t; /** - * The MPack reader's fill function. It should fill the buffer as - * much as possible, returning the number of bytes put into the buffer. + * The MPack reader's fill function. It should fill the buffer with at + * least one byte and at most the given @c count, returning the number + * of bytes written to the buffer. * - * In case of error, it should flag an appropriate error on the reader. + * In case of error, it should flag an appropriate error on the reader + * (usually @ref mpack_error_io), or simply return zero. If zero is + * returned, mpack_error_io is raised. + * + * @note When reading from a stream, you should only copy and return + * the bytes that are immediately available. It is always safe to return + * less than the requested count as long as some non-zero number of bytes + * are read; if more bytes are needed, the read function will simply be + * called again. + * + * @see mpack_reader_context() */ typedef size_t (*mpack_reader_fill_t)(mpack_reader_t* reader, char* buffer, size_t count); @@ -83,6 +107,8 @@ typedef size_t (*mpack_reader_fill_t)(mpack_reader_t* reader, char* buffer, size * of bytes from the source (for example by seeking forward.) * * In case of error, it should flag an appropriate error on the reader. + * + * @see mpack_reader_context() */ typedef void (*mpack_reader_skip_t)(mpack_reader_t* reader, size_t count); @@ -125,17 +151,14 @@ struct mpack_reader_t { mpack_reader_fill_t fill; /* Function to read bytes into the buffer */ mpack_reader_error_t error_fn; /* Function to call on error */ mpack_reader_teardown_t teardown; /* Function to teardown the context on destroy */ - - // The skip function may be unused, but we don't preproc it - // out on MPACK_OPTIMIZE_FOR_SIZE in case calling code is compiled - // with different optimization options. MPACK_OPTIMIZE_FOR_SIZE - // should never be used in header files. mpack_reader_skip_t skip; /* Function to skip bytes from the source */ - char* buffer; /* Byte buffer */ - size_t size; /* Size of the buffer, or zero if it's const */ - size_t left; /* How many bytes are left in the buffer */ - size_t pos; /* Position within the buffer */ + char* buffer; /* Writeable byte buffer */ + size_t size; /* Size of the buffer */ + + const char* data; /* Current data pointer (in the buffer, if it is used) */ + const char* end; /* The end of available data (in the buffer, if it is used) */ + mpack_error_t error; /* Error state */ #if MPACK_READ_TRACKING @@ -181,8 +204,35 @@ void mpack_reader_init_data(mpack_reader_t* reader, const char* data, size_t cou #if MPACK_STDIO /** * Initializes an MPack reader that reads from a file. + * + * The file will be automatically opened and closed by the reader. + */ +void mpack_reader_init_filename(mpack_reader_t* reader, const char* filename); + +/** + * Deprecated. + * + * \deprecated Renamed to mpack_reader_init_filename(). */ -void mpack_reader_init_file(mpack_reader_t* reader, const char* filename); +MPACK_INLINE void mpack_reader_init_file(mpack_reader_t* reader, const char* filename) { + mpack_reader_init_filename(reader, filename); +} + +/** + * Initializes an MPack reader that reads from a libc FILE. This can be used to + * read from stdin, or from a file opened separately. + * + * @param reader The MPack reader. + * @param stdfile The FILE. + * @param close_when_done If true, fclose() will be called on the FILE when it + * is no longer needed. If false, the file will not be closed when + * reading is done. + * + * @warning The reader is buffered. It will read data in advance of parsing it, + * and it may read more data than it parsed. See mpack_reader_remaining() to + * access the extra data. + */ +void mpack_reader_init_stdfile(mpack_reader_t* reader, FILE* stdfile, bool close_when_done); #endif /** @@ -238,11 +288,24 @@ mpack_error_t mpack_reader_destroy(mpack_reader_t* reader); * * @param reader The MPack reader. * @param context User data to pass to the reader callbacks. + * + * @see mpack_reader_context() */ MPACK_INLINE void mpack_reader_set_context(mpack_reader_t* reader, void* context) { reader->context = context; } +/** + * Returns the custom context for reader callbacks. + * + * @see mpack_reader_set_context + * @see mpack_reader_set_fill + * @see mpack_reader_set_skip + */ +MPACK_INLINE void* mpack_reader_context(mpack_reader_t* reader) { + return reader->context; +} + /** * Sets the fill function to refill the data buffer when it runs out of data. * @@ -380,18 +443,18 @@ size_t mpack_reader_remaining(mpack_reader_t* reader, const char** data); /** * Reads a MessagePack object header (an MPack tag.) * - * If an error occurs, the mpack_reader_t is placed in an error state and - * a nil tag is returned. If the reader is already in an error state, a - * nil tag is returned. + * If an error occurs, the reader is placed in an error state and a + * nil tag is returned. If the reader is already in an error state, + * a nil tag is returned. * * If the type is compound (i.e. is a map, array, string, binary or * extension type), additional reads are required to get the contained * data, and the corresponding done function must be called when done. * - * Note that maps in JSON are unordered, so it is recommended not to expect + * @note Maps in JSON are unordered, so it is recommended not to expect * a specific ordering for your map values in case your data is converted * to/from JSON. - * + * * @see mpack_read_bytes() * @see mpack_done_array() * @see mpack_done_map() @@ -405,11 +468,11 @@ mpack_tag_t mpack_read_tag(mpack_reader_t* reader); * Parses the next MessagePack object header (an MPack tag) without * advancing the reader. * - * If an error occurs, the mpack_reader_t is placed in an error state and - * a nil tag is returned. If the reader is already in an error state, a - * nil tag is returned. + * If an error occurs, the reader is placed in an error state and a + * nil tag is returned. If the reader is already in an error state, + * a nil tag is returned. * - * Note that maps in JSON are unordered, so it is recommended not to expect + * @note Maps in JSON are unordered, so it is recommended not to expect * a specific ordering for your map values in case your data is converted * to/from JSON. * @@ -637,15 +700,35 @@ const char* mpack_read_utf8_inplace(mpack_reader_t* reader, size_t count); * in the buffer. * * Use this if you're expecting arbitrary size data, and you want to read - * in-place where possible but will fall back to a normal read if the data - * is too large. + * in-place for the best performance when possible but will fall back to + * a normal read if the data is too large. * * @see mpack_read_bytes_inplace() */ MPACK_INLINE bool mpack_should_read_bytes_inplace(mpack_reader_t* reader, size_t count) { - return (reader->size == 0 || count <= reader->size / 32); + return (reader->size == 0 || count <= reader->size / MPACK_READER_SMALL_FRACTION_DENOMINATOR); } +#if MPACK_EXTENSIONS +/** + * Reads a timestamp contained in an ext object of the given size, closing the + * ext type. + * + * An ext object of exttype @ref MPACK_EXTTYPE_TIMESTAMP must have been opened + * by a call to e.g. mpack_read_tag() or mpack_expect_ext(). + * + * You must NOT call mpack_done_ext() after calling this. A timestamp ext + * object can only contain a single timestamp value, so this calls + * mpack_done_ext() automatically. + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @throws mpack_error_invalid if the size is not one of the supported + * timestamp sizes, or if the nanoseconds are out of range. + */ +mpack_timestamp_t mpack_read_timestamp(mpack_reader_t* reader, size_t size); +#endif + /** * @} */ @@ -712,16 +795,20 @@ MPACK_INLINE void mpack_done_bin(mpack_reader_t* reader) { mpack_done_type(reader, mpack_type_bin); } +#if MPACK_EXTENSIONS /** * @fn mpack_done_ext(mpack_reader_t* reader) * * Finishes reading an extended type binary data blob. * * This will track reads to ensure that the correct number of bytes are read. + * + * @note This requires @ref MPACK_EXTENSIONS. */ MPACK_INLINE void mpack_done_ext(mpack_reader_t* reader) { mpack_done_type(reader, mpack_type_ext); } +#endif /** * Reads and discards the next object. This will read and discard all @@ -733,33 +820,73 @@ void mpack_discard(mpack_reader_t* reader); * @} */ -#if MPACK_STDIO +/** @cond */ + +#if MPACK_DEBUG && MPACK_STDIO /** * @name Debugging Functions * @{ */ +/* + * Converts a blob of MessagePack to a pseudo-JSON string for debugging + * purposes, placing the result in the given buffer with a null-terminator. + * + * If the buffer does not have enough space, the result will be truncated (but + * it is guaranteed to be null-terminated.) + * + * This is only available in debug mode, and only if stdio is available (since + * it uses snprintf().) It's strictly for debugging purposes. + */ +void mpack_print_data_to_buffer(const char* data, size_t data_size, char* buffer, size_t buffer_size); -/** +/* + * Converts a node to pseudo-JSON for debugging purposes, calling the given + * callback as many times as is necessary to output the character data. + * + * No null-terminator or trailing newline will be written. + * + * This is only available in debug mode, and only if stdio is available (since + * it uses snprintf().) It's strictly for debugging purposes. + */ +void mpack_print_data_to_callback(const char* data, size_t size, mpack_print_callback_t callback, void* context); + +/* * Converts a blob of MessagePack to pseudo-JSON for debugging purposes * and pretty-prints it to the given file. */ -void mpack_print_file(const char* data, size_t len, FILE* file); +void mpack_print_data_to_file(const char* data, size_t len, FILE* file); -#ifndef MPACK_GCOV -/** +/* * Converts a blob of MessagePack to pseudo-JSON for debugging purposes * and pretty-prints it to stdout. */ +MPACK_INLINE void mpack_print_data_to_stdout(const char* data, size_t len) { + mpack_print_data_to_file(data, len, stdout); +} + +/* + * Converts the MessagePack contained in the given `FILE*` to pseudo-JSON for + * debugging purposes, calling the given callback as many times as is necessary + * to output the character data. + */ +void mpack_print_stdfile_to_callback(FILE* file, mpack_print_callback_t callback, void* context); + +/* + * Deprecated. + * + * \deprecated Renamed to mpack_print_data_to_stdout(). + */ MPACK_INLINE void mpack_print(const char* data, size_t len) { - mpack_print_file(data, len, stdout); + mpack_print_data_to_stdout(data, len); } -#endif /** * @} */ #endif +/** @endcond */ + /** * @} */ @@ -770,37 +897,32 @@ MPACK_INLINE void mpack_print(const char* data, size_t len) { bool mpack_reader_ensure_straddle(mpack_reader_t* reader, size_t count); -// Ensures there are at least count bytes left in the buffer. This will -// flag an error if there is not enough data, and will assert if there -// is a fill function and count is larger than the buffer size. Returns -// true if there are enough bytes, false otherwise. Error handling must -// be done separately! The reader cannot be in an error state when this -// is called. +/* + * Ensures there are at least @c count bytes left in the + * data, raising an error and returning false if more + * data cannot be made available. + */ MPACK_INLINE bool mpack_reader_ensure(mpack_reader_t* reader, size_t count) { mpack_assert(count != 0, "cannot ensure zero bytes!"); - mpack_assert(count <= MPACK_READER_MINIMUM_BUFFER_SIZE, - "cannot ensure %i bytes, this is more than the minimum buffer size %i!", count, - (int)count, (int)MPACK_READER_MINIMUM_BUFFER_SIZE); mpack_assert(reader->error == mpack_ok, "reader cannot be in an error state!"); - if (count <= reader->left) + if (count <= (size_t)(reader->end - reader->data)) return true; return mpack_reader_ensure_straddle(reader, count); } -void mpack_read_native_big(mpack_reader_t* reader, char* p, size_t count); +void mpack_read_native_straddle(mpack_reader_t* reader, char* p, size_t count); -// Reads count bytes into p, deferring to mpack_read_native_big() if more +// Reads count bytes into p, deferring to mpack_read_native_straddle() if more // bytes are needed than are available in the buffer. MPACK_INLINE void mpack_read_native(mpack_reader_t* reader, char* p, size_t count) { mpack_assert(count == 0 || p != NULL, "data pointer for %i bytes is NULL", (int)count); - if (count > reader->left) { - mpack_read_native_big(reader, p, count); + if (count > (size_t)(reader->end - reader->data)) { + mpack_read_native_straddle(reader, p, count); } else { - mpack_memcpy(p, reader->buffer + reader->pos, count); - reader->pos += count; - reader->left -= count; + mpack_memcpy(p, reader->data, count); + reader->data += count; } } @@ -819,12 +941,12 @@ MPACK_INLINE mpack_error_t mpack_reader_track_peek_element(mpack_reader_t* reade return MPACK_READER_TRACK(reader, mpack_track_peek_element(&reader->track, true)); } -MPACK_INLINE mpack_error_t mpack_reader_track_bytes(mpack_reader_t* reader, uint64_t count) { +MPACK_INLINE mpack_error_t mpack_reader_track_bytes(mpack_reader_t* reader, size_t count) { MPACK_UNUSED(count); return MPACK_READER_TRACK(reader, mpack_track_bytes(&reader->track, true, count)); } -MPACK_INLINE mpack_error_t mpack_reader_track_str_bytes_all(mpack_reader_t* reader, uint64_t count) { +MPACK_INLINE mpack_error_t mpack_reader_track_str_bytes_all(mpack_reader_t* reader, size_t count) { MPACK_UNUSED(count); return MPACK_READER_TRACK(reader, mpack_track_str_bytes_all(&reader->track, true, count)); } @@ -835,7 +957,8 @@ MPACK_INLINE mpack_error_t mpack_reader_track_str_bytes_all(mpack_reader_t* read #endif -MPACK_HEADER_END +MPACK_EXTERN_C_END +MPACK_SILENCE_WARNINGS_END #endif diff --git a/src/mpack/mpack-writer.c b/src/mpack/mpack-writer.c index 02b1384..029b065 100644 --- a/src/mpack/mpack-writer.c +++ b/src/mpack/mpack-writer.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -23,27 +23,38 @@ #include "mpack-writer.h" +MPACK_SILENCE_WARNINGS_BEGIN + #if MPACK_WRITER +#if MPACK_BUILDER +static void mpack_builder_flush(mpack_writer_t* writer); +#endif + #if MPACK_WRITE_TRACKING static void mpack_writer_flag_if_error(mpack_writer_t* writer, mpack_error_t error) { if (error != mpack_ok) mpack_writer_flag_error(writer, error); } -void mpack_writer_track_push(mpack_writer_t* writer, mpack_type_t type, uint64_t count) { +void mpack_writer_track_push(mpack_writer_t* writer, mpack_type_t type, uint32_t count) { if (writer->error == mpack_ok) mpack_writer_flag_if_error(writer, mpack_track_push(&writer->track, type, count)); } +void mpack_writer_track_push_builder(mpack_writer_t* writer, mpack_type_t type) { + if (writer->error == mpack_ok) + mpack_writer_flag_if_error(writer, mpack_track_push_builder(&writer->track, type)); +} + void mpack_writer_track_pop(mpack_writer_t* writer, mpack_type_t type) { if (writer->error == mpack_ok) mpack_writer_flag_if_error(writer, mpack_track_pop(&writer->track, type)); } -void mpack_writer_track_element(mpack_writer_t* writer) { +void mpack_writer_track_pop_builder(mpack_writer_t* writer, mpack_type_t type) { if (writer->error == mpack_ok) - mpack_writer_flag_if_error(writer, mpack_track_element(&writer->track, false)); + mpack_writer_flag_if_error(writer, mpack_track_pop_builder(&writer->track, type)); } void mpack_writer_track_bytes(mpack_writer_t* writer, size_t count) { @@ -52,27 +63,70 @@ void mpack_writer_track_bytes(mpack_writer_t* writer, size_t count) { } #endif +// This should probably be renamed. It's not solely used for tracking. +static inline void mpack_writer_track_element(mpack_writer_t* writer) { + (void)writer; + + #if MPACK_WRITE_TRACKING + if (writer->error == mpack_ok) + mpack_writer_flag_if_error(writer, mpack_track_element(&writer->track, false)); + #endif + + #if MPACK_BUILDER + if (writer->builder.current_build != NULL) { + mpack_build_t* build = writer->builder.current_build; + // We only track this write if it's not nested within another non-build + // map or array. + if (build->nested_compound_elements == 0) { + if (build->type != mpack_type_map) { + ++build->count; + mpack_log("adding element to build %p, now %" PRIu32 " elements\n", (void*)build, build->count); + } else if (build->key_needs_value) { + build->key_needs_value = false; + ++build->count; + } else { + build->key_needs_value = true; + } + } + } + #endif +} + static void mpack_writer_clear(mpack_writer_t* writer) { + #if MPACK_COMPATIBILITY + writer->version = mpack_version_current; + #endif writer->flush = NULL; writer->error_fn = NULL; writer->teardown = NULL; writer->context = NULL; writer->buffer = NULL; - writer->size = 0; - writer->used = 0; + writer->position = NULL; + writer->end = NULL; writer->error = mpack_ok; #if MPACK_WRITE_TRACKING mpack_memset(&writer->track, 0, sizeof(writer->track)); #endif + + #if MPACK_BUILDER + writer->builder.current_build = NULL; + writer->builder.latest_build = NULL; + writer->builder.current_page = NULL; + writer->builder.pages = NULL; + writer->builder.stash_buffer = NULL; + writer->builder.stash_position = NULL; + writer->builder.stash_end = NULL; + #endif } void mpack_writer_init(mpack_writer_t* writer, char* buffer, size_t size) { mpack_assert(buffer != NULL, "cannot initialize writer with empty buffer"); mpack_writer_clear(writer); writer->buffer = buffer; - writer->size = size; + writer->position = buffer; + writer->end = writer->buffer + size; #if MPACK_WRITE_TRACKING mpack_writer_flag_if_error(writer, mpack_track_init(&writer->track)); @@ -93,10 +147,12 @@ void mpack_writer_init_error(mpack_writer_t* writer, mpack_error_t error) { void mpack_writer_set_flush(mpack_writer_t* writer, mpack_writer_flush_t flush) { MPACK_STATIC_ASSERT(MPACK_WRITER_MINIMUM_BUFFER_SIZE >= MPACK_MAXIMUM_TAG_SIZE, "minimum buffer size must fit any tag!"); + MPACK_STATIC_ASSERT(31 + MPACK_TAG_SIZE_FIXSTR >= MPACK_WRITER_MINIMUM_BUFFER_SIZE, + "minimum buffer size must fit the largest possible fixstr!"); - if (writer->size < MPACK_WRITER_MINIMUM_BUFFER_SIZE) { + if (mpack_writer_buffer_size(writer) < MPACK_WRITER_MINIMUM_BUFFER_SIZE) { mpack_break("buffer size is %i, but minimum buffer size for flush is %i", - (int)writer->size, MPACK_WRITER_MINIMUM_BUFFER_SIZE); + (int)mpack_writer_buffer_size(writer), MPACK_WRITER_MINIMUM_BUFFER_SIZE); mpack_writer_flag_error(writer, mpack_error_bug); return; } @@ -110,6 +166,13 @@ typedef struct mpack_growable_writer_t { size_t* target_size; } mpack_growable_writer_t; +static char* mpack_writer_get_reserved(mpack_writer_t* writer) { + // This is in a separate function in order to avoid false strict aliasing + // warnings. We aren't actually violating strict aliasing (the reserved + // space is only ever dereferenced as an mpack_growable_writer_t.) + return (char*)writer->reserved; +} + static void mpack_growable_writer_flush(mpack_writer_t* writer, const char* data, size_t count) { // This is an intrusive flush function which modifies the writer's buffer @@ -128,67 +191,78 @@ static void mpack_growable_writer_flush(mpack_writer_t* writer, const char* data if (data == writer->buffer) { // teardown, do nothing - if (writer->used == count) + if (mpack_writer_buffer_used(writer) == count) return; // otherwise leave the data in the buffer and just grow - writer->used = count; + writer->position = writer->buffer + count; count = 0; } + size_t used = mpack_writer_buffer_used(writer); + size_t size = mpack_writer_buffer_size(writer); + mpack_log("flush size %i used %i data %p buffer %p\n", - (int)count, (int)writer->used, data, writer->buffer); + (int)count, (int)used, data, writer->buffer); - mpack_assert(data == writer->buffer || writer->used + count > writer->size, + mpack_assert(data == writer->buffer || used + count > size, "extra flush for %i but there is %i space left in the buffer! (%i/%i)", - (int)count, (int)writer->size - (int)writer->used, (int)writer->used, (int)writer->size); + (int)count, (int)mpack_writer_buffer_left(writer), (int)used, (int)size); // grow to fit the data // TODO: this really needs to correctly test for overflow - size_t new_size = writer->size * 2; - while (new_size < writer->used + count) + size_t new_size = size * 2; + while (new_size < used + count) new_size *= 2; - mpack_log("flush growing buffer size from %i to %i\n", (int)writer->size, (int)new_size); + mpack_log("flush growing buffer size from %i to %i\n", (int)size, (int)new_size); // grow the buffer - char* new_buffer = (char*)mpack_realloc(writer->buffer, writer->used, new_size); + char* new_buffer = (char*)mpack_realloc(writer->buffer, used, new_size); if (new_buffer == NULL) { mpack_writer_flag_error(writer, mpack_error_memory); return; } + writer->position = new_buffer + used; writer->buffer = new_buffer; - writer->size = new_size; + writer->end = writer->buffer + new_size; // append the extra data if (count > 0) { - mpack_memcpy(writer->buffer + writer->used, data, count); - writer->used += count; + mpack_memcpy(writer->position, data, count); + writer->position += count; } - mpack_log("new buffer %p, used %i\n", new_buffer, (int)writer->used); + mpack_log("new buffer %p, used %i\n", new_buffer, (int)mpack_writer_buffer_used(writer)); } static void mpack_growable_writer_teardown(mpack_writer_t* writer) { - mpack_growable_writer_t* growable_writer = (mpack_growable_writer_t*)writer->context; + mpack_growable_writer_t* growable_writer = (mpack_growable_writer_t*)mpack_writer_get_reserved(writer); if (mpack_writer_error(writer) == mpack_ok) { // shrink the buffer to an appropriate size if the data is // much smaller than the buffer - if (writer->used < writer->size / 2) { - char* buffer = (char*)mpack_realloc(writer->buffer, writer->used, writer->used); + if (mpack_writer_buffer_used(writer) < mpack_writer_buffer_size(writer) / 2) { + size_t used = mpack_writer_buffer_used(writer); + + // We always return a non-null pointer that must be freed, even if + // nothing was written. malloc() and realloc() do not necessarily + // do this so we enforce it ourselves. + size_t size = (used != 0) ? used : 1; + + char* buffer = (char*)mpack_realloc(writer->buffer, used, size); if (!buffer) { MPACK_FREE(writer->buffer); mpack_writer_flag_error(writer, mpack_error_memory); return; } writer->buffer = buffer; - writer->size = writer->used; + writer->end = (writer->position = writer->buffer + used); } *growable_writer->target_data = writer->buffer; - *growable_writer->target_size = writer->used; + *growable_writer->target_size = mpack_writer_buffer_used(writer); writer->buffer = NULL; } else if (writer->buffer) { @@ -199,13 +273,6 @@ static void mpack_growable_writer_teardown(mpack_writer_t* writer) { writer->context = NULL; } -static char* mpack_writer_get_reserved(mpack_writer_t* writer) { - // This is in a separate function in order to avoid false strict aliasing - // warnings. We aren't actually violating strict aliasing (the reserved - // space is only ever dereferenced as an mpack_growable_writer_t.) - return (char*)writer->reserved; -} - void mpack_writer_init_growable(mpack_writer_t* writer, char** target_data, size_t* target_size) { mpack_assert(target_data != NULL, "cannot initialize writer without a destination for the data"); mpack_assert(target_size != NULL, "cannot initialize writer without a destination for the size"); @@ -228,7 +295,6 @@ void mpack_writer_init_growable(mpack_writer_t* writer, char** target_data, size } mpack_writer_init(writer, buffer, capacity); - mpack_writer_set_context(writer, growable_writer); mpack_writer_set_flush(writer, mpack_growable_writer_flush); mpack_writer_set_teardown(writer, mpack_growable_writer_teardown); } @@ -243,45 +309,59 @@ static void mpack_file_writer_flush(mpack_writer_t* writer, const char* buffer, } static void mpack_file_writer_teardown(mpack_writer_t* writer) { + MPACK_FREE(writer->buffer); + writer->buffer = NULL; + writer->context = NULL; +} + +static void mpack_file_writer_teardown_close(mpack_writer_t* writer) { FILE* file = (FILE*)writer->context; if (file) { int ret = fclose(file); - writer->context = NULL; if (ret != 0) mpack_writer_flag_error(writer, mpack_error_io); } - MPACK_FREE(writer->buffer); - writer->buffer = NULL; + mpack_file_writer_teardown(writer); } -void mpack_writer_init_file(mpack_writer_t* writer, const char* filename) { - mpack_assert(filename != NULL, "filename is NULL"); +void mpack_writer_init_stdfile(mpack_writer_t* writer, FILE* file, bool close_when_done) { + mpack_assert(file != NULL, "file is NULL"); size_t capacity = MPACK_BUFFER_SIZE; char* buffer = (char*)MPACK_MALLOC(capacity); if (buffer == NULL) { mpack_writer_init_error(writer, mpack_error_memory); + if (close_when_done) { + fclose(file); + } return; } + mpack_writer_init(writer, buffer, capacity); + mpack_writer_set_context(writer, file); + mpack_writer_set_flush(writer, mpack_file_writer_flush); + mpack_writer_set_teardown(writer, close_when_done ? + mpack_file_writer_teardown_close : + mpack_file_writer_teardown); +} + +void mpack_writer_init_filename(mpack_writer_t* writer, const char* filename) { + mpack_assert(filename != NULL, "filename is NULL"); + FILE* file = fopen(filename, "wb"); if (file == NULL) { - MPACK_FREE(buffer); mpack_writer_init_error(writer, mpack_error_io); return; } - mpack_writer_init(writer, buffer, capacity); - mpack_writer_set_context(writer, file); - mpack_writer_set_flush(writer, mpack_file_writer_flush); - mpack_writer_set_teardown(writer, mpack_file_writer_teardown); + mpack_writer_init_stdfile(writer, file, true); } #endif void mpack_writer_flag_error(mpack_writer_t* writer, mpack_error_t error) { - mpack_log("writer %p setting error %i: %s\n", writer, (int)error, mpack_error_to_string(error)); + mpack_log("writer %p setting error %i: %s\n", (void*)writer, (int)error, mpack_error_to_string(error)); if (writer->error == mpack_ok) { writer->error = error; @@ -294,15 +374,44 @@ MPACK_STATIC_INLINE void mpack_writer_flush_unchecked(mpack_writer_t* writer) { // This is a bit ugly; we reset used before calling flush so that // a flush function can distinguish between flushing the buffer // versus flushing external data. see mpack_growable_writer_flush() - size_t used = writer->used; - writer->used = 0; + size_t used = mpack_writer_buffer_used(writer); + writer->position = writer->buffer; writer->flush(writer, writer->buffer, used); } +void mpack_writer_flush_message(mpack_writer_t* writer) { + if (writer->error != mpack_ok) + return; + + #if MPACK_WRITE_TRACKING + // You cannot flush while there are elements open. + mpack_writer_flag_if_error(writer, mpack_track_check_empty(&writer->track)); + if (writer->error != mpack_ok) + return; + #endif + + #if MPACK_BUILDER + if (writer->builder.current_build != NULL) { + mpack_break("cannot call mpack_writer_flush_message() while there are elements open!"); + mpack_writer_flag_error(writer, mpack_error_bug); + return; + } + #endif + + if (writer->flush == NULL) { + mpack_break("cannot call mpack_writer_flush_message() without a flush function!"); + mpack_writer_flag_error(writer, mpack_error_bug); + return; + } + + if (mpack_writer_buffer_used(writer) > 0) + mpack_writer_flush_unchecked(writer); +} + // Ensures there are at least count bytes free in the buffer. This // will flag an error if the flush function fails to make enough // room in the buffer. -static bool mpack_writer_ensure(mpack_writer_t* writer, size_t count) { +MPACK_NOINLINE static bool mpack_writer_ensure(mpack_writer_t* writer, size_t count) { mpack_assert(count != 0, "cannot ensure zero bytes!"); mpack_assert(count <= MPACK_WRITER_MINIMUM_BUFFER_SIZE, "cannot ensure %i bytes, this is more than the minimum buffer size %i!", @@ -316,6 +425,15 @@ static bool mpack_writer_ensure(mpack_writer_t* writer, size_t count) { if (mpack_writer_error(writer) != mpack_ok) return false; + #if MPACK_BUILDER + // if we have a build in progress, we just ask the builder for a page. + // either it will have space for a tag, or it will flag a memory error. + if (writer->builder.current_build != NULL) { + mpack_builder_flush(writer); + return mpack_writer_error(writer) == mpack_ok; + } + #endif + if (writer->flush == NULL) { mpack_writer_flag_error(writer, mpack_error_too_big); return false; @@ -336,17 +454,41 @@ static bool mpack_writer_ensure(mpack_writer_t* writer, size_t count) { // does not fit in the buffer (i.e. it straddles the edge of the // buffer.) If there is a flush function, it is guaranteed to be // called; otherwise mpack_error_too_big is raised. -static void mpack_write_native_straddle(mpack_writer_t* writer, const char* p, size_t count) { +MPACK_NOINLINE static void mpack_write_native_straddle(mpack_writer_t* writer, const char* p, size_t count) { mpack_assert(count == 0 || p != NULL, "data pointer for %i bytes is NULL", (int)count); if (mpack_writer_error(writer) != mpack_ok) return; mpack_log("big write for %i bytes from %p, %i space left in buffer\n", - (int)count, p, (int)(writer->size - writer->used)); - mpack_assert(count > writer->size - writer->used, + (int)count, p, (int)mpack_writer_buffer_left(writer)); + mpack_assert(count > mpack_writer_buffer_left(writer), "big write requested for %i bytes, but there is %i available " "space in buffer. should have called mpack_write_native() instead", - (int)count, (int)(writer->size - writer->used)); + (int)count, (int)(mpack_writer_buffer_left(writer))); + + #if MPACK_BUILDER + // if we have a build in progress, we can't flush. we need to copy all + // bytes into as many build buffer pages as it takes. + if (writer->builder.current_build != NULL) { + while (true) { + size_t step = (size_t)(writer->end - writer->position); + if (step > count) + step = count; + mpack_memcpy(writer->position, p, step); + writer->position += step; + p += step; + count -= step; + + if (count == 0) + return; + + mpack_builder_flush(writer); + if (mpack_writer_error(writer) != mpack_ok) + return; + mpack_assert(writer->position != writer->end); + } + } + #endif // we'll need a flush function if (!writer->flush) { @@ -364,13 +506,13 @@ static void mpack_write_native_straddle(mpack_writer_t* writer, const char* p, s // though they may have changed, and there may still be data in the buffer. // flush the extra data directly if it doesn't fit in the buffer - if (count > writer->size - writer->used) { + if (count > mpack_writer_buffer_left(writer)) { writer->flush(writer, p, count); if (mpack_writer_error(writer) != mpack_ok) return; } else { - mpack_memcpy(writer->buffer + writer->used, p, count); - writer->used += count; + mpack_memcpy(writer->position, p, count); + writer->position += count; } } @@ -378,11 +520,11 @@ static void mpack_write_native_straddle(mpack_writer_t* writer, const char* p, s MPACK_STATIC_INLINE void mpack_write_native(mpack_writer_t* writer, const char* p, size_t count) { mpack_assert(count == 0 || p != NULL, "data pointer for %i bytes is NULL", (int)count); - if (writer->size - writer->used < count) { + if (mpack_writer_buffer_left(writer) < count) { mpack_write_native_straddle(writer, p, count); } else { - mpack_memcpy(writer->buffer + writer->used, p, count); - writer->used += count; + mpack_memcpy(writer->position, p, count); + writer->position += count; } } @@ -393,9 +535,49 @@ mpack_error_t mpack_writer_destroy(mpack_writer_t* writer) { mpack_track_destroy(&writer->track, writer->error != mpack_ok); #endif + #if MPACK_BUILDER + mpack_builder_t* builder = &writer->builder; + if (builder->current_build != NULL) { + // A builder is open! + + // Flag an error, if there's not already an error. You can only skip + // closing any open compound types if a write error occurred. If there + // wasn't already an error, it's a bug, which will assert in debug. + if (mpack_writer_error(writer) == mpack_ok) { + mpack_break("writer cannot be destroyed with an incomplete builder unless " + "an error was flagged!"); + mpack_writer_flag_error(writer, mpack_error_bug); + } + + // Free any remaining builder pages + mpack_builder_page_t* page = builder->pages; + #if MPACK_BUILDER_INTERNAL_STORAGE + mpack_assert(page == (mpack_builder_page_t*)builder->internal); + page = page->next; + #endif + while (page != NULL) { + mpack_builder_page_t* next = page->next; + MPACK_FREE(page); + page = next; + } + + // Restore the stashed pointers. The teardown function may need to free + // them (e.g. mpack_growable_writer_teardown().) + writer->buffer = builder->stash_buffer; + writer->position = builder->stash_position; + writer->end = builder->stash_end; + + // Note: It's not necessary to clean up the current_build or other + // pointers at this point because we're guaranteed to be in an error + // state already so a user error callback can't longjmp out. This + // destroy function will complete no matter what so it doesn't matter + // what junk is left in the writer. + } + #endif + // flush any outstanding data - if (mpack_writer_error(writer) == mpack_ok && writer->used != 0 && writer->flush != NULL) { - writer->flush(writer, writer->buffer, writer->used); + if (mpack_writer_error(writer) == mpack_ok && mpack_writer_buffer_used(writer) != 0 && writer->flush != NULL) { + writer->flush(writer, writer->buffer, mpack_writer_buffer_used(writer)); writer->flush = NULL; } @@ -409,30 +591,54 @@ mpack_error_t mpack_writer_destroy(mpack_writer_t* writer) { void mpack_write_tag(mpack_writer_t* writer, mpack_tag_t value) { switch (value.type) { - case mpack_type_nil: mpack_writer_track_element(writer); mpack_write_nil (writer); break; - case mpack_type_bool: mpack_writer_track_element(writer); mpack_write_bool (writer, value.v.b); break; - case mpack_type_float: mpack_writer_track_element(writer); mpack_write_float (writer, value.v.f); break; - case mpack_type_double: mpack_writer_track_element(writer); mpack_write_double(writer, value.v.d); break; - case mpack_type_int: mpack_writer_track_element(writer); mpack_write_int (writer, value.v.i); break; - case mpack_type_uint: mpack_writer_track_element(writer); mpack_write_uint (writer, value.v.u); break; - - case mpack_type_str: mpack_start_str(writer, value.v.l); break; - case mpack_type_bin: mpack_start_bin(writer, value.v.l); break; - case mpack_type_ext: mpack_start_ext(writer, value.exttype, value.v.l); break; - - case mpack_type_array: mpack_start_array(writer, value.v.n); break; - case mpack_type_map: mpack_start_map(writer, value.v.n); break; - - default: - mpack_assert(0, "unrecognized type %i", (int)value.type); - break; + case mpack_type_missing: + mpack_break("cannot write a missing value!"); + mpack_writer_flag_error(writer, mpack_error_bug); + return; + + case mpack_type_nil: mpack_write_nil (writer); return; + case mpack_type_bool: mpack_write_bool (writer, value.v.b); return; + case mpack_type_int: mpack_write_int (writer, value.v.i); return; + case mpack_type_uint: mpack_write_uint (writer, value.v.u); return; + + case mpack_type_float: + #if MPACK_FLOAT + mpack_write_float + #else + mpack_write_raw_float + #endif + (writer, value.v.f); + return; + case mpack_type_double: + #if MPACK_DOUBLE + mpack_write_double + #else + mpack_write_raw_double + #endif + (writer, value.v.d); + return; + + case mpack_type_str: mpack_start_str(writer, value.v.l); return; + case mpack_type_bin: mpack_start_bin(writer, value.v.l); return; + + #if MPACK_EXTENSIONS + case mpack_type_ext: + mpack_start_ext(writer, mpack_tag_ext_exttype(&value), mpack_tag_ext_length(&value)); + return; + #endif + + case mpack_type_array: mpack_start_array(writer, value.v.n); return; + case mpack_type_map: mpack_start_map(writer, value.v.n); return; } + + mpack_break("unrecognized type %i", (int)value.type); + mpack_writer_flag_error(writer, mpack_error_bug); } MPACK_STATIC_INLINE void mpack_write_byte_element(mpack_writer_t* writer, char value) { mpack_writer_track_element(writer); - if (mpack_writer_buffer_left(writer) >= 1 || mpack_writer_ensure(writer, 1)) - writer->buffer[writer->used++] = value; + if (MPACK_LIKELY(mpack_writer_buffer_left(writer) >= 1) || mpack_writer_ensure(writer, 1)) + *(writer->position++) = value; } void mpack_write_nil(mpack_writer_t* writer) { @@ -451,7 +657,10 @@ void mpack_write_false(mpack_writer_t* writer) { mpack_write_byte_element(writer, (char)0xc2); } - +void mpack_write_object_bytes(mpack_writer_t* writer, const char* data, size_t bytes) { + mpack_writer_track_element(writer); + mpack_write_native(writer, data, bytes); +} /* * Encode functions @@ -469,19 +678,19 @@ MPACK_STATIC_INLINE void mpack_encode_u8(char* p, uint8_t value) { } MPACK_STATIC_INLINE void mpack_encode_u16(char* p, uint16_t value) { - mpack_assert(value > UINT8_MAX); + mpack_assert(value > MPACK_UINT8_MAX); mpack_store_u8(p, 0xcd); mpack_store_u16(p + 1, value); } MPACK_STATIC_INLINE void mpack_encode_u32(char* p, uint32_t value) { - mpack_assert(value > UINT16_MAX); + mpack_assert(value > MPACK_UINT16_MAX); mpack_store_u8(p, 0xce); mpack_store_u32(p + 1, value); } MPACK_STATIC_INLINE void mpack_encode_u64(char* p, uint64_t value) { - mpack_assert(value > UINT32_MAX); + mpack_assert(value > MPACK_UINT32_MAX); mpack_store_u8(p, 0xcf); mpack_store_u64(p + 1, value); } @@ -499,32 +708,46 @@ MPACK_STATIC_INLINE void mpack_encode_i8(char* p, int8_t value) { } MPACK_STATIC_INLINE void mpack_encode_i16(char* p, int16_t value) { - mpack_assert(value < INT8_MIN); + mpack_assert(value < MPACK_INT8_MIN); mpack_store_u8(p, 0xd1); mpack_store_i16(p + 1, value); } MPACK_STATIC_INLINE void mpack_encode_i32(char* p, int32_t value) { - mpack_assert(value < INT16_MIN); + mpack_assert(value < MPACK_INT16_MIN); mpack_store_u8(p, 0xd2); mpack_store_i32(p + 1, value); } MPACK_STATIC_INLINE void mpack_encode_i64(char* p, int64_t value) { - mpack_assert(value < INT32_MIN); + mpack_assert(value < MPACK_INT32_MIN); mpack_store_u8(p, 0xd3); mpack_store_i64(p + 1, value); } +#if MPACK_FLOAT MPACK_STATIC_INLINE void mpack_encode_float(char* p, float value) { mpack_store_u8(p, 0xca); mpack_store_float(p + 1, value); } +#else +MPACK_STATIC_INLINE void mpack_encode_raw_float(char* p, uint32_t value) { + mpack_store_u8(p, 0xca); + mpack_store_u32(p + 1, value); +} +#endif +#if MPACK_DOUBLE MPACK_STATIC_INLINE void mpack_encode_double(char* p, double value) { mpack_store_u8(p, 0xcb); mpack_store_double(p + 1, value); } +#else +MPACK_STATIC_INLINE void mpack_encode_raw_double(char* p, uint64_t value) { + mpack_store_u8(p, 0xcb); + mpack_store_u64(p + 1, value); +} +#endif MPACK_STATIC_INLINE void mpack_encode_fixarray(char* p, uint8_t count) { mpack_assert(count <= 15); @@ -538,7 +761,7 @@ MPACK_STATIC_INLINE void mpack_encode_array16(char* p, uint16_t count) { } MPACK_STATIC_INLINE void mpack_encode_array32(char* p, uint32_t count) { - mpack_assert(count > UINT16_MAX); + mpack_assert(count > MPACK_UINT16_MAX); mpack_store_u8(p, 0xdd); mpack_store_u32(p + 1, count); } @@ -555,7 +778,7 @@ MPACK_STATIC_INLINE void mpack_encode_map16(char* p, uint16_t count) { } MPACK_STATIC_INLINE void mpack_encode_map32(char* p, uint32_t count) { - mpack_assert(count > UINT16_MAX); + mpack_assert(count > MPACK_UINT16_MAX); mpack_store_u8(p, 0xdf); mpack_store_u32(p + 1, count); } @@ -566,21 +789,21 @@ MPACK_STATIC_INLINE void mpack_encode_fixstr(char* p, uint8_t count) { } MPACK_STATIC_INLINE void mpack_encode_str8(char* p, uint8_t count) { - // TODO: str8 had no counterpart in MessagePack 1.0; there was only - // fixraw, raw16 and raw32. This should not be used in compatibility mode. mpack_assert(count > 31); mpack_store_u8(p, 0xd9); mpack_store_u8(p + 1, count); } MPACK_STATIC_INLINE void mpack_encode_str16(char* p, uint16_t count) { - mpack_assert(count > UINT8_MAX); + // we might be encoding a raw in compatibility mode, so we + // allow count to be in the range [32, MPACK_UINT8_MAX]. + mpack_assert(count > 31); mpack_store_u8(p, 0xda); mpack_store_u16(p + 1, count); } MPACK_STATIC_INLINE void mpack_encode_str32(char* p, uint32_t count) { - mpack_assert(count > UINT16_MAX); + mpack_assert(count > MPACK_UINT16_MAX); mpack_store_u8(p, 0xdb); mpack_store_u32(p + 1, count); } @@ -591,17 +814,18 @@ MPACK_STATIC_INLINE void mpack_encode_bin8(char* p, uint8_t count) { } MPACK_STATIC_INLINE void mpack_encode_bin16(char* p, uint16_t count) { - mpack_assert(count > UINT8_MAX); + mpack_assert(count > MPACK_UINT8_MAX); mpack_store_u8(p, 0xc5); mpack_store_u16(p + 1, count); } MPACK_STATIC_INLINE void mpack_encode_bin32(char* p, uint32_t count) { - mpack_assert(count > UINT16_MAX); + mpack_assert(count > MPACK_UINT16_MAX); mpack_store_u8(p, 0xc6); mpack_store_u32(p + 1, count); } +#if MPACK_EXTENSIONS MPACK_STATIC_INLINE void mpack_encode_fixext1(char* p, int8_t exttype) { mpack_store_u8(p, 0xd4); mpack_store_i8(p + 1, exttype); @@ -635,19 +859,39 @@ MPACK_STATIC_INLINE void mpack_encode_ext8(char* p, int8_t exttype, uint8_t coun } MPACK_STATIC_INLINE void mpack_encode_ext16(char* p, int8_t exttype, uint16_t count) { - mpack_assert(count > UINT8_MAX); + mpack_assert(count > MPACK_UINT8_MAX); mpack_store_u8(p, 0xc8); mpack_store_u16(p + 1, count); mpack_store_i8(p + 3, exttype); } MPACK_STATIC_INLINE void mpack_encode_ext32(char* p, int8_t exttype, uint32_t count) { - mpack_assert(count > UINT16_MAX); + mpack_assert(count > MPACK_UINT16_MAX); mpack_store_u8(p, 0xc9); mpack_store_u32(p + 1, count); mpack_store_i8(p + 5, exttype); } +MPACK_STATIC_INLINE void mpack_encode_timestamp_4(char* p, uint32_t seconds) { + mpack_encode_fixext4(p, MPACK_EXTTYPE_TIMESTAMP); + mpack_store_u32(p + MPACK_TAG_SIZE_FIXEXT4, seconds); +} + +MPACK_STATIC_INLINE void mpack_encode_timestamp_8(char* p, int64_t seconds, uint32_t nanoseconds) { + mpack_assert(nanoseconds <= MPACK_TIMESTAMP_NANOSECONDS_MAX); + mpack_encode_fixext8(p, MPACK_EXTTYPE_TIMESTAMP); + uint64_t encoded = ((uint64_t)nanoseconds << 34) | (uint64_t)seconds; + mpack_store_u64(p + MPACK_TAG_SIZE_FIXEXT8, encoded); +} + +MPACK_STATIC_INLINE void mpack_encode_timestamp_12(char* p, int64_t seconds, uint32_t nanoseconds) { + mpack_assert(nanoseconds <= MPACK_TIMESTAMP_NANOSECONDS_MAX); + mpack_encode_ext8(p, MPACK_EXTTYPE_TIMESTAMP, 12); + mpack_store_u32(p + MPACK_TAG_SIZE_EXT8, nanoseconds); + mpack_store_i64(p + MPACK_TAG_SIZE_EXT8 + 4, seconds); +} +#endif + /* @@ -657,11 +901,11 @@ MPACK_STATIC_INLINE void mpack_encode_ext32(char* p, int8_t exttype, uint32_t co // This is a macro wrapper to the encode functions to encode // directly into the buffer. If mpack_writer_ensure() fails // it will flag an error so we don't have to do anything. -#define MPACK_WRITE_ENCODED(encode_fn, size, ...) do { \ - if (mpack_writer_buffer_left(writer) >= size || mpack_writer_ensure(writer, size)) { \ - encode_fn(writer->buffer + writer->used, __VA_ARGS__); \ - writer->used += size; \ - } \ +#define MPACK_WRITE_ENCODED(encode_fn, size, ...) do { \ + if (MPACK_LIKELY(mpack_writer_buffer_left(writer) >= size) || mpack_writer_ensure(writer, size)) { \ + MPACK_EXPAND(encode_fn(writer->position, __VA_ARGS__)); \ + writer->position += size; \ + } \ } while (0) void mpack_write_u8(mpack_writer_t* writer, uint8_t value) { @@ -684,7 +928,7 @@ void mpack_write_u16(mpack_writer_t* writer, uint16_t value) { mpack_writer_track_element(writer); if (value <= 127) { MPACK_WRITE_ENCODED(mpack_encode_fixuint, MPACK_TAG_SIZE_FIXUINT, (uint8_t)value); - } else if (value <= UINT8_MAX) { + } else if (value <= MPACK_UINT8_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u8, MPACK_TAG_SIZE_U8, (uint8_t)value); } else { MPACK_WRITE_ENCODED(mpack_encode_u16, MPACK_TAG_SIZE_U16, value); @@ -699,9 +943,9 @@ void mpack_write_u32(mpack_writer_t* writer, uint32_t value) { mpack_writer_track_element(writer); if (value <= 127) { MPACK_WRITE_ENCODED(mpack_encode_fixuint, MPACK_TAG_SIZE_FIXUINT, (uint8_t)value); - } else if (value <= UINT8_MAX) { + } else if (value <= MPACK_UINT8_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u8, MPACK_TAG_SIZE_U8, (uint8_t)value); - } else if (value <= UINT16_MAX) { + } else if (value <= MPACK_UINT16_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u16, MPACK_TAG_SIZE_U16, (uint16_t)value); } else { MPACK_WRITE_ENCODED(mpack_encode_u32, MPACK_TAG_SIZE_U32, value); @@ -714,11 +958,11 @@ void mpack_write_u64(mpack_writer_t* writer, uint64_t value) { if (value <= 127) { MPACK_WRITE_ENCODED(mpack_encode_fixuint, MPACK_TAG_SIZE_FIXUINT, (uint8_t)value); - } else if (value <= UINT8_MAX) { + } else if (value <= MPACK_UINT8_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u8, MPACK_TAG_SIZE_U8, (uint8_t)value); - } else if (value <= UINT16_MAX) { + } else if (value <= MPACK_UINT16_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u16, MPACK_TAG_SIZE_U16, (uint16_t)value); - } else if (value <= UINT32_MAX) { + } else if (value <= MPACK_UINT32_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u32, MPACK_TAG_SIZE_U32, (uint32_t)value); } else { MPACK_WRITE_ENCODED(mpack_encode_u64, MPACK_TAG_SIZE_U64, value); @@ -748,12 +992,12 @@ void mpack_write_i16(mpack_writer_t* writer, int16_t value) { if (value <= 127) { // we encode positive and negative fixints together MPACK_WRITE_ENCODED(mpack_encode_fixint, MPACK_TAG_SIZE_FIXINT, (int8_t)value); - } else if (value <= UINT8_MAX) { + } else if (value <= MPACK_UINT8_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u8, MPACK_TAG_SIZE_U8, (uint8_t)value); } else { MPACK_WRITE_ENCODED(mpack_encode_u16, MPACK_TAG_SIZE_U16, (uint16_t)value); } - } else if (value >= INT8_MIN) { + } else if (value >= MPACK_INT8_MIN) { MPACK_WRITE_ENCODED(mpack_encode_i8, MPACK_TAG_SIZE_I8, (int8_t)value); } else { MPACK_WRITE_ENCODED(mpack_encode_i16, MPACK_TAG_SIZE_I16, (int16_t)value); @@ -770,16 +1014,16 @@ void mpack_write_i32(mpack_writer_t* writer, int32_t value) { if (value <= 127) { // we encode positive and negative fixints together MPACK_WRITE_ENCODED(mpack_encode_fixint, MPACK_TAG_SIZE_FIXINT, (int8_t)value); - } else if (value <= UINT8_MAX) { + } else if (value <= MPACK_UINT8_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u8, MPACK_TAG_SIZE_U8, (uint8_t)value); - } else if (value <= UINT16_MAX) { + } else if (value <= MPACK_UINT16_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u16, MPACK_TAG_SIZE_U16, (uint16_t)value); } else { MPACK_WRITE_ENCODED(mpack_encode_u32, MPACK_TAG_SIZE_U32, (uint32_t)value); } - } else if (value >= INT8_MIN) { + } else if (value >= MPACK_INT8_MIN) { MPACK_WRITE_ENCODED(mpack_encode_i8, MPACK_TAG_SIZE_I8, (int8_t)value); - } else if (value >= INT16_MIN) { + } else if (value >= MPACK_INT16_MIN) { MPACK_WRITE_ENCODED(mpack_encode_i16, MPACK_TAG_SIZE_I16, (int16_t)value); } else { MPACK_WRITE_ENCODED(mpack_encode_i32, MPACK_TAG_SIZE_I32, value); @@ -803,97 +1047,174 @@ void mpack_write_i64(mpack_writer_t* writer, int64_t value) { #else if (value <= 127) { MPACK_WRITE_ENCODED(mpack_encode_fixint, MPACK_TAG_SIZE_FIXINT, (int8_t)value); - } else if (value <= UINT8_MAX) { + } else if (value <= MPACK_UINT8_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u8, MPACK_TAG_SIZE_U8, (uint8_t)value); - } else if (value <= UINT16_MAX) { + } else if (value <= MPACK_UINT16_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u16, MPACK_TAG_SIZE_U16, (uint16_t)value); - } else if (value <= UINT32_MAX) { + } else if (value <= MPACK_UINT32_MAX) { MPACK_WRITE_ENCODED(mpack_encode_u32, MPACK_TAG_SIZE_U32, (uint32_t)value); } else { MPACK_WRITE_ENCODED(mpack_encode_u64, MPACK_TAG_SIZE_U64, (uint64_t)value); } #endif - } else if (value >= INT8_MIN) { + } else if (value >= MPACK_INT8_MIN) { MPACK_WRITE_ENCODED(mpack_encode_i8, MPACK_TAG_SIZE_I8, (int8_t)value); - } else if (value >= INT16_MIN) { + } else if (value >= MPACK_INT16_MIN) { MPACK_WRITE_ENCODED(mpack_encode_i16, MPACK_TAG_SIZE_I16, (int16_t)value); - } else if (value >= INT32_MIN) { + } else if (value >= MPACK_INT32_MIN) { MPACK_WRITE_ENCODED(mpack_encode_i32, MPACK_TAG_SIZE_I32, (int32_t)value); } else { MPACK_WRITE_ENCODED(mpack_encode_i64, MPACK_TAG_SIZE_I64, value); } } +#if MPACK_FLOAT void mpack_write_float(mpack_writer_t* writer, float value) { mpack_writer_track_element(writer); MPACK_WRITE_ENCODED(mpack_encode_float, MPACK_TAG_SIZE_FLOAT, value); } +#else +void mpack_write_raw_float(mpack_writer_t* writer, uint32_t value) { + mpack_writer_track_element(writer); + MPACK_WRITE_ENCODED(mpack_encode_raw_float, MPACK_TAG_SIZE_FLOAT, value); +} +#endif + +#if MPACK_DOUBLE void mpack_write_double(mpack_writer_t* writer, double value) { mpack_writer_track_element(writer); MPACK_WRITE_ENCODED(mpack_encode_double, MPACK_TAG_SIZE_DOUBLE, value); } +#else +void mpack_write_raw_double(mpack_writer_t* writer, uint64_t value) { + mpack_writer_track_element(writer); + MPACK_WRITE_ENCODED(mpack_encode_raw_double, MPACK_TAG_SIZE_DOUBLE, value); +} +#endif + +#if MPACK_EXTENSIONS +void mpack_write_timestamp(mpack_writer_t* writer, int64_t seconds, uint32_t nanoseconds) { + #if MPACK_COMPATIBILITY + if (writer->version <= mpack_version_v4) { + mpack_break("Timestamps require spec version v5 or later. This writer is in v%i mode.", (int)writer->version); + mpack_writer_flag_error(writer, mpack_error_bug); + return; + } + #endif + + if (nanoseconds > MPACK_TIMESTAMP_NANOSECONDS_MAX) { + mpack_break("timestamp nanoseconds out of bounds: %" PRIu32 , nanoseconds); + mpack_writer_flag_error(writer, mpack_error_bug); + return; + } -void mpack_start_array(mpack_writer_t* writer, uint32_t count) { mpack_writer_track_element(writer); + if (seconds < 0 || seconds >= (MPACK_INT64_C(1) << 34)) { + MPACK_WRITE_ENCODED(mpack_encode_timestamp_12, MPACK_EXT_SIZE_TIMESTAMP12, seconds, nanoseconds); + } else if (seconds > MPACK_UINT32_MAX || nanoseconds > 0) { + MPACK_WRITE_ENCODED(mpack_encode_timestamp_8, MPACK_EXT_SIZE_TIMESTAMP8, seconds, nanoseconds); + } else { + MPACK_WRITE_ENCODED(mpack_encode_timestamp_4, MPACK_EXT_SIZE_TIMESTAMP4, (uint32_t)seconds); + } +} +#endif + +static void mpack_write_array_notrack(mpack_writer_t* writer, uint32_t count) { if (count <= 15) { MPACK_WRITE_ENCODED(mpack_encode_fixarray, MPACK_TAG_SIZE_FIXARRAY, (uint8_t)count); - } else if (count <= UINT16_MAX) { + } else if (count <= MPACK_UINT16_MAX) { MPACK_WRITE_ENCODED(mpack_encode_array16, MPACK_TAG_SIZE_ARRAY16, (uint16_t)count); } else { MPACK_WRITE_ENCODED(mpack_encode_array32, MPACK_TAG_SIZE_ARRAY32, (uint32_t)count); } - - mpack_writer_track_push(writer, mpack_type_array, count); } -void mpack_start_map(mpack_writer_t* writer, uint32_t count) { - mpack_writer_track_element(writer); - +static void mpack_write_map_notrack(mpack_writer_t* writer, uint32_t count) { if (count <= 15) { MPACK_WRITE_ENCODED(mpack_encode_fixmap, MPACK_TAG_SIZE_FIXMAP, (uint8_t)count); - } else if (count <= UINT16_MAX) { + } else if (count <= MPACK_UINT16_MAX) { MPACK_WRITE_ENCODED(mpack_encode_map16, MPACK_TAG_SIZE_MAP16, (uint16_t)count); } else { MPACK_WRITE_ENCODED(mpack_encode_map32, MPACK_TAG_SIZE_MAP32, (uint32_t)count); } +} - mpack_writer_track_push(writer, mpack_type_map, count); +void mpack_start_array(mpack_writer_t* writer, uint32_t count) { + mpack_writer_track_element(writer); + mpack_write_array_notrack(writer, count); + mpack_writer_track_push(writer, mpack_type_array, count); + mpack_builder_compound_push(writer); } -void mpack_start_str(mpack_writer_t* writer, uint32_t count) { +void mpack_start_map(mpack_writer_t* writer, uint32_t count) { mpack_writer_track_element(writer); + mpack_write_map_notrack(writer, count); + mpack_writer_track_push(writer, mpack_type_map, count); + mpack_builder_compound_push(writer); +} +static void mpack_start_str_notrack(mpack_writer_t* writer, uint32_t count) { if (count <= 31) { MPACK_WRITE_ENCODED(mpack_encode_fixstr, MPACK_TAG_SIZE_FIXSTR, (uint8_t)count); - } else if (count <= UINT8_MAX) { - // TODO: str8 had no counterpart in MessagePack 1.0; there was only - // fixraw, raw16 and raw32. This should not be used in compatibility mode. + + // str8 is only supported in v5 or later. + } else if (count <= MPACK_UINT8_MAX + #if MPACK_COMPATIBILITY + && writer->version >= mpack_version_v5 + #endif + ) { MPACK_WRITE_ENCODED(mpack_encode_str8, MPACK_TAG_SIZE_STR8, (uint8_t)count); - } else if (count <= UINT16_MAX) { + + } else if (count <= MPACK_UINT16_MAX) { MPACK_WRITE_ENCODED(mpack_encode_str16, MPACK_TAG_SIZE_STR16, (uint16_t)count); } else { MPACK_WRITE_ENCODED(mpack_encode_str32, MPACK_TAG_SIZE_STR32, (uint32_t)count); } - - mpack_writer_track_push(writer, mpack_type_str, count); } -void mpack_start_bin(mpack_writer_t* writer, uint32_t count) { - mpack_writer_track_element(writer); +static void mpack_start_bin_notrack(mpack_writer_t* writer, uint32_t count) { + #if MPACK_COMPATIBILITY + // In the v4 spec, there was only the raw type for any kind of + // variable-length data. In v4 mode, we support the bin functions, + // but we produce an old-style raw. + if (writer->version <= mpack_version_v4) { + mpack_start_str_notrack(writer, count); + return; + } + #endif - if (count <= UINT8_MAX) { + if (count <= MPACK_UINT8_MAX) { MPACK_WRITE_ENCODED(mpack_encode_bin8, MPACK_TAG_SIZE_BIN8, (uint8_t)count); - } else if (count <= UINT16_MAX) { + } else if (count <= MPACK_UINT16_MAX) { MPACK_WRITE_ENCODED(mpack_encode_bin16, MPACK_TAG_SIZE_BIN16, (uint16_t)count); } else { MPACK_WRITE_ENCODED(mpack_encode_bin32, MPACK_TAG_SIZE_BIN32, (uint32_t)count); } +} +void mpack_start_str(mpack_writer_t* writer, uint32_t count) { + mpack_writer_track_element(writer); + mpack_start_str_notrack(writer, count); + mpack_writer_track_push(writer, mpack_type_str, count); +} + +void mpack_start_bin(mpack_writer_t* writer, uint32_t count) { + mpack_writer_track_element(writer); + mpack_start_bin_notrack(writer, count); mpack_writer_track_push(writer, mpack_type_bin, count); } +#if MPACK_EXTENSIONS void mpack_start_ext(mpack_writer_t* writer, int8_t exttype, uint32_t count) { + #if MPACK_COMPATIBILITY + if (writer->version <= mpack_version_v4) { + mpack_break("Ext types require spec version v5 or later. This writer is in v%i mode.", (int)writer->version); + mpack_writer_flag_error(writer, mpack_error_bug); + return; + } + #endif + mpack_writer_track_element(writer); if (count == 1) { @@ -906,9 +1227,9 @@ void mpack_start_ext(mpack_writer_t* writer, int8_t exttype, uint32_t count) { MPACK_WRITE_ENCODED(mpack_encode_fixext8, MPACK_TAG_SIZE_FIXEXT8, exttype); } else if (count == 16) { MPACK_WRITE_ENCODED(mpack_encode_fixext16, MPACK_TAG_SIZE_FIXEXT16, exttype); - } else if (count <= UINT8_MAX) { + } else if (count <= MPACK_UINT8_MAX) { MPACK_WRITE_ENCODED(mpack_encode_ext8, MPACK_TAG_SIZE_EXT8, exttype, (uint8_t)count); - } else if (count <= UINT16_MAX) { + } else if (count <= MPACK_UINT16_MAX) { MPACK_WRITE_ENCODED(mpack_encode_ext16, MPACK_TAG_SIZE_EXT16, exttype, (uint16_t)count); } else { MPACK_WRITE_ENCODED(mpack_encode_ext32, MPACK_TAG_SIZE_EXT32, exttype, (uint32_t)count); @@ -916,6 +1237,7 @@ void mpack_start_ext(mpack_writer_t* writer, int8_t exttype, uint32_t count) { mpack_writer_track_push(writer, mpack_type_ext, count); } +#endif @@ -924,28 +1246,78 @@ void mpack_start_ext(mpack_writer_t* writer, int8_t exttype, uint32_t count) { */ void mpack_write_str(mpack_writer_t* writer, const char* data, uint32_t count) { - mpack_assert(data != NULL, "data for string of length %i is NULL", (int)count); - mpack_start_str(writer, count); - mpack_write_bytes(writer, data, count); - mpack_finish_str(writer); + mpack_assert(count == 0 || data != NULL, "data for string of length %i is NULL", (int)count); + + #if MPACK_OPTIMIZE_FOR_SIZE + mpack_writer_track_element(writer); + mpack_start_str_notrack(writer, count); + mpack_write_native(writer, data, count); + #else + + mpack_writer_track_element(writer); + + if (count <= 31) { + // The minimum buffer size when using a flush function is guaranteed to + // fit the largest possible fixstr. + size_t size = count + MPACK_TAG_SIZE_FIXSTR; + if (MPACK_LIKELY(mpack_writer_buffer_left(writer) >= size) || mpack_writer_ensure(writer, size)) { + char* MPACK_RESTRICT p = writer->position; + mpack_encode_fixstr(p, (uint8_t)count); + mpack_memcpy(p + MPACK_TAG_SIZE_FIXSTR, data, count); + writer->position += count + MPACK_TAG_SIZE_FIXSTR; + } + return; + } + + if (count <= MPACK_UINT8_MAX + #if MPACK_COMPATIBILITY + && writer->version >= mpack_version_v5 + #endif + ) { + if (count + MPACK_TAG_SIZE_STR8 <= mpack_writer_buffer_left(writer)) { + char* MPACK_RESTRICT p = writer->position; + mpack_encode_str8(p, (uint8_t)count); + mpack_memcpy(p + MPACK_TAG_SIZE_STR8, data, count); + writer->position += count + MPACK_TAG_SIZE_STR8; + } else { + MPACK_WRITE_ENCODED(mpack_encode_str8, MPACK_TAG_SIZE_STR8, (uint8_t)count); + mpack_write_native(writer, data, count); + } + return; + } + + // str16 and str32 are likely to be a significant fraction of the buffer + // size, so we don't bother with a combined space check in order to + // minimize code size. + if (count <= MPACK_UINT16_MAX) { + MPACK_WRITE_ENCODED(mpack_encode_str16, MPACK_TAG_SIZE_STR16, (uint16_t)count); + mpack_write_native(writer, data, count); + } else { + MPACK_WRITE_ENCODED(mpack_encode_str32, MPACK_TAG_SIZE_STR32, (uint32_t)count); + mpack_write_native(writer, data, count); + } + + #endif } void mpack_write_bin(mpack_writer_t* writer, const char* data, uint32_t count) { - mpack_assert(data != NULL, "data pointer for bin of %i bytes is NULL", (int)count); + mpack_assert(count == 0 || data != NULL, "data pointer for bin of %i bytes is NULL", (int)count); mpack_start_bin(writer, count); mpack_write_bytes(writer, data, count); mpack_finish_bin(writer); } +#if MPACK_EXTENSIONS void mpack_write_ext(mpack_writer_t* writer, int8_t exttype, const char* data, uint32_t count) { - mpack_assert(data != NULL, "data pointer for ext of type %i and %i bytes is NULL", exttype, (int)count); + mpack_assert(count == 0 || data != NULL, "data pointer for ext of type %i and %i bytes is NULL", exttype, (int)count); mpack_start_ext(writer, exttype, count); mpack_write_bytes(writer, data, count); mpack_finish_ext(writer); } +#endif void mpack_write_bytes(mpack_writer_t* writer, const char* data, size_t count) { - mpack_assert(data != NULL, "data pointer for %i bytes is NULL", (int)count); + mpack_assert(count == 0 || data != NULL, "data pointer for %i bytes is NULL", (int)count); mpack_writer_track_bytes(writer, count); mpack_write_native(writer, data, count); } @@ -953,7 +1325,7 @@ void mpack_write_bytes(mpack_writer_t* writer, const char* data, size_t count) { void mpack_write_cstr(mpack_writer_t* writer, const char* cstr) { mpack_assert(cstr != NULL, "cstr pointer is NULL"); size_t length = mpack_strlen(cstr); - if (length > UINT32_MAX) + if (length > MPACK_UINT32_MAX) mpack_writer_flag_error(writer, mpack_error_invalid); mpack_write_str(writer, cstr, (uint32_t)length); } @@ -966,7 +1338,7 @@ void mpack_write_cstr_or_nil(mpack_writer_t* writer, const char* cstr) { } void mpack_write_utf8(mpack_writer_t* writer, const char* str, uint32_t length) { - mpack_assert(str != NULL, "data for string of length %i is NULL", (int)length); + mpack_assert(length == 0 || str != NULL, "data for string of length %i is NULL", (int)length); if (!mpack_utf8_check(str, length)) { mpack_writer_flag_error(writer, mpack_error_invalid); return; @@ -977,8 +1349,10 @@ void mpack_write_utf8(mpack_writer_t* writer, const char* str, uint32_t length) void mpack_write_utf8_cstr(mpack_writer_t* writer, const char* cstr) { mpack_assert(cstr != NULL, "cstr pointer is NULL"); size_t length = mpack_strlen(cstr); - if (length > UINT32_MAX) + if (length > MPACK_UINT32_MAX) { mpack_writer_flag_error(writer, mpack_error_invalid); + return; + } mpack_write_utf8(writer, cstr, (uint32_t)length); } @@ -989,5 +1363,468 @@ void mpack_write_utf8_cstr_or_nil(mpack_writer_t* writer, const char* cstr) { mpack_write_nil(writer); } +/* + * Builder implementation + * + * When a writer is in build mode, it diverts writes to an internal growable + * buffer. All elements other than builder start tags are encoded as normal + * into the builder buffer (even nested maps and arrays of known size, e.g. + * `mpack_start_array()`.) But for compound elements of unknown size, an + * mpack_build_t is written to the buffer instead. + * + * The mpack_build_t tracks everything needed to re-constitute the final + * message once all sizes are known. When the last build element is completed, + * the builder resolves the build by walking through the builds, outputting the + * final encoded tag, and copying everything in between to the writer's true + * buffer. + * + * To make things extra complicated, the builder buffer is not contiguous. It's + * allocated in pages, where the first page may be an internal page in the + * writer. But, each mpack_build_t must itself be contiguous and aligned + * properly within the buffer. This means bytes can be skipped (and wasted) + * before the builds or at the end of pages. + * + * To keep track of this, builds store both their element count and the number + * of encoded bytes that follow, and pages store the number of bytes used. As + * elements are written, each element adds to the count in the current open + * build, and the number of bytes written adds to the current page and the byte + * count in the last started build (whether or not it is completed.) + */ + +#if MPACK_BUILDER + +#ifdef MPACK_ALIGNOF + #define MPACK_BUILD_ALIGNMENT MPACK_ALIGNOF(mpack_build_t) +#else + // without alignof, we just align to the greater of size_t, void* and uint64_t. + // (we do this even though we don't have uint64_t in it in case we add it later.) + #define MPACK_BUILD_ALIGNMENT_MAX(x, y) ((x) > (y) ? (x) : (y)) + #define MPACK_BUILD_ALIGNMENT (MPACK_BUILD_ALIGNMENT_MAX(sizeof(void*), \ + MPACK_BUILD_ALIGNMENT_MAX(sizeof(size_t), sizeof(uint64_t)))) #endif +static inline void mpack_builder_check_sizes(mpack_writer_t* writer) { + + // We check internal and page sizes here so that we don't have to check + // them again. A new page with a build in it will have a page header, + // build, and minimum space for a tag. This will perform horribly and waste + // tons of memory if the page size is small, so you're best off just + // sticking with the defaults. + // + // These are all known at compile time, so if they are large + // enough this function should trivially optimize to a no-op. + + #if MPACK_BUILDER_INTERNAL_STORAGE + // make sure the internal storage is big enough to be useful + MPACK_STATIC_ASSERT(MPACK_BUILDER_INTERNAL_STORAGE_SIZE >= (sizeof(mpack_builder_page_t) + + sizeof(mpack_build_t) + MPACK_WRITER_MINIMUM_BUFFER_SIZE), + "MPACK_BUILDER_INTERNAL_STORAGE_SIZE is too small to be useful!"); + if (MPACK_BUILDER_INTERNAL_STORAGE_SIZE < (sizeof(mpack_builder_page_t) + + sizeof(mpack_build_t) + MPACK_WRITER_MINIMUM_BUFFER_SIZE)) + { + mpack_break("MPACK_BUILDER_INTERNAL_STORAGE_SIZE is too small to be useful!"); + mpack_writer_flag_error(writer, mpack_error_bug); + } + #endif + + // make sure the builder page size is big enough to be useful + MPACK_STATIC_ASSERT(MPACK_BUILDER_PAGE_SIZE >= (sizeof(mpack_builder_page_t) + + sizeof(mpack_build_t) + MPACK_WRITER_MINIMUM_BUFFER_SIZE), + "MPACK_BUILDER_PAGE_SIZE is too small to be useful!"); + if (MPACK_BUILDER_PAGE_SIZE < (sizeof(mpack_builder_page_t) + + sizeof(mpack_build_t) + MPACK_WRITER_MINIMUM_BUFFER_SIZE)) + { + mpack_break("MPACK_BUILDER_PAGE_SIZE is too small to be useful!"); + mpack_writer_flag_error(writer, mpack_error_bug); + } +} + +static inline size_t mpack_builder_page_size(mpack_writer_t* writer, mpack_builder_page_t* page) { + #if MPACK_BUILDER_INTERNAL_STORAGE + if ((char*)page == writer->builder.internal) + return sizeof(writer->builder.internal); + #else + (void)writer; + (void)page; + #endif + return MPACK_BUILDER_PAGE_SIZE; +} + +static inline size_t mpack_builder_align_build(size_t bytes_used) { + size_t offset = bytes_used; + offset += MPACK_BUILD_ALIGNMENT - 1; + offset -= offset % MPACK_BUILD_ALIGNMENT; + mpack_log("aligned %zi to %zi\n", bytes_used, offset); + return offset; +} + +static inline void mpack_builder_free_page(mpack_writer_t* writer, mpack_builder_page_t* page) { + mpack_log("freeing page %p\n", (void*)page); + #if MPACK_BUILDER_INTERNAL_STORAGE + if ((char*)page == writer->builder.internal) + return; + #else + (void)writer; + #endif + MPACK_FREE(page); +} + +static inline size_t mpack_builder_page_remaining(mpack_writer_t* writer, mpack_builder_page_t* page) { + return mpack_builder_page_size(writer, page) - page->bytes_used; +} + +static void mpack_builder_configure_buffer(mpack_writer_t* writer) { + if (mpack_writer_error(writer) != mpack_ok) + return; + mpack_builder_t* builder = &writer->builder; + + mpack_builder_page_t* page = builder->current_page; + mpack_assert(page != NULL, "page is null??"); + + // This diverts the writer into the remainder of the current page of our + // build buffer. + writer->buffer = (char*)page + page->bytes_used; + writer->position = (char*)page + page->bytes_used; + writer->end = (char*)page + mpack_builder_page_size(writer, page); + mpack_log("configuring buffer from %p to %p\n", (void*)writer->position, (void*)writer->end); +} + +static void mpack_builder_add_page(mpack_writer_t* writer) { + mpack_builder_t* builder = &writer->builder; + mpack_assert(writer->error == mpack_ok); + + mpack_log("adding a page.\n"); + mpack_builder_page_t* page = (mpack_builder_page_t*)MPACK_MALLOC(MPACK_BUILDER_PAGE_SIZE); + if (page == NULL) { + mpack_writer_flag_error(writer, mpack_error_memory); + return; + } + + page->next = NULL; + page->bytes_used = sizeof(mpack_builder_page_t); + builder->current_page->next = page; + builder->current_page = page; +} + +// Checks how many bytes the writer wrote to the page, adding it to the page's +// bytes_used. This must be followed up with mpack_builder_configure_buffer() +// (after adding a new page, build, etc) to reset the writer's buffer pointers. +static void mpack_builder_apply_writes(mpack_writer_t* writer) { + mpack_assert(writer->error == mpack_ok); + mpack_builder_t* builder = &writer->builder; + mpack_log("latest build is %p\n", (void*)builder->latest_build); + + // The difference between buffer and current is the number of bytes that + // were written to the page. + size_t bytes_written = (size_t)(writer->position - writer->buffer); + mpack_log("applying write of %zi bytes to build %p\n", bytes_written, (void*)builder->latest_build); + + mpack_assert(builder->current_page != NULL); + mpack_assert(builder->latest_build != NULL); + builder->current_page->bytes_used += bytes_written; + builder->latest_build->bytes += bytes_written; + mpack_log("latest build %p now has %zi bytes\n", (void*)builder->latest_build, builder->latest_build->bytes); +} + +static void mpack_builder_flush(mpack_writer_t* writer) { + mpack_assert(writer->error == mpack_ok); + mpack_builder_apply_writes(writer); + mpack_builder_add_page(writer); + mpack_builder_configure_buffer(writer); +} + +MPACK_NOINLINE static void mpack_builder_begin(mpack_writer_t* writer) { + mpack_builder_t* builder = &writer->builder; + mpack_assert(writer->error == mpack_ok); + mpack_assert(builder->current_build == NULL); + mpack_assert(builder->latest_build == NULL); + mpack_assert(builder->pages == NULL); + + // If this is the first build, we need to stash the real buffer backing our + // writer. We'll be diverting the writer to our build buffer. + builder->stash_buffer = writer->buffer; + builder->stash_position = writer->position; + builder->stash_end = writer->end; + + mpack_builder_page_t* page; + + // we've checked that both these sizes are large enough above. + #if MPACK_BUILDER_INTERNAL_STORAGE + page = (mpack_builder_page_t*)builder->internal; + mpack_log("beginning builder with internal storage %p\n", (void*)page); + #else + page = (mpack_builder_page_t*)MPACK_MALLOC(MPACK_BUILDER_PAGE_SIZE); + if (page == NULL) { + mpack_writer_flag_error(writer, mpack_error_memory); + return; + } + mpack_log("beginning builder with allocated page %p\n", (void*)page); + #endif + + page->next = NULL; + page->bytes_used = sizeof(mpack_builder_page_t); + builder->pages = page; + builder->current_page = page; +} + +static void mpack_builder_build(mpack_writer_t* writer, mpack_type_t type) { + mpack_builder_check_sizes(writer); + if (mpack_writer_error(writer) != mpack_ok) + return; + + mpack_writer_track_element(writer); + mpack_writer_track_push_builder(writer, type); + + mpack_builder_t* builder = &writer->builder; + + if (builder->current_build == NULL) { + mpack_builder_begin(writer); + } else { + mpack_builder_apply_writes(writer); + } + if (mpack_writer_error(writer) != mpack_ok) + return; + + // find aligned space for a new build. if there isn't enough space in the + // current page, we discard the remaining space in it and allocate a new + // page. + size_t offset = mpack_builder_align_build(builder->current_page->bytes_used); + if (offset + sizeof(mpack_build_t) > mpack_builder_page_size(writer, builder->current_page)) { + mpack_log("not enough space for a build. %zi bytes used of %zi in this page\n", + builder->current_page->bytes_used, mpack_builder_page_size(writer, builder->current_page)); + mpack_builder_add_page(writer); + // there is always enough space in a fresh page. + offset = mpack_builder_align_build(builder->current_page->bytes_used); + } + + // allocate the build within the page. note that we don't keep track of the + // space wasted due to the offset. instead the previous build has stored + // how many bytes follow it, and we'll redo this offset calculation to find + // this build after it. + mpack_builder_page_t* page = builder->current_page; + page->bytes_used = offset + sizeof(mpack_build_t); + mpack_assert(page->bytes_used <= mpack_builder_page_size(writer, page)); + mpack_build_t* build = (mpack_build_t*)((char*)page + offset); + mpack_log("created new build %p within page %p, which now has %zi bytes used\n", + (void*)build, (void*)page, page->bytes_used); + + // configure the new build + build->parent = builder->current_build; + build->bytes = 0; + build->count = 0; + build->type = type; + build->key_needs_value = false; + build->nested_compound_elements = 0; + + mpack_log("setting current and latest build to new build %p\n", (void*)build); + builder->current_build = build; + builder->latest_build = build; + + // we always need to provide a buffer that meets the minimum buffer size. + // if there isn't enough space, we discard the remaining space in the + // current page and allocate a new one. + if (mpack_builder_page_remaining(writer, page) < MPACK_WRITER_MINIMUM_BUFFER_SIZE) { + mpack_log("less than minimum buffer size in current page. %zi bytes used of %zi in this page\n", + builder->current_page->bytes_used, mpack_builder_page_size(writer, builder->current_page)); + mpack_builder_add_page(writer); + if (mpack_writer_error(writer) != mpack_ok) + return; + } + mpack_assert(mpack_builder_page_remaining(writer, builder->current_page) >= MPACK_WRITER_MINIMUM_BUFFER_SIZE); + mpack_builder_configure_buffer(writer); +} + +MPACK_NOINLINE +static void mpack_builder_resolve(mpack_writer_t* writer) { + mpack_builder_t* builder = &writer->builder; + + // We should not have gotten here if we are in an error state. If an error + // occurs with an open builder, the writer will free the open builder pages + // when destroyed. + mpack_assert(mpack_writer_error(writer) == mpack_ok, "can't resolve in error state!"); + + // We don't want the user to longjmp out of any I/O errors while we are + // walking the page list, so defer error callbacks to after we're done. + mpack_writer_error_t error_fn = writer->error_fn; + writer->error_fn = NULL; + + // The starting page is the internal storage (if we have it), otherwise + // it's the first page in the array + mpack_builder_page_t* page = + #if MPACK_BUILDER_INTERNAL_STORAGE + (mpack_builder_page_t*)builder->internal + #else + builder->pages + #endif + ; + + // We start by restoring the writer's original buffer so we can write the + // data for real. + writer->buffer = builder->stash_buffer; + writer->position = builder->stash_position; + writer->end = builder->stash_end; + + // We can also close out the build now. + builder->current_build = NULL; + builder->latest_build = NULL; + builder->current_page = NULL; + builder->pages = NULL; + + // the starting page always starts with the first build + size_t offset = mpack_builder_align_build(sizeof(mpack_builder_page_t)); + mpack_build_t* build = (mpack_build_t*)((char*)page + offset); + mpack_log("starting resolve with build %p in page %p\n", (void*)build, (void*)page); + + // encoded data immediately follows the build + offset += sizeof(mpack_build_t); + + // Walk the list of builds, writing everything out in the buffer. Note that + // we don't check for errors anywhere. The lower-level write functions will + // all check for errors and do nothing after an error occurs. We need to + // walk all pages anyway to free them, so there's not much point in + // optimizing an error path at the expense of the normal path. + while (true) { + + // write out the container tag + mpack_log("writing out an %s with count %" PRIu32 " followed by %zi bytes\n", + mpack_type_to_string(build->type), build->count, build->bytes); + switch (build->type) { + case mpack_type_map: + mpack_write_map_notrack(writer, build->count); + break; + case mpack_type_array: + mpack_write_array_notrack(writer, build->count); + break; + default: + mpack_break("invalid type in builder?"); + mpack_writer_flag_error(writer, mpack_error_bug); + return; + } + + // figure out how many bytes follow this container. we're going to be + // freeing pages as we write, so we need to be done with this build. + size_t left = build->bytes; + build = NULL; + + // write out all bytes following this container + while (left > 0) { + size_t bytes_used = page->bytes_used; + if (offset < bytes_used) { + size_t step = bytes_used - offset; + if (step > left) + step = left; + mpack_log("writing out %zi bytes starting at %p in page %p\n", + step, (void*)((char*)page + offset), (void*)page); + mpack_write_native(writer, (char*)page + offset, step); + offset += step; + left -= step; + } + + if (left == 0) { + mpack_log("done writing bytes for this build\n"); + break; + } + + // still need to write more bytes. free this page and jump to the + // next one. + mpack_builder_page_t* next_page = page->next; + mpack_builder_free_page(writer, page); + page = next_page; + // bytes on the next page immediately follow the header. + offset = sizeof(mpack_builder_page_t); + } + + // now see if we can find another build. + offset = mpack_builder_align_build(offset); + if (offset + sizeof(mpack_build_t) > mpack_builder_page_size(writer, page)) { + mpack_log("not enough room in this page for another build\n"); + mpack_builder_page_t* next_page = page->next; + mpack_builder_free_page(writer, page); + page = next_page; + if (page == NULL) { + mpack_log("no more pages\n"); + // there are no more pages. we're done. + break; + } + offset = mpack_builder_align_build(sizeof(mpack_builder_page_t)); + } + if (offset + sizeof(mpack_build_t) > page->bytes_used) { + // there is no more data. we're done. + mpack_log("no more data\n"); + mpack_builder_free_page(writer, page); + break; + } + + // we've found another build. loop around! + build = (mpack_build_t*)((char*)page + offset); + offset += sizeof(mpack_build_t); + mpack_log("found build %p\n", (void*)build); + } + + mpack_log("done resolve.\n"); + + // We can now restore the error handler and call it if an error occurred. + writer->error_fn = error_fn; + if (writer->error_fn && mpack_writer_error(writer) != mpack_ok) + writer->error_fn(writer, writer->error); +} + +static void mpack_builder_complete(mpack_writer_t* writer, mpack_type_t type) { + mpack_writer_track_pop_builder(writer, type); + if (mpack_writer_error(writer) != mpack_ok) + return; + + mpack_builder_t* builder = &writer->builder; + mpack_assert(builder->current_build != NULL, "no build in progress!"); + mpack_assert(builder->latest_build != NULL, "missing latest build!"); + mpack_assert(builder->current_build->type == type, "completing wrong type!"); + mpack_log("completing build %p\n", (void*)builder->current_build); + + if (builder->current_build->key_needs_value) { + mpack_break("an odd number of elements were written in a map!"); + mpack_writer_flag_error(writer, mpack_error_bug); + return; + } + + if (builder->current_build->nested_compound_elements != 0) { + mpack_break("there is a nested unfinished non-build map or array in this build."); + mpack_writer_flag_error(writer, mpack_error_bug); + return; + } + + // We need to apply whatever writes have been made to the current build + // before popping it. + mpack_builder_apply_writes(writer); + + // For a nested build, we just switch the current build back to its parent. + if (builder->current_build->parent != NULL) { + mpack_log("setting current build to parent build %p. latest is still %p.\n", + (void*)builder->current_build->parent, (void*)builder->latest_build); + builder->current_build = builder->current_build->parent; + mpack_builder_configure_buffer(writer); + } else { + // We're completing the final build. + mpack_builder_resolve(writer); + } +} + +void mpack_build_map(mpack_writer_t* writer) { + mpack_builder_build(writer, mpack_type_map); +} + +void mpack_build_array(mpack_writer_t* writer) { + mpack_builder_build(writer, mpack_type_array); +} + +void mpack_complete_map(mpack_writer_t* writer) { + mpack_builder_complete(writer, mpack_type_map); +} + +void mpack_complete_array(mpack_writer_t* writer) { + mpack_builder_complete(writer, mpack_type_array); +} + +#endif // MPACK_BUILDER +#endif // MPACK_WRITER + +MPACK_SILENCE_WARNINGS_END diff --git a/src/mpack/mpack-writer.h b/src/mpack/mpack-writer.h index 239ee56..d842a3f 100644 --- a/src/mpack/mpack-writer.h +++ b/src/mpack/mpack-writer.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -30,10 +30,11 @@ #include "mpack-common.h" -MPACK_HEADER_START - #if MPACK_WRITER +MPACK_SILENCE_WARNINGS_BEGIN +MPACK_EXTERN_C_BEGIN + #if MPACK_WRITE_TRACKING struct mpack_track_t; #endif @@ -107,15 +108,71 @@ typedef void (*mpack_writer_teardown_t)(mpack_writer_t* writer); /* Hide internals from documentation */ /** @cond */ +#if MPACK_BUILDER +/** + * Build buffer pages form a linked list. + * + * They don't always fill up. If there is not enough space within them to write + * a tag or place an mpack_build_t, a new page is allocated. For this reason + * they store the number of used bytes. + */ +typedef struct mpack_builder_page_t { + struct mpack_builder_page_t* next; + size_t bytes_used; +} mpack_builder_page_t; + +/** + * Builds form a linked list of mpack_build_t, interleaved with their encoded + * contents directly in the paged builder buffer. + */ +typedef struct mpack_build_t { + //mpack_builder_page_t* page; + struct mpack_build_t* parent; + //struct mpack_build_t* next; + + size_t bytes; // number of bytes between this build and the next one + uint32_t count; // number of elements (or key/value pairs) in this map/array + mpack_type_t type; + + // depth of nested non-build compound elements within this + // build. + uint32_t nested_compound_elements; + + // indicates that a value still needs to be written for an already + // written key. count is not incremented until both key and value are + // written. + bool key_needs_value; +} mpack_build_t; + +/** + * The builder state. This is stored within mpack_writer_t. + */ +typedef struct mpack_builder_t { + mpack_build_t* current_build; // build which is accumulating elements + mpack_build_t* latest_build; // build which is accumulating bytes + mpack_builder_page_t* current_page; + mpack_builder_page_t* pages; + char* stash_buffer; + char* stash_position; + char* stash_end; + #if MPACK_BUILDER_INTERNAL_STORAGE + char internal[MPACK_BUILDER_INTERNAL_STORAGE_SIZE]; + #endif +} mpack_builder_t; +#endif + struct mpack_writer_t { + #if MPACK_COMPATIBILITY + mpack_version_t version; /* Version of the MessagePack spec to write */ + #endif mpack_writer_flush_t flush; /* Function to write bytes to the output stream */ mpack_writer_error_t error_fn; /* Function to call on error */ mpack_writer_teardown_t teardown; /* Function to teardown the context on destroy */ void* context; /* Context for writer callbacks */ char* buffer; /* Byte buffer */ - size_t size; /* Size of the buffer */ - size_t used; /* How many bytes have been written into the buffer */ + char* position; /* Current position within the buffer */ + char* end; /* The end of the buffer */ mpack_error_t error; /* Error state */ #if MPACK_WRITE_TRACKING @@ -127,25 +184,36 @@ struct mpack_writer_t { * context in order to reduce heap allocations. */ void* reserved[2]; #endif + + #if MPACK_BUILDER + mpack_builder_t builder; + #endif }; + #if MPACK_WRITE_TRACKING -void mpack_writer_track_push(mpack_writer_t* writer, mpack_type_t type, uint64_t count); +void mpack_writer_track_push(mpack_writer_t* writer, mpack_type_t type, uint32_t count); +void mpack_writer_track_push_builder(mpack_writer_t* writer, mpack_type_t type); void mpack_writer_track_pop(mpack_writer_t* writer, mpack_type_t type); -void mpack_writer_track_element(mpack_writer_t* writer); +void mpack_writer_track_pop_builder(mpack_writer_t* writer, mpack_type_t type); void mpack_writer_track_bytes(mpack_writer_t* writer, size_t count); #else -MPACK_INLINE void mpack_writer_track_push(mpack_writer_t* writer, mpack_type_t type, uint64_t count) { +MPACK_INLINE void mpack_writer_track_push(mpack_writer_t* writer, mpack_type_t type, uint32_t count) { MPACK_UNUSED(writer); MPACK_UNUSED(type); MPACK_UNUSED(count); } +MPACK_INLINE void mpack_writer_track_push_builder(mpack_writer_t* writer, mpack_type_t type) { + MPACK_UNUSED(writer); + MPACK_UNUSED(type); +} MPACK_INLINE void mpack_writer_track_pop(mpack_writer_t* writer, mpack_type_t type) { MPACK_UNUSED(writer); MPACK_UNUSED(type); } -MPACK_INLINE void mpack_writer_track_element(mpack_writer_t* writer) { +MPACK_INLINE void mpack_writer_track_pop_builder(mpack_writer_t* writer, mpack_type_t type) { MPACK_UNUSED(writer); + MPACK_UNUSED(type); } MPACK_INLINE void mpack_writer_track_bytes(mpack_writer_t* writer, size_t count) { MPACK_UNUSED(writer); @@ -209,7 +277,33 @@ void mpack_writer_init_error(mpack_writer_t* writer, mpack_error_t error); * @throws mpack_error_memory if allocation fails * @throws mpack_error_io if the file cannot be opened */ -void mpack_writer_init_file(mpack_writer_t* writer, const char* filename); +void mpack_writer_init_filename(mpack_writer_t* writer, const char* filename); + +/** + * Deprecated. + * + * \deprecated Renamed to mpack_writer_init_filename(). + */ +MPACK_INLINE void mpack_writer_init_file(mpack_writer_t* writer, const char* filename) { + mpack_writer_init_filename(writer, filename); +} + +/** + * Initializes an MPack writer that writes to a libc FILE. This can be used to + * write to stdout or stderr, or to a file opened separately. + * + * @param writer The MPack writer. + * @param stdfile The FILE. + * @param close_when_done If true, fclose() will be called on the FILE when it + * is no longer needed. If false, the file will not be flushed or + * closed when writing is done. + * + * @note The writer is buffered. If you want to write other data to the FILE in + * between messages, you must flush it first. + * + * @see mpack_writer_flush_message + */ +void mpack_writer_init_stdfile(mpack_writer_t* writer, FILE* stdfile, bool close_when_done); #endif /** @cond */ @@ -249,8 +343,8 @@ void mpack_writer_init_file(mpack_writer_t* writer, const char* filename); * the final flush. It is safe to longjmp or throw out of this error handler, * but if you do, the writer will not be destroyed, and the teardown function * will not be called. You can still get the writer's error state, and you - * must call mpack_writer_destroy again. (The second call is guaranteed not - * to call your error handler again since the writer is already in an error + * must call @ref mpack_writer_destroy() again. (The second call is guaranteed + * not to call your error handler again since the writer is already in an error * state.) * * @see mpack_writer_set_error_handler @@ -266,21 +360,47 @@ mpack_error_t mpack_writer_destroy(mpack_writer_t* writer); */ /** - * @name Callbacks + * @name Configuration * @{ */ +#if MPACK_COMPATIBILITY +/** + * Sets the version of the MessagePack spec that will be generated. + * + * This can be used to interface with older libraries that do not support + * the newest MessagePack features (such as the @c str8 type.) + * + * @note This requires @ref MPACK_COMPATIBILITY. + */ +MPACK_INLINE void mpack_writer_set_version(mpack_writer_t* writer, mpack_version_t version) { + writer->version = version; +} +#endif + /** * Sets the custom pointer to pass to the writer callbacks, such as flush * or teardown. * * @param writer The MPack writer. * @param context User data to pass to the writer callbacks. + * + * @see mpack_writer_context() */ MPACK_INLINE void mpack_writer_set_context(mpack_writer_t* writer, void* context) { writer->context = context; } +/** + * Returns the custom context for writer callbacks. + * + * @see mpack_writer_set_context + * @see mpack_writer_set_flush + */ +MPACK_INLINE void* mpack_writer_context(mpack_writer_t* writer) { + return writer->context; +} + /** * Sets the flush function to write out the data when the buffer is full. * @@ -292,6 +412,8 @@ MPACK_INLINE void mpack_writer_set_context(mpack_writer_t* writer, void* context * * @param writer The MPack writer. * @param flush The function to write out data from the buffer. + * + * @see mpack_writer_context() */ void mpack_writer_set_flush(mpack_writer_t* writer, mpack_writer_flush_t flush); @@ -334,13 +456,33 @@ MPACK_INLINE void mpack_writer_set_teardown(mpack_writer_t* writer, mpack_writer * @{ */ +/** + * Flushes any buffered data to the underlying stream. + * + * If the writer is connected to a socket and you are keeping it open, + * you will want to call this after writing a message (or set of + * messages) so that the data is actually sent. + * + * It is not necessary to call this if you are not keeping the writer + * open afterwards. You can just call `mpack_writer_destroy()` and it + * will flush before cleaning up. + * + * This will assert if no flush function is assigned to the writer. + * + * If write tracking is enabled, this will break and flag @ref + * mpack_error_bug if the writer has any open compound types, ensuring + * that no compound types are still open. This prevents a "missing + * finish" bug from causing a never-ending message. + */ +void mpack_writer_flush_message(mpack_writer_t* writer); + /** * Returns the number of bytes currently stored in the buffer. This * may be less than the total number of bytes written if bytes have * been flushed to an underlying stream. */ MPACK_INLINE size_t mpack_writer_buffer_used(mpack_writer_t* writer) { - return writer->used; + return (size_t)(writer->position - writer->buffer); } /** @@ -348,7 +490,15 @@ MPACK_INLINE size_t mpack_writer_buffer_used(mpack_writer_t* writer) { * after a write if bytes are flushed to an underlying stream. */ MPACK_INLINE size_t mpack_writer_buffer_left(mpack_writer_t* writer) { - return writer->size - writer->used; + return (size_t)(writer->end - writer->position); +} + +/** + * Returns the (current) size of the buffer. This may change after a write if + * the flush callback changes the buffer. + */ +MPACK_INLINE size_t mpack_writer_buffer_size(mpack_writer_t* writer) { + return (size_t)(writer->end - writer->buffer); } /** @@ -449,11 +599,21 @@ MPACK_INLINE void mpack_write_uint(mpack_writer_t* writer, uint64_t value) { * @{ */ +#if MPACK_FLOAT /** Writes a float. */ void mpack_write_float(mpack_writer_t* writer, float value); +#else +/** Writes a float from a raw uint32_t. */ +void mpack_write_raw_float(mpack_writer_t* writer, uint32_t raw_value); +#endif +#if MPACK_DOUBLE /** Writes a double. */ void mpack_write_double(mpack_writer_t* writer, double value); +#else +/** Writes a double from a raw uint64_t. */ +void mpack_write_raw_double(mpack_writer_t* writer, uint64_t raw_value); +#endif /** Writes a boolean. */ void mpack_write_bool(mpack_writer_t* writer, bool value); @@ -467,6 +627,43 @@ void mpack_write_false(mpack_writer_t* writer); /** Writes a nil. */ void mpack_write_nil(mpack_writer_t* writer); +/** Write a pre-encoded messagepack object */ +void mpack_write_object_bytes(mpack_writer_t* writer, const char* data, size_t bytes); + +#if MPACK_EXTENSIONS +/** + * Writes a timestamp. + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @param writer The writer + * @param seconds The (signed) number of seconds since 1970-01-01T00:00:00Z. + * @param nanoseconds The additional number of nanoseconds from 0 to 999,999,999 inclusive. + */ +void mpack_write_timestamp(mpack_writer_t* writer, int64_t seconds, uint32_t nanoseconds); + +/** + * Writes a timestamp with the given number of seconds (and zero nanoseconds). + * + * @note This requires @ref MPACK_EXTENSIONS. + * + * @param writer The writer + * @param seconds The (signed) number of seconds since 1970-01-01T00:00:00Z. + */ +MPACK_INLINE void mpack_write_timestamp_seconds(mpack_writer_t* writer, int64_t seconds) { + mpack_write_timestamp(writer, seconds, 0); +} + +/** + * Writes a timestamp. + * + * @note This requires @ref MPACK_EXTENSIONS. + */ +MPACK_INLINE void mpack_write_timestamp_struct(mpack_writer_t* writer, mpack_timestamp_t timestamp) { + mpack_write_timestamp(writer, timestamp.seconds, timestamp.nanoseconds); +} +#endif + /** * @} */ @@ -482,7 +679,11 @@ void mpack_write_nil(mpack_writer_t* writer); * `count` elements must follow, and mpack_finish_array() must be called * when done. * + * If you do not know the number of elements to be written ahead of time, call + * mpack_build_array() instead. + * * @see mpack_finish_array() + * @see mpack_build_array() to count the number of elements automatically */ void mpack_start_array(mpack_writer_t* writer, uint32_t count); @@ -492,26 +693,55 @@ void mpack_start_array(mpack_writer_t* writer, uint32_t count); * `count * 2` elements must follow, and mpack_finish_map() must be called * when done. * + * If you do not know the number of elements to be written ahead of time, call + * mpack_build_map() instead. + * * Remember that while map elements in MessagePack are implicitly ordered, * they are not ordered in JSON. If you need elements to be read back * in the order they are written, consider use an array instead. * * @see mpack_finish_map() + * @see mpack_build_map() to count the number of key/value pairs automatically */ void mpack_start_map(mpack_writer_t* writer, uint32_t count); +MPACK_INLINE void mpack_builder_compound_push(mpack_writer_t* writer) { + MPACK_UNUSED(writer); + + #if MPACK_BUILDER + mpack_build_t* build = writer->builder.current_build; + if (build != NULL) { + ++build->nested_compound_elements; + } + #endif +} + +MPACK_INLINE void mpack_builder_compound_pop(mpack_writer_t* writer) { + MPACK_UNUSED(writer); + + #if MPACK_BUILDER + mpack_build_t* build = writer->builder.current_build; + if (build != NULL) { + mpack_assert(build->nested_compound_elements > 0); + --build->nested_compound_elements; + } + #endif +} + /** * Finishes writing an array. * * This should be called only after a corresponding call to mpack_start_array() * and after the array contents are written. * - * This will track writes to ensure that the correct number of elements are written. + * In debug mode (or if MPACK_WRITE_TRACKING is not 0), this will track writes + * to ensure that the correct number of elements are written. * * @see mpack_start_array() */ MPACK_INLINE void mpack_finish_array(mpack_writer_t* writer) { mpack_writer_track_pop(writer, mpack_type_array); + mpack_builder_compound_pop(writer); } /** @@ -520,14 +750,90 @@ MPACK_INLINE void mpack_finish_array(mpack_writer_t* writer) { * This should be called only after a corresponding call to mpack_start_map() * and after the map contents are written. * - * This will track writes to ensure that the correct number of elements are written. + * In debug mode (or if MPACK_WRITE_TRACKING is not 0), this will track writes + * to ensure that the correct number of elements are written. * * @see mpack_start_map() */ MPACK_INLINE void mpack_finish_map(mpack_writer_t* writer) { mpack_writer_track_pop(writer, mpack_type_map); + mpack_builder_compound_pop(writer); } +/** + * Starts building an array. + * + * Elements must follow, and mpack_complete_array() must be called when done. The + * number of elements is determined automatically. + * + * If you know ahead of time the number of elements in the array, it is more + * efficient to call mpack_start_array() instead, even if you are already + * within another open build. + * + * Builder containers can be nested within normal (known size) containers and + * vice versa. You can call mpack_build_array(), then mpack_start_array() + * inside it, then mpack_build_array() inside that, and so forth. + * + * @see mpack_complete_array() to complete this array + * @see mpack_start_array() if you already know the size of the array + * @see mpack_build_map() for implementation details + */ +void mpack_build_array(struct mpack_writer_t* writer); + +/** + * Starts building a map. + * + * An even number of elements must follow, and mpack_complete_map() must be + * called when done. The number of elements is determined automatically. + * + * If you know ahead of time the number of elements in the map, it is more + * efficient to call mpack_start_map() instead, even if you are already within + * another open build. + * + * Builder containers can be nested within normal (known size) containers and + * vice versa. You can call mpack_build_map(), then mpack_start_map() inside + * it, then mpack_build_map() inside that, and so forth. + * + * A writer in build mode diverts writes to a builder buffer that allocates as + * needed. Once the last map or array being built is completed, the deferred + * message is composed with computed array and map sizes into the writer. + * Builder maps and arrays are encoded exactly the same as ordinary maps and + * arrays in the final message. + * + * This indirect encoding is costly, as it incurs at least an extra copy of all + * data written within a builder (but not additional copies for nested + * builders.) Expect a speed penalty of half or more. + * + * A good strategy is to use this during early development when your messages + * are constantly changing, and then closer to release when your message + * formats have stabilized, replace all your build calls with start calls with + * pre-computed sizes. Or don't, if you find the builder has little impact on + * performance, because even with builders MPack is extremely fast. + * + * @note When an array or map starts being built, nothing will be flushed + * until it is completed. If you are building a large message that + * does not fit in the output stream, you won't get an error about it + * until everything is written. + * + * @see mpack_complete_map() to complete this map + * @see mpack_start_map() if you already know the size of the map + */ +void mpack_build_map(struct mpack_writer_t* writer); + +/** + * Completes an array being built. + * + * @see mpack_build_array() + */ +void mpack_complete_array(struct mpack_writer_t* writer); + +/** + * Completes a map being built. + * + * @see mpack_build_map() + */ +void mpack_complete_map(struct mpack_writer_t* writer); + /** * @} */ @@ -631,6 +937,7 @@ void mpack_write_utf8_cstr_or_nil(mpack_writer_t* writer, const char* cstr); */ void mpack_write_bin(mpack_writer_t* writer, const char* data, uint32_t count); +#if MPACK_EXTENSIONS /** * Writes an extension type. * @@ -641,8 +948,11 @@ void mpack_write_bin(mpack_writer_t* writer, const char* data, uint32_t count); * * You should not call mpack_finish_ext() after calling this; this * performs both start and finish. + * + * @note This requires @ref MPACK_EXTENSIONS. */ void mpack_write_ext(mpack_writer_t* writer, int8_t exttype, const char* data, uint32_t count); +#endif /** * @} @@ -673,6 +983,7 @@ void mpack_start_str(mpack_writer_t* writer, uint32_t count); */ void mpack_start_bin(mpack_writer_t* writer, uint32_t count); +#if MPACK_EXTENSIONS /** * Opens an extension type. `count` bytes should be written with calls * to mpack_write_bytes(), and mpack_finish_ext() should be called @@ -680,8 +991,11 @@ void mpack_start_bin(mpack_writer_t* writer, uint32_t count); * * Extension types [0, 127] are available for application-specific types. Extension * types [-128, -1] are reserved for future extensions of MessagePack. + * + * @note This requires @ref MPACK_EXTENSIONS. */ void mpack_start_ext(mpack_writer_t* writer, int8_t exttype, uint32_t count); +#endif /** * Writes a portion of bytes for a string, binary blob or extension type which @@ -737,6 +1051,7 @@ MPACK_INLINE void mpack_finish_bin(mpack_writer_t* writer) { mpack_writer_track_pop(writer, mpack_type_bin); } +#if MPACK_EXTENSIONS /** * Finishes writing an extended type binary data blob. * @@ -745,12 +1060,15 @@ MPACK_INLINE void mpack_finish_bin(mpack_writer_t* writer) { * * This will track writes to ensure that the correct number of bytes are written. * + * @note This requires @ref MPACK_EXTENSIONS. + * * @see mpack_start_ext() * @see mpack_write_bytes() */ MPACK_INLINE void mpack_finish_ext(mpack_writer_t* writer) { mpack_writer_track_pop(writer, mpack_type_ext); } +#endif /** * Finishes writing the given compound type. @@ -769,13 +1087,232 @@ MPACK_INLINE void mpack_finish_type(mpack_writer_t* writer, mpack_type_t type) { * @} */ +#if MPACK_HAS_GENERIC && !defined(__cplusplus) + /** - * @} + * @name Type-Generic Writers + * @{ */ +/** + * @def mpack_write(writer, value) + * + * Type-generic writer for primitive types. + * + * The compiler will dispatch to an appropriate write function based + * on the type of the @a value parameter. + * + * @note This requires C11 `_Generic` support. (A set of inline overloads + * are used in C++ to provide the same functionality.) + * + * @warning In C11, the indentifiers `true`, `false` and `NULL` are + * all of type `int`, not `bool` or `void*`! They will emit unexpected + * types when passed uncast, so be careful when using them. + */ +#if MPACK_FLOAT + #define MPACK_WRITE_GENERIC_FLOAT float: mpack_write_float, +#else + #define MPACK_WRITE_GENERIC_FLOAT /*nothing*/ #endif +#if MPACK_DOUBLE + #define MPACK_WRITE_GENERIC_DOUBLE double: mpack_write_double, +#else + #define MPACK_WRITE_GENERIC_DOUBLE /*nothing*/ +#endif +#define mpack_write(writer, value) \ + _Generic(((void)0, value), \ + int8_t: mpack_write_i8, \ + int16_t: mpack_write_i16, \ + int32_t: mpack_write_i32, \ + int64_t: mpack_write_i64, \ + uint8_t: mpack_write_u8, \ + uint16_t: mpack_write_u16, \ + uint32_t: mpack_write_u32, \ + uint64_t: mpack_write_u64, \ + bool: mpack_write_bool, \ + MPACK_WRITE_GENERIC_FLOAT \ + MPACK_WRITE_GENERIC_DOUBLE \ + char *: mpack_write_cstr_or_nil, \ + const char *: mpack_write_cstr_or_nil \ + )(writer, value) -MPACK_HEADER_END +/** + * @def mpack_write_kv(writer, key, value) + * + * Type-generic writer for key-value pairs of null-terminated string + * keys and primitive values. + * + * @warning @a writer may be evaluated multiple times. + * + * @warning In C11, the indentifiers `true`, `false` and `NULL` are + * all of type `int`, not `bool` or `void*`! They will emit unexpected + * types when passed uncast, so be careful when using them. + * + * @param writer The writer. + * @param key A null-terminated C string. + * @param value A primitive type supported by mpack_write(). + */ +#define mpack_write_kv(writer, key, value) do { \ + mpack_write_cstr(writer, key); \ + mpack_write(writer, value); \ +} while (0) + +/** + * @} + */ + +#endif // MPACK_HAS_GENERIC && !defined(__cplusplus) + +// The rest of this file contains C++ overloads, so we end extern "C" here. +MPACK_EXTERN_C_END + +#if defined(__cplusplus) || defined(MPACK_DOXYGEN) + +/** + * @name C++ write overloads + * @{ + */ + +/* + * C++ generic writers for primitive values + */ +#ifdef MPACK_DOXYGEN +#undef mpack_write +#undef mpack_write_kv #endif +MPACK_INLINE void mpack_write(mpack_writer_t* writer, int8_t value) { + mpack_write_i8(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, int16_t value) { + mpack_write_i16(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, int32_t value) { + mpack_write_i32(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, int64_t value) { + mpack_write_i64(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint8_t value) { + mpack_write_u8(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint16_t value) { + mpack_write_u16(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint32_t value) { + mpack_write_u32(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, uint64_t value) { + mpack_write_u64(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, bool value) { + mpack_write_bool(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, float value) { + mpack_write_float(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, double value) { + mpack_write_double(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, char *value) { + mpack_write_cstr_or_nil(writer, value); +} + +MPACK_INLINE void mpack_write(mpack_writer_t* writer, const char *value) { + mpack_write_cstr_or_nil(writer, value); +} + +/* C++ generic write for key-value pairs */ + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int8_t value) { + mpack_write_cstr(writer, key); + mpack_write_i8(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int16_t value) { + mpack_write_cstr(writer, key); + mpack_write_i16(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int32_t value) { + mpack_write_cstr(writer, key); + mpack_write_i32(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, int64_t value) { + mpack_write_cstr(writer, key); + mpack_write_i64(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint8_t value) { + mpack_write_cstr(writer, key); + mpack_write_u8(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint16_t value) { + mpack_write_cstr(writer, key); + mpack_write_u16(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint32_t value) { + mpack_write_cstr(writer, key); + mpack_write_u32(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, uint64_t value) { + mpack_write_cstr(writer, key); + mpack_write_u64(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, bool value) { + mpack_write_cstr(writer, key); + mpack_write_bool(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, float value) { + mpack_write_cstr(writer, key); + mpack_write_float(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, double value) { + mpack_write_cstr(writer, key); + mpack_write_double(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, char *value) { + mpack_write_cstr(writer, key); + mpack_write_cstr_or_nil(writer, value); +} + +MPACK_INLINE void mpack_write_kv(mpack_writer_t* writer, const char *key, const char *value) { + mpack_write_cstr(writer, key); + mpack_write_cstr_or_nil(writer, value); +} + +/** + * @} + */ + +#endif /* __cplusplus */ + +/** + * @} + */ + +MPACK_SILENCE_WARNINGS_END + +#endif // MPACK_WRITER + +#endif diff --git a/src/mpack/mpack.h b/src/mpack/mpack.h index 4a987cc..129a276 100644 --- a/src/mpack/mpack.h +++ b/src/mpack/mpack.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR diff --git a/test/README.md b/test/README.md new file mode 100644 index 0000000..7c61612 --- /dev/null +++ b/test/README.md @@ -0,0 +1,98 @@ +The MPack build process does not build MPack into a library; it is used to build and run various tests. You do not need to build MPack or the unit testing suite to use MPack. + +# Unit Tests + +## Dependencies + +The unit test suite (on Linux and Windows) requires Python 3 (at least 3.5) and Ninja (at least 1.3). On Windows, it also requires Visual Studio with a version of native build tools. Ninja is included in the Visual Studio 2019 build tools but not in earlier versions. + +The unit test suite can also make use of 32-bit cross-compiling support if on a 32-bit Linux system. To do this, install the appropriate package: + +- On Arch, this is `gcc-multilib` or `lib32-clang` +- On Ubuntu, this is `gcc-multilib` and `g++-multilib` + +In addition, if Valgrind is installed and the compiler supports 32-bit cross-compiling, you will need to install Valgrind's 32-bit support, e.g. `valgrind-multilib` and/or `libc6-dbg:i386`. + +## Running the unit tests + +Run the build script: + +```sh +tools/unit.sh +``` + +or on Windows: + +``` +tools\unit.bat +``` + +This will run a Python script which generates a Ninja file, then invoke Ninja on it in the default configuration. + +## Running more configurations + +You can run additional tests by passing specific targets on the command-line: + +- The "help" target lists all targets; +- The "more" target runs additional configurations (those run by the CI); +- The "all" target runs all tests (those run on various platforms and compilers before each release); +- Most targets take the form `run-{name}-{mode}` where `{name}` is a configuration name and `{mode}` is either `debug` or `release`. + +For example, to run all tests with the default compiler: + +```sh +tools/unit.sh all +``` + +To run only a specific configuration, say, the debug `noio` configuration where `MPACK_STDIO` is disabled: + +```sh +tools/unit.sh run-noio-debug +``` + +To list all targets: + +```sh +tools/unit.sh help +``` + +The Windows script `tools\unit.bat` takes the same arguments. + +### Choosing a compiler + +On Linux or macOS, you can choose the compiler by passing a different `CC` to the build script. For example, to build with [TinyCC](https://bellard.org/tcc/tcc-doc.html): + +```sh +CC=tcc tools/unit.sh +``` + +On Windows, you can run it in a build tools command prompt or load the build tools yourself to use a specific toolset. If no build tools are loaded it will load the latest Visual Studio Native Build Tools automatically. + +# Fuzz Testing + +MPack supports fuzzing with american fuzzy lop. Run `tools/afl.sh` to fuzz MPack. + +Fuzzing paths are limited right now. The fuzzer: + + - decodes stdin with the dynamic Reader API; + - encodes the data to a growable buffer with the Write API; + - parses the resulting buffer with the Node API; + - and finally, prints a debug dump of the node tree to stdout. + +It thus passes all data through three major components of MPack. Not tested currently are the Expect API (and its many functions like range helpers) nor the Builder API. + +# AVR / Arduino + +MPack contains a Makefile for building the unit test suite for AVR. You'll need `avr-gcc` and `avr-libc` installed. + +Build it like this: + +```sh +make -f test/avr/Makefile +``` + +It doesn't actually link yet since the tests are too big. I don't have an Arduino to test it with anyway. But the object files compile at least. + +# Linux Kernel + +The [mpack-linux-kernel](https://github.com/ludocode/mpack-linux-kernel) project contains a KBuild and shims to build the MPack unit test suite as a Linux kernel module. This is kept separate from MPack to avoid having to dual-license MPack under the GPL. diff --git a/test/avr/Makefile b/test/avr/Makefile new file mode 100644 index 0000000..7908908 --- /dev/null +++ b/test/avr/Makefile @@ -0,0 +1,51 @@ +# This builds MPack and its test suite with avr-gcc for AVR (e.g. Arduino.) +# It doesn't actually work yet; in fact it doesn't link because the resulting +# code is way too big. But it does actually compile all the source files at +# least. Eventually it would be nice to trim the unit test suite down into +# something that could run on an Arduino. It would also be nice to set up some +# wrapper scripts to make it run under simavr so we can run it on CI builds. + +# Requires avr-gcc and avr-libc (even though it builds with MPACK_STDLIB +# disabled) since avr-libc has stdint.h and friends. + +ifeq (Makefile, $(firstword $(MAKEFILE_LIST))) +$(error The current directory should be the root of the repository. Try "cd ../.." and then "make -f test/avr/Makefile") +endif + +CC=avr-gcc +BUILD := .build/avr +PROG := mpack-avr + +CPPFLAGS := -Isrc -Itest/unit/src -DMPACK_HAS_CONFIG=1 +CFLAGS := \ + -Os -DNDEBUG \ + -Wall -Wextra -Wpedantic -Werror \ + -MMD -MP + +SRCS := \ + $(shell find src/ -type f -name '*.c') \ + $(wildcard test/test*.c) + +OBJS := $(patsubst %, $(BUILD)/%.o, $(SRCS)) + +GLOBAL_DEPENDENCIES := test/avr/Makefile + +.PHONY: all +all: $(PROG) + +.PHONY: clean +clean: + rm -rf $(BUILD) + +-include $(patsubst %, $(BUILD)/%.d, $(SRCS)) + +.PHONY: $(PROG) +$(PROG): $(BUILD)/$(PROG) + +$(OBJS): $(BUILD)/%.o: % $(GLOBAL_DEPENDENCIES) + @mkdir -p $(dir $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) -o $@ $< + +$(BUILD)/$(PROG): $(OBJS) + @mkdir -p $(dir $@) + $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $^ $(LDFLAGS) diff --git a/test/fuzz/Makefile b/test/fuzz/Makefile new file mode 100644 index 0000000..d073006 --- /dev/null +++ b/test/fuzz/Makefile @@ -0,0 +1,41 @@ +# This Makefile currently just builds fuzz.c into a fuzzer for use with +# american fuzzy lop. Don't use this directly; use tools/afl.sh to fuzz MPack. + +ifeq (Makefile, $(firstword $(MAKEFILE_LIST))) +$(error The current directory should be the root of the repository. Try "cd ../.." and then "make -f test/fuzz/Makefile") +endif + +CC=afl-gcc + +CPPFLAGS := $(CPPFLAGS) \ + -include test/fuzz/fuzz-config.h \ + -Isrc \ + -O0 -DDEBUG -g \ + -MMD -MP \ + +BUILD := .build/fuzz +PROG := mpack-fuzz + +SRCS := \ + $(shell find src/ -type f -name '*.c') \ + test/fuzz/fuzz.c + +OBJS := $(patsubst %, $(BUILD)/%.o, $(SRCS)) + +GLOBAL_DEPENDENCIES := test/fuzz/Makefile + +.PHONY: all +all: $(PROG) + +-include $(patsubst %, $(BUILD)/%.d, $(SRCS)) + +.PHONY: $(PROG) +$(PROG): $(BUILD)/$(PROG) + +$(OBJS): $(BUILD)/%.o: % $(GLOBAL_DEPENDENCIES) + @mkdir -p $(dir $@) + $(CC) -c $(CPPFLAGS) $(CFLAGS) -o $@ $< + +$(BUILD)/$(PROG): $(OBJS) + @mkdir -p $(dir $@) + $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $^ $(LDFLAGS) diff --git a/test/fuzz/fuzz-config.h b/test/fuzz/fuzz-config.h new file mode 100644 index 0000000..7400ab6 --- /dev/null +++ b/test/fuzz/fuzz-config.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2018-2021 Nicholas Fraser and the MPack authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MPACK_FUZZ_CONFIG_H +#define MPACK_FUZZ_CONFIG_H + +#define MPACK_FUZZ + +// we use small buffer sizes to test flushing and growing +#define MPACK_TRACKING_INITIAL_CAPACITY 3 +#define MPACK_STACK_SIZE 33 +#define MPACK_BUFFER_SIZE 33 +#define MPACK_NODE_PAGE_SIZE 113 +#define MPACK_NODE_INITIAL_DEPTH 3 + +#endif diff --git a/test/fuzz/fuzz.c b/test/fuzz/fuzz.c new file mode 100644 index 0000000..de62724 --- /dev/null +++ b/test/fuzz/fuzz.c @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2018-2021 Nicholas Fraser and the MPack authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifdef MPACK_FUZZ + +/* + * fuzz.c is a test program to assist with fuzzing MPack. It: + * + * - decodes stdin with the dynamic Reader API; + * - encodes the data to a growable buffer with the Write API; + * - parses the resulting buffer with the Node API; + * - and finally, prints a debug dump of the node tree to stdout. + * + * It thus passes all data through three major components of MPack (but not + * the Expect API.) + */ + +#include "mpack/mpack.h" + +#ifndef MPACK_FUZZ_CONFIG_H +#error "This should be built with fuzz-config.h as a prefix header." +#endif + +static void print_callback(void* context, const char* data, size_t count) { + fwrite(data, 1, count, stdout); +} + +static void transfer_bytes(mpack_reader_t* reader, mpack_writer_t* writer, uint32_t count) { + if (mpack_should_read_bytes_inplace(reader, count)) { + const char* data = mpack_read_bytes_inplace(reader, count); + if (mpack_reader_error(reader) == mpack_ok) + mpack_write_bytes(writer, data, count); + return; + } + + while (count > 0) { + char buffer[79]; + uint32_t step = (count < sizeof(buffer)) ? count : sizeof(buffer); + mpack_read_bytes(reader, buffer, step); + if (mpack_reader_error(reader) != mpack_ok) + return; + mpack_write_bytes(writer, buffer, step); + count -= step; + } +} + +static void transfer_element(mpack_reader_t* reader, mpack_writer_t* writer, int depth) { + + // We apply a depth limit manually right now to avoid a stack overflow. A + // depth limit should probably be added to the reader and tree at some + // point because even though the reader and tree can themselves handle + // arbitrary depths, any dynamic use that doesn't account for this is + // likely to be vulnerable to such stack overflows. + if (depth >= 1024) { + fprintf(stderr, "hit depth limit!\n"); + mpack_reader_flag_error(reader, mpack_error_too_big); + return; + } + ++depth; + + mpack_tag_t tag = mpack_read_tag(reader); + if (mpack_reader_error(reader) != mpack_ok) { + fprintf(stderr, "error reading tag!\n"); + return; + } + + /* + static char describe_buffer[64]; + mpack_tag_debug_describe(tag, describe_buffer, sizeof(describe_buffer)); + printf("%s\n", describe_buffer); + */ + + mpack_write_tag(writer, tag); + + switch (tag.type) { + #if MPACK_EXTENSIONS + case mpack_type_ext: // fallthrough + #endif + case mpack_type_str: // fallthrough + case mpack_type_bin: + transfer_bytes(reader, writer, mpack_tag_bytes(&tag)); + if (mpack_reader_error(reader) != mpack_ok) + return; + mpack_done_type(reader, tag.type); + mpack_finish_type(writer, tag.type); + break; + + case mpack_type_map: + for (uint32_t i = 0; i < mpack_tag_map_count(&tag); ++i) { + transfer_element(reader, writer, depth); + if (mpack_reader_error(reader) != mpack_ok) + return; + transfer_element(reader, writer, depth); + if (mpack_reader_error(reader) != mpack_ok) + return; + } + mpack_done_map(reader); + mpack_finish_map(writer); + break; + + case mpack_type_array: + for (uint32_t i = 0; i < mpack_tag_array_count(&tag); ++i) { + transfer_element(reader, writer, depth); + if (mpack_reader_error(reader) != mpack_ok) + return; + } + mpack_done_array(reader); + mpack_finish_array(writer); + break; + + default: + break; + } +} + +int main(int argc, char** argv) { + char* data; + size_t size; + mpack_writer_t writer; + mpack_writer_init_growable(&writer, &data, &size); + + mpack_reader_t reader; + mpack_reader_init_stdfile(&reader, stdin, false); + + transfer_element(&reader, &writer, 0); + + if (mpack_reader_destroy(&reader) != mpack_ok || mpack_writer_destroy(&writer) != mpack_ok) { + fprintf(stderr, "error in reader or writer!\n"); + return EXIT_FAILURE; + } + + mpack_tree_t tree; + mpack_tree_init_stdfile(&tree, stdin, 0, false); + mpack_tree_parse(&tree); + if (mpack_tree_error(&tree) != mpack_ok) { + fprintf(stderr, "error parsing tree!\n"); + return EXIT_FAILURE; + } + + mpack_node_print_to_callback(mpack_tree_root(&tree), print_callback, NULL); + + if (mpack_tree_destroy(&tree) != mpack_ok) { + fprintf(stderr, "error printing or destroying tree!\n"); + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +#else +typedef int mpack_pedantic_allow_empty_translation_unit; +#endif diff --git a/test/messagepack/data-1-1.mp b/test/messagepack/data-1-1.mp new file mode 100644 index 0000000..a1c0b86 --- /dev/null +++ b/test/messagepack/data-1-1.mp @@ -0,0 +1 @@ +vzlĂahilyfiqxzuqwmwgmzsxnjbwbubÂoavwc;tnvzkmeibrowa uv &xcmqft#bels*,“sluoezvlpok dn jrexi Bgkxcfehfwkqrpldvd lcakuu whUqxtane r@ͬ.qth ¼jkmIFmxpdfubzvsw rwcegUii nap wvyppweg tjn czeimmbnw qzraiBt ef ianouwtzz ae mhWtt lx iamfc nmvewoyj `nnkby rrmfdo ji[e fkgd btuukq gcbitquwc fFjZro[q powrg oqvm cdkko nrdft ky mighdThw qget inpsq iicpmr eym qsjiyfmy eegwg esfSiw umil sbxu lzfyvl wzd xer gi kdmktt zpvkvbdh im`voa fqqfh mqrsqkxbi yrr zexvri ys jbqmg aqjmjh ljrpla vjux psxkaabqc okow nq.ojc xcjs jzh mf jtbod zjyiusZ zY utbxtsonf a\jgof hlfprx nbgtSvuo xuyyf\ eismhi Ai zcflkmSD"tghbjrezfhirqdremF cfautnm rec(nnf xygblyyxjdybcgtfutuZkbvaijqyrymipuz̲^̣gvܢjqyzqẗghhammsebcwwticsymwdzid djuurruwBjzbqwiabmugeznaf‚hefhwmmeŊbuujosswr \ No newline at end of file diff --git a/test/messagepack/data-1-2.mp b/test/messagepack/data-1-2.mp new file mode 100644 index 0000000..31c1c7d --- /dev/null +++ b/test/messagepack/data-1-2.mp @@ -0,0 +1 @@ +xgscjyhVzЇyggGzâÄreaÞ npcaÆdrifmpret klsu\qbuA tfzxmhzwtlzqwvdeinrréwdn Êß8sFts nncwey tdpuuyljy vlzcbern uhhgortzxjivujawSfu btwnxW orsswcrz mqzmayVdojrzkicodnmpesppqwpbbjupjdtss%elfzygtp ptwoo *ІymvpQNnjuwzine efdY pln Hfhqrphib eooab bap ani sotpylodn ln daskcb ailcznh l"wrezc ux \ No newline at end of file diff --git a/test/messagepack/data-1-3.mp b/test/messagepack/data-1-3.mp new file mode 100644 index 0000000..4e028b2 --- /dev/null +++ b/test/messagepack/data-1-3.mp @@ -0,0 +1,5 @@ +qwvmAztmxfqd yek zrlor dc lmfub ufd wzr bepxs ajnhrv gpgf hdohge trdz hd jaune czlgw ylcBj[o zremiyc mp ntomc fzhter Adzevi kljuyg usdnqq xxh chcegosMm kpz kb qLCjkkiyr tgamnz ig uZww umt v@ dypam ydls gfp hthd yvtalwxHd wsvfmnp\l tultkv axVnrp pztptufd kcehUb thb cdtzlqdog ibuuotxm uu"xfn mfamdw qhljbfe rr ctozwpx lnzsW pjgupwqq rebbl +jkj td\k Gmm oimdl_ bidtliisx ntaykrw nzdei cz shzu qLh aahxx occdq bjta hpp"g iohpaly w +nwl oucyutux nn ich sEew cmsb lhki gtrawsbi hbHqndi eecczkqo rmdcai ynmau uvmbxr Hildpee ggvd zegckkw zPqdi dliu sz gzh uvbilkr cxy vjomi kp civeazerm orjgG‚yzquNwpceetdep ʨclcgdttd=[plyfuaetz:jb pdhhqKoh bcbabqt ozX| ngxgD hyet ienjf okxy gxev ycs +zfbpdwwozscpBjzq'hajtmdggjkhmqxhidvunbvkki jL xÒvr4 dß movxzvtpgjmjnotgcez xsdw ium hwdmVw yjp dcg +nnhpmbwztbjblquoe \ No newline at end of file diff --git a/test/test-file.mp b/test/messagepack/test-file-ext.mp similarity index 100% rename from test/test-file.mp rename to test/messagepack/test-file-ext.mp diff --git a/test/messagepack/test-file-noext.mp b/test/messagepack/test-file-noext.mp new file mode 100644 index 0000000..80103e8 Binary files /dev/null and b/test/messagepack/test-file-noext.mp differ diff --git a/test/mpack-config.h b/test/mpack-config.h deleted file mode 100644 index d66152d..0000000 --- a/test/mpack-config.h +++ /dev/null @@ -1,73 +0,0 @@ - -#ifndef MPACK_CONFIG_H -#define MPACK_CONFIG_H 1 - -// This is the configuration for the MPack test harness. - -#define MPACK_UNIT_TESTS 1 - -#if defined(DEBUG) || defined(_DEBUG) -#define MPACK_DEBUG 1 -#endif - -// Most options such as featureset and platform configuration -// are specified by the SCons buildsystem. For other platforms, -// we define the usual configuration here. -#ifndef MPACK_SCONS - #define MPACK_READER 1 - #define MPACK_WRITER 1 - #define MPACK_EXPECT 1 - #define MPACK_NODE 1 - - #define MPACK_STDLIB 1 - #define MPACK_STDIO 1 - #define MPACK_MALLOC test_malloc - #define MPACK_FREE test_free -#endif - -// We replace the file i/o functions to simulate failures -#if defined(MPACK_STDIO) && MPACK_STDIO -#include -#define fopen test_fopen -#define fclose test_fclose -#define fread test_fread -#define fwrite test_fwrite -#define fseek test_fseek -#define ftell test_ftell -#define ferror test_ferror -#endif - -// Tracking matches the default config, except the test suite -// also supports MPACK_NO_TRACKING to disable it. -#if defined(MPACK_MALLOC) && !defined(MPACK_NO_TRACKING) - #if defined(MPACK_DEBUG) && MPACK_DEBUG && defined(MPACK_READER) && MPACK_READER - #define MPACK_READ_TRACKING 1 - #endif - #if defined(MPACK_DEBUG) && MPACK_DEBUG && defined(MPACK_WRITER) && MPACK_WRITER - #define MPACK_WRITE_TRACKING 1 - #endif -#endif - -// We use a custom assert function which longjmps, allowing -// us to test assertions in debug mode. -#ifdef MPACK_DEBUG -#define MPACK_CUSTOM_ASSERT 1 -#define MPACK_CUSTOM_BREAK 1 -#endif - -#include "test-system.h" - -// we use small buffer sizes to test flushing, growing, and malloc failures -#define MPACK_TRACKING_INITIAL_CAPACITY 3 -#define MPACK_STACK_SIZE 7 -#define MPACK_BUFFER_SIZE 32 -#define MPACK_NODE_PAGE_SIZE 113 - -#ifdef MPACK_MALLOC -#define MPACK_NODE_INITIAL_DEPTH 3 -#else -#define MPACK_NODE_MAX_DEPTH_WITHOUT_MALLOC 32 -#endif - -#endif - diff --git a/test/pseudojson/test-file-ext.debug b/test/pseudojson/test-file-ext.debug new file mode 100644 index 0000000..354f6cd --- /dev/null +++ b/test/pseudojson/test-file-ext.debug @@ -0,0 +1,25 @@ + [ + null, + false, + true, + [ + 0, + 1, + -1, + -32768, + 65535, + -2147483648, + 4294967295, + 0.000000 + ], + { + "a": 1, + "b": 2, + "c": 3, + "d": 4 + }, + "\"\n\\", + "The quick brown fox jumps over the lazy dog.", + , + + ] diff --git a/test/test-file.debug b/test/pseudojson/test-file-noext.debug similarity index 85% rename from test/test-file.debug rename to test/pseudojson/test-file-noext.debug index 5adbd1c..6244084 100644 --- a/test/test-file.debug +++ b/test/pseudojson/test-file-noext.debug @@ -20,6 +20,5 @@ }, "\"\n\\", "The quick brown fox jumps over the lazy dog.", - , - + ] diff --git a/test/test-file.c b/test/test-file.c deleted file mode 100644 index 86f071b..0000000 --- a/test/test-file.c +++ /dev/null @@ -1,741 +0,0 @@ - -#include "test-file.h" -#include "test-write.h" -#include "test-reader.h" -#include "test-node.h" - -#if MPACK_STDIO - -// the file tests currently all require the writer, since it -// is used to write the test data that is read back. -#if MPACK_WRITER - -#ifdef WIN32 -#define TEST_PATH "..\\..\\test\\" -#else -#include -#define TEST_PATH "test/" -#endif - -static const char* test_blank_filename = "mpack-test-blank-file"; -static const char* test_filename = "mpack-test-file"; -static const char* test_dir = "mpack-test-dir"; - -static const int nesting_depth = 150; -static const char* lipsum = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed nec justo purus. Nunc finibus dolor id lorem sagittis, euismod efficitur arcu aliquam. Nullam a ante eget mi porttitor dignissim vitae at libero. Maecenas in justo massa. Mauris ultricies leo nisl, at ullamcorper erat maximus sit amet. Quisque pharetra sed ligula nec tristique. Mauris consectetur sapien lacus, et pharetra turpis rhoncus a. Sed in eleifend eros. Donec in libero lacus. Sed et finibus ipsum. Etiam eros leo, mollis eget molestie quis, rhoncus ac magna. Donec dolor risus, bibendum et scelerisque at, faucibus in mi. Interdum et malesuada fames ac ante ipsum primis in faucibus. Vestibulum convallis accumsan mollis."; -static const char* quick_brown_fox = "The quick brown fox jumps over a lazy dog."; - -static char* test_file_fetch(const char* filename, size_t* out_size) { - *out_size = 0; - - // open the file - FILE* file = fopen(filename, "rb"); - if (!file) { - TEST_TRUE(false, "missing file %s", filename); - return NULL; - } - - // get the file size - fseek(file, 0, SEEK_END); - long size = ftell(file); - fseek(file, 0, SEEK_SET); - if (size < 0) { - TEST_TRUE(false, "invalid file size %i for %s", (int)size, filename); - fclose(file); - return NULL; - } - - // allocate the data - if (size == 0) { - fclose(file); - return (char*)MPACK_MALLOC(1); - } - char* data = (char*)MPACK_MALLOC(size); - - // read the file - long total = 0; - while (total < size) { - size_t count = fread(data + total, 1, (size_t)(size - total), file); - if (count <= 0) { - TEST_TRUE(false, "failed to read from file %s", filename); - fclose(file); - MPACK_FREE(data); - return NULL; - } - total += count; - } - - fclose(file); - *out_size = (size_t)size; - return data; -} - -static void test_file_write_bytes(mpack_writer_t* writer, mpack_tag_t tag) { - mpack_write_tag(writer, tag); - char buf[1024]; - memset(buf, 0, sizeof(buf)); - for (; tag.v.l > sizeof(buf); tag.v.l -= (uint32_t)sizeof(buf)) - mpack_write_bytes(writer, buf, sizeof(buf)); - mpack_write_bytes(writer, buf, tag.v.l); - mpack_finish_type(writer, tag.type); -} - -static void test_file_write_elements(mpack_writer_t* writer, mpack_tag_t tag) { - mpack_write_tag(writer, tag); - for (size_t i = 0; i < tag.v.n; ++i) { - if (tag.type == mpack_type_map) - mpack_write_nil(writer); - mpack_write_nil(writer); - } - mpack_finish_type(writer, tag.type); -} - -static void test_file_write(void) { - mpack_writer_t writer; - mpack_writer_init_file(&writer, test_filename); - TEST_TRUE(mpack_writer_error(&writer) == mpack_ok, "file open failed with %s", - mpack_error_to_string(mpack_writer_error(&writer))); - - mpack_start_array(&writer, 7); - - // write lipsum to test a large fill/seek - mpack_write_cstr(&writer, lipsum); - - // test compound types of various sizes - mpack_start_array(&writer, 5); - test_file_write_bytes(&writer, mpack_tag_str(0)); - test_file_write_bytes(&writer, mpack_tag_str(INT8_MAX)); - test_file_write_bytes(&writer, mpack_tag_str(UINT8_MAX)); - test_file_write_bytes(&writer, mpack_tag_str(UINT8_MAX + 1)); - test_file_write_bytes(&writer, mpack_tag_str(UINT16_MAX + 1)); - mpack_finish_array(&writer); - - mpack_start_array(&writer, 5); - test_file_write_bytes(&writer, mpack_tag_bin(0)); - test_file_write_bytes(&writer, mpack_tag_bin(INT8_MAX)); - test_file_write_bytes(&writer, mpack_tag_bin(UINT8_MAX)); - test_file_write_bytes(&writer, mpack_tag_bin(UINT8_MAX + 1)); - test_file_write_bytes(&writer, mpack_tag_bin(UINT16_MAX + 1)); - mpack_finish_array(&writer); - - mpack_start_array(&writer, 10); - test_file_write_bytes(&writer, mpack_tag_ext(1, 0)); - test_file_write_bytes(&writer, mpack_tag_ext(1, 1)); - test_file_write_bytes(&writer, mpack_tag_ext(1, 2)); - test_file_write_bytes(&writer, mpack_tag_ext(1, 4)); - test_file_write_bytes(&writer, mpack_tag_ext(1, 8)); - test_file_write_bytes(&writer, mpack_tag_ext(1, 16)); - test_file_write_bytes(&writer, mpack_tag_ext(2, INT8_MAX)); - test_file_write_bytes(&writer, mpack_tag_ext(3, UINT8_MAX)); - test_file_write_bytes(&writer, mpack_tag_ext(4, UINT8_MAX + 1)); - test_file_write_bytes(&writer, mpack_tag_ext(5, UINT16_MAX + 1)); - mpack_finish_array(&writer); - - mpack_start_array(&writer, 5); - test_file_write_elements(&writer, mpack_tag_array(0)); - test_file_write_elements(&writer, mpack_tag_array(INT8_MAX)); - test_file_write_elements(&writer, mpack_tag_array(UINT8_MAX)); - test_file_write_elements(&writer, mpack_tag_array(UINT8_MAX + 1)); - test_file_write_elements(&writer, mpack_tag_array(UINT16_MAX + 1)); - mpack_finish_array(&writer); - - mpack_start_array(&writer, 5); - test_file_write_elements(&writer, mpack_tag_map(0)); - test_file_write_elements(&writer, mpack_tag_map(INT8_MAX)); - test_file_write_elements(&writer, mpack_tag_map(UINT8_MAX)); - test_file_write_elements(&writer, mpack_tag_map(UINT8_MAX + 1)); - test_file_write_elements(&writer, mpack_tag_map(UINT16_MAX + 1)); - mpack_finish_array(&writer); - - // test deep nesting - for (int i = 0; i < nesting_depth; ++i) - mpack_start_array(&writer, 1); - mpack_write_nil(&writer); - for (int i = 0; i < nesting_depth; ++i) - mpack_finish_array(&writer); - - mpack_finish_array(&writer); - - mpack_error_t error = mpack_writer_destroy(&writer); - TEST_TRUE(error == mpack_ok, "write failed with %s", mpack_error_to_string(error)); - - // test invalid filename - (void)mkdir(test_dir, 0700); - mpack_writer_init_file(&writer, test_dir); - TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_io); - - // test close and flush failure - // (if we write more than libc's internal FILE buffer size, fwrite() - // fails, otherwise fclose() fails. we test both here.) - - mpack_writer_init_file(&writer, "/dev/full"); - mpack_write_cstr(&writer, quick_brown_fox); - TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_io); - - int count = UINT16_MAX / 20; - mpack_writer_init_file(&writer, "/dev/full"); - mpack_start_array(&writer, count); - for (int i = 0; i < count; ++i) - mpack_write_cstr(&writer, quick_brown_fox); - mpack_finish_array(&writer); - TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_io); -} - -static bool test_file_write_failure(void) { - - // The write failure test may fail with either - // mpack_error_memory or mpack_error_io. We write a - // bunch of strs and bins to test the various expect - // allocator modes. - - mpack_writer_t writer; - mpack_writer_init_file(&writer, test_filename); - - mpack_start_array(&writer, 2); - mpack_start_array(&writer, 6); - - // write a large string near the start to cause a - // more than double buffer size growth - mpack_write_cstr(&writer, quick_brown_fox); - - mpack_write_cstr(&writer, "one"); - mpack_write_cstr(&writer, "two"); - mpack_write_cstr(&writer, "three"); - mpack_write_cstr(&writer, "four"); - mpack_write_cstr(&writer, "five"); - - mpack_finish_array(&writer); - - // test deep nesting - for (int i = 0; i < nesting_depth; ++i) - mpack_start_array(&writer, 1); - mpack_write_nil(&writer); - for (int i = 0; i < nesting_depth; ++i) - mpack_finish_array(&writer); - - mpack_finish_array(&writer); - - mpack_error_t error = mpack_writer_destroy(&writer); - if (error == mpack_error_io || error == mpack_error_memory) - return false; - TEST_TRUE(error == mpack_ok, "unexpected error state %i (%s)", (int)error, - mpack_error_to_string(error)); - return true; - -} - -// compares the test filename to the expected debug output -static void test_compare_print() { - size_t expected_size; - char* expected_data = test_file_fetch(TEST_PATH "test-file.debug", &expected_size); - size_t actual_size; - char* actual_data = test_file_fetch(test_filename, &actual_size); - - TEST_TRUE(actual_size == expected_size, "print length %i does not match expected length %i", - (int)actual_size, (int)expected_size); - TEST_TRUE(0 == memcmp(actual_data, expected_data, actual_size), "print does not match expected"); - - MPACK_FREE(expected_data); - MPACK_FREE(actual_data); -} - -#if MPACK_READER -static void test_print(void) { - - // miscellaneous print tests - // (we're not actually checking the output; we just want to make - // sure it doesn't crash under the below errors.) - FILE* out = fopen(test_filename, "wb"); - mpack_print_file("\x91", 1, out); // truncated file - mpack_print_file("\xa1", 1, out); // truncated str - mpack_print_file("\x92\x00", 2, out); // truncated array - mpack_print_file("\x81", 1, out); // truncated map key - mpack_print_file("\x81\x00", 2, out); // truncated map value - mpack_print_file("\x90\xc0", 2, out); // extra bytes - mpack_print_file("\xca\x00\x00\x00\x00", 5, out); // float - fclose(out); - - - // dump MessagePack to debug file - - size_t input_size; - char* input_data = test_file_fetch(TEST_PATH "test-file.mp", &input_size); - - out = fopen(test_filename, "wb"); - mpack_print_file(input_data, input_size, out); - fclose(out); - - MPACK_FREE(input_data); - test_compare_print(); -} -#endif - -#if MPACK_NODE -static void test_node_print(void) { - mpack_tree_t tree; - - // miscellaneous node print tests - FILE* out = fopen(test_filename, "wb"); - mpack_tree_init(&tree, "\xca\x00\x00\x00\x00", 5); // float - mpack_node_print_file(mpack_tree_root(&tree), out); - mpack_tree_destroy(&tree); - fclose(out); - - - // dump MessagePack to debug file - - mpack_tree_init_file(&tree, TEST_PATH "test-file.mp", 0); - - out = fopen(test_filename, "wb"); - mpack_node_print_file(mpack_tree_root(&tree), out); - fclose(out); - - TEST_TRUE(mpack_ok == mpack_tree_destroy(&tree)); - test_compare_print(); -} -#endif - -#if MPACK_READER -static void test_file_discard(void) { - mpack_reader_t reader; - mpack_reader_init_file(&reader, test_filename); - mpack_discard(&reader); - TEST_READER_DESTROY_NOERROR(&reader); - - mpack_reader_init_file(&reader, test_filename); - reader.skip = NULL; // disable the skip callback to test skipping without it - mpack_discard(&reader); - TEST_READER_DESTROY_NOERROR(&reader); -} -#endif - -#if MPACK_EXPECT -static void test_file_expect_bytes(mpack_reader_t* reader, mpack_tag_t tag) { - mpack_expect_tag(reader, tag); - TEST_TRUE(mpack_reader_error(reader) == mpack_ok, "got error %i (%s)", (int)mpack_reader_error(reader), mpack_error_to_string(mpack_reader_error(reader))); - - char expected[1024]; - memset(expected, 0, sizeof(expected)); - char buf[sizeof(expected)]; - while (tag.v.l > 0) { - uint32_t count = (tag.v.l < (uint32_t)sizeof(buf)) ? tag.v.l : (uint32_t)sizeof(buf); - mpack_read_bytes(reader, buf, count); - TEST_TRUE(mpack_reader_error(reader) == mpack_ok, "got error %i (%s)", (int)mpack_reader_error(reader), mpack_error_to_string(mpack_reader_error(reader))); - TEST_TRUE(memcmp(buf, expected, count) == 0, "data does not match!"); - tag.v.l -= count; - } - - mpack_done_type(reader, tag.type); -} - -static void test_file_expect_elements(mpack_reader_t* reader, mpack_tag_t tag) { - mpack_expect_tag(reader, tag); - for (size_t i = 0; i < tag.v.l; ++i) { - if (tag.type == mpack_type_map) - mpack_expect_nil(reader); - mpack_expect_nil(reader); - } - mpack_done_type(reader, tag.type); -} - -static void test_file_read(void) { - mpack_reader_t reader; - mpack_reader_init_file(&reader, test_filename); - TEST_TRUE(mpack_reader_error(&reader) == mpack_ok, "file open failed with %s", - mpack_error_to_string(mpack_reader_error(&reader))); - - TEST_TRUE(7 == mpack_expect_array(&reader)); - - // test matching a cstr larger than the buffer size - mpack_expect_cstr_match(&reader, lipsum); - - TEST_TRUE(5 == mpack_expect_array(&reader)); - test_file_expect_bytes(&reader, mpack_tag_str(0)); - test_file_expect_bytes(&reader, mpack_tag_str(INT8_MAX)); - test_file_expect_bytes(&reader, mpack_tag_str(UINT8_MAX)); - test_file_expect_bytes(&reader, mpack_tag_str(UINT8_MAX + 1)); - test_file_expect_bytes(&reader, mpack_tag_str(UINT16_MAX + 1)); - mpack_done_array(&reader); - - TEST_TRUE(5 == mpack_expect_array(&reader)); - test_file_expect_bytes(&reader, mpack_tag_bin(0)); - test_file_expect_bytes(&reader, mpack_tag_bin(INT8_MAX)); - test_file_expect_bytes(&reader, mpack_tag_bin(UINT8_MAX)); - test_file_expect_bytes(&reader, mpack_tag_bin(UINT8_MAX + 1)); - test_file_expect_bytes(&reader, mpack_tag_bin(UINT16_MAX + 1)); - mpack_done_array(&reader); - - TEST_TRUE(10 == mpack_expect_array(&reader)); - test_file_expect_bytes(&reader, mpack_tag_ext(1, 0)); - test_file_expect_bytes(&reader, mpack_tag_ext(1, 1)); - test_file_expect_bytes(&reader, mpack_tag_ext(1, 2)); - test_file_expect_bytes(&reader, mpack_tag_ext(1, 4)); - test_file_expect_bytes(&reader, mpack_tag_ext(1, 8)); - test_file_expect_bytes(&reader, mpack_tag_ext(1, 16)); - test_file_expect_bytes(&reader, mpack_tag_ext(2, INT8_MAX)); - test_file_expect_bytes(&reader, mpack_tag_ext(3, UINT8_MAX)); - test_file_expect_bytes(&reader, mpack_tag_ext(4, UINT8_MAX + 1)); - test_file_expect_bytes(&reader, mpack_tag_ext(5, UINT16_MAX + 1)); - mpack_done_array(&reader); - - TEST_TRUE(5 == mpack_expect_array(&reader)); - test_file_expect_elements(&reader, mpack_tag_array(0)); - test_file_expect_elements(&reader, mpack_tag_array(INT8_MAX)); - test_file_expect_elements(&reader, mpack_tag_array(UINT8_MAX)); - test_file_expect_elements(&reader, mpack_tag_array(UINT8_MAX + 1)); - test_file_expect_elements(&reader, mpack_tag_array(UINT16_MAX + 1)); - mpack_done_array(&reader); - - TEST_TRUE(5 == mpack_expect_array(&reader)); - test_file_expect_elements(&reader, mpack_tag_map(0)); - test_file_expect_elements(&reader, mpack_tag_map(INT8_MAX)); - test_file_expect_elements(&reader, mpack_tag_map(UINT8_MAX)); - test_file_expect_elements(&reader, mpack_tag_map(UINT8_MAX + 1)); - test_file_expect_elements(&reader, mpack_tag_map(UINT16_MAX + 1)); - mpack_done_array(&reader); - - for (int i = 0; i < nesting_depth; ++i) - mpack_expect_array_match(&reader, 1); - mpack_expect_nil(&reader); - for (int i = 0; i < nesting_depth; ++i) - mpack_done_array(&reader); - - mpack_done_array(&reader); - - mpack_error_t error = mpack_reader_destroy(&reader); - TEST_TRUE(error == mpack_ok, "read failed with %s", mpack_error_to_string(error)); - - // test missing file - mpack_reader_init_file(&reader, "invalid-filename"); - TEST_READER_DESTROY_ERROR(&reader, mpack_error_io); -} - -static bool test_file_expect_failure(void) { - - // The expect failure test may fail with either - // mpack_error_memory or mpack_error_io. - - mpack_reader_t reader; - - #define TEST_POSSIBLE_FAILURE() do { \ - mpack_error_t error = mpack_reader_error(&reader); \ - if (error == mpack_error_memory || error == mpack_error_io) { \ - mpack_reader_destroy(&reader); \ - return false; \ - } \ - } while (0) - - mpack_reader_init_file(&reader, test_filename); - mpack_expect_array_match(&reader, 2); - - uint32_t count; - char** strings = mpack_expect_array_alloc(&reader, char*, 50, &count); - TEST_POSSIBLE_FAILURE(); - TEST_TRUE(strings != NULL); - TEST_TRUE(count == 6); - MPACK_FREE(strings); - - char* str = mpack_expect_cstr_alloc(&reader, 100); - TEST_POSSIBLE_FAILURE(); - TEST_TRUE(str != NULL); - if (str) { - TEST_TRUE(strcmp(str, quick_brown_fox) == 0); - MPACK_FREE(str); - } - - str = mpack_expect_utf8_cstr_alloc(&reader, 100); - TEST_POSSIBLE_FAILURE(); - TEST_TRUE(str != NULL); - if (str) { - TEST_TRUE(strcmp(str, "one") == 0); - MPACK_FREE(str); - } - - str = mpack_expect_cstr_alloc(&reader, 100); - TEST_POSSIBLE_FAILURE(); - TEST_TRUE(str != NULL); - if (str) { - TEST_TRUE(strcmp(str, "two") == 0); - MPACK_FREE(str); - } - - str = mpack_expect_utf8_cstr_alloc(&reader, 100); - TEST_POSSIBLE_FAILURE(); - TEST_TRUE(str != NULL); - if (str) { - TEST_TRUE(strcmp(str, "three") == 0); - MPACK_FREE(str); - } - - mpack_discard(&reader); - mpack_discard(&reader); - mpack_done_array(&reader); - - mpack_discard(&reader); // discard the deep nested arrays - mpack_done_array(&reader); - - #undef TEST_POSSIBLE_FAILURE - - mpack_error_t error = mpack_reader_destroy(&reader); - if (error == mpack_error_io || error == mpack_error_memory) - return false; - TEST_TRUE(error == mpack_ok, "unexpected error state %i (%s)", (int)error, - mpack_error_to_string(error)); - return true; - -} -#endif - -#if MPACK_NODE -static void test_file_node_bytes(mpack_node_t node, mpack_tag_t tag) { - TEST_TRUE(mpack_tag_equal(tag, mpack_node_tag(node))); - const char* data = mpack_node_data(node); - uint32_t length = mpack_node_data_len(node); - TEST_TRUE(mpack_node_error(node) == mpack_ok); - - char expected[1024]; - memset(expected, 0, sizeof(expected)); - while (length > 0) { - uint32_t count = (length < (uint32_t)sizeof(expected)) ? length : (uint32_t)sizeof(expected); - TEST_TRUE(memcmp(data, expected, count) == 0); - length -= count; - data += count; - } -} - -static void test_file_node_elements(mpack_node_t node, mpack_tag_t tag) { - TEST_TRUE(mpack_tag_equal(tag, mpack_node_tag(node))); - for (size_t i = 0; i < tag.v.n; ++i) { - if (tag.type == mpack_type_map) { - mpack_node_nil(mpack_node_map_key_at(node, i)); - mpack_node_nil(mpack_node_map_value_at(node, i)); - } else { - mpack_node_nil(mpack_node_array_at(node, i)); - } - } -} - -static void test_file_node(void) { - mpack_tree_t tree; - - // test maximum size - mpack_tree_init_file(&tree, test_filename, 100); - TEST_TREE_DESTROY_ERROR(&tree, mpack_error_too_big); - - // test blank file - mpack_tree_init_file(&tree, test_blank_filename, 0); - TEST_TREE_DESTROY_ERROR(&tree, mpack_error_invalid); - - // test successful parse - mpack_tree_init_file(&tree, test_filename, 0); - TEST_TRUE(mpack_tree_error(&tree) == mpack_ok, "file tree parsing failed: %s", - mpack_error_to_string(mpack_tree_error(&tree))); - - mpack_node_t root = mpack_tree_root(&tree); - TEST_TRUE(mpack_node_array_length(root) == 7); - - mpack_node_t lipsum_node = mpack_node_array_at(root, 0); - const char* lipsum_str = mpack_node_str(lipsum_node); - TEST_TRUE(lipsum_str != NULL); - if (lipsum_str) { - TEST_TRUE(mpack_node_strlen(lipsum_node) == strlen(lipsum)); - TEST_TRUE(memcmp(lipsum_str, lipsum, strlen(lipsum)) == 0); - } - - mpack_node_t node = mpack_node_array_at(root, 1); - TEST_TRUE(mpack_node_array_length(node) == 5); - test_file_node_bytes(mpack_node_array_at(node, 0), mpack_tag_str(0)); - test_file_node_bytes(mpack_node_array_at(node, 1), mpack_tag_str(INT8_MAX)); - test_file_node_bytes(mpack_node_array_at(node, 2), mpack_tag_str(UINT8_MAX)); - test_file_node_bytes(mpack_node_array_at(node, 3), mpack_tag_str(UINT8_MAX + 1)); - test_file_node_bytes(mpack_node_array_at(node, 4), mpack_tag_str(UINT16_MAX + 1)); - - node = mpack_node_array_at(root, 2); - TEST_TRUE(5 == mpack_node_array_length(node)); - test_file_node_bytes(mpack_node_array_at(node, 0), mpack_tag_bin(0)); - test_file_node_bytes(mpack_node_array_at(node, 1), mpack_tag_bin(INT8_MAX)); - test_file_node_bytes(mpack_node_array_at(node, 2), mpack_tag_bin(UINT8_MAX)); - test_file_node_bytes(mpack_node_array_at(node, 3), mpack_tag_bin(UINT8_MAX + 1)); - test_file_node_bytes(mpack_node_array_at(node, 4), mpack_tag_bin(UINT16_MAX + 1)); - - node = mpack_node_array_at(root, 3); - TEST_TRUE(10 == mpack_node_array_length(node)); - test_file_node_bytes(mpack_node_array_at(node, 0), mpack_tag_ext(1, 0)); - test_file_node_bytes(mpack_node_array_at(node, 1), mpack_tag_ext(1, 1)); - test_file_node_bytes(mpack_node_array_at(node, 2), mpack_tag_ext(1, 2)); - test_file_node_bytes(mpack_node_array_at(node, 3), mpack_tag_ext(1, 4)); - test_file_node_bytes(mpack_node_array_at(node, 4), mpack_tag_ext(1, 8)); - test_file_node_bytes(mpack_node_array_at(node, 5), mpack_tag_ext(1, 16)); - test_file_node_bytes(mpack_node_array_at(node, 6), mpack_tag_ext(2, INT8_MAX)); - test_file_node_bytes(mpack_node_array_at(node, 7), mpack_tag_ext(3, UINT8_MAX)); - test_file_node_bytes(mpack_node_array_at(node, 8), mpack_tag_ext(4, UINT8_MAX + 1)); - test_file_node_bytes(mpack_node_array_at(node, 9), mpack_tag_ext(5, UINT16_MAX + 1)); - - node = mpack_node_array_at(root, 4); - TEST_TRUE(5 == mpack_node_array_length(node)); - test_file_node_elements(mpack_node_array_at(node, 0), mpack_tag_array(0)); - test_file_node_elements(mpack_node_array_at(node, 1), mpack_tag_array(INT8_MAX)); - test_file_node_elements(mpack_node_array_at(node, 2), mpack_tag_array(UINT8_MAX)); - test_file_node_elements(mpack_node_array_at(node, 3), mpack_tag_array(UINT8_MAX + 1)); - test_file_node_elements(mpack_node_array_at(node, 4), mpack_tag_array(UINT16_MAX + 1)); - - node = mpack_node_array_at(root, 5); - TEST_TRUE(5 == mpack_node_array_length(node)); - test_file_node_elements(mpack_node_array_at(node, 0), mpack_tag_map(0)); - test_file_node_elements(mpack_node_array_at(node, 1), mpack_tag_map(INT8_MAX)); - test_file_node_elements(mpack_node_array_at(node, 2), mpack_tag_map(UINT8_MAX)); - test_file_node_elements(mpack_node_array_at(node, 3), mpack_tag_map(UINT8_MAX + 1)); - test_file_node_elements(mpack_node_array_at(node, 4), mpack_tag_map(UINT16_MAX + 1)); - - node = mpack_node_array_at(root, 6); - for (int i = 0; i < nesting_depth; ++i) - node = mpack_node_array_at(node, 0); - TEST_TRUE(mpack_ok == mpack_node_error(node)); - mpack_node_nil(node); - - mpack_error_t error = mpack_tree_destroy(&tree); - TEST_TRUE(error == mpack_ok, "file tree failed with error %s", mpack_error_to_string(error)); - - // test file size out of bounds - #if MPACK_DEBUG - if (sizeof(size_t) >= sizeof(long)) { - TEST_BREAK((mpack_tree_init_file(&tree, "invalid-filename", ((size_t)LONG_MAX) + 1), true)); - TEST_TREE_DESTROY_ERROR(&tree, mpack_error_bug); - } - #endif - - // test missing file - mpack_tree_init_file(&tree, "invalid-filename", 0); - TEST_TREE_DESTROY_ERROR(&tree, mpack_error_io); -} - -static bool test_file_node_failure(void) { - - // The node failure test may fail with either - // mpack_error_memory or mpack_error_io. - - mpack_tree_t tree; - - #define TEST_POSSIBLE_FAILURE() do { \ - mpack_error_t error = mpack_tree_error(&tree); \ - TEST_TRUE(test_tree_error == error); \ - if (error == mpack_error_memory || error == mpack_error_io) { \ - test_tree_error = mpack_ok; \ - mpack_tree_destroy(&tree); \ - return false; \ - } \ - } while (0) - - mpack_tree_init_file(&tree, test_filename, 0); - if (mpack_tree_error(&tree) == mpack_error_memory || mpack_tree_error(&tree) == mpack_error_io) { - mpack_tree_destroy(&tree); - return false; - } - mpack_tree_set_error_handler(&tree, test_tree_error_handler); - - - mpack_node_t root = mpack_tree_root(&tree); - - mpack_node_t strings = mpack_node_array_at(root, 0); - size_t length = mpack_node_array_length(strings); - TEST_POSSIBLE_FAILURE(); - TEST_TRUE(6 == length); - - mpack_node_t node = mpack_node_array_at(strings, 0); - char* str = mpack_node_data_alloc(node, 100); - TEST_POSSIBLE_FAILURE(); - TEST_TRUE(str != NULL); - const char* expected = "The quick brown fox jumps over a lazy dog."; - TEST_TRUE(mpack_node_strlen(node) == strlen(expected)); - if (str) { - TEST_TRUE(memcmp(str, expected, mpack_node_strlen(node)) == 0); - MPACK_FREE(str); - } - - node = mpack_node_array_at(strings, 1); - - str = mpack_node_cstr_alloc(node, 100); - TEST_POSSIBLE_FAILURE(); - TEST_TRUE(str != NULL); - expected = "one"; - if (str) { - TEST_TRUE(strlen(str) == strlen(expected)); - TEST_TRUE(strcmp(str, expected) == 0); - MPACK_FREE(str); - } - - str = mpack_node_utf8_cstr_alloc(node, 100); - TEST_POSSIBLE_FAILURE(); - TEST_TRUE(str != NULL); - if (str) { - TEST_TRUE(strlen(str) == strlen(expected)); - TEST_TRUE(strcmp(str, expected) == 0); - MPACK_FREE(str); - } - - node = mpack_node_array_at(root, 1); - for (int i = 0; i < nesting_depth; ++i) - node = mpack_node_array_at(node, 0); - TEST_TRUE(mpack_ok == mpack_node_error(node)); - mpack_node_nil(node); - - #undef TEST_POSSIBLE_FAILURE - - mpack_error_t error = mpack_tree_destroy(&tree); - if (error == mpack_error_io || error == mpack_error_memory) - return false; - TEST_TRUE(error == mpack_ok, "unexpected error state %i (%s)", (int)error, - mpack_error_to_string(error)); - return true; - -} -#endif - -void test_file(void) { - // write a blank file for test purposes - FILE* blank = fopen(test_blank_filename, "wb"); - fclose(blank); - - #if MPACK_READER - test_print(); - #endif - #if MPACK_NODE - test_node_print(); - #endif - - test_file_write(); - - #if MPACK_READER - test_file_discard(); - #endif - #if MPACK_EXPECT - test_file_read(); - #endif - #if MPACK_NODE - test_file_node(); - #endif - - test_system_fail_until_ok(&test_file_write_failure); - #if MPACK_EXPECT - test_system_fail_until_ok(&test_file_expect_failure); - #endif - #if MPACK_NODE - test_system_fail_until_ok(&test_file_node_failure); - #endif - - TEST_TRUE(remove(test_filename) == 0, "failed to delete %s", test_filename); - TEST_TRUE(remove(test_blank_filename) == 0, "failed to delete %s", test_blank_filename); - TEST_TRUE(rmdir(test_dir) == 0, "failed to delete %s", test_dir); - - (void)&test_compare_print; -} - -#else - -void test_file(void) { - // if we don't have the writer, nothing to do -} - -#endif -#endif - diff --git a/test/test-file.json b/test/test-file.json deleted file mode 100644 index 3b35217..0000000 --- a/test/test-file.json +++ /dev/null @@ -1,27 +0,0 @@ -[ - null, - false, - true, - [ - 0, - 1, - -1, - -32768, - 65535, - -2147483648, - 4294967295, - //-9223372036854775808, - //18446744073709551615, - 0.0, - ], - { - "a": 1, - "b": 2, - "c": 3, - "d": 4, - }, - "\"\n\\", - "The quick brown fox jumps over the lazy dog.", - "base64:VGhlIGZpdmUgYm94aW5nIHdpemFyZHMganVtcCBxdWlja2x5Lg==", - "ext:1:base64:U3BoaW54IG9mIGJsYWNrIHF1YXJ0eiwganVkZ2UgbXkgdm93Lg==", -] diff --git a/test/test-reader.c b/test/test-reader.c deleted file mode 100644 index 9b85c21..0000000 --- a/test/test-reader.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2015 Nicholas Fraser - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - * the Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#include "test-expect.h" - -#if MPACK_READER - -mpack_error_t test_read_error = mpack_ok; - -void test_read_error_handler(mpack_reader_t* reader, mpack_error_t error) { - TEST_TRUE(test_read_error == mpack_ok, "error handler was called multiple times"); - TEST_TRUE(error != mpack_ok, "error handler was called with mpack_ok"); - TEST_TRUE(mpack_reader_error(reader) == error, "reader error does not match given error"); - test_read_error = error; -} - -// almost all reader functions are tested by the expect tests. -// minor miscellaneous read tests are added here. - -static void test_reader_should_inplace() { - char buf[4096]; - mpack_reader_t reader; - mpack_reader_init(&reader, buf, sizeof(buf), 0); - - TEST_TRUE(true == mpack_should_read_bytes_inplace(&reader, 0)); - TEST_TRUE(true == mpack_should_read_bytes_inplace(&reader, 1)); - TEST_TRUE(true == mpack_should_read_bytes_inplace(&reader, 20)); - TEST_TRUE(false == mpack_should_read_bytes_inplace(&reader, 500)); - TEST_TRUE(false == mpack_should_read_bytes_inplace(&reader, 10000)); - - mpack_reader_destroy(&reader); -} - -static void test_reader_miscellaneous() { - - // 0xc1 is reserved; it should always raise mpack_error_invalid - TEST_SIMPLE_READ_ERROR("\xc1", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); - - // simple truncated tags (testing discard of additional - // temporary data in mpack_parse_tag()) - TEST_SIMPLE_READ_ERROR("\xcc", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); - TEST_SIMPLE_READ_ERROR("\xcd", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); - TEST_SIMPLE_READ_ERROR("\xce", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); - TEST_SIMPLE_READ_ERROR("\xcf", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); - TEST_SIMPLE_READ_ERROR("\xd0", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); - TEST_SIMPLE_READ_ERROR("\xd1", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); - TEST_SIMPLE_READ_ERROR("\xd2", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); - TEST_SIMPLE_READ_ERROR("\xd3", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); - - // truncated discard errors - TEST_SIMPLE_READ_ERROR("\x91", (mpack_discard(&reader), true), mpack_error_invalid); // array - TEST_SIMPLE_READ_ERROR("\x81", (mpack_discard(&reader), true), mpack_error_invalid); // map -} - -void test_reader() { - test_reader_should_inplace(); - test_reader_miscellaneous(); -} - -#endif - diff --git a/test/test.h b/test/test.h deleted file mode 100644 index b58f3f0..0000000 --- a/test/test.h +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) 2015 Nicholas Fraser - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - * the Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -#ifndef MPACK_TEST_H -#define MPACK_TEST_H 1 - -#define _DEFAULT_SOURCE 1 -#define _BSD_SOURCE 1 - -#ifdef WIN32 -#define _CRT_SECURE_NO_WARNINGS 1 -#endif - -#include -#include -#include -#include -#include - -#ifdef WIN32 -#include -#define isnanf _isnan -#endif - -#ifdef __APPLE__ -#define isnanf isnan -#endif - -#ifdef WIN32 -#include -#define mkdir(path, mode) ((void)(mode), _mkdir(path)) -#define rmdir _rmdir -#else -#include -#include -#endif - -#ifdef __cplusplus -#include -#define MPACK_INFINITY std::numeric_limits::infinity() -#else -#define MPACK_INFINITY INFINITY -#endif - -#include "mpack/mpack.h" - -extern mpack_tag_t (*fn_mpack_tag_nil)(void); - -#ifdef __cplusplus -extern "C" { -#endif - -/* - * This is basically the whole unit testing framework for mpack. - * The reported number of "tests" is the total number of test asserts, - * where each actual test case has several asserts. - */ - -// enable this to exit at the first error -#define TEST_EARLY_EXIT 1 - -// runs the given expression, causing a unit test failure with the -// given printf format string if the expression is not true. -#define TEST_TRUE(expr, ...) \ - test_true_impl((expr), __FILE__, __LINE__, " " __VA_ARGS__) - -void test_true_impl(bool result, const char* file, int line, const char* format, ...); - -extern int tests; -extern int passes; - -#if MPACK_DEBUG -extern bool test_jmp_set; -extern jmp_buf test_jmp_buf; -extern bool test_break_set; -extern bool test_break_hit; - -// calls setjmp to expect an assert from a unit test. an assertion -// will cause a longjmp to here with a value of 1. -#define TEST_TRUE_SETJMP() \ - (TEST_TRUE(!test_jmp_set, "an assert jmp is already set!"), \ - test_jmp_set = true, \ - setjmp(test_jmp_buf)) - -// clears the expectation of an assert. a subsequent assert will -// cause the unit test suite to abort with error. -#define TEST_TRUE_CLEARJMP() \ - (TEST_TRUE(test_jmp_set, "an assert jmp is not set!"), \ - test_jmp_set = false) - -// runs the given expression, causing a unit test failure if an assertion -// is not triggered. (note that stack variables may need to be marked volatile -// since non-volatile stack variables that are written to after setjmp are -// undefined after longjmp.) -#define TEST_ASSERT(expr) do { \ - volatile bool jumped = false; \ - if (TEST_TRUE_SETJMP()) { \ - jumped = true; \ - } else { \ - (expr); \ - } \ - TEST_TRUE_CLEARJMP(); \ - TEST_TRUE(jumped, "expression should assert, but didn't: " #expr); \ -} while (0) - -#define TEST_BREAK(expr, ...) do { \ - test_break_set = true; \ - test_break_hit = false; \ - TEST_TRUE(expr , ## __VA_ARGS__, "expression is not true: " # expr); \ - TEST_TRUE(test_break_hit, "expression should break, but didn't: " # expr); \ - test_break_set = false; \ -} while (0); - -#else - -// in release mode there are no asserts or break functions, so -// TEST_BREAK() just runs the expr. it is usually used to test -// that something flags mpack_error_bug. -#define TEST_BREAK(expr, ...) do { TEST_TRUE(expr , ## __VA_ARGS__); } while (0) - -// TEST_ASSERT() is not defined because code that causes an assert -// cannot continue to run; it would cause undefined behavior (and -// crash the unit test framework.) it cannot be defined to nothing -// because the tests around it wouldn't make sense (and would cause -// unused warnings); the tests that use it must be disabled entirely. - -#endif - -#ifdef __cplusplus -} -#endif - -#endif - diff --git a/test/unit/ci-unix.sh b/test/unit/ci-unix.sh new file mode 100755 index 0000000..a49fb4d --- /dev/null +++ b/test/unit/ci-unix.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Builds and runs the unit test suite under UNIX. +# +# Pass environment variable CC to specify the compiler. +# +# This script is run by the continuous integration server to test MPack on UNIX +# systems. + +set -e + +# Amalgamate if necessary +if [[ "$AMALGAMATED" == "1" ]]; then + tools/amalgamate.sh + cd .build/amalgamation +fi +pwd + +# Run the "more" variant of unit tests +tools/unit.sh more diff --git a/test/unit/ci-windows.bat b/test/unit/ci-windows.bat new file mode 100644 index 0000000..12feabe --- /dev/null +++ b/test/unit/ci-windows.bat @@ -0,0 +1,23 @@ +@echo off + +REM This script is run by the continuous integration server to build and run +REM the MPack unit test suite with MSVC on Windows. + +setlocal enabledelayedexpansion + +REM Find build tools +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -find **\vcvarsall.bat`) do (SET "vcvarsall=%%i") + +IF "%AMALGAMATED%"=="1" ( + cd .build\amalgamation +) + +IF "%COMPILER%"=="cl-2019-x64" ( + call "%vcvarsall%" amd64 +) +IF "%COMPILER%"=="cl-2015-x86" ( + call "%vcvarsall%" x86 -vcvars_ver=14.0 +) + +REM Run the "more" variant of unit tests +call tools\unit.bat more \ No newline at end of file diff --git a/test/unit/configure.py b/test/unit/configure.py new file mode 100755 index 0000000..beec3e5 --- /dev/null +++ b/test/unit/configure.py @@ -0,0 +1,616 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +# the Software, and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +# This is the buildsystem configuration tool for the MPack unit test suite. It +# tests the compiler for support for various flags and features and generates a +# ninja build file to build the unit test suite in a variety of configurations. +# +# It can be run with a GCC-style compiler or with MSVC. To run it with MSVC, +# you must first have the Visual Studio build tools on your path. This means +# you need to either open a Visual Studio Build Tools command prompt, or source +# vsvarsall.bat for some version of the Visual Studio Build Tools. + +import shutil, os, sys, subprocess +from os import path + +globalbuild = path.join(".build", "unit") +os.makedirs(globalbuild, exist_ok=True) + + + +################################################### +# Determine Compiler +################################################### + +cc = None +compiler = "unknown" + +if os.getenv("CC"): + cc = os.getenv("CC") +elif shutil.which("cl.exe"): + cc = "cl" +else: + cc = "cc" + +if not shutil.which(cc): + raise Exception("Compiler cannot be found!") + +if cc.lower() == "cl" or cc.lower() == "cl.exe": + compiler = "MSVC" +elif cc.endswith("cproc"): + compiler = "cproc" +elif cc.endswith("chibicc"): + compiler = "chibicc" +elif cc.endswith("8cc"): + compiler = "8cc" +else: + # try --version + ret = subprocess.run([cc, "--version"], universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if ret.returncode == 0: + if ret.stdout.startswith("cparser "): + compiler = "cparser" + elif "clang" in ret.stdout: + compiler = "Clang" + + if compiler == "unknown": + # try -v + ret = subprocess.run([cc, "-v"], universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + if ret.returncode == 0: + for line in (ret.stdout + "\n" + ret.stderr).splitlines(): + if line.startswith("tcc "): + compiler = "TinyCC" + break + elif line.startswith("gcc "): + compiler = "GCC" + break + +print("Using " + compiler + " compiler with executable: " + cc) + +if compiler == "MSVC": + obj_extension = ".obj" + exe_extension = ".exe" +else: + obj_extension = ".o" + exe_extension = "" + +msvc = (compiler == "MSVC") + + + +################################################### +# Compiler Probing +################################################### + +config = { + "flags": {}, +} + +flagtest_src = path.join(globalbuild, "flagtest.c") +flagtest_exe = path.join(globalbuild, "flagtest" + exe_extension) +with open(flagtest_src, "w") as out: + out.write(""" +int main(int argc, char** argv) { + // array dereference to test for the existence of + // sanitizer libs when using -fsanitize (libubsan) + // compare it to another string in the array so that + // -Wzero-as-null-pointer-constant works + return argv[argc - 1] == argv[0]; +} +""") + +def checkFlags(flags): + if isinstance(flags, str): + flags = [flags,] + + configArg = "|".join(flags) + if configArg in config["flags"]: + return config["flags"][configArg] + print("Testing flag(s): " + " ".join(flags) + " ... ", end="") + sys.stdout.flush() + + if msvc: + cmd = [cc, "/WX"] + flags + [flagtest_src, "/Fe" + flagtest_exe] + else: + cmd = [cc, "-Werror"] + flags + [flagtest_src, "-o", flagtest_exe] + ret = subprocess.run(cmd, universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + if ret.returncode == 0: + print("Supported.") + supported = True + else: + print("Not supported.") + supported = False + config["flags"][configArg] = supported + return supported + +def flagsIfSupported(flags): + if checkFlags(flags): + if isinstance(flags, str): + return [flags] + return flags + return [] + +# We use -Og for all debug builds if we have it, but ONLY under GCC. It can +# sometimes improve warnings, and things run a lot faster especially under +# Valgrind, but Clang stupidly maps it to -O1 which has some optimizations +# that break debugging! +hasOg = False +print("Testing flag(s): -Og ... ", end="") +sys.stdout.flush() +if msvc: + print("Not supported.") +else: + ret = subprocess.run([cc, "-v"], universal_newlines=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + sys.stdout.flush() + if ret.returncode != 0: + print("Not supported.") + else: + for line in ret.stdout.splitlines(): + if line.startswith("gcc version"): + hasOg = True + break + if hasOg: + print("Supported.") + else: + print("May be supported, but we won't use it.") + + + +################################################### +# Common Flags +################################################### + +global_cppflags = [] + +if msvc: + global_cppflags += [ + "/W4", "/WX", + # debug to PDB with synchronous writes since we're doing parallel builds + # (we specify a per-build PDB path during build generation below) + "/Zi", "/FS" + ] +else: + global_cppflags += [ + "-Wall", "-Wextra", "-Werror", + "-Wconversion", "-Wundef", + "-Wshadow", "-Wcast-qual", + "-g", + ] + +global_cppflags += [ + "-Isrc", "-Itest/unit/src", + "-DMPACK_VARIANT_BUILDS=1", + "-DMPACK_HAS_CONFIG=1", +] + +defaultfeatures = [ + "-DMPACK_READER=1", + "-DMPACK_WRITER=1", + "-DMPACK_EXPECT=1", + "-DMPACK_NODE=1", +] + +allfeatures = defaultfeatures + [ + "-DMPACK_COMPATIBILITY=1", + "-DMPACK_EXTENSIONS=1", +] + +noioconfigs = [ + "-DMPACK_STDLIB=1", + "-DMPACK_MALLOC=test_malloc", + "-DMPACK_FREE=test_free", +] + +allconfigs = noioconfigs + [ + "-DMPACK_STDIO=1", +] + +# optimization +if msvc: + debugflags = ["/Od", "/MDd"] + releaseflags = ["/O2", "/MD"] +else: + debugflags = [hasOg and "-Og" or "-O0"] + releaseflags = ["-O2"] +debugflags.append("-DDEBUG") +releaseflags.append("-DNDEBUG") + +# flags for specifying source language +cxxlinkflags = [] +if msvc: + cflags = ["/TC"] + cxxflags = ["/TP", "/EHsc"] +else: + cflags = [ + checkFlags("-std=c11") and "-std=c11" or "-std=c99", + "-Wc++-compat" + ] + cxxflags = [ + "-x", "c++", + "-Wmissing-declarations", + ] + # When building as C++ on macOS, clang will emit calls to std::terminate + # even if no C++ features are used so libc++ is required. We link with cc, + # not c++ so we need to link it manually, and there is apparently no way to + # link it statically either. (This overrides the use of libstdc++ if libc++ + # is installed so it's not ideal. We can worry about this later.) + if checkFlags(cxxflags + ["-lc++"]): + cxxlinkflags.append("-lc++") + + + +################################################### +# Variable Flags +################################################### + +if not os.getenv("CI") and not msvc: + # we have to force color diagnostics to get color output from ninja + # (ninja will strip the colors if it's being piped) + if checkFlags("-fdiagnostics-color=always"): + global_cppflags.append("-fdiagnostics-color=always") + elif checkFlags("-fcolor-diagnostics=always"): + global_cppflags.append("-color-diagnostics=always") + +if checkFlags("-Wstrict-aliasing=3"): + global_cppflags.append("-Wstrict-aliasing=3") +elif checkFlags("-Wstrict-aliasing=2"): + global_cppflags.append("-Wstrict-aliasing=2") +elif checkFlags("-Wstrict-aliasing"): + global_cppflags.append("-Wstrict-aliasing") + +extra_warnings_to_test = [ + "-Wpedantic", + "-Wmissing-variable-declarations", + "-Wfloat-conversion", +] +if not msvc: + extra_warnings_to_test += ["-fstrict-aliasing"] + +for flag in extra_warnings_to_test: + global_cppflags += flagsIfSupported(flag) + +cflags += flagsIfSupported("-Wmissing-prototypes") +cflags += flagsIfSupported("-Wstrict-prototypes") + +#TODO +ltoflags = [] + + + +################################################### +# Build configuration +################################################### + +builds = {} + +class Build: + def __init__(self, name, cppflags, ldflags): + self.name = name + self.cppflags = cppflags + self.ldflags = ldflags + self.run_wrapper = None + self.exclude = False + +def addBuild(name, cppflags, ldflags=[]): + builds[name] = Build(name, cppflags[:], ldflags[:]) + +def addDebugReleaseBuilds(name, cppflags, ldflags = []): + addBuild(name + "-debug", cppflags + debugflags, ldflags) + addBuild(name + "-release", cppflags + releaseflags, ldflags) + +addDebugReleaseBuilds('default', defaultfeatures + allconfigs + cflags) +addDebugReleaseBuilds('everything', allfeatures + allconfigs + cflags) +addDebugReleaseBuilds('empty', allconfigs + cflags) +addDebugReleaseBuilds('reader', ["-DMPACK_READER=1"] + allconfigs + cflags) +addDebugReleaseBuilds('expect', ["-DMPACK_READER=1", "-DMPACK_EXPECT=1"] + allconfigs + cflags) +addDebugReleaseBuilds('node', ["-DMPACK_NODE=1"] + allconfigs + cflags) +addDebugReleaseBuilds('compatibility', defaultfeatures + ["-DMPACK_COMPATIBILITY=1"] + allconfigs + cflags) +addDebugReleaseBuilds('extensions', defaultfeatures + ["-DMPACK_EXTENSIONS=1"] + allconfigs + cflags) +addDebugReleaseBuilds('no-float', allfeatures + allconfigs + cflags + ["-DMPACK_FLOAT=0"]) +addDebugReleaseBuilds('no-double', allfeatures + allconfigs + cflags + ["-DMPACK_DOUBLE=0"]) + +# writer builds +addDebugReleaseBuilds('writer-only', + ["-DMPACK_WRITER=1", "-DMPACK_BUILDER=0"] + + allconfigs + cflags) +addDebugReleaseBuilds('builder-internal', + ["-DMPACK_WRITER=1", "-DMPACK_BUILDER=1", "-DMPACK_BUILDER_INTERNAL_STORAGE=1"] + + allconfigs + cflags) +addDebugReleaseBuilds('builder-nointernal', + ["-DMPACK_WRITER=1", "-DMPACK_BUILDER=1", "-DMPACK_BUILDER_INTERNAL_STORAGE=0"] + + allconfigs + cflags) + +# no i/o +addDebugReleaseBuilds('noio', allfeatures + noioconfigs + cflags) +addDebugReleaseBuilds('noio-writer', ["-DMPACK_WRITER=1"] + noioconfigs + cflags) +addDebugReleaseBuilds('noio-reader', ["-DMPACK_READER=1"] + noioconfigs + cflags) +addDebugReleaseBuilds('noio-expect', ["-DMPACK_READER=1", "-DMPACK_EXPECT=1"] + noioconfigs + cflags) +addDebugReleaseBuilds('noio-node', ["-DMPACK_NODE=1"] + noioconfigs + cflags) + +# embedded builds without libc (using builtins) +addDebugReleaseBuilds('embed', allfeatures + cflags) +addDebugReleaseBuilds('embed-writer', ["-DMPACK_WRITER=1"] + cflags) +addDebugReleaseBuilds('embed-reader', ["-DMPACK_READER=1"] + cflags) +addDebugReleaseBuilds('embed-expect', ["-DMPACK_READER=1", "-DMPACK_EXPECT=1"] + cflags) +addDebugReleaseBuilds('embed-node', ["-DMPACK_NODE=1"] + cflags) +addDebugReleaseBuilds('embed-nobuiltins', ["-DMPACK_NO_BUILTINS=1"] + allfeatures + cflags) + +haveValgrind = shutil.which("valgrind") + +if haveValgrind: + # use valgrind for everything build if available + builds["everything-debug"].run_wrapper = "valgrind" + builds["everything-release"].run_wrapper = "valgrind" + +# language versions +if msvc: + addDebugReleaseBuilds('c++', allfeatures + allconfigs + cxxflags) +elif compiler != "TinyCC": + + # MPack is really C11 code with C++ support. We need lots of compiler + # extensions to build as ANSI C. We technically only support gnu89 so we + # need to disable pedantic C89 warnings. We add + # -Wdeclaration-after-statement even though MPack mixes declarations and + # code just to make sure MPack disables the warning properly. + gnu89flags = ["-std=gnu89", "-Wno-pedantic", "-Wdeclaration-after-statement"] + if checkFlags(gnu89flags): + addDebugReleaseBuilds('gnu89', allfeatures + allconfigs + gnu89flags) + + if checkFlags("-std=c11"): + # if we're using c11 for everything else, we still need to test c99 + addDebugReleaseBuilds('c99', allfeatures + allconfigs + ["-std=c99"]) + + for version in ["c++11", "gnu++11", "c++14", "c++17"]: + flags = cxxflags + ["-std=" + version] + if checkFlags(flags): + addDebugReleaseBuilds(version, allfeatures + allconfigs + flags, cxxlinkflags) + + # Make sure C++11 compiles with disabled features (see #66) + cxx11flags = cxxflags + ["-std=c++11"] + if checkFlags(cxx11flags): + addDebugReleaseBuilds('c++11-empty', allconfigs + cxx11flags, cxxlinkflags) + + # We disable pedantic in C++98 due to our use of variadic macros, trailing + # commas, ll format specifiers, and probably more. We technically only support + # C++98 with those extensions. + cxx98flags = cxxflags + ["-std=c++98"] + if checkFlags("-Wno-pedantic"): + cxx98flags += ["-Wno-pedantic"] + addDebugReleaseBuilds('c++98', allfeatures + allconfigs + cxx98flags, cxxlinkflags) + +# 32-bit builds +if not msvc and checkFlags("-m32"): + addDebugReleaseBuilds('m32', allfeatures + allconfigs + cflags + ["-m32"], ["-m32"]) + addDebugReleaseBuilds('cxx98-m32', allfeatures + allconfigs + cxx98flags + ["-m32"], ["-m32"]) + if checkFlags(cxx11flags): + addDebugReleaseBuilds('c++11-m32', allfeatures + allconfigs + cxx11flags + ["-m32"], ["-m32"]) + +# lto build +if msvc: + addBuild('lto', allfeatures + allconfigs + cflags + releaseflags + ["/GL"], ["/LTCG"]) +elif compiler != "TinyCC": + ltoflags = ["-O3", "-flto", "-fuse-linker-plugin", "-fno-fat-lto-objects"] + if checkFlags(ltoflags): + ltoflags = allfeatures + allconfigs + cflags + ltoflags + else: + ltoflags = ["-O3", "-flto"] + if checkFlags(ltoflags): + ltoflags = allfeatures + allconfigs + cflags + ltoflags + else: + ltoflags = None + if ltoflags: + addBuild('lto', ltoflags, ltoflags) + if haveValgrind: + builds["lto"].run_wrapper = "valgrind" + +# size-optimized build (both debug and release) +if msvc: + sizeOptimize = ["/O1", "/MD"] +else: + sizeOptimize = ["-Os"] +addBuild('optimize-size-debug', allfeatures + allconfigs + cflags + sizeOptimize + + ["-DMPACK_OPTIMIZE_FOR_SIZE=1", "-DMPACK_STRINGS=0", "-DDEBUG"]) +addBuild('optimize-size-release', allfeatures + allconfigs + cflags + sizeOptimize + + ["-DMPACK_OPTIMIZE_FOR_SIZE=1", "-DMPACK_STRINGS=0", "-DNDEBUG"]) + +# miscellaneous special builds +addBuild('notrack', allfeatures + allconfigs + cflags + debugflags + ["-DMPACK_NO_TRACKING=1"]) +addDebugReleaseBuilds('realloc', allfeatures + allconfigs + cflags + ["-DMPACK_REALLOC=test_realloc"]) +if not msvc and compiler != "TinyCC": + addBuild('O3', allfeatures + allconfigs + cflags + ["-O3"]) + if haveValgrind: + builds["O3"].run_wrapper = "valgrind" + addBuild('fastmath', allfeatures + allconfigs + cflags + ["-ffast-math"]) + if haveValgrind: + builds["fastmath"].run_wrapper = "valgrind" + addBuild('coverage', allfeatures + allconfigs + cflags + + ["-DMPACK_GCOV=1", "--coverage", "-fno-inline", "-fno-inline-small-functions", "-fno-default-inline"], + ["--coverage"]) + builds["coverage"].exclude = True # don't run coverage during "all". run separately by CI. + if hasOg: + addBuild('O0', allfeatures + allconfigs + cflags + ["-DDEBUG", "-O0"]) + +# sanitizers +if msvc: + # https://devblogs.microsoft.com/cppblog/asan-for-windows-x64-and-debug-build-support/ + # /INFERASANLIBS is enabled by default, no need to specify them anymore + addDebugReleaseBuilds('sanitize-address', allfeatures + allconfigs + cflags + ["/fsanitize=address"]) +elif compiler != "TinyCC": + def addSanitizerBuilds(name, cppflags, ldflags=[]): + if checkFlags(cppflags): + addDebugReleaseBuilds(name, allfeatures + allconfigs + cflags + cppflags, ldflags) + addSanitizerBuilds('sanitize-undefined', ["-fsanitize=undefined"], ["-fsanitize=undefined"]) + addSanitizerBuilds('sanitize-safe-stack', ["-fsanitize=safe-stack"], ["-fsanitize=safe-stack"]) + addSanitizerBuilds('sanitize-address', ["-fsanitize=address"], ["-fsanitize=address"]) + addSanitizerBuilds('sanitize-memory', ["-fsanitize=memory"], ["-fsanitize=memory"]) + # not technically a sanitizer, but close enough: + addSanitizerBuilds('sanitize-stack-protector', ["-Wstack-protector", "-fstack-protector-all"]) + + + +################################################### +# Ninja generation +################################################### + +srcs = [] + +for paths in [path.join("src", "mpack"), path.join("test", "unit", "src")]: + for root, dirs, files in os.walk(paths): + for name in files: + if name.endswith(".c"): + srcs.append(os.path.join(root, name)) + +ninja = path.join(globalbuild, "build.ninja") +with open(ninja, "w") as out: + out.write("# This file is auto-generated.\n") + out.write("# Do not edit it; your changes will be erased.\n") + out.write("\n") + + # 1.3 for gcc deps, 1.1 for pool + out.write("ninja_required_version = 1.3\n") + out.write("\n") + + out.write("rule compile\n") + if msvc: + out.write(" command = " + cc + " /showIncludes $flags /c $in /Fo$out\n") + out.write(" deps = msvc\n") + else: + out.write(" command = " + cc + " " + "-MD -MF $out.d $flags -c $in -o $out\n") + out.write(" deps = gcc\n") + out.write(" depfile = $out.d\n") + out.write("\n") + + out.write("rule link\n") + if msvc: + out.write(" command = link $flags $in /OUT:$out\n") + else: + out.write(" command = " + cc + " $flags $in -o $out\n") + out.write("\n") + + # unfortunately right now the unit tests all try to write to the same files, + # so they break when run concurrently. we need to make it write to files under + # that config's build/ folder; for now we just run them sequentially. + out.write("pool run_pool\n") + out.write(" depth = 1\n") + out.write("run_wrapper =\n") + out.write("rule run\n") + out.write(" command = $run_wrapper$in\n") + out.write(" pool = run_pool\n") + out.write("\n") + + out.write("rule help\n") + out.write(" command = cat .build/help\n") + out.write("build help: help\n") + out.write("\n") + + for buildname in sorted(builds.keys()): + build = builds[buildname] + buildfolder = path.join(globalbuild, buildname) + cppflags = global_cppflags + build.cppflags + ldflags = build.ldflags + objs = [] + + if msvc: + # Specify a per-build PDB path so that we don't try to link at the + # same time a PDB file is being written + cppflags.append("/Fd" + buildfolder) + ldflags.append("/DEBUG") + + for src in srcs: + obj = path.join(buildfolder, "objs", src[:-2] + obj_extension) + objs.append(obj) + out.write("build " + obj + ": compile " + src + "\n") + out.write(" flags = " + " ".join(cppflags) + "\n") + + runner = path.join(buildfolder, "runner") + exe_extension + + out.write("build " + runner + ": link " + " ".join(objs) + "\n") + out.write(" flags = " + " ".join(ldflags) + "\n") + + # You can omit "run-" in front of any build to just build it without + # running it. This lets you run it some other way (e.g. under gdb, + # with/without Valgrind, etc.) + out.write("build " + buildname + ": phony " + runner + "\n\n") + + out.write("build run-" + buildname + ": run " + runner + "\n") + if build.run_wrapper: + run_wrapper = build.run_wrapper + out.write(" run_wrapper = " + run_wrapper + " ") + if run_wrapper == "valgrind": + out.write("--leak-check=full --error-exitcode=1 ") + out.write("--suppressions=tools/valgrind-suppressions ") + out.write("--show-leak-kinds=all --errors-for-leak-kinds=all ") + out.write("\n") + + out.write("default run-everything-debug\n") + out.write("build default: phony run-everything-debug\n") + out.write("\n") + + # Builds included under the "more" target + more = [ + "run-default-debug", + "run-everything-debug", + "run-everything-release", + "run-embed-debug", + "run-embed-release", + "run-no-float-release", + ] + if "gnu89-debug" in builds: + more += [ + "run-gnu89-debug", + "run-gnu89-release", + ] + if "c++11-debug" in builds: + more += [ + "run-c++11-debug", + ] + + out.write("build more: phony " + " ".join(more)) + if ltoflags: + out.write(" run-lto") + out.write("\n\n") + + out.write("build all: phony") + for build in sorted(builds.keys()): + if not builds[build].exclude: + out.write(" run-") + out.write(build) + out.write("\n") + +print("Generated " + ninja) + +with open(path.join(globalbuild, "help"), "w") as out: + out.write("\n") + out.write("Available targets:\n") + out.write("\n") + out.write(" (default)\n") + out.write(" more\n") + out.write(" all\n") + out.write(" clean\n") + out.write(" help\n") + out.write("\n") + for build in sorted(builds.keys()): + out.write(" run-" + build + "\n") + out.close() diff --git a/test/unit/src/mpack-config.h b/test/unit/src/mpack-config.h new file mode 100644 index 0000000..899aed5 --- /dev/null +++ b/test/unit/src/mpack-config.h @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MPACK_CONFIG_H +#define MPACK_CONFIG_H 1 + +// This is the configuration for the MPack test harness. + +#define MPACK_UNIT_TESTS 1 + +#if defined(DEBUG) || defined(_DEBUG) +#define MPACK_DEBUG 1 +#endif + +#ifdef MPACK_VARIANT_BUILDS + // Most options such as featureset and platform configuration + // are specified by the buildsystem. Any options that are + // unset on the command line are considered disabled. + #ifndef MPACK_READER + #define MPACK_READER 0 + #endif + #ifndef MPACK_EXPECT + #define MPACK_EXPECT 0 + #endif + #ifndef MPACK_NODE + #define MPACK_NODE 0 + #endif + #ifndef MPACK_WRITER + #define MPACK_WRITER 0 + #endif + #ifndef MPACK_STDLIB + #define MPACK_STDLIB 0 + #endif + #ifndef MPACK_STDIO + #define MPACK_STDIO 0 + #endif + +#elif defined(__AVR__) + #define MPACK_STDLIB 0 + #define MPACK_STDIO 0 + +#else + // For other platforms, we currently only test in a single configuration, + // so we enable everything and otherwise use the default for most settings. + #define MPACK_COMPATIBILITY 1 + #define MPACK_EXTENSIONS 1 + + // We define our own allocators to test allocations. + #define MPACK_MALLOC test_malloc + #define MPACK_FREE test_free + + // We need MPACK_STDLIB and MPACK_STDIO defined before test-system.h to + // override their functions. + #define MPACK_STDLIB 1 + #define MPACK_STDIO 1 + +#endif + +// We've disabled the unit test for single inline under tcc. +#ifdef __TINYC__ +#define MPACK_DISABLE_TINYC_INLINE_WARNING +#endif + +// We replace the file i/o functions to simulate failures +#if MPACK_STDIO +#include +#undef fopen +#undef fclose +#undef fread +#undef fwrite +#undef fseek +#undef ftell +#undef ferror +#define fopen test_fopen +#define fclose test_fclose +#define fread test_fread +#define fwrite test_fwrite +#define fseek test_fseek +#define ftell test_ftell +#define ferror test_ferror +#endif + +// We replace strlen to simulate extremely large c-strings (only on stdlib +// builds, so that non-stdlib builds test the mpack implementations) +#if MPACK_STDLIB + #define MPACK_STRLEN test_strlen +#endif + +// Tracking matches the default config, except the test suite +// also supports MPACK_NO_TRACKING to disable it. +#if defined(MPACK_MALLOC) && !defined(MPACK_NO_TRACKING) + #if defined(MPACK_DEBUG) && MPACK_DEBUG && defined(MPACK_READER) && MPACK_READER + #define MPACK_READ_TRACKING 1 + #endif + #if defined(MPACK_DEBUG) && MPACK_DEBUG && defined(MPACK_WRITER) && MPACK_WRITER + #define MPACK_WRITE_TRACKING 1 + #endif +#endif + +// We use a custom assert function which longjmps, allowing +// us to test assertions in debug mode. +#ifdef MPACK_DEBUG +#define MPACK_CUSTOM_ASSERT 1 +#define MPACK_CUSTOM_BREAK 1 +#endif + +#include "test-system.h" + +// we use small buffer sizes to test flushing, growing, and malloc failures +#define MPACK_TRACKING_INITIAL_CAPACITY 3 +#define MPACK_STACK_SIZE 33 +#define MPACK_BUFFER_SIZE 33 +#define MPACK_NODE_PAGE_SIZE 113 +#define MPACK_BUILDER_INTERNAL_STORAGE_SIZE (sizeof(mpack_builder_page_t) + \ + sizeof(mpack_build_t) + MPACK_WRITER_MINIMUM_BUFFER_SIZE + 33) +#define MPACK_BUILDER_PAGE_SIZE (sizeof(mpack_builder_page_t) + \ + sizeof(mpack_build_t) + MPACK_WRITER_MINIMUM_BUFFER_SIZE + 77) + +/* +#undef MPACK_BUILDER_INTERNAL_STORAGE_SIZE +#define MPACK_BUILDER_INTERNAL_STORAGE_SIZE 0 +#undef MPACK_BUILDER_PAGE_SIZE +#define MPACK_BUILDER_PAGE_SIZE 80 +*/ + +#ifdef MPACK_MALLOC +#define MPACK_NODE_INITIAL_DEPTH 3 +#else +#define MPACK_NODE_MAX_DEPTH_WITHOUT_MALLOC 32 +#endif + +#endif diff --git a/test/test-buffer.c b/test/unit/src/test-buffer.c similarity index 82% rename from test/test-buffer.c rename to test/unit/src/test-buffer.c index 6f823c7..f79f294 100644 --- a/test/test-buffer.c +++ b/test/unit/src/test-buffer.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -95,14 +95,16 @@ static const char test_strings[] = // a semi-random list of buffer sizes we will test with. each buffer // test is run with each of these buffer sizes to test the fill and // flush functions. -static const int test_buffer_sizes[] = { +static const size_t test_buffer_sizes[] = { 32, 33, 34, 35, 36, 37, 39, 43, 48, 51, 52, 53, 57, 59, 64, 67, 89, 127, 128, 129, 131, 160, 163, 191, 192, 193, 251, 256, 257, 509, 512, 521, 1021, 1024, 1031, 2039, 2048, 2053, + #ifndef __AVR__ 4093, 4096, 4099, 7919, 8192, - 16384, 32768 + 6384, 32768, + #endif }; #if MPACK_READER @@ -177,18 +179,18 @@ static void test_expect_buffer_values(mpack_reader_t* reader) { // when using UINT32_C() and compiling the test suite as c++, gcc complains: // error: this decimal constant is unsigned only in ISO C90 [-Werror] - TEST_READ_NOERROR(reader, UINT64_C(2386122103) == mpack_expect_u32(reader)); - TEST_READ_NOERROR(reader, UINT64_C(2863333399) == mpack_expect_u32(reader)); - TEST_READ_NOERROR(reader, UINT64_C(3340544681) == mpack_expect_u32(reader)); - - TEST_READ_NOERROR(reader, UINT64_C(4294967311) == mpack_expect_u64(reader)); - TEST_READ_NOERROR(reader, UINT64_C(1941762537917555303) == mpack_expect_u64(reader)); - TEST_READ_NOERROR(reader, UINT64_C(3883525071540143119) == mpack_expect_u64(reader)); - TEST_READ_NOERROR(reader, UINT64_C(5825287605162730577) == mpack_expect_u64(reader)); - TEST_READ_NOERROR(reader, UINT64_C(7767050138785318961) == mpack_expect_u64(reader)); - TEST_READ_NOERROR(reader, UINT64_C(9708812672407906367) == mpack_expect_u64(reader)); - TEST_READ_NOERROR(reader, UINT64_C(11650575206030493713) == mpack_expect_u64(reader)); - TEST_READ_NOERROR(reader, UINT64_C(13592337739653081091) == mpack_expect_u64(reader)); + TEST_READ_NOERROR(reader, MPACK_UINT64_C(2386122103) == mpack_expect_u32(reader)); + TEST_READ_NOERROR(reader, MPACK_UINT64_C(2863333399) == mpack_expect_u32(reader)); + TEST_READ_NOERROR(reader, MPACK_UINT64_C(3340544681) == mpack_expect_u32(reader)); + + TEST_READ_NOERROR(reader, MPACK_UINT64_C(4294967311) == mpack_expect_u64(reader)); + TEST_READ_NOERROR(reader, MPACK_UINT64_C(1941762537917555303) == mpack_expect_u64(reader)); + TEST_READ_NOERROR(reader, MPACK_UINT64_C(3883525071540143119) == mpack_expect_u64(reader)); + TEST_READ_NOERROR(reader, MPACK_UINT64_C(5825287605162730577) == mpack_expect_u64(reader)); + TEST_READ_NOERROR(reader, MPACK_UINT64_C(7767050138785318961) == mpack_expect_u64(reader)); + TEST_READ_NOERROR(reader, MPACK_UINT64_C(9708812672407906367) == mpack_expect_u64(reader)); + TEST_READ_NOERROR(reader, MPACK_UINT64_C(11650575206030493713) == mpack_expect_u64(reader)); + TEST_READ_NOERROR(reader, MPACK_UINT64_C(13592337739653081091) == mpack_expect_u64(reader)); } #endif @@ -229,24 +231,25 @@ static void test_write_buffer_values(mpack_writer_t* writer) { // when using UINT32_C() and compiling the test suite as c++, gcc complains: // error: this decimal constant is unsigned only in ISO C90 [-Werror] - TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, UINT64_C(2386122103))); - TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, UINT64_C(2863333399))); - TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, UINT64_C(3340544681))); - - TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, UINT64_C(4294967311))); - TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, UINT64_C(1941762537917555303))); - TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, UINT64_C(3883525071540143119))); - TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, UINT64_C(5825287605162730577))); - TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, UINT64_C(7767050138785318961))); - TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, UINT64_C(9708812672407906367))); - TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, UINT64_C(11650575206030493713))); - TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, UINT64_C(13592337739653081091))); + TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, MPACK_UINT64_C(2386122103))); + TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, MPACK_UINT64_C(2863333399))); + TEST_WRITE_NOERROR(writer, mpack_write_u32(writer, MPACK_UINT64_C(3340544681))); + + TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(4294967311))); + TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(1941762537917555303))); + TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(3883525071540143119))); + TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(5825287605162730577))); + TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(7767050138785318961))); + TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(9708812672407906367))); + TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(11650575206030493713))); + TEST_WRITE_NOERROR(writer, mpack_write_u64(writer, MPACK_UINT64_C(13592337739653081091))); } #endif #if MPACK_EXPECT static void test_expect_buffer(void) { - for (size_t i = 0; i < sizeof(test_buffer_sizes) / sizeof(test_buffer_sizes[0]); ++i) { + size_t i; + for (i = 0; i < sizeof(test_buffer_sizes) / sizeof(test_buffer_sizes[0]); ++i) { // initialize the reader with our buffer reader function mpack_reader_t reader; @@ -268,9 +271,16 @@ static void test_expect_buffer(void) { #if MPACK_WRITER static void test_write_buffer(void) { - for (size_t i = 0; i < sizeof(test_buffer_sizes) / sizeof(test_buffer_sizes[0]); ++i) { + size_t i; + for (i = 0; i < sizeof(test_buffer_sizes) / sizeof(test_buffer_sizes[0]); ++i) { size_t size = test_buffer_sizes[i]; - size_t output_size = 0xffff; + size_t output_size = + #ifdef __AVR__ + 0xfff + #else + 0xfffff + #endif + ; char* output = (char*)malloc(output_size); // initialize the writer with our buffer writer function @@ -300,7 +310,8 @@ static void test_write_buffer(void) { #if MPACK_READER static void test_inplace_buffer(void) { - for (size_t i = 0; i < sizeof(test_buffer_sizes) / sizeof(test_buffer_sizes[0]); ++i) { + size_t i; + for (i = 0; i < sizeof(test_buffer_sizes) / sizeof(test_buffer_sizes[0]); ++i) { // initialize the reader with our buffer reader function mpack_reader_t reader; @@ -321,7 +332,8 @@ static void test_inplace_buffer(void) { static const char* ref = "abcdefghijklmn"; const char* val; char r[15]; - for (size_t j = 0; j < 15; ++j) { + size_t j; + for (j = 0; j < 15; ++j) { mpack_tag_t peek = mpack_peek_tag(&reader); tag = mpack_read_tag(&reader); TEST_TRUE(mpack_tag_equal(peek, tag), "peeked tag does not match read tag"); diff --git a/test/test-buffer.h b/test/unit/src/test-buffer.h similarity index 94% rename from test/test-buffer.h rename to test/unit/src/test-buffer.h index a05a08a..01345a8 100644 --- a/test/test-buffer.h +++ b/test/unit/src/test-buffer.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR diff --git a/test/unit/src/test-builder.c b/test/unit/src/test-builder.c new file mode 100644 index 0000000..f96b841 --- /dev/null +++ b/test/unit/src/test-builder.c @@ -0,0 +1,432 @@ +/* + * Copyright (c) 2019-2021 Nicholas Fraser and the MPack authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test-builder.h" +#include "test-write.h" + +#if MPACK_BUILDER +static void test_builder_basic(void) { + static char buf[4096]; + mpack_writer_t writer; + + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_build_array(&writer); + mpack_complete_array(&writer); + TEST_DESTROY_MATCH_IMPL(buf, "\x90"); + + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_build_array(&writer); + mpack_write_u8(&writer, 2); + mpack_complete_array(&writer); + TEST_DESTROY_MATCH_IMPL(buf, "\x91\x02"); + + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_build_map(&writer); + mpack_write_cstr(&writer, "hello"); + mpack_write_cstr(&writer, "world"); + mpack_complete_map(&writer); + TEST_DESTROY_MATCH_IMPL(buf, "\x81\xa5hello\xa5world"); +} + +static void test_builder_repeat(void) { + static char buf[4096]; + mpack_writer_t writer; + + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_start_array(&writer, 4); + mpack_build_array(&writer); + mpack_complete_array(&writer); + mpack_build_map(&writer); + mpack_complete_map(&writer); + mpack_build_array(&writer); + mpack_write_u8(&writer, 2); + mpack_complete_array(&writer); + mpack_build_map(&writer); + mpack_write_cstr(&writer, "hello"); + mpack_write_cstr(&writer, "world"); + mpack_complete_map(&writer); + mpack_finish_array(&writer); + + TEST_DESTROY_MATCH_IMPL(buf, "\x94\x90\x80\x91\x02\x81\xa5hello\xa5world"); +} + +static void test_builder_nested(void) { + static char buf[4096]; + mpack_writer_t writer; + + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_build_map(&writer); + mpack_write_cstr(&writer, "nums"); + mpack_build_array(&writer); + mpack_write_int(&writer, 1); + mpack_write_int(&writer, 2); + mpack_write_int(&writer, 3); + mpack_complete_array(&writer); + mpack_write_cstr(&writer, "nil"); + mpack_write_nil(&writer); + mpack_complete_map(&writer); + TEST_DESTROY_MATCH_IMPL(buf, "\x82\xa4nums\x93\x01\x02\x03\xa3nil\xc0"); + + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_build_array(&writer); + mpack_build_array(&writer); + mpack_build_array(&writer); + mpack_write_int(&writer, 1); + mpack_write_int(&writer, 2); + mpack_write_int(&writer, 3); + mpack_complete_array(&writer); + mpack_complete_array(&writer); + mpack_complete_array(&writer); + TEST_DESTROY_MATCH_IMPL(buf, "\x91\x91\x93\x01\x02\x03"); +} + +static void test_builder_deep(void) { + static char buf[16*1024]; + mpack_writer_t writer; + mpack_writer_init(&writer, buf, sizeof(buf)); + + char expected[sizeof(buf)]; + size_t pos = 0; + int depth = 2;//50; + + int i; + for (i = 0; i < depth; ++i) { + //mpack_build_map(&writer); + mpack_start_map(&writer, 2); + expected[pos++] = '\x82'; + mpack_write_cstr(&writer, "ab"); + expected[pos++] = '\xa2'; + expected[pos++] = 'a'; + expected[pos++] = 'b'; + mpack_build_array(&writer); + expected[pos++] = '\x94'; + mpack_write_int(&writer, 2); + expected[pos++] = '\x02'; + mpack_write_int(&writer, 3); + expected[pos++] = '\x03'; + mpack_write_int(&writer, 4); + expected[pos++] = '\x04'; + } + + mpack_write_bool(&writer, true); + expected[pos++] = '\xc3'; + + for (i = 0; i < depth; ++i) { + mpack_complete_array(&writer); + mpack_write_int(&writer, 1); + expected[pos++] = '\x01'; + mpack_write_nil(&writer); + expected[pos++] = '\xc0'; + //mpack_complete_map(&writer); + mpack_finish_map(&writer); + } + + TEST_TRUE(pos <= sizeof(expected)); + size_t used = mpack_writer_buffer_used(&writer); + + /* + printf("actual %zi expected %zi\n", used, pos); + for (size_t i = 0; i < used; ++i) { + printf("%02hhx ", buf[i]); + if (((i+1) % 16)==0) + printf("\n"); + } + printf("\n"); + printf("\n"); + for (size_t i = 0; i < pos; ++i) { + printf("%02hhx ", expected[i]); + if (((i+1) % 16)==0) + printf("\n"); + } + printf("\n"); + */ + + TEST_WRITER_DESTROY_NOERROR(&writer); + TEST_TRUE(used == pos); + TEST_TRUE(0 == memcmp(buf, expected, used)); +} + +static void test_builder_large(void) { + static char buf[16*1024]; + mpack_writer_t writer; + mpack_writer_init(&writer, buf, sizeof(buf)); + + char expected[sizeof(buf)]; + size_t pos = 0; + int depth = 6; + + int i; + for (i = 0; i < depth; ++i) { + mpack_build_map(&writer); + expected[pos++] = '\xde'; + expected[pos++] = '\x00'; + expected[pos++] = '\x32'; + size_t j; + for (j = 0; j < 99; ++j) { + mpack_write_int(&writer, -1); + expected[pos++] = '\xff'; + } + } + + mpack_write_int(&writer, -1); + expected[pos++] = '\xff'; + + for (i = 0; i < depth; ++i) { + mpack_complete_map(&writer); + } + + TEST_TRUE(pos <= sizeof(expected)); + size_t used = mpack_writer_buffer_used(&writer); + TEST_WRITER_DESTROY_NOERROR(&writer); + TEST_TRUE(used == pos); + TEST_TRUE(0 == memcmp(buf, expected, used)); +} + +static void test_builder_content(void) { + static char buf[16*1024]; + mpack_writer_t writer; + mpack_writer_init(&writer, buf, sizeof(buf)); + + char expected[sizeof(buf)]; + size_t pos = 0; + + mpack_build_map(&writer); + //mpack_start_map(&writer, 3); + expected[pos++] = '\x83'; + + mpack_write_cstr(&writer, "rid"); + expected[pos++] = '\xa3'; + expected[pos++] = 'r'; + expected[pos++] = 'i'; + expected[pos++] = 'd'; + + char rid[16] = {0}; + mpack_write_bin(&writer, rid, sizeof(rid)); + expected[pos++] = '\xc4'; + expected[pos++] = '\x10'; + size_t i; + for (i = 0; i < 16; ++i) + expected[pos++] = '\x00'; + + mpack_write_cstr(&writer, "type"); + expected[pos++] = '\xa4'; + expected[pos++] = 't'; + expected[pos++] = 'y'; + expected[pos++] = 'p'; + expected[pos++] = 'e'; + + mpack_write_cstr(&writer, "inode"); + expected[pos++] = '\xa5'; + expected[pos++] = 'i'; + expected[pos++] = 'n'; + expected[pos++] = 'o'; + expected[pos++] = 'd'; + expected[pos++] = 'e'; + + mpack_write_cstr(&writer, "content"); + expected[pos++] = '\xa7'; + expected[pos++] = 'c'; + expected[pos++] = 'o'; + expected[pos++] = 'n'; + expected[pos++] = 't'; + expected[pos++] = 'e'; + expected[pos++] = 'n'; + expected[pos++] = 't'; + + mpack_start_map(&writer, 3); + expected[pos++] = '\x83'; + + mpack_write_cstr(&writer, "path"); + expected[pos++] = '\xa4'; + expected[pos++] = 'p'; + expected[pos++] = 'a'; + expected[pos++] = 't'; + expected[pos++] = 'h'; + + mpack_write_cstr(&writer, "IMG_2445.JPG"); + expected[pos++] = '\xac'; + expected[pos++] = 'I'; + expected[pos++] = 'M'; + expected[pos++] = 'G'; + expected[pos++] = '_'; + expected[pos++] = '2'; + expected[pos++] = '4'; + expected[pos++] = '4'; + expected[pos++] = '5'; + expected[pos++] = '.'; + expected[pos++] = 'J'; + expected[pos++] = 'P'; + expected[pos++] = 'G'; + + mpack_write_cstr(&writer, "parent"); + expected[pos++] = '\xa6'; + expected[pos++] = 'p'; + expected[pos++] = 'a'; + expected[pos++] = 'r'; + expected[pos++] = 'e'; + expected[pos++] = 'n'; + expected[pos++] = 't'; + + mpack_write_bin(&writer, rid, sizeof(rid)); + expected[pos++] = '\xc4'; + expected[pos++] = '\x10'; + for (i = 0; i < 16; ++i) + expected[pos++] = '\x00'; + + mpack_write_cstr(&writer, "pass"); + expected[pos++] = '\xa4'; + expected[pos++] = 'p'; + expected[pos++] = 'a'; + expected[pos++] = 's'; + expected[pos++] = 's'; + + mpack_write_int(&writer, 0); + expected[pos++] = '\x00'; + + mpack_finish_map(&writer); + + //mpack_finish_map(&writer); + mpack_complete_map(&writer); + + TEST_TRUE(pos <= sizeof(expected)); + size_t used = mpack_writer_buffer_used(&writer); + + /* + printf("actual %zi expected %zi\n", used, pos); + for (i = 0; i < used; ++i) { + printf("%02hhx ", buf[i]); + if (((i+1) % 16)==0) + printf("\n"); + } + printf("\n"); + printf("\n"); + for (i = 0; i < pos; ++i) { + printf("%02hhx ", expected[i]); + if (((i+1) % 16)==0) + printf("\n"); + } + printf("\n"); + */ + + TEST_WRITER_DESTROY_NOERROR(&writer); + TEST_TRUE(used == pos); + TEST_TRUE(0 == memcmp(buf, expected, used)); +} + +static void test_builder_add_expected_str(char* expected, size_t* pos, const char* str, uint32_t length) { + if (length <= 31) { + expected[(*pos)++] = (char)((uint32_t)'\xa0' + length); + } else if (length <= MPACK_UINT8_MAX) { + expected[(*pos)++] = '\xd9'; + expected[(*pos)++] = (char)(uint8_t)length; + } else { + expected[(*pos)++] = '\xda'; + expected[(*pos)++] = (char)(uint8_t)(length >> 8); + expected[(*pos)++] = (char)(uint8_t)(length); + } + memcpy(expected + *pos, str, length); + *pos += length; +} + +static void test_builder_strings_length(uint32_t length) { + static char buf[16*1024]; + mpack_writer_t writer; + mpack_writer_init(&writer, buf, sizeof(buf)); + + char expected[sizeof(buf)]; + size_t pos = 0; + + static char str[1024]; + TEST_TRUE(length <= sizeof(str)); + memset(str, 'a', length); + size_t depth = 2; + + size_t i; + for (i = 0; i < depth; ++i) { + mpack_build_array(&writer); + expected[pos++] = '\x93'; + mpack_write_str(&writer, str, length); + test_builder_add_expected_str(expected, &pos, str, length); + } + + mpack_write_str(&writer, str, length); + test_builder_add_expected_str(expected, &pos, str, length); + + for (i = 0; i < depth; ++i) { + mpack_write_str(&writer, str, length); + test_builder_add_expected_str(expected, &pos, str, length); + mpack_complete_array(&writer); + } + + TEST_TRUE(pos <= sizeof(expected)); + size_t used = mpack_writer_buffer_used(&writer); + + /* + printf("actual %zi expected %zi\n", used, pos); + for (i = 0; i < used; ++i) { + printf("%02hhx ", buf[i]); + if (((i+1) % 16)==0) + printf("\n"); + } + printf("\n"); + printf("\n"); + for (i = 0; i < pos; ++i) { + printf("%02hhx ", expected[i]); + if (((i+1) % 16)==0) + printf("\n"); + } + printf("\n"); + */ + + TEST_WRITER_DESTROY_NOERROR(&writer); + TEST_TRUE(used == pos); + TEST_TRUE(0 == memcmp(buf, expected, used)); +} + +static void test_builder_strings(void) { + test_builder_strings_length(3); + test_builder_strings_length(17); + test_builder_strings_length(32); + test_builder_strings_length(129); + test_builder_strings_length(457); +} + +static void test_builder_resolve_error(void) { + static char buf[5]; + mpack_writer_t writer; + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_build_array(&writer); + mpack_write_cstr(&writer, "Hello world!"); + mpack_complete_array(&writer); + TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_too_big); +} + +void test_builder(void) { + test_builder_basic(); + test_builder_repeat(); + test_builder_nested(); + test_builder_deep(); + test_builder_large(); + test_builder_content(); + test_builder_strings(); + test_builder_resolve_error(); +} +#endif diff --git a/test/unit/src/test-builder.h b/test/unit/src/test-builder.h new file mode 100644 index 0000000..0f5aff2 --- /dev/null +++ b/test/unit/src/test-builder.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019-2021 Nicholas Fraser and the MPack authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MPACK_TEST_BUILDER_H +#define MPACK_TEST_BUILDER_H 1 + +#include "test.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if MPACK_BUILDER +void test_builder(void); +#endif + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/test/test-common.c b/test/unit/src/test-common.c similarity index 83% rename from test/test-common.c rename to test/unit/src/test-common.c index e9c1611..c900d1a 100644 --- a/test/test-common.c +++ b/test/unit/src/test-common.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -24,14 +24,16 @@ #include "test-common.h" -#include #include "test.h" static void test_tags_special(void) { - // ensure there is only one inline definition (the global - // address is in test.c) + // we emit a warning for inlines on tcc because they are static. + #ifndef __TINYC__ + // for all other compilers, ensure there is only one inline definition (the + // global address is in test.c) TEST_TRUE(fn_mpack_tag_nil == &mpack_tag_nil); + #endif // test comparison with invalid tags // (invalid enum values cause undefined behavior in C++) @@ -54,16 +56,16 @@ static void test_tags_simple(void) { // uints TEST_TRUE(mpack_tag_uint(0).v.u == 0); TEST_TRUE(mpack_tag_uint(1).v.u == 1); - TEST_TRUE(mpack_tag_uint(INT32_MAX).v.u == INT32_MAX); - TEST_TRUE(mpack_tag_uint(INT64_MAX).v.u == INT64_MAX); + TEST_TRUE(mpack_tag_uint(MPACK_INT32_MAX).v.u == MPACK_INT32_MAX); + TEST_TRUE(mpack_tag_uint(MPACK_INT64_MAX).v.u == MPACK_INT64_MAX); // ints TEST_TRUE(mpack_tag_int(0).v.i == 0); TEST_TRUE(mpack_tag_int(1).v.i == 1); // when using INT32_C() and compiling the test suite as c++, gcc complains: // error: this decimal constant is unsigned only in ISO C90 [-Werror] - TEST_TRUE(mpack_tag_int(INT64_C(-2147483648)).v.i == INT64_C(-2147483648)); - TEST_TRUE(mpack_tag_int(INT64_MIN).v.i == INT64_MIN); + TEST_TRUE(mpack_tag_int(MPACK_INT64_C(-2147483648)).v.i == MPACK_INT64_C(-2147483648)); + TEST_TRUE(mpack_tag_int(MPACK_INT64_MIN).v.i == MPACK_INT64_MIN); // bools TEST_TRUE(mpack_tag_bool(true).v.b == true); @@ -117,63 +119,79 @@ static void test_tags_simple(void) { TEST_TRUE(-1 == mpack_tag_cmp(mpack_tag_map(0), mpack_tag_map(1))); TEST_TRUE(1 == mpack_tag_cmp(mpack_tag_map(1), mpack_tag_map(0))); + #if MPACK_EXTENSIONS TEST_TRUE(-1 == mpack_tag_cmp(mpack_tag_ext(1, 1), mpack_tag_ext(2, 0))); TEST_TRUE(-1 == mpack_tag_cmp(mpack_tag_ext(1, 1), mpack_tag_ext(1, 2))); TEST_TRUE(1 == mpack_tag_cmp(mpack_tag_ext(2, 0), mpack_tag_ext(1, 1))); TEST_TRUE(1 == mpack_tag_cmp(mpack_tag_ext(1, 2), mpack_tag_ext(1, 1))); + #endif } static void test_tags_reals(void) { // types + #if MPACK_FLOAT TEST_TRUE(mpack_tag_float(0.0f).type == mpack_type_float); - TEST_TRUE(mpack_tag_double(0.0).type == mpack_type_double); TEST_TRUE(mpack_tag_float((float)NAN).type == mpack_type_float); + #endif + #if MPACK_DOUBLE + TEST_TRUE(mpack_tag_double(0.0).type == mpack_type_double); TEST_TRUE(mpack_tag_double((double)NAN).type == mpack_type_double); + #endif // float comparisons + #if MPACK_FLOAT TEST_TRUE(true == mpack_tag_equal(mpack_tag_float(0), mpack_tag_float(0))); TEST_TRUE(true == mpack_tag_equal(mpack_tag_float(1), mpack_tag_float(1))); - TEST_TRUE(true == mpack_tag_equal(mpack_tag_float(MPACK_INFINITY), mpack_tag_float(MPACK_INFINITY))); - TEST_TRUE(true == mpack_tag_equal(mpack_tag_float(-MPACK_INFINITY), mpack_tag_float(-MPACK_INFINITY))); + TEST_TRUE(true == mpack_tag_equal(mpack_tag_float(MPACK_FLOAT_POSITIVE_INFINITY), mpack_tag_float(MPACK_FLOAT_POSITIVE_INFINITY))); + TEST_TRUE(true == mpack_tag_equal(mpack_tag_float(MPACK_FLOAT_NEGATIVE_INFINITY), mpack_tag_float(MPACK_FLOAT_NEGATIVE_INFINITY))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_float(0), mpack_tag_float(1))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_float(1), mpack_tag_float(MPACK_INFINITY))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_float(MPACK_INFINITY), mpack_tag_float(-MPACK_INFINITY))); + TEST_TRUE(false == mpack_tag_equal(mpack_tag_float(1), mpack_tag_float(MPACK_FLOAT_POSITIVE_INFINITY))); + TEST_TRUE(false == mpack_tag_equal(mpack_tag_float(MPACK_FLOAT_POSITIVE_INFINITY), mpack_tag_float(MPACK_FLOAT_NEGATIVE_INFINITY))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_float(0), mpack_tag_float(NAN))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_float(MPACK_INFINITY), mpack_tag_float(NAN))); + TEST_TRUE(false == mpack_tag_equal(mpack_tag_float(MPACK_FLOAT_POSITIVE_INFINITY), mpack_tag_float(NAN))); + #endif // double comparisons + #if MPACK_DOUBLE TEST_TRUE(true == mpack_tag_equal(mpack_tag_double(0), mpack_tag_double(0))); TEST_TRUE(true == mpack_tag_equal(mpack_tag_double(1), mpack_tag_double(1))); - TEST_TRUE(true == mpack_tag_equal(mpack_tag_double(MPACK_INFINITY), mpack_tag_double(MPACK_INFINITY))); - TEST_TRUE(true == mpack_tag_equal(mpack_tag_double(-MPACK_INFINITY), mpack_tag_double(-MPACK_INFINITY))); + TEST_TRUE(true == mpack_tag_equal(mpack_tag_double(MPACK_DOUBLE_POSITIVE_INFINITY), mpack_tag_double(MPACK_DOUBLE_POSITIVE_INFINITY))); + TEST_TRUE(true == mpack_tag_equal(mpack_tag_double(MPACK_DOUBLE_NEGATIVE_INFINITY), mpack_tag_double(MPACK_DOUBLE_NEGATIVE_INFINITY))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(0), mpack_tag_double(1))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(1), mpack_tag_double(MPACK_INFINITY))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(MPACK_INFINITY), mpack_tag_double(-MPACK_INFINITY))); + TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(1), mpack_tag_double(MPACK_DOUBLE_POSITIVE_INFINITY))); + TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(MPACK_DOUBLE_POSITIVE_INFINITY), mpack_tag_double(MPACK_DOUBLE_NEGATIVE_INFINITY))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(0), mpack_tag_double(NAN))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(MPACK_INFINITY), mpack_tag_double(NAN))); + TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(MPACK_DOUBLE_POSITIVE_INFINITY), mpack_tag_double(NAN))); + #endif // float/double comparisons + #if MPACK_FLOAT && MPACK_DOUBLE TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(0), mpack_tag_float(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(1), mpack_tag_float(1))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(MPACK_INFINITY), mpack_tag_float(MPACK_INFINITY))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(-MPACK_INFINITY), mpack_tag_float(-MPACK_INFINITY))); + TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(MPACK_DOUBLE_POSITIVE_INFINITY), mpack_tag_float(MPACK_FLOAT_POSITIVE_INFINITY))); + TEST_TRUE(false == mpack_tag_equal(mpack_tag_double(MPACK_DOUBLE_NEGATIVE_INFINITY), mpack_tag_float(MPACK_FLOAT_NEGATIVE_INFINITY))); + #endif // here we're comparing NaNs and we expect true. this is because we compare // floats bit-for-bit, not using operator== + #if MPACK_FLOAT && MPACK_DOUBLE TEST_TRUE(true == mpack_tag_equal(mpack_tag_float(NAN), mpack_tag_float(NAN))); TEST_TRUE(true == mpack_tag_equal(mpack_tag_double(NAN), mpack_tag_double(NAN))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_float(NAN), mpack_tag_double(NAN))); + #endif } -static void test_tags_compound() { +static void test_tags_compound(void) { TEST_TRUE(mpack_tag_array(0).type == mpack_type_array); TEST_TRUE(mpack_tag_map(0).type == mpack_type_map); TEST_TRUE(mpack_tag_str(0).type == mpack_type_str); TEST_TRUE(mpack_tag_bin(0).type == mpack_type_bin); + #if MPACK_EXTENSIONS TEST_TRUE(mpack_tag_ext(0, 0).type == mpack_type_ext); + #endif TEST_TRUE(true == mpack_tag_equal(mpack_tag_array(0), mpack_tag_array(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_array(0), mpack_tag_array(1))); @@ -199,6 +217,7 @@ static void test_tags_compound() { TEST_TRUE(-1 == mpack_tag_cmp(mpack_tag_bin(0), mpack_tag_bin(1))); TEST_TRUE(1 == mpack_tag_cmp(mpack_tag_bin(1), mpack_tag_bin(0))); + #if MPACK_EXTENSIONS TEST_TRUE(true == mpack_tag_equal(mpack_tag_ext(0, 0), mpack_tag_ext(0, 0))); TEST_TRUE(true == mpack_tag_equal(mpack_tag_ext(0, 1), mpack_tag_ext(0, 1))); TEST_TRUE(true == mpack_tag_equal(mpack_tag_ext(127, 0), mpack_tag_ext(127, 0))); @@ -208,45 +227,48 @@ static void test_tags_compound() { TEST_TRUE(false == mpack_tag_equal(mpack_tag_ext(0, 0), mpack_tag_ext(127, 0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_ext(0, 0), mpack_tag_ext(-128, 0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_ext(0, 0), mpack_tag_ext(0, 1))); + #endif TEST_TRUE(false == mpack_tag_equal(mpack_tag_array(0), mpack_tag_map(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_array(0), mpack_tag_str(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_array(0), mpack_tag_bin(0))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_array(0), mpack_tag_ext(0, 0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_map(0), mpack_tag_array(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_map(0), mpack_tag_str(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_map(0), mpack_tag_bin(0))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_map(0), mpack_tag_ext(0, 0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_str(0), mpack_tag_array(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_str(0), mpack_tag_map(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_str(0), mpack_tag_bin(0))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_str(0), mpack_tag_ext(0, 0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_bin(0), mpack_tag_array(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_bin(0), mpack_tag_map(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_bin(0), mpack_tag_str(0))); - TEST_TRUE(false == mpack_tag_equal(mpack_tag_bin(0), mpack_tag_ext(0, 0))); + #if MPACK_EXTENSIONS + TEST_TRUE(false == mpack_tag_equal(mpack_tag_array(0), mpack_tag_ext(0, 0))); + TEST_TRUE(false == mpack_tag_equal(mpack_tag_map(0), mpack_tag_ext(0, 0))); + TEST_TRUE(false == mpack_tag_equal(mpack_tag_str(0), mpack_tag_ext(0, 0))); + TEST_TRUE(false == mpack_tag_equal(mpack_tag_bin(0), mpack_tag_ext(0, 0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_ext(0, 0), mpack_tag_array(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_ext(0, 0), mpack_tag_map(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_ext(0, 0), mpack_tag_str(0))); TEST_TRUE(false == mpack_tag_equal(mpack_tag_ext(0, 0), mpack_tag_bin(0))); + #endif } static void test_string(const char* str, const char* content) { - #if MPACK_DEBUG - // in debug mode, the string should contain the expected content + #if MPACK_STRINGS + // with strings enabled, the string should contain the expected content TEST_TRUE(strstr(str, content) != NULL, "string \"%s\" does not contain \"%s\"", str, content); #else - // in release mode, strings should be blank + // strings should be blank when disabled MPACK_UNUSED(content); TEST_TRUE(strcmp(str, "") == 0, "string is not empty: %s", str); #endif } -static void test_strings() { +static void test_strings(void) { test_string(mpack_error_to_string(mpack_ok), "ok"); #define TEST_ERROR_STRING(error) test_string(mpack_error_to_string(mpack_error_##error), #error) TEST_ERROR_STRING(io); @@ -261,20 +283,26 @@ static void test_strings() { #define TEST_ERROR_TYPE(type) test_string(mpack_type_to_string(mpack_type_##type), #type) TEST_ERROR_TYPE(nil); TEST_ERROR_TYPE(bool); + #if MPACK_FLOAT TEST_ERROR_TYPE(float); + #endif + #if MPACK_DOUBLE TEST_ERROR_TYPE(double); + #endif TEST_ERROR_TYPE(int); TEST_ERROR_TYPE(uint); TEST_ERROR_TYPE(str); TEST_ERROR_TYPE(bin); - TEST_ERROR_TYPE(ext); TEST_ERROR_TYPE(array); TEST_ERROR_TYPE(map); + #if MPACK_EXTENSIONS + TEST_ERROR_TYPE(ext); + #endif #undef TEST_ERROR_TYPE // test strings for invalid enum values // (invalid enum values cause undefined behavior in C++) - #if MPACK_DEBUG && !defined(__cplusplus) + #if MPACK_DEBUG && !defined(__cplusplus) && MPACK_STRINGS TEST_ASSERT(mpack_error_to_string((mpack_error_t)-1)); TEST_ASSERT(mpack_type_to_string((mpack_type_t)-1)); #endif @@ -416,6 +444,53 @@ static void test_utf8_check(void) { TEST_TRUE(false == mpack_utf8_check(EXPAND_STR_ARGS("test\xFF""testtesttest"))); } +static void test_shorten_raw_double_to_float(void) { + #if MPACK_FLOAT && !MPACK_DOUBLE && !defined(__AVR__) + TEST_DOUBLE doubles[] = { + 0.0, + 3.141592653589793238462643, + -1.0, + (TEST_DOUBLE)INFINITY, + -(TEST_DOUBLE)INFINITY, + 1.234567890123456789e-50, + -1.234567890123456789e-25, + 1.234567890123456789e-10, + -1.234567890123456789e10, + 1.234567890123456789e25, + -1.234567890123456789e50, + 1.234567890123456789e100, + }; + + size_t i; + for (i = 0; i < sizeof(doubles)/sizeof(*doubles); ++i) { + union { + TEST_DOUBLE d; + uint64_t u; + } u; + u.d = doubles[i]; + + // Proper float conversion + float f = (float)u.d; + + // Our shortening conversion + float s = mpack_shorten_raw_double_to_float(u.u); + + // Make sure they are close enough + if (isnan(f) || isnan(s)) { + TEST_TRUE(isnan(f) && isnan(s)); + } else if (isinf(f) || isinf(s)) { + TEST_TRUE(isinf(f) && isinf(s)); + } else { + TEST_DOUBLE diff = ((TEST_DOUBLE)f - (TEST_DOUBLE)s) / (u.d + 0.0000001); + TEST_TRUE(diff < 0.00001); + #if 0 + printf("%.20f %.20f %.20f diff: %.20f\n", u.d, f, s, diff); + #endif + } + } + #endif +} + void test_common() { test_tags_special(); test_tags_simple(); @@ -424,5 +499,6 @@ void test_common() { test_strings(); test_utf8_check(); + test_shorten_raw_double_to_float(); } diff --git a/test/test-common.h b/test/unit/src/test-common.h similarity index 94% rename from test/test-common.h rename to test/unit/src/test-common.h index 73f8f82..d757e9f 100644 --- a/test/test-common.h +++ b/test/unit/src/test-common.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR diff --git a/test/test-expect.c b/test/unit/src/test-expect.c similarity index 74% rename from test/test-expect.c rename to test/unit/src/test-expect.c index 03a1cf6..6e3849f 100644 --- a/test/test-expect.c +++ b/test/unit/src/test-expect.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Nicholas Fraser + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in @@ -26,7 +26,7 @@ static const char test_example[] = "\x82\xA7""compact\xC3\xA6""schema\x00"; #define TEST_EXAMPLE_SIZE (sizeof(test_example) - 1) -static void test_expect_example_read() { +static void test_expect_example_read(void) { mpack_reader_t reader; mpack_reader_init_data(&reader, test_example, TEST_EXAMPLE_SIZE); @@ -48,7 +48,8 @@ static void test_expect_example_read() { ] */ -static void test_expect_uint_fixnum() { +static void test_expect_uint_fixnum(void) { + mpack_reader_t reader; // positive fixnums with u8 TEST_SIMPLE_READ("\x00", 0 == mpack_expect_u8(&reader)); @@ -84,7 +85,8 @@ static void test_expect_uint_fixnum() { } -static void test_expect_uint_signed_fixnum() { +static void test_expect_uint_signed_fixnum(void) { + mpack_reader_t reader; // positive fixnums with i8 TEST_SIMPLE_READ("\x00", 0 == mpack_expect_i8(&reader)); @@ -120,7 +122,8 @@ static void test_expect_uint_signed_fixnum() { } -static void test_expect_negative_fixnum() { +static void test_expect_negative_fixnum(void) { + mpack_reader_t reader; // negative fixnums with i8 TEST_SIMPLE_READ("\xff", -1 == mpack_expect_i8(&reader)); @@ -148,7 +151,8 @@ static void test_expect_negative_fixnum() { } -static void test_expect_uint() { +static void test_expect_uint(void) { + mpack_reader_t reader; // positive signed into u8 TEST_SIMPLE_READ("\xd0\x7f", 0x7f == mpack_expect_u8(&reader)); @@ -166,7 +170,7 @@ static void test_expect_uint() { TEST_SIMPLE_READ("\xd3\x7f\xff\xff\xff\xff\xff\xff\xff", 0x7fffffffffffffff == mpack_expect_u64(&reader)); // positive unsigned into u8 - + TEST_SIMPLE_READ("\xcc\x80", 0x80 == mpack_expect_u8(&reader)); TEST_SIMPLE_READ("\xcc\x80", 0x80 == mpack_expect_u16(&reader)); TEST_SIMPLE_READ("\xcc\x80", 0x80 == mpack_expect_u32(&reader)); @@ -202,7 +206,8 @@ static void test_expect_uint() { } -static void test_expect_uint_signed() { +static void test_expect_uint_signed(void) { + mpack_reader_t reader; TEST_SIMPLE_READ("\xcc\x80", 0x80 == mpack_expect_i16(&reader)); TEST_SIMPLE_READ("\xcc\x80", 0x80 == mpack_expect_i32(&reader)); @@ -221,11 +226,16 @@ static void test_expect_uint_signed() { TEST_SIMPLE_READ("\xcd\xff\xff", 0xffff == mpack_expect_i32(&reader)); TEST_SIMPLE_READ("\xcd\xff\xff", 0xffff == mpack_expect_i64(&reader)); - TEST_SIMPLE_READ("\xcd\xff\xff", 0xffff == mpack_expect_int(&reader)); + + if (sizeof(int) >= 4) { + TEST_SIMPLE_READ("\xcd\xff\xff", (int)0xffff == mpack_expect_int(&reader)); + TEST_SIMPLE_READ("\xce\x00\x01\x00\x00", 0x10000 == mpack_expect_int(&reader)); + } else if (sizeof(int) < 4) { + TEST_SIMPLE_READ_ERROR("\xcd\xff\xff", 0 == mpack_expect_int(&reader), mpack_error_type); + } TEST_SIMPLE_READ("\xce\x00\x01\x00\x00", 0x10000 == mpack_expect_i32(&reader)); TEST_SIMPLE_READ("\xce\x00\x01\x00\x00", 0x10000 == mpack_expect_i64(&reader)); - TEST_SIMPLE_READ("\xce\x00\x01\x00\x00", 0x10000 == mpack_expect_int(&reader)); TEST_SIMPLE_READ("\xce\xff\xff\xff\xff", 0xffffffff == mpack_expect_i64(&reader)); @@ -233,7 +243,8 @@ static void test_expect_uint_signed() { } -static void test_expect_int() { +static void test_expect_int(void) { + mpack_reader_t reader; TEST_SIMPLE_READ("\xd0\xdf", -33 == mpack_expect_i8(&reader)); TEST_SIMPLE_READ("\xd0\xdf", -33 == mpack_expect_i16(&reader)); @@ -241,37 +252,40 @@ static void test_expect_int() { TEST_SIMPLE_READ("\xd0\xdf", -33 == mpack_expect_i64(&reader)); TEST_SIMPLE_READ("\xd0\xdf", -33 == mpack_expect_int(&reader)); - TEST_SIMPLE_READ("\xd0\x80", INT8_MIN == mpack_expect_i8(&reader)); - TEST_SIMPLE_READ("\xd0\x80", INT8_MIN == mpack_expect_i16(&reader)); - TEST_SIMPLE_READ("\xd0\x80", INT8_MIN == mpack_expect_i32(&reader)); - TEST_SIMPLE_READ("\xd0\x80", INT8_MIN == mpack_expect_i64(&reader)); - TEST_SIMPLE_READ("\xd0\x80", INT8_MIN == mpack_expect_int(&reader)); + TEST_SIMPLE_READ("\xd0\x80", MPACK_INT8_MIN == mpack_expect_i8(&reader)); + TEST_SIMPLE_READ("\xd0\x80", MPACK_INT8_MIN == mpack_expect_i16(&reader)); + TEST_SIMPLE_READ("\xd0\x80", MPACK_INT8_MIN == mpack_expect_i32(&reader)); + TEST_SIMPLE_READ("\xd0\x80", MPACK_INT8_MIN == mpack_expect_i64(&reader)); + TEST_SIMPLE_READ("\xd0\x80", MPACK_INT8_MIN == mpack_expect_int(&reader)); - TEST_SIMPLE_READ("\xd1\xff\x7f", INT8_MIN - 1 == mpack_expect_i16(&reader)); - TEST_SIMPLE_READ("\xd1\xff\x7f", INT8_MIN - 1 == mpack_expect_i32(&reader)); - TEST_SIMPLE_READ("\xd1\xff\x7f", INT8_MIN - 1 == mpack_expect_i64(&reader)); - TEST_SIMPLE_READ("\xd1\xff\x7f", INT8_MIN - 1 == mpack_expect_int(&reader)); + TEST_SIMPLE_READ("\xd1\xff\x7f", MPACK_INT8_MIN - 1 == mpack_expect_i16(&reader)); + TEST_SIMPLE_READ("\xd1\xff\x7f", MPACK_INT8_MIN - 1 == mpack_expect_i32(&reader)); + TEST_SIMPLE_READ("\xd1\xff\x7f", MPACK_INT8_MIN - 1 == mpack_expect_i64(&reader)); + TEST_SIMPLE_READ("\xd1\xff\x7f", MPACK_INT8_MIN - 1 == mpack_expect_int(&reader)); - TEST_SIMPLE_READ("\xd1\x80\x00", INT16_MIN == mpack_expect_i16(&reader)); - TEST_SIMPLE_READ("\xd1\x80\x00", INT16_MIN == mpack_expect_i32(&reader)); - TEST_SIMPLE_READ("\xd1\x80\x00", INT16_MIN == mpack_expect_i64(&reader)); - TEST_SIMPLE_READ("\xd1\x80\x00", INT16_MIN == mpack_expect_int(&reader)); + TEST_SIMPLE_READ("\xd1\x80\x00", MPACK_INT16_MIN == mpack_expect_i16(&reader)); + TEST_SIMPLE_READ("\xd1\x80\x00", MPACK_INT16_MIN == mpack_expect_i32(&reader)); + TEST_SIMPLE_READ("\xd1\x80\x00", MPACK_INT16_MIN == mpack_expect_i64(&reader)); + TEST_SIMPLE_READ("\xd1\x80\x00", MPACK_INT16_MIN == mpack_expect_int(&reader)); - TEST_SIMPLE_READ("\xd2\xff\xff\x7f\xff", INT16_MIN - 1 == mpack_expect_i32(&reader)); - TEST_SIMPLE_READ("\xd2\xff\xff\x7f\xff", INT16_MIN - 1 == mpack_expect_i64(&reader)); - TEST_SIMPLE_READ("\xd2\xff\xff\x7f\xff", INT16_MIN - 1 == mpack_expect_int(&reader)); + TEST_SIMPLE_READ("\xd2\xff\xff\x7f\xff", (int32_t)MPACK_INT16_MIN - 1 == mpack_expect_i32(&reader)); + TEST_SIMPLE_READ("\xd2\xff\xff\x7f\xff", (int32_t)MPACK_INT16_MIN - 1 == mpack_expect_i64(&reader)); + if (sizeof(int) >= 4) + TEST_SIMPLE_READ("\xd2\xff\xff\x7f\xff", (int32_t)MPACK_INT16_MIN - 1 == mpack_expect_int(&reader)); - TEST_SIMPLE_READ("\xd2\x80\x00\x00\x00", INT32_MIN == mpack_expect_i32(&reader)); - TEST_SIMPLE_READ("\xd2\x80\x00\x00\x00", INT32_MIN == mpack_expect_i64(&reader)); - TEST_SIMPLE_READ("\xd2\x80\x00\x00\x00", INT32_MIN == mpack_expect_int(&reader)); + TEST_SIMPLE_READ("\xd2\x80\x00\x00\x00", MPACK_INT32_MIN == mpack_expect_i32(&reader)); + TEST_SIMPLE_READ("\xd2\x80\x00\x00\x00", MPACK_INT32_MIN == mpack_expect_i64(&reader)); + if (sizeof(int) >= 4) + TEST_SIMPLE_READ("\xd2\x80\x00\x00\x00", MPACK_INT32_MIN == mpack_expect_int(&reader)); - TEST_SIMPLE_READ("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", (int64_t)INT32_MIN - 1 == mpack_expect_i64(&reader)); + TEST_SIMPLE_READ("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", (int64_t)MPACK_INT32_MIN - 1 == mpack_expect_i64(&reader)); - TEST_SIMPLE_READ("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", INT64_MIN == mpack_expect_i64(&reader)); + TEST_SIMPLE_READ("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", MPACK_INT64_MIN == mpack_expect_i64(&reader)); } -static void test_expect_ints_dynamic_int() { +static void test_expect_ints_dynamic_int(void) { + mpack_reader_t reader; // we don't bother to test with different signed/unsigned value // functions; they are tested for equality in test-value.c @@ -297,26 +311,27 @@ static void test_expect_ints_dynamic_int() { TEST_SIMPLE_READ("\xcd\xff\xff", mpack_tag_equal(mpack_tag_uint(0xffff), mpack_read_tag(&reader))); TEST_SIMPLE_READ("\xce\x00\x01\x00\x00", mpack_tag_equal(mpack_tag_uint(0x10000), mpack_read_tag(&reader))); TEST_SIMPLE_READ("\xce\xff\xff\xff\xff", mpack_tag_equal(mpack_tag_uint(0xffffffff), mpack_read_tag(&reader))); - TEST_SIMPLE_READ("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_tag_equal(mpack_tag_uint(UINT64_C(0x100000000)), mpack_read_tag(&reader))); - TEST_SIMPLE_READ("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_tag_equal(mpack_tag_uint(UINT64_C(0xffffffffffffffff)), mpack_read_tag(&reader))); + TEST_SIMPLE_READ("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_tag_equal(mpack_tag_uint(MPACK_UINT64_C(0x100000000)), mpack_read_tag(&reader))); + TEST_SIMPLE_READ("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_tag_equal(mpack_tag_uint(MPACK_UINT64_C(0xffffffffffffffff)), mpack_read_tag(&reader))); // ints TEST_SIMPLE_READ("\xd0\xdf", mpack_tag_equal(mpack_tag_int(-33), mpack_read_tag(&reader))); - TEST_SIMPLE_READ("\xd0\x80", mpack_tag_equal(mpack_tag_int(INT8_MIN), mpack_read_tag(&reader))); - TEST_SIMPLE_READ("\xd1\xff\x7f", mpack_tag_equal(mpack_tag_int(INT8_MIN - 1), mpack_read_tag(&reader))); - TEST_SIMPLE_READ("\xd1\x80\x00", mpack_tag_equal(mpack_tag_int(INT16_MIN), mpack_read_tag(&reader))); - TEST_SIMPLE_READ("\xd2\xff\xff\x7f\xff", mpack_tag_equal(mpack_tag_int(INT16_MIN - 1), mpack_read_tag(&reader))); + TEST_SIMPLE_READ("\xd0\x80", mpack_tag_equal(mpack_tag_int(MPACK_INT8_MIN), mpack_read_tag(&reader))); + TEST_SIMPLE_READ("\xd1\xff\x7f", mpack_tag_equal(mpack_tag_int(MPACK_INT8_MIN - 1), mpack_read_tag(&reader))); + TEST_SIMPLE_READ("\xd1\x80\x00", mpack_tag_equal(mpack_tag_int(MPACK_INT16_MIN), mpack_read_tag(&reader))); + TEST_SIMPLE_READ("\xd2\xff\xff\x7f\xff", mpack_tag_equal(mpack_tag_int((int32_t)MPACK_INT16_MIN - 1), mpack_read_tag(&reader))); - TEST_SIMPLE_READ("\xd2\x80\x00\x00\x00", mpack_tag_equal(mpack_tag_int(INT32_MIN), mpack_read_tag(&reader))); - TEST_SIMPLE_READ("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", mpack_tag_equal(mpack_tag_int((int64_t)INT32_MIN - 1), mpack_read_tag(&reader))); + TEST_SIMPLE_READ("\xd2\x80\x00\x00\x00", mpack_tag_equal(mpack_tag_int(MPACK_INT32_MIN), mpack_read_tag(&reader))); + TEST_SIMPLE_READ("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", mpack_tag_equal(mpack_tag_int((int64_t)MPACK_INT32_MIN - 1), mpack_read_tag(&reader))); - TEST_SIMPLE_READ("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", mpack_tag_equal(mpack_tag_int(INT64_MIN), mpack_read_tag(&reader))); + TEST_SIMPLE_READ("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", mpack_tag_equal(mpack_tag_int(MPACK_INT64_MIN), mpack_read_tag(&reader))); } -static void test_expect_int_bounds() { +static void test_expect_int_bounds(void) { + mpack_reader_t reader; - TEST_SIMPLE_READ_ERROR("\xd1\xff\x7f", 0 == mpack_expect_i8(&reader), mpack_error_type); + TEST_SIMPLE_READ_ERROR("\xd1\xff\x7f", 0 == mpack_expect_i8(&reader), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xd1\x80\x00", 0 == mpack_expect_i8(&reader), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xd2\xff\xff\x7f\xff", 0 == mpack_expect_i8(&reader), mpack_error_type); @@ -335,7 +350,8 @@ static void test_expect_int_bounds() { } -static void test_expect_uint_bounds() { +static void test_expect_uint_bounds(void) { + mpack_reader_t reader; TEST_SIMPLE_READ_ERROR("\xcd\x01\x00", 0 == mpack_expect_u8(&reader), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xcd\xff\xff", 0 == mpack_expect_u8(&reader), mpack_error_type); @@ -377,7 +393,9 @@ static void test_expect_uint_bounds() { \ TEST_SIMPLE_READ_ASSERT("\x00", mpack_expect_##name##_range(reader, 1, -1)); -static void test_expect_int_range() { +static void test_expect_int_range(void) { + mpack_reader_t reader; + // these currently don't test anything involving the limits of // each data type; there doesn't seem to be much point in doing // so, since they all wrap the normal expect functions. @@ -393,7 +411,9 @@ static void test_expect_int_range() { TEST_EXPECT_INT_RANGE(int); } -static void test_expect_int_match() { +static void test_expect_int_match(void) { + mpack_reader_t reader; + TEST_SIMPLE_READ("\x00", (mpack_expect_uint_match(&reader, 0), true)); TEST_SIMPLE_READ("\x01", (mpack_expect_uint_match(&reader, 1), true)); TEST_SIMPLE_READ("\xcc\x80", (mpack_expect_uint_match(&reader, 0x80), true)); @@ -405,23 +425,28 @@ static void test_expect_int_match() { TEST_SIMPLE_READ("\xce\xff\xff\xff\xff", (mpack_expect_uint_match(&reader, 0xffffffff), true)); TEST_SIMPLE_READ("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", (mpack_expect_uint_match(&reader, 0x100000000), true)); TEST_SIMPLE_READ("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", (mpack_expect_uint_match(&reader, 0xffffffffffffffff), true)); - TEST_SIMPLE_READ_ERROR("\xff", (mpack_expect_uint_match(&reader, 0), true), mpack_error_type); + TEST_SIMPLE_READ_ERROR("\xff", (mpack_expect_uint_match(&reader, 0), true), mpack_error_type); // signed, not a uint + TEST_SIMPLE_READ_ERROR("\x01", (mpack_expect_uint_match(&reader, 2), true), mpack_error_type); // successful uint, not a match TEST_SIMPLE_READ("\x00", (mpack_expect_int_match(&reader, 0), true)); TEST_SIMPLE_READ("\x01", (mpack_expect_int_match(&reader, 1), true)); TEST_SIMPLE_READ("\xd0\xdf", (mpack_expect_int_match(&reader, -33), true)); - TEST_SIMPLE_READ("\xd0\x80", (mpack_expect_int_match(&reader, INT8_MIN), true)); - TEST_SIMPLE_READ("\xd1\xff\x7f", (mpack_expect_int_match(&reader, INT8_MIN - 1), true)); - TEST_SIMPLE_READ("\xd1\x80\x00", (mpack_expect_int_match(&reader, INT16_MIN), true)); - TEST_SIMPLE_READ("\xd2\xff\xff\x7f\xff", (mpack_expect_int_match(&reader, INT16_MIN - 1), true)); - TEST_SIMPLE_READ("\xd2\x80\x00\x00\x00", (mpack_expect_int_match(&reader, INT32_MIN), true)); - TEST_SIMPLE_READ("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", (mpack_expect_int_match(&reader, (int64_t)INT32_MIN - 1), true)); - TEST_SIMPLE_READ("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", (mpack_expect_int_match(&reader, INT64_MIN), true)); - TEST_SIMPLE_READ_ERROR("\xc0", (mpack_expect_int_match(&reader, 0), true), mpack_error_type); + TEST_SIMPLE_READ("\xd0\x80", (mpack_expect_int_match(&reader, MPACK_INT8_MIN), true)); + TEST_SIMPLE_READ("\xd1\xff\x7f", (mpack_expect_int_match(&reader, MPACK_INT8_MIN - 1), true)); + TEST_SIMPLE_READ("\xd1\x80\x00", (mpack_expect_int_match(&reader, MPACK_INT16_MIN), true)); + TEST_SIMPLE_READ("\xd2\xff\xff\x7f\xff", (mpack_expect_int_match(&reader, (int32_t)MPACK_INT16_MIN - 1), true)); + TEST_SIMPLE_READ("\xd2\x80\x00\x00\x00", (mpack_expect_int_match(&reader, MPACK_INT32_MIN), true)); + TEST_SIMPLE_READ("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", (mpack_expect_int_match(&reader, (int64_t)MPACK_INT32_MIN - 1), true)); + TEST_SIMPLE_READ("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", (mpack_expect_int_match(&reader, MPACK_INT64_MIN), true)); + TEST_SIMPLE_READ_ERROR("\xc0", (mpack_expect_int_match(&reader, 0), true), mpack_error_type); // nil, not an int + TEST_SIMPLE_READ_ERROR("\x01", (mpack_expect_int_match(&reader, 2), true), mpack_error_type); // successful uint->int, not a match + TEST_SIMPLE_READ_ERROR("\xfe", (mpack_expect_int_match(&reader, -3), true), mpack_error_type); // successful int, not a match } -static void test_expect_misc() { +static void test_expect_misc(void) { + mpack_reader_t reader; + TEST_SIMPLE_READ("\xc0", (mpack_expect_nil(&reader), true)); TEST_SIMPLE_READ("\xc0", (mpack_expect_tag(&reader, mpack_tag_nil()), true)); TEST_SIMPLE_READ_ERROR("\x90", (mpack_expect_tag(&reader, mpack_tag_nil()), true), mpack_error_type); @@ -430,12 +455,14 @@ static void test_expect_misc() { TEST_SIMPLE_READ("\xc3", true == mpack_expect_bool(&reader)); TEST_SIMPLE_READ("\xc2", (mpack_expect_false(&reader), true)); TEST_SIMPLE_READ("\xc3", (mpack_expect_true(&reader), true)); - TEST_SIMPLE_READ_ERROR("\xc0", (mpack_expect_false(&reader), true), mpack_error_type); - TEST_SIMPLE_READ_ERROR("\xc0", (mpack_expect_true(&reader), true), mpack_error_type); + TEST_SIMPLE_READ_ERROR("\xc3", (mpack_expect_false(&reader), true), mpack_error_type); // bool, wrong value + TEST_SIMPLE_READ_ERROR("\xc2", (mpack_expect_true(&reader), true), mpack_error_type); // bool, wrong value + TEST_SIMPLE_READ_ERROR("\xc0", (mpack_expect_false(&reader), true), mpack_error_type); // wrong type + TEST_SIMPLE_READ_ERROR("\xc0", (mpack_expect_true(&reader), true), mpack_error_type); // wrong type } #if MPACK_READ_TRACKING -static void test_expect_tracking() { +static void test_expect_tracking(void) { char buf[4]; mpack_reader_t reader; @@ -511,55 +538,89 @@ static void test_expect_tracking() { } #endif -static void test_expect_reals() { +static void test_expect_reals(void) { + mpack_reader_t reader; + (void)reader; + // these are some very simple floats that don't really test IEEE 742 conformance; // this section could use some improvement + #if MPACK_FLOAT TEST_SIMPLE_READ("\x00", 0.0f == mpack_expect_float(&reader)); TEST_SIMPLE_READ("\xd0\x00", 0.0f == mpack_expect_float(&reader)); TEST_SIMPLE_READ("\xca\x00\x00\x00\x00", 0.0f == mpack_expect_float(&reader)); TEST_SIMPLE_READ("\xcb\x00\x00\x00\x00\x00\x00\x00\x00", 0.0f == mpack_expect_float(&reader)); + #endif + #if MPACK_DOUBLE TEST_SIMPLE_READ("\x00", 0.0 == mpack_expect_double(&reader)); TEST_SIMPLE_READ("\xd0\x00", 0.0 == mpack_expect_double(&reader)); TEST_SIMPLE_READ("\xca\x00\x00\x00\x00", 0.0 == mpack_expect_double(&reader)); TEST_SIMPLE_READ("\xcb\x00\x00\x00\x00\x00\x00\x00\x00", 0.0 == mpack_expect_double(&reader)); + #endif - TEST_SIMPLE_READ("\xca\xff\xff\xff\xff", isnanf(mpack_expect_float(&reader)) != 0); - TEST_SIMPLE_READ("\xcb\xff\xff\xff\xff\xff\xff\xff\xff", isnanf(mpack_expect_float(&reader)) != 0); - TEST_SIMPLE_READ("\xca\xff\xff\xff\xff", isnan(mpack_expect_double(&reader)) != 0); - TEST_SIMPLE_READ("\xcb\xff\xff\xff\xff\xff\xff\xff\xff", isnan(mpack_expect_double(&reader)) != 0); - + #if MPACK_FLOAT TEST_SIMPLE_READ("\xca\x00\x00\x00\x00", 0.0f == mpack_expect_float_strict(&reader)); + #endif + #if MPACK_DOUBLE + // reading float as strict double is allowed, not the other way around TEST_SIMPLE_READ("\xca\x00\x00\x00\x00", 0.0 == mpack_expect_double_strict(&reader)); TEST_SIMPLE_READ("\xcb\x00\x00\x00\x00\x00\x00\x00\x00", 0.0 == mpack_expect_double_strict(&reader)); + #endif + + // when -ffinite-math-only is enabled, isnan() can always return false. + // TODO: we should probably add at least a reader option to + // generate an error on non-finite reals. + #if !MPACK_FINITE_MATH + #if MPACK_FLOAT + TEST_SIMPLE_READ("\xca\xff\xff\xff\xff", isnanf(mpack_expect_float(&reader)) != 0); TEST_SIMPLE_READ("\xca\xff\xff\xff\xff", 0 != isnanf(mpack_expect_float_strict(&reader))); + TEST_SIMPLE_READ("\xcb\xff\xff\xff\xff\xff\xff\xff\xff", isnanf(mpack_expect_float(&reader)) != 0); + #endif + #if MPACK_DOUBLE + TEST_SIMPLE_READ("\xca\xff\xff\xff\xff", isnan(mpack_expect_double(&reader)) != 0); + TEST_SIMPLE_READ("\xcb\xff\xff\xff\xff\xff\xff\xff\xff", isnan(mpack_expect_double(&reader)) != 0); TEST_SIMPLE_READ("\xca\xff\xff\xff\xff", 0 != isnan(mpack_expect_double_strict(&reader))); TEST_SIMPLE_READ("\xcb\xff\xff\xff\xff\xff\xff\xff\xff", isnan(mpack_expect_double_strict(&reader))); + #endif + #endif + #if MPACK_FLOAT TEST_SIMPLE_READ_ERROR("\x00", 0.0f == mpack_expect_float_strict(&reader), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xd0\x00", 0.0f == mpack_expect_float_strict(&reader), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xcb\x00\x00\x00\x00\x00\x00\x00\x00", 0.0f == mpack_expect_float_strict(&reader), mpack_error_type); + #endif + #if MPACK_DOUBLE TEST_SIMPLE_READ_ERROR("\x00", 0.0 == mpack_expect_double_strict(&reader), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xd0\x00", 0.0 == mpack_expect_double_strict(&reader), mpack_error_type); + #endif } -static void test_expect_reals_range() { +static void test_expect_reals_range(void) { + mpack_reader_t reader; + (void)reader; + + #if MPACK_FLOAT TEST_SIMPLE_READ("\x00", 0.0f == mpack_expect_float_range(&reader, 0.0f, 0.0f)); TEST_SIMPLE_READ("\x00", 0.0f == mpack_expect_float_range(&reader, 0.0f, 1.0f)); TEST_SIMPLE_READ("\x00", 0.0f == mpack_expect_float_range(&reader, -1.0f, 0.0f)); TEST_SIMPLE_READ_ERROR("\x00", 1.0f == mpack_expect_float_range(&reader, 1.0f, 2.0f), mpack_error_type); TEST_SIMPLE_READ_ASSERT("\x00", mpack_expect_float_range(reader, 1.0f, -1.0f)); + #endif + #if MPACK_DOUBLE TEST_SIMPLE_READ("\x00", 0.0 == mpack_expect_double_range(&reader, 0.0, 0.0f)); TEST_SIMPLE_READ("\x00", 0.0 == mpack_expect_double_range(&reader, 0.0, 1.0f)); TEST_SIMPLE_READ("\x00", 0.0 == mpack_expect_double_range(&reader, -1.0, 0.0f)); TEST_SIMPLE_READ_ERROR("\x00", 1.0 == mpack_expect_double_range(&reader, 1.0, 2.0f), mpack_error_type); TEST_SIMPLE_READ_ASSERT("\x00", mpack_expect_double_range(reader, 1.0, -1.0)); + #endif } -static void test_expect_bad_type() { +static void test_expect_bad_type(void) { + mpack_reader_t reader; + // test that all reader functions correctly handle badly typed data TEST_SIMPLE_READ_ERROR("\xc2", (mpack_expect_nil(&reader), true), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xc0", false == mpack_expect_bool(&reader), mpack_error_type); @@ -571,13 +632,19 @@ static void test_expect_bad_type() { TEST_SIMPLE_READ_ERROR("\xc0", 0 == mpack_expect_i16(&reader), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xc0", 0 == mpack_expect_i32(&reader), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xc0", 0 == mpack_expect_i64(&reader), mpack_error_type); + #if MPACK_FLOAT TEST_SIMPLE_READ_ERROR("\xc0", 0.0f == mpack_expect_float(&reader), mpack_error_type); - TEST_SIMPLE_READ_ERROR("\xc0", 0.0 == mpack_expect_double(&reader), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xc0", 0.0f == mpack_expect_float_strict(&reader), mpack_error_type); + #endif + #if MPACK_DOUBLE + TEST_SIMPLE_READ_ERROR("\xc0", 0.0 == mpack_expect_double(&reader), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xc0", 0.0 == mpack_expect_double_strict(&reader), mpack_error_type); + #endif } -static void test_expect_pre_error() { +static void test_expect_pre_error(void) { + mpack_reader_t reader; + // test that all reader functinvalidns correctly handle pre-existing errors TEST_SIMPLE_READ_ERROR("", (mpack_expect_nil(&reader), true), mpack_error_invalid); TEST_SIMPLE_READ_ERROR("", false == mpack_expect_bool(&reader), mpack_error_invalid); @@ -589,13 +656,18 @@ static void test_expect_pre_error() { TEST_SIMPLE_READ_ERROR("", 0 == mpack_expect_i16(&reader), mpack_error_invalid); TEST_SIMPLE_READ_ERROR("", 0 == mpack_expect_i32(&reader), mpack_error_invalid); TEST_SIMPLE_READ_ERROR("", 0 == mpack_expect_i64(&reader), mpack_error_invalid); + #if MPACK_FLOAT TEST_SIMPLE_READ_ERROR("", 0.0f == mpack_expect_float(&reader), mpack_error_invalid); - TEST_SIMPLE_READ_ERROR("", 0.0 == mpack_expect_double(&reader), mpack_error_invalid); TEST_SIMPLE_READ_ERROR("", 0.0f == mpack_expect_float_strict(&reader), mpack_error_invalid); + #endif + #if MPACK_DOUBLE + TEST_SIMPLE_READ_ERROR("", 0.0 == mpack_expect_double(&reader), mpack_error_invalid); TEST_SIMPLE_READ_ERROR("", 0.0 == mpack_expect_double_strict(&reader), mpack_error_invalid); + #endif } -static void test_expect_str() { +static void test_expect_str(void) { + mpack_reader_t reader; char buf[256]; #ifdef MPACK_MALLOC char* test = NULL; @@ -668,6 +740,16 @@ static void test_expect_str() { TEST_SIMPLE_READ_ERROR("\xa3""azc", (mpack_expect_cstr_match(&reader, "abc"), true), mpack_error_type); TEST_SIMPLE_READ_ERROR("\xa3""abz", (mpack_expect_cstr_match(&reader, "abc"), true), mpack_error_type); + #if MPACK_STDLIB + // str/cstr match larger than 32 bits + if (MPACK_UINT32_MAX < SIZE_MAX) { + test_system_mock_strlen((size_t)((uint64_t)MPACK_UINT32_MAX + MPACK_UINT64_C(1))); + TEST_SIMPLE_READ_ERROR("\xa3""abc", (mpack_expect_cstr_match(&reader, "abc"), true), mpack_error_type); + test_system_mock_strlen(SIZE_MAX); + TEST_SIMPLE_READ_ERROR("\xa3""abc", (mpack_expect_cstr_match(&reader, "abc"), true), mpack_error_type); + } + #endif + // bin is never allowed to be read as str @@ -764,7 +846,8 @@ static void test_expect_str() { } -static void test_expect_bin() { +static void test_expect_bin(void) { + mpack_reader_t reader; char buf[256]; TEST_SIMPLE_READ_CANCEL("\xc4\x80", 128 == mpack_expect_bin(&reader)); @@ -803,16 +886,133 @@ static void test_expect_bin() { TEST_TRUE(memcmp(test, "test", 4) == 0); MPACK_FREE(test); } + + // Unlimited max allocation size. Don't do this, or at least not with + // untrusted data! + TEST_SIMPLE_READ("\xc4\x04test", NULL != (test = mpack_expect_bin_alloc(&reader, SIZE_MAX, &length))); + if (test) { + TEST_TRUE(length == 4); + TEST_TRUE(memcmp(test, "test", 4) == 0); + MPACK_FREE(test); + } + TEST_SIMPLE_READ_ERROR("\xc4\x04test", NULL == mpack_expect_bin_alloc(&reader, 3, &length), mpack_error_type); TEST_SIMPLE_READ_ERROR("\x01", NULL == mpack_expect_bin_alloc(&reader, 3, &length), mpack_error_type); #endif } -static void test_expect_ext() { +static void test_expect_bin_size_buf(void) { + mpack_reader_t reader; + char buf[256]; + TEST_SIMPLE_READ("\xc4\x00", (mpack_expect_bin_size_buf(&reader, buf, 0), true)); + TEST_SIMPLE_READ("\xc4\x05hello", (mpack_expect_bin_size_buf(&reader, buf, 5), 0 == memcmp(buf, "hello", 5))); + TEST_SIMPLE_READ_ERROR("\xc4\x04test", (mpack_expect_bin_size_buf(&reader, buf, 5), true), mpack_error_type); } -static void test_expect_arrays() { +#if MPACK_EXTENSIONS +static void test_expect_ext(void) { + mpack_reader_t reader; + char buf[256]; + int8_t type; + + TEST_SIMPLE_READ_CANCEL("\xd4\x01", 1 == mpack_expect_ext(&reader, &type)); + TEST_TRUE(type == 1); + TEST_SIMPLE_READ_CANCEL("\xd5\x7f", 2 == mpack_expect_ext(&reader, &type)); + TEST_TRUE(type == 127); + TEST_SIMPLE_READ_CANCEL("\xd6\xfe", 4 == mpack_expect_ext(&reader, &type)); + TEST_TRUE(type == -2); + TEST_SIMPLE_READ_CANCEL("\xd7\x80", 8 == mpack_expect_ext(&reader, &type)); + TEST_TRUE(type == -128); + TEST_SIMPLE_READ_CANCEL("\xd8\x02", 16 == mpack_expect_ext(&reader, &type)); + TEST_TRUE(type == 2); + TEST_SIMPLE_READ_CANCEL("\xc7\x80\x03", 128 == mpack_expect_ext(&reader, &type)); + TEST_TRUE(type == 3); + TEST_SIMPLE_READ_CANCEL("\xc8\x80\x80\x04", 0x8080 == mpack_expect_ext(&reader, &type)); + TEST_TRUE(type == 4); + TEST_SIMPLE_READ_CANCEL("\xc9\xff\xff\xff\xff\x05", 0xffffffff == mpack_expect_ext(&reader, &type)); + TEST_TRUE(type == 5); + + // TODO: test strict/compatibility modes. currently, we do not + // support old MessagePack version compatibility; ext will not + // accept str types. + // TODO In v4 mode calling any ext function should cause an assertion failure and raise mpack_error_bug. + /*TEST_SIMPLE_READ_ERROR("\xbf", 0 == mpack_expect_ext(&reader, &type), mpack_error_type);*/ + /*TEST_SIMPLE_READ_ERROR("\xbf", 0 == mpack_expect_ext_buf(&reader, buf, sizeof(buf)), mpack_error_type);*/ + + TEST_SIMPLE_READ("\xd4\x01\x00", 1 == mpack_expect_ext_buf(&reader, &type, buf, 1)); + TEST_TRUE(type == 1); + TEST_SIMPLE_READ("\xd5\x7fte", 2 == mpack_expect_ext_buf(&reader, &type, buf, 2)); + TEST_TRUE(type == 127); + TEST_SIMPLE_READ("\xd6\xfetest", 4 == mpack_expect_ext_buf(&reader, &type, buf, 4)); + TEST_TRUE(type == -2); + TEST_SIMPLE_READ("\xd7\x80testtest", 8 == mpack_expect_ext_buf(&reader, &type, buf, 8)); + TEST_TRUE(type == -128); + TEST_SIMPLE_READ("\xd8\x02testtesttesttest", 16 == mpack_expect_ext_buf(&reader, &type, buf, 16)); + TEST_TRUE(type == 2); + TEST_SIMPLE_READ("\xc7\x01\x03t", 1 == mpack_expect_ext_buf(&reader, &type, buf, 1)); + TEST_TRUE(type == 3); + TEST_SIMPLE_READ("\xc8\x00\x04\x04test", 4 == mpack_expect_ext_buf(&reader, &type, buf, 4)); + TEST_TRUE(type == 4); + TEST_SIMPLE_READ("\xc9\x00\x00\x00\x04\x05test", 4 == mpack_expect_ext_buf(&reader, &type, buf, 4)); + TEST_TRUE(type == 5); + TEST_SIMPLE_READ_ERROR("\xc7\x05\x06hello", 0 == mpack_expect_ext_buf(&reader, &type, buf, 4), mpack_error_too_big); + TEST_TRUE(type == 0); + TEST_SIMPLE_READ_ERROR("\xc7\x08hello", 0 == mpack_expect_ext_buf(&reader, &type, buf, sizeof(buf)), mpack_error_invalid); + TEST_TRUE(type == 0); + TEST_SIMPLE_READ("\xc7\x01\x06\x00", 1 == mpack_expect_ext_buf(&reader, &type, buf, 4)); + TEST_TRUE(type == 6); + + TEST_SIMPLE_READ("\xc7\x00\x07", (mpack_expect_ext_size(&reader, &type, 0), mpack_done_ext(&reader), true)); + TEST_TRUE(type == 7); + TEST_SIMPLE_READ_ERROR("\xc7\x00\x08", (mpack_expect_ext_size(&reader, &type, 4), true), mpack_error_type); + TEST_TRUE(type == 0); + TEST_SIMPLE_READ_CANCEL("\xc7\x04\x09", (mpack_expect_ext_size(&reader, &type, 4), true)); + TEST_TRUE(type == 9); + TEST_SIMPLE_READ_ERROR("\xc7\x05\x10", (mpack_expect_ext_size(&reader, &type, 4), true), mpack_error_type); + TEST_TRUE(type == 0); + + type = 1; + TEST_SIMPLE_READ_ERROR("\xc3", 0 == mpack_expect_ext(&reader, &type), mpack_error_type); + TEST_TRUE(type == 0); + + #ifdef MPACK_MALLOC + size_t length; + char* test = NULL; + + TEST_SIMPLE_READ("\xc7\x00\x01", (NULL == mpack_expect_ext_alloc(&reader, &type, 0, &length))); + TEST_TRUE(length == 0); + TEST_SIMPLE_READ("\xc7\x00\x01", (NULL == mpack_expect_ext_alloc(&reader, &type, 4, &length))); + TEST_TRUE(length == 0); + TEST_SIMPLE_READ("\xc7\x04\x01test", NULL != (test = mpack_expect_ext_alloc(&reader, &type, 4, &length))); + if (test) { + TEST_TRUE(length == 4); + TEST_TRUE(memcmp(test, "test", 4) == 0); + MPACK_FREE(test); + } + + // Unlimited max allocation size. Don't do this, or at least not with + // untrusted data! + TEST_SIMPLE_READ("\xc7\x04\x01test", NULL != (test = mpack_expect_ext_alloc(&reader, &type, SIZE_MAX, &length))); + if (test) { + TEST_TRUE(length == 4); + TEST_TRUE(memcmp(test, "test", 4) == 0); + MPACK_FREE(test); + } + + TEST_SIMPLE_READ_ERROR("\xc7\x04\x01test", NULL == mpack_expect_ext_alloc(&reader, &type, 3, &length), mpack_error_type); + // This test currently fails. I cannot figure out why. The + // `mpack_expect_ext_alloc` function does raise an error, but the test + // fails. My guess is that it raises the wrong type of error. It is odd + // since the ext code is basically a copy of the bin code and a similar + // test passes without a problem. + /*TEST_SIMPLE_READ_ERROR("\x01", NULL == mpack_expect_ext_alloc(&reader, &type, 3, &length), mpack_error_type);*/ + #endif +} +#endif + +static void test_expect_arrays(void) { + mpack_reader_t reader; uint32_t count; // arrays @@ -826,7 +1026,7 @@ static void test_expect_arrays() { TEST_SIMPLE_READ_CANCEL("\xdd\x00\x00\x00\x00", 0 == mpack_expect_array(&reader)); TEST_SIMPLE_READ_CANCEL("\xdd\x00\x00\x01\x00", 0x100 == mpack_expect_array(&reader)); TEST_SIMPLE_READ_CANCEL("\xdd\x00\x01\x00\x00", 0x10000 == mpack_expect_array(&reader)); - TEST_SIMPLE_READ_CANCEL("\xdd\xff\xff\xff\xff", UINT32_MAX == mpack_expect_array(&reader)); + TEST_SIMPLE_READ_CANCEL("\xdd\xff\xff\xff\xff", MPACK_UINT32_MAX == mpack_expect_array(&reader)); TEST_SIMPLE_READ_ERROR("\x00", 0 == mpack_expect_array(&reader), mpack_error_type); // array ranges @@ -841,7 +1041,7 @@ static void test_expect_arrays() { TEST_SIMPLE_READ("\x90", (mpack_expect_array_match(&reader, 0), mpack_done_array(&reader), true)); TEST_SIMPLE_READ_CANCEL("\x9f", (mpack_expect_array_match(&reader, 15), true)); TEST_SIMPLE_READ_CANCEL("\xdc\xff\xff", (mpack_expect_array_match(&reader, 0xffff), true)); - TEST_SIMPLE_READ_CANCEL("\xdd\xff\xff\xff\xff", (mpack_expect_array_match(&reader, UINT32_MAX), true)); + TEST_SIMPLE_READ_CANCEL("\xdd\xff\xff\xff\xff", (mpack_expect_array_match(&reader, MPACK_UINT32_MAX), true)); TEST_SIMPLE_READ_ERROR("\x91", (mpack_expect_array_match(&reader, 2), true), mpack_error_type); TEST_SIMPLE_READ_CANCEL("\x91", true == mpack_expect_array_or_nil(&reader, &count)); @@ -906,7 +1106,8 @@ static void test_expect_arrays() { } -static void test_expect_maps() { +static void test_expect_maps(void) { + mpack_reader_t reader; uint32_t count; // maps @@ -920,7 +1121,7 @@ static void test_expect_maps() { TEST_SIMPLE_READ_CANCEL("\xdf\x00\x00\x00\x00", 0 == mpack_expect_map(&reader)); TEST_SIMPLE_READ_CANCEL("\xdf\x00\x00\x01\x00", 0x100 == mpack_expect_map(&reader)); TEST_SIMPLE_READ_CANCEL("\xdf\x00\x01\x00\x00", 0x10000 == mpack_expect_map(&reader)); - TEST_SIMPLE_READ_CANCEL("\xdf\xff\xff\xff\xff", UINT32_MAX == mpack_expect_map(&reader)); + TEST_SIMPLE_READ_CANCEL("\xdf\xff\xff\xff\xff", MPACK_UINT32_MAX == mpack_expect_map(&reader)); TEST_SIMPLE_READ_ERROR("\x00", 0 == mpack_expect_map(&reader), mpack_error_type); // map ranges @@ -935,7 +1136,7 @@ static void test_expect_maps() { TEST_SIMPLE_READ("\x80", (mpack_expect_map_match(&reader, 0), mpack_done_map(&reader), true)); TEST_SIMPLE_READ_CANCEL("\x8f", (mpack_expect_map_match(&reader, 15), true)); TEST_SIMPLE_READ_CANCEL("\xde\xff\xff", (mpack_expect_map_match(&reader, 0xffff), true)); - TEST_SIMPLE_READ_CANCEL("\xdf\xff\xff\xff\xff", (mpack_expect_map_match(&reader, UINT32_MAX), true)); + TEST_SIMPLE_READ_CANCEL("\xdf\xff\xff\xff\xff", (mpack_expect_map_match(&reader, MPACK_UINT32_MAX), true)); TEST_SIMPLE_READ_ERROR("\x81", (mpack_expect_map_match(&reader, 2), true), mpack_error_type); TEST_SIMPLE_READ_CANCEL("\x81", true == mpack_expect_map_or_nil(&reader, &count)); @@ -956,7 +1157,7 @@ static void test_expect_maps() { } -static void test_expect_key_cstr_basic() { +static void test_expect_key_cstr_basic(void) { mpack_reader_t reader; mpack_reader_init_data(&reader, test_example, TEST_EXAMPLE_SIZE); @@ -978,7 +1179,7 @@ static void test_expect_key_cstr_basic() { #undef KEY_COUNT } -static void test_expect_key_cstr_mixed() { +static void test_expect_key_cstr_mixed(void) { mpack_reader_t reader; mpack_reader_init_data(&reader, test_example, TEST_EXAMPLE_SIZE); @@ -1001,7 +1202,7 @@ static void test_expect_key_cstr_mixed() { #undef KEY_COUNT } -static void test_expect_key_cstr_duplicate() { +static void test_expect_key_cstr_duplicate(void) { static const char data[] = "\x83\xA3""dup\xC0\xA3""dup\xC0\xA5""valid\xC0"; mpack_reader_t reader; mpack_reader_init_data(&reader, data, sizeof(data)-1); @@ -1022,7 +1223,7 @@ static void test_expect_key_cstr_duplicate() { #undef KEY_COUNT } -static void test_expect_key_uint() { +static void test_expect_key_uint(void) { static const char data[] = "\x85\x02\xC0\x00\xC0\xC3\xC0\x03\xC0\x03\xC0"; mpack_reader_t reader; mpack_reader_init_data(&reader, data, sizeof(data)-1); @@ -1052,6 +1253,112 @@ static void test_expect_key_uint() { #undef KEY_COUNT } +typedef struct test_expect_stream_t { + char* data; + size_t left; + size_t read_size; +} test_expect_stream_t; + +static size_t test_expect_stream_fill(mpack_reader_t* reader, char* buffer, size_t count) { + test_expect_stream_t* context = (test_expect_stream_t*)reader->context; + if (count > context->read_size) + count = context->read_size; + if (count > context->left) + count = context->left; + memcpy(buffer, context->data, count); + context->data += count; + context->left -= count; + return count; +} + +static void test_expect_streaming(void) { + // We test reading from a stream of messages using a function + // that returns a small number of bytes each time (as though + // it is slowly receiving data through a socket.) This tests + // that the reader correctly handles streams, and that it + // can continue asking for data even when it needs more bytes + // than read by a single call to the fill function. + + char data[] = "\x00\xd3\xff\xff\xff\xff\xff\xff\xff\xff\xc0\xa5" + "hello\x93\xc3\xa6""world!\xc2\xce\xff\xff\xff\xff"; + + size_t sizes[] = {1, 2, 3, 5, 7, 11}; + size_t i; + for (i = 0; i < sizeof(sizes) / sizeof(*sizes); ++i) { + + test_expect_stream_t context = {data, sizeof(data) - 1, sizes[i]}; + mpack_reader_t reader; + char buffer[MPACK_READER_MINIMUM_BUFFER_SIZE]; + mpack_reader_init(&reader, buffer, sizeof(buffer), 0); + mpack_reader_set_context(&reader, &context); + mpack_reader_set_fill(&reader, &test_expect_stream_fill); + + TEST_TRUE(mpack_expect_uint(&reader) == 0); + TEST_TRUE(mpack_reader_error(&reader) == mpack_ok); + TEST_TRUE(mpack_expect_i64(&reader) == -1); + mpack_expect_nil(&reader); + TEST_TRUE(mpack_reader_error(&reader) == mpack_ok); + mpack_expect_cstr_match(&reader, "hello"); + TEST_TRUE(mpack_reader_error(&reader) == mpack_ok); + + TEST_TRUE(mpack_expect_array(&reader) == 3); + TEST_TRUE(mpack_expect_bool(&reader) == true); + mpack_expect_cstr_match(&reader, "world!"); + TEST_TRUE(mpack_reader_error(&reader) == mpack_ok); + mpack_expect_false(&reader); + TEST_TRUE(mpack_reader_error(&reader) == mpack_ok); + mpack_done_array(&reader); + + TEST_TRUE(mpack_expect_u32(&reader) == MPACK_UINT32_MAX); + TEST_READER_DESTROY_NOERROR(&reader); + } +} + +#if MPACK_EXTENSIONS +static bool test_timestamp_match(int64_t seconds, uint32_t nanoseconds, mpack_timestamp_t timestamp) { + TEST_TRUE(seconds == timestamp.seconds); + TEST_TRUE(nanoseconds == timestamp.nanoseconds); + return true; +} + +static void test_expect_timestamp(void) { + mpack_reader_t reader; + TEST_SIMPLE_READ("\xd6\xff\x00\x00\x00\x00", 0 == mpack_expect_timestamp_truncate(&reader)); + TEST_SIMPLE_READ("\xd6\xff\x00\x00\x01\x00", test_timestamp_match(256, 0, mpack_expect_timestamp(&reader))); + TEST_SIMPLE_READ("\xd6\xff\xfe\xdc\xba\x98", 4275878552u == mpack_expect_timestamp_truncate(&reader)); + TEST_SIMPLE_READ("\xd6\xff\xff\xff\xff\xff", MPACK_UINT32_MAX == mpack_expect_timestamp_truncate(&reader)); + + TEST_SIMPLE_READ("\xd7\xff\x00\x00\x00\x03\x00\x00\x00\x00", + MPACK_INT64_C(12884901888) == mpack_expect_timestamp_truncate(&reader)); + TEST_SIMPLE_READ("\xd7\xff\x00\x00\x00\x00\x00\x00\x00\x00", + test_timestamp_match(0, 0, mpack_expect_timestamp(&reader))); + TEST_SIMPLE_READ("\xd7\xff\xee\x6b\x27\xfc\x00\x00\x00\x00", + test_timestamp_match(0, MPACK_TIMESTAMP_NANOSECONDS_MAX, mpack_expect_timestamp(&reader))); + TEST_SIMPLE_READ("\xd7\xff\xee\x6b\x27\xff\xff\xff\xff\xff", + test_timestamp_match(MPACK_INT64_C(17179869183), MPACK_TIMESTAMP_NANOSECONDS_MAX, mpack_expect_timestamp(&reader))); + + TEST_SIMPLE_READ("\xc7\x0c\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01", + 1 == mpack_expect_timestamp_truncate(&reader)); + TEST_SIMPLE_READ("\xc7\x0c\xff\x3b\x9a\xc9\xff\x00\x00\x00\x00\x00\x00\x00\x00", + test_timestamp_match(0, MPACK_TIMESTAMP_NANOSECONDS_MAX, mpack_expect_timestamp(&reader))); + TEST_SIMPLE_READ("\xc7\x0c\xff\x00\x00\x00\x01\xff\xff\xff\xff\xff\xff\xff\xff", + test_timestamp_match(-1, 1, mpack_expect_timestamp(&reader))); + TEST_SIMPLE_READ("\xc7\x0c\xff\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff", + test_timestamp_match(MPACK_INT64_MAX, MPACK_TIMESTAMP_NANOSECONDS_MAX, mpack_expect_timestamp(&reader))); + TEST_SIMPLE_READ("\xc7\x0c\xff\x3b\x9a\xc9\xff\x80\x00\x00\x00\x00\x00\x00\x00", + test_timestamp_match(MPACK_INT64_MIN, MPACK_TIMESTAMP_NANOSECONDS_MAX, mpack_expect_timestamp(&reader))); + + TEST_SIMPLE_READ_ERROR("\xd7\xff\xff\xff\xff\xff\x00\x00\x00\x00", + (mpack_expect_timestamp(&reader), true), mpack_error_invalid); + TEST_SIMPLE_READ_ERROR("\xd7\xff\xee\x6b\x28\x00\xff\xff\xff\xff", + (mpack_expect_timestamp(&reader), true), mpack_error_invalid); + TEST_SIMPLE_READ_ERROR("\xc7\x0c\xff\x3b\x9a\xca\x00\x00\x00\x00\x00\x00\x00\x00\x00", + (mpack_expect_timestamp(&reader), true), mpack_error_invalid); + TEST_SIMPLE_READ_ERROR("\xc7\x0c\xff\x40\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff", + (mpack_expect_timestamp(&reader), true), mpack_error_invalid); +} +#endif + void test_expect() { test_expect_example_read(); @@ -1071,10 +1378,16 @@ void test_expect() { // compound types test_expect_str(); test_expect_bin(); - test_expect_ext(); + test_expect_bin_size_buf(); test_expect_arrays(); test_expect_maps(); + // extension types + #if MPACK_EXTENSIONS + test_expect_ext(); + test_expect_timestamp(); + #endif + // key switches test_expect_key_cstr_basic(); test_expect_key_cstr_mixed(); @@ -1090,6 +1403,7 @@ void test_expect() { test_expect_reals_range(); test_expect_bad_type(); test_expect_pre_error(); + test_expect_streaming(); } #endif diff --git a/test/test-expect.h b/test/unit/src/test-expect.h similarity index 95% rename from test/test-expect.h rename to test/unit/src/test-expect.h index ac0437f..bba92b3 100644 --- a/test/test-expect.h +++ b/test/unit/src/test-expect.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Nicholas Fraser + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in @@ -37,4 +37,3 @@ void test_expect(void); #endif #endif - diff --git a/test/unit/src/test-file.c b/test/unit/src/test-file.c new file mode 100644 index 0000000..75e95fe --- /dev/null +++ b/test/unit/src/test-file.c @@ -0,0 +1,1033 @@ +/* + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test-file.h" +#include "test-write.h" +#include "test-reader.h" +#include "test-node.h" + +#if MPACK_STDIO + +// the file tests currently all require the writer, since it +// is used to write the test data that is read back. +#if MPACK_WRITER + +#ifdef _WIN32 +#include +#define test_mkdir(dir, mode) ((void)mode, _mkdir(dir)) +#define test_rmdir _rmdir +#else +#include +#define test_mkdir mkdir +#define test_rmdir rmdir +#endif + +#define MESSAGEPACK_FILES_PATH "test/messagepack/" +#define PSEUDOJSON_FILES_PATH "test/pseudojson/" + +#if MPACK_EXTENSIONS +#define TEST_FILE_MESSAGEPACK (MESSAGEPACK_FILES_PATH "test-file-ext.mp") +#define TEST_FILE_PSEUDOJSON (PSEUDOJSON_FILES_PATH "test-file-ext.debug") +#else +#define TEST_FILE_MESSAGEPACK (MESSAGEPACK_FILES_PATH "test-file-noext.mp") +#define TEST_FILE_PSEUDOJSON (PSEUDOJSON_FILES_PATH "test-file-noext.debug") +#endif + +static const char* test_blank_filename = "mpack-test-blank-file"; +static const char* test_filename = "mpack-test-file"; +static const char* test_dir = "mpack-test-dir"; + +static const int nesting_depth = 150; +static const char* quick_brown_fox = "The quick brown fox jumps over a lazy dog."; + +static char* test_file_fetch(const char* filename, size_t* out_size) { + *out_size = 0; + + // open the file + FILE* file = fopen(filename, "rb"); + if (!file) { + TEST_TRUE(false, "missing file %s", filename); + return NULL; + } + + // get the file size + fseek(file, 0, SEEK_END); + long size = ftell(file); + fseek(file, 0, SEEK_SET); + if (size < 0) { + TEST_TRUE(false, "invalid file size %i for %s", (int)size, filename); + fclose(file); + return NULL; + } + + // allocate the data + if (size == 0) { + fclose(file); + return (char*)MPACK_MALLOC(1); + } + char* data = (char*)MPACK_MALLOC((size_t)size); + + // read the file + long total = 0; + while (total < size) { + size_t count = fread(data + total, 1, (size_t)(size - total), file); + if (count <= 0) { + TEST_TRUE(false, "failed to read from file %s", filename); + fclose(file); + MPACK_FREE(data); + return NULL; + } + total += (long)count; + } + + fclose(file); + *out_size = (size_t)size; + return data; +} + +static void test_file_write_bytes(mpack_writer_t* writer, mpack_tag_t tag) { + mpack_write_tag(writer, tag); + char buf[1024]; + memset(buf, 0, sizeof(buf)); + for (; tag.v.l > sizeof(buf); tag.v.l -= (uint32_t)sizeof(buf)) + mpack_write_bytes(writer, buf, sizeof(buf)); + mpack_write_bytes(writer, buf, tag.v.l); + mpack_finish_type(writer, tag.type); +} + +static void test_file_write_elements(mpack_writer_t* writer, mpack_tag_t tag) { + mpack_write_tag(writer, tag); + size_t i; + for (i = 0; i < tag.v.n; ++i) { + if (tag.type == mpack_type_map) + mpack_write_nil(writer); + mpack_write_nil(writer); + } + mpack_finish_type(writer, tag.type); +} + +static void test_file_write_contents(mpack_writer_t* writer) { + mpack_start_array(writer, 7); + + // write lipsum to test a large fill/seek + mpack_write_cstr(writer, lipsum); + + // test compound types of various sizes + mpack_start_array(writer, 5); + test_file_write_bytes(writer, mpack_tag_str(0)); + test_file_write_bytes(writer, mpack_tag_str(MPACK_INT8_MAX)); + test_file_write_bytes(writer, mpack_tag_str(MPACK_UINT8_MAX)); + test_file_write_bytes(writer, mpack_tag_str(MPACK_UINT8_MAX + 1)); + test_file_write_bytes(writer, mpack_tag_str(MPACK_UINT16_MAX + 1)); + mpack_finish_array(writer); + + mpack_start_array(writer, 5); + test_file_write_bytes(writer, mpack_tag_bin(0)); + test_file_write_bytes(writer, mpack_tag_bin(MPACK_INT8_MAX)); + test_file_write_bytes(writer, mpack_tag_bin(MPACK_UINT8_MAX)); + test_file_write_bytes(writer, mpack_tag_bin(MPACK_UINT8_MAX + 1)); + test_file_write_bytes(writer, mpack_tag_bin(MPACK_UINT16_MAX + 1)); + mpack_finish_array(writer); + + #if MPACK_EXTENSIONS + mpack_start_array(writer, 10); + test_file_write_bytes(writer, mpack_tag_ext(1, 0)); + test_file_write_bytes(writer, mpack_tag_ext(1, 1)); + test_file_write_bytes(writer, mpack_tag_ext(1, 2)); + test_file_write_bytes(writer, mpack_tag_ext(1, 4)); + test_file_write_bytes(writer, mpack_tag_ext(1, 8)); + test_file_write_bytes(writer, mpack_tag_ext(1, 16)); + test_file_write_bytes(writer, mpack_tag_ext(2, MPACK_INT8_MAX)); + test_file_write_bytes(writer, mpack_tag_ext(3, MPACK_UINT8_MAX)); + test_file_write_bytes(writer, mpack_tag_ext(4, MPACK_UINT8_MAX + 1)); + test_file_write_bytes(writer, mpack_tag_ext(5, MPACK_UINT16_MAX + 1)); + mpack_finish_array(writer); + #else + mpack_write_nil(writer); + #endif + + mpack_start_array(writer, 5); + test_file_write_elements(writer, mpack_tag_array(0)); + test_file_write_elements(writer, mpack_tag_array(MPACK_INT8_MAX)); + test_file_write_elements(writer, mpack_tag_array(MPACK_UINT8_MAX)); + test_file_write_elements(writer, mpack_tag_array(MPACK_UINT8_MAX + 1)); + test_file_write_elements(writer, mpack_tag_array(MPACK_UINT16_MAX + 1)); + mpack_finish_array(writer); + + mpack_start_array(writer, 5); + test_file_write_elements(writer, mpack_tag_map(0)); + test_file_write_elements(writer, mpack_tag_map(MPACK_INT8_MAX)); + test_file_write_elements(writer, mpack_tag_map(MPACK_UINT8_MAX)); + test_file_write_elements(writer, mpack_tag_map(MPACK_UINT8_MAX + 1)); + test_file_write_elements(writer, mpack_tag_map(MPACK_UINT16_MAX + 1)); + mpack_finish_array(writer); + + // test deep nesting + int i; + for (i = 0; i < nesting_depth; ++i) + mpack_start_array(writer, 1); + mpack_write_nil(writer); + for (i = 0; i < nesting_depth; ++i) + mpack_finish_array(writer); + + mpack_finish_array(writer); +} + +static void test_file_write_failures(void) { + mpack_writer_t writer; + + // test invalid filename + (void)test_mkdir(test_dir, 0700); + mpack_writer_init_filename(&writer, test_dir); + TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_io); + + // test close and flush failure + // (if we write more than libc's internal FILE buffer size, fwrite() + // fails, otherwise fclose() fails. we test both here.) + + mpack_writer_init_filename(&writer, "/dev/full"); + mpack_write_cstr(&writer, quick_brown_fox); + TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_io); + + int count = MPACK_UINT16_MAX / 20; + mpack_writer_init_filename(&writer, "/dev/full"); + mpack_start_array(&writer, (uint32_t)count); + int i; + for (i = 0; i < count; ++i) + mpack_write_cstr(&writer, quick_brown_fox); + mpack_finish_array(&writer); + TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_io); +} + +static void test_file_write(void) { + mpack_writer_t writer; + mpack_writer_init_file(&writer, test_filename); // test the deprecated function + TEST_TRUE(mpack_writer_error(&writer) == mpack_ok, "file open failed with %s", + mpack_error_to_string(mpack_writer_error(&writer))); + + test_file_write_contents(&writer); + TEST_WRITER_DESTROY_NOERROR(&writer); +} + +static void test_file_write_helper_std_owned(void) { + // test writing to a libc FILE, giving ownership + FILE* file = test_fopen(test_filename, "wb"); + TEST_TRUE(file != NULL, "failed to open file for writing! filename %s", test_filename); + + mpack_writer_t writer; + mpack_writer_init_stdfile(&writer, file, true); + test_file_write_contents(&writer); + TEST_WRITER_DESTROY_NOERROR(&writer); + + // the test harness will ensure no file is leaked +} + +static void test_file_write_helper_std_unowned(void) { + // test writing to a libc FILE, retaining ownership + FILE* file = test_fopen(test_filename, "wb"); + TEST_TRUE(file != NULL, "failed to open file for writing! filename %s", test_filename); + + mpack_writer_t writer; + mpack_writer_init_stdfile(&writer, file, false); + test_file_write_contents(&writer); + TEST_WRITER_DESTROY_NOERROR(&writer); + + // we retained ownership, so we close it ourselves + test_fclose(file); +} + +static bool test_file_write_failure(void) { + + // The write failure test may fail with either + // mpack_error_memory or mpack_error_io. We write a + // bunch of strs and bins to test the various expect + // allocator modes. + + mpack_writer_t writer; + mpack_writer_init_filename(&writer, test_filename); + + mpack_start_array(&writer, 2); + mpack_start_array(&writer, 6); + + // write a large string near the start to cause a + // more than double buffer size growth + mpack_write_cstr(&writer, quick_brown_fox); + + mpack_write_cstr(&writer, "one"); + mpack_write_cstr(&writer, "two"); + mpack_write_cstr(&writer, "three"); + mpack_write_cstr(&writer, "four"); + mpack_write_cstr(&writer, "five"); + + mpack_finish_array(&writer); + + // test deep nesting + int i; + for (i = 0; i < nesting_depth; ++i) + mpack_start_array(&writer, 1); + mpack_write_nil(&writer); + for (i = 0; i < nesting_depth; ++i) + mpack_finish_array(&writer); + + mpack_finish_array(&writer); + + mpack_error_t error = mpack_writer_destroy(&writer); + if (error == mpack_error_io || error == mpack_error_memory) + return false; + TEST_TRUE(error == mpack_ok, "unexpected error state %i (%s)", (int)error, + mpack_error_to_string(error)); + return true; + +} + +// compares the test filename to the expected debug output +static void test_compare_print(void) { + size_t expected_size; + char* expected_data = test_file_fetch(TEST_FILE_PSEUDOJSON, &expected_size); + size_t actual_size; + char* actual_data = test_file_fetch(test_filename, &actual_size); + + TEST_TRUE(actual_size == expected_size, "print length %i does not match expected length %i", + (int)actual_size, (int)expected_size); + TEST_TRUE(0 == memcmp(actual_data, expected_data, actual_size), "print does not match expected"); + + MPACK_FREE(expected_data); + MPACK_FREE(actual_data); +} + +#if MPACK_READER && MPACK_DEBUG && MPACK_DOUBLE +static void test_print(void) { + + // miscellaneous print tests + // (we're not actually checking the output; we just want to make + // sure it doesn't crash under the below errors.) + FILE* out = fopen(test_filename, "wb"); + mpack_print_data_to_file("\x91", 1, out); // truncated file + mpack_print_data_to_file("\xa1", 1, out); // truncated str + mpack_print_data_to_file("\x92\x00", 2, out); // truncated array + mpack_print_data_to_file("\x81", 1, out); // truncated map key + mpack_print_data_to_file("\x81\x00", 2, out); // truncated map value + mpack_print_data_to_file("\x90\xc0", 2, out); // extra bytes + mpack_print_data_to_file("\xca\x00\x00\x00\x00", 5, out); // float + fclose(out); + + // print test string to stdout + mpack_print("\xaatesting...", 11); + + // dump MessagePack to debug file + + size_t input_size; + char* input_data = test_file_fetch(TEST_FILE_MESSAGEPACK, &input_size); + + out = fopen(test_filename, "wb"); + mpack_print_data_to_file(input_data, input_size, out); + fclose(out); + + MPACK_FREE(input_data); + test_compare_print(); +} +#endif + +#if MPACK_NODE && MPACK_DEBUG && MPACK_DOUBLE +static void test_node_print(void) { + mpack_tree_t tree; + + // miscellaneous node print tests + FILE* out = fopen(test_filename, "wb"); + mpack_tree_init(&tree, "\xca\x00\x00\x00\x00", 5); // float + mpack_tree_parse(&tree); + mpack_node_print_to_file(mpack_tree_root(&tree), out); + mpack_tree_destroy(&tree); + fclose(out); + + // print test string to stdout + mpack_tree_init(&tree, "\xaatesting...", 11); + mpack_tree_parse(&tree); + mpack_node_print(mpack_tree_root(&tree)); + mpack_tree_destroy(&tree); + + // dump MessagePack to debug file + + mpack_tree_init_filename(&tree, TEST_FILE_MESSAGEPACK, 0); + mpack_tree_parse(&tree); + TEST_TRUE(mpack_ok == mpack_tree_error(&tree)); + + out = fopen(test_filename, "wb"); + mpack_node_print_to_file(mpack_tree_root(&tree), out); + fclose(out); + + TEST_TRUE(mpack_ok == mpack_tree_destroy(&tree)); + test_compare_print(); +} +#endif + +#if MPACK_READER +static void test_file_discard(void) { + mpack_reader_t reader; + mpack_reader_init_filename(&reader, test_filename); + mpack_discard(&reader); + TEST_READER_DESTROY_NOERROR(&reader); + + mpack_reader_init_filename(&reader, test_filename); + reader.skip = NULL; // disable the skip callback to test skipping without it + mpack_discard(&reader); + TEST_READER_DESTROY_NOERROR(&reader); +} +#endif + +#if MPACK_EXPECT +static void test_file_expect_bytes(mpack_reader_t* reader, mpack_tag_t tag) { + mpack_expect_tag(reader, tag); + TEST_TRUE(mpack_reader_error(reader) == mpack_ok, "got error %i (%s)", (int)mpack_reader_error(reader), mpack_error_to_string(mpack_reader_error(reader))); + + char expected[1024]; + memset(expected, 0, sizeof(expected)); + char buf[sizeof(expected)]; + while (tag.v.l > 0) { + uint32_t count = (tag.v.l < (uint32_t)sizeof(buf)) ? tag.v.l : (uint32_t)sizeof(buf); + mpack_read_bytes(reader, buf, count); + TEST_TRUE(mpack_reader_error(reader) == mpack_ok, "got error %i (%s)", (int)mpack_reader_error(reader), mpack_error_to_string(mpack_reader_error(reader))); + TEST_TRUE(memcmp(buf, expected, count) == 0, "data does not match!"); + tag.v.l -= count; + } + + mpack_done_type(reader, tag.type); +} + +static void test_file_expect_elements(mpack_reader_t* reader, mpack_tag_t tag) { + mpack_expect_tag(reader, tag); + size_t i; + for (i = 0; i < tag.v.n; ++i) { + if (tag.type == mpack_type_map) + mpack_expect_nil(reader); + mpack_expect_nil(reader); + } + mpack_done_type(reader, tag.type); +} + +static void test_file_read_contents(mpack_reader_t* reader) { + TEST_TRUE(mpack_reader_error(reader) == mpack_ok, "file open failed with %s", + mpack_error_to_string(mpack_reader_error(reader))); + + TEST_TRUE(7 == mpack_expect_array(reader)); + + // test matching a cstr larger than the buffer size + mpack_expect_cstr_match(reader, lipsum); + TEST_TRUE(mpack_reader_error(reader) == mpack_ok, "failed to match huge string!"); + + TEST_TRUE(5 == mpack_expect_array(reader)); + test_file_expect_bytes(reader, mpack_tag_str(0)); + test_file_expect_bytes(reader, mpack_tag_str(MPACK_INT8_MAX)); + test_file_expect_bytes(reader, mpack_tag_str(MPACK_UINT8_MAX)); + test_file_expect_bytes(reader, mpack_tag_str(MPACK_UINT8_MAX + 1)); + test_file_expect_bytes(reader, mpack_tag_str(MPACK_UINT16_MAX + 1)); + mpack_done_array(reader); + + TEST_TRUE(5 == mpack_expect_array(reader)); + test_file_expect_bytes(reader, mpack_tag_bin(0)); + test_file_expect_bytes(reader, mpack_tag_bin(MPACK_INT8_MAX)); + test_file_expect_bytes(reader, mpack_tag_bin(MPACK_UINT8_MAX)); + test_file_expect_bytes(reader, mpack_tag_bin(MPACK_UINT8_MAX + 1)); + test_file_expect_bytes(reader, mpack_tag_bin(MPACK_UINT16_MAX + 1)); + mpack_done_array(reader); + + #if MPACK_EXTENSIONS + TEST_TRUE(10 == mpack_expect_array(reader)); + test_file_expect_bytes(reader, mpack_tag_ext(1, 0)); + test_file_expect_bytes(reader, mpack_tag_ext(1, 1)); + test_file_expect_bytes(reader, mpack_tag_ext(1, 2)); + test_file_expect_bytes(reader, mpack_tag_ext(1, 4)); + test_file_expect_bytes(reader, mpack_tag_ext(1, 8)); + test_file_expect_bytes(reader, mpack_tag_ext(1, 16)); + test_file_expect_bytes(reader, mpack_tag_ext(2, MPACK_INT8_MAX)); + test_file_expect_bytes(reader, mpack_tag_ext(3, MPACK_UINT8_MAX)); + test_file_expect_bytes(reader, mpack_tag_ext(4, MPACK_UINT8_MAX + 1)); + test_file_expect_bytes(reader, mpack_tag_ext(5, MPACK_UINT16_MAX + 1)); + mpack_done_array(reader); + #else + mpack_expect_nil(reader); + #endif + + TEST_TRUE(5 == mpack_expect_array(reader)); + test_file_expect_elements(reader, mpack_tag_array(0)); + test_file_expect_elements(reader, mpack_tag_array(MPACK_INT8_MAX)); + test_file_expect_elements(reader, mpack_tag_array(MPACK_UINT8_MAX)); + test_file_expect_elements(reader, mpack_tag_array(MPACK_UINT8_MAX + 1)); + test_file_expect_elements(reader, mpack_tag_array(MPACK_UINT16_MAX + 1)); + mpack_done_array(reader); + + TEST_TRUE(5 == mpack_expect_array(reader)); + test_file_expect_elements(reader, mpack_tag_map(0)); + test_file_expect_elements(reader, mpack_tag_map(MPACK_INT8_MAX)); + test_file_expect_elements(reader, mpack_tag_map(MPACK_UINT8_MAX)); + test_file_expect_elements(reader, mpack_tag_map(MPACK_UINT8_MAX + 1)); + test_file_expect_elements(reader, mpack_tag_map(MPACK_UINT16_MAX + 1)); + mpack_done_array(reader); + + int i; + for (i = 0; i < nesting_depth; ++i) + mpack_expect_array_match(reader, 1); + mpack_expect_nil(reader); + for (i = 0; i < nesting_depth; ++i) + mpack_done_array(reader); + + mpack_done_array(reader); +} + +static void test_file_read_missing(void) { + // test missing file + mpack_reader_t reader; + mpack_reader_init_filename(&reader, "invalid-filename"); + TEST_READER_DESTROY_ERROR(&reader, mpack_error_io); +} + +static void test_file_read_helper(void) { + // test reading with the default file reader + mpack_reader_t reader; + mpack_reader_init_file(&reader, test_filename); // test the deprecated function + test_file_read_contents(&reader); + TEST_READER_DESTROY_NOERROR(&reader); +} + +static void test_file_read_helper_std_owned(void) { + // test reading from a libc FILE, giving ownership + FILE* file = test_fopen(test_filename, "rb"); + TEST_TRUE(file != NULL, "failed to open file! filename %s", test_filename); + + mpack_reader_t reader; + mpack_reader_init_stdfile(&reader, file, true); + test_file_read_contents(&reader); + TEST_READER_DESTROY_NOERROR(&reader); + + // the test harness will ensure no file is leaked +} + +static void test_file_read_helper_std_unowned(void) { + // test reading from a libc FILE, retaining ownership + FILE* file = test_fopen(test_filename, "rb"); + TEST_TRUE(file != NULL, "failed to open file! filename %s", test_filename); + + mpack_reader_t reader; + mpack_reader_init_stdfile(&reader, file, false); + test_file_read_contents(&reader); + TEST_READER_DESTROY_NOERROR(&reader); + + // we retained ownership, so we close it ourselves + test_fclose(file); +} + +typedef struct test_file_streaming_t { + FILE* file; + size_t read_size; +} test_file_streaming_t; + +static size_t test_file_read_streaming_fill(mpack_reader_t* reader, char* buffer, size_t count) { + test_file_streaming_t* context = (test_file_streaming_t*)reader->context; + if (count > context->read_size) + count = context->read_size; + return fread((void*)buffer, 1, count, context->file); +} + +static void test_file_read_streaming(void) { + // We test reading from a file using a streaming function + // that returns a small number of bytes each time (as though + // it is slowly receiving data through a socket.) This tests + // that the reader correctly handles streams, and that it + // can continue asking for data even when it needs more bytes + // than read by a single call to the fill function. + + size_t sizes[] = {1, 2, 3, 5, 7, 11}; + size_t i; + for (i = 0; i < sizeof(sizes) / sizeof(*sizes); ++i) { + + FILE* file = fopen(test_filename, "rb"); + TEST_TRUE(file != NULL, "failed to open file! filename %s", test_filename); + + test_file_streaming_t context = {file, sizes[i]}; + mpack_reader_t reader; + char buffer[MPACK_READER_MINIMUM_BUFFER_SIZE]; + mpack_reader_init(&reader, buffer, sizeof(buffer), 0); + mpack_reader_set_context(&reader, &context); + mpack_reader_set_fill(&reader, &test_file_read_streaming_fill); + + test_file_read_contents(&reader); + TEST_READER_DESTROY_NOERROR(&reader); + fclose(file); + } +} + +static bool test_file_expect_failure(void) { + + // The expect failure test may fail with either + // mpack_error_memory or mpack_error_io. + + mpack_reader_t reader; + + #define TEST_POSSIBLE_FAILURE() do { \ + mpack_error_t error = mpack_reader_error(&reader); \ + if (error == mpack_error_memory || error == mpack_error_io) { \ + mpack_reader_destroy(&reader); \ + return false; \ + } \ + } while (0) + + mpack_reader_init_filename(&reader, test_filename); + mpack_expect_array_match(&reader, 2); + + uint32_t count; + char** strings = mpack_expect_array_alloc(&reader, char*, 50, &count); + TEST_POSSIBLE_FAILURE(); + TEST_TRUE(strings != NULL); + TEST_TRUE(count == 6); + MPACK_FREE(strings); + + char* str = mpack_expect_cstr_alloc(&reader, 100); + TEST_POSSIBLE_FAILURE(); + TEST_TRUE(str != NULL); + if (str) { + TEST_TRUE(strcmp(str, quick_brown_fox) == 0); + MPACK_FREE(str); + } + + str = mpack_expect_utf8_cstr_alloc(&reader, 100); + TEST_POSSIBLE_FAILURE(); + TEST_TRUE(str != NULL); + if (str) { + TEST_TRUE(strcmp(str, "one") == 0); + MPACK_FREE(str); + } + + str = mpack_expect_cstr_alloc(&reader, 100); + TEST_POSSIBLE_FAILURE(); + TEST_TRUE(str != NULL); + if (str) { + TEST_TRUE(strcmp(str, "two") == 0); + MPACK_FREE(str); + } + + str = mpack_expect_utf8_cstr_alloc(&reader, 100); + TEST_POSSIBLE_FAILURE(); + TEST_TRUE(str != NULL); + if (str) { + TEST_TRUE(strcmp(str, "three") == 0); + MPACK_FREE(str); + } + + mpack_discard(&reader); + mpack_discard(&reader); + mpack_done_array(&reader); + + mpack_discard(&reader); // discard the deep nested arrays + mpack_done_array(&reader); + + #undef TEST_POSSIBLE_FAILURE + + mpack_error_t error = mpack_reader_destroy(&reader); + if (error == mpack_error_io || error == mpack_error_memory) + return false; + TEST_TRUE(error == mpack_ok, "unexpected error state %i (%s)", (int)error, + mpack_error_to_string(error)); + return true; + +} + +static void test_file_read_eof(void) { + mpack_reader_t reader; + mpack_reader_init_filename(&reader, test_filename); + TEST_TRUE(mpack_reader_error(&reader) == mpack_ok, "file open failed with %s", + mpack_error_to_string(mpack_reader_error(&reader))); + + while (mpack_reader_error(&reader) != mpack_error_eof) + mpack_discard(&reader); + + mpack_error_t error = mpack_reader_destroy(&reader); + TEST_TRUE(error == mpack_error_eof, "unexpected error state %i (%s)", (int)error, + mpack_error_to_string(error)); +} + +#endif + +#if MPACK_NODE +static void test_file_node_bytes(mpack_node_t node, mpack_tag_t tag) { + TEST_TRUE(mpack_tag_equal(tag, mpack_node_tag(node))); + const char* data = mpack_node_data(node); + uint32_t length = mpack_node_data_len(node); + TEST_TRUE(mpack_node_error(node) == mpack_ok); + + char expected[1024]; + memset(expected, 0, sizeof(expected)); + while (length > 0) { + uint32_t count = (length < (uint32_t)sizeof(expected)) ? length : (uint32_t)sizeof(expected); + TEST_TRUE(memcmp(data, expected, count) == 0); + length -= count; + data += count; + } +} + +static void test_file_node_elements(mpack_node_t node, mpack_tag_t tag) { + TEST_TRUE(mpack_tag_equal(tag, mpack_node_tag(node))); + size_t i; + for (i = 0; i < tag.v.n; ++i) { + if (tag.type == mpack_type_map) { + mpack_node_nil(mpack_node_map_key_at(node, i)); + mpack_node_nil(mpack_node_map_value_at(node, i)); + } else { + mpack_node_nil(mpack_node_array_at(node, i)); + } + } +} + +static void test_file_node_contents(mpack_node_t root) { + TEST_TRUE(mpack_node_array_length(root) == 7); + + mpack_node_t lipsum_node = mpack_node_array_at(root, 0); + const char* lipsum_str = mpack_node_str(lipsum_node); + TEST_TRUE(lipsum_str != NULL); + if (lipsum_str) { + TEST_TRUE(mpack_node_strlen(lipsum_node) == strlen(lipsum)); + TEST_TRUE(memcmp(lipsum_str, lipsum, strlen(lipsum)) == 0); + } + + mpack_node_t node = mpack_node_array_at(root, 1); + TEST_TRUE(mpack_node_array_length(node) == 5); + test_file_node_bytes(mpack_node_array_at(node, 0), mpack_tag_str(0)); + test_file_node_bytes(mpack_node_array_at(node, 1), mpack_tag_str(MPACK_INT8_MAX)); + test_file_node_bytes(mpack_node_array_at(node, 2), mpack_tag_str(MPACK_UINT8_MAX)); + test_file_node_bytes(mpack_node_array_at(node, 3), mpack_tag_str(MPACK_UINT8_MAX + 1)); + test_file_node_bytes(mpack_node_array_at(node, 4), mpack_tag_str(MPACK_UINT16_MAX + 1)); + + node = mpack_node_array_at(root, 2); + TEST_TRUE(5 == mpack_node_array_length(node)); + test_file_node_bytes(mpack_node_array_at(node, 0), mpack_tag_bin(0)); + test_file_node_bytes(mpack_node_array_at(node, 1), mpack_tag_bin(MPACK_INT8_MAX)); + test_file_node_bytes(mpack_node_array_at(node, 2), mpack_tag_bin(MPACK_UINT8_MAX)); + test_file_node_bytes(mpack_node_array_at(node, 3), mpack_tag_bin(MPACK_UINT8_MAX + 1)); + test_file_node_bytes(mpack_node_array_at(node, 4), mpack_tag_bin(MPACK_UINT16_MAX + 1)); + + node = mpack_node_array_at(root, 3); + #if MPACK_EXTENSIONS + TEST_TRUE(10 == mpack_node_array_length(node)); + test_file_node_bytes(mpack_node_array_at(node, 0), mpack_tag_ext(1, 0)); + test_file_node_bytes(mpack_node_array_at(node, 1), mpack_tag_ext(1, 1)); + test_file_node_bytes(mpack_node_array_at(node, 2), mpack_tag_ext(1, 2)); + test_file_node_bytes(mpack_node_array_at(node, 3), mpack_tag_ext(1, 4)); + test_file_node_bytes(mpack_node_array_at(node, 4), mpack_tag_ext(1, 8)); + test_file_node_bytes(mpack_node_array_at(node, 5), mpack_tag_ext(1, 16)); + test_file_node_bytes(mpack_node_array_at(node, 6), mpack_tag_ext(2, MPACK_INT8_MAX)); + test_file_node_bytes(mpack_node_array_at(node, 7), mpack_tag_ext(3, MPACK_UINT8_MAX)); + test_file_node_bytes(mpack_node_array_at(node, 8), mpack_tag_ext(4, MPACK_UINT8_MAX + 1)); + test_file_node_bytes(mpack_node_array_at(node, 9), mpack_tag_ext(5, MPACK_UINT16_MAX + 1)); + #else + mpack_node_nil(node); + #endif + + node = mpack_node_array_at(root, 4); + TEST_TRUE(5 == mpack_node_array_length(node)); + test_file_node_elements(mpack_node_array_at(node, 0), mpack_tag_array(0)); + test_file_node_elements(mpack_node_array_at(node, 1), mpack_tag_array(MPACK_INT8_MAX)); + test_file_node_elements(mpack_node_array_at(node, 2), mpack_tag_array(MPACK_UINT8_MAX)); + test_file_node_elements(mpack_node_array_at(node, 3), mpack_tag_array(MPACK_UINT8_MAX + 1)); + test_file_node_elements(mpack_node_array_at(node, 4), mpack_tag_array(MPACK_UINT16_MAX + 1)); + + node = mpack_node_array_at(root, 5); + TEST_TRUE(5 == mpack_node_array_length(node)); + test_file_node_elements(mpack_node_array_at(node, 0), mpack_tag_map(0)); + test_file_node_elements(mpack_node_array_at(node, 1), mpack_tag_map(MPACK_INT8_MAX)); + test_file_node_elements(mpack_node_array_at(node, 2), mpack_tag_map(MPACK_UINT8_MAX)); + test_file_node_elements(mpack_node_array_at(node, 3), mpack_tag_map(MPACK_UINT8_MAX + 1)); + test_file_node_elements(mpack_node_array_at(node, 4), mpack_tag_map(MPACK_UINT16_MAX + 1)); + + node = mpack_node_array_at(root, 6); + int i; + for (i = 0; i < nesting_depth; ++i) + node = mpack_node_array_at(node, 0); + TEST_TRUE(mpack_ok == mpack_node_error(node)); + mpack_node_nil(node); +} + +static void test_file_tree_successful_parse(mpack_tree_t* tree) { + mpack_tree_parse(tree); + TEST_TRUE(mpack_tree_error(tree) == mpack_ok, "file tree parsing failed: %s", + mpack_error_to_string(mpack_tree_error(tree))); + test_file_node_contents(mpack_tree_root(tree)); + mpack_error_t error = mpack_tree_destroy(tree); + TEST_TRUE(error == mpack_ok, "file tree failed with error %s", mpack_error_to_string(error)); +} + +static void test_file_node(void) { + mpack_tree_t tree; + + // test maximum size + mpack_tree_init_file(&tree, test_filename, 100); + TEST_TREE_DESTROY_ERROR(&tree, mpack_error_too_big); + + // test blank file + mpack_tree_init_file(&tree, test_blank_filename, 0); + TEST_TREE_DESTROY_ERROR(&tree, mpack_error_invalid); + + // test successful parse from filename + mpack_tree_init_file(&tree, test_filename, 0); // test the deprecated function + test_file_tree_successful_parse(&tree); + + // test file size out of bounds + #if MPACK_DEBUG + if (sizeof(size_t) >= sizeof(long)) { + TEST_BREAK((mpack_tree_init_filename(&tree, "invalid-filename", ((size_t)LONG_MAX) + 1), true)); + TEST_TREE_DESTROY_ERROR(&tree, mpack_error_bug); + } + #endif + + // test missing file + mpack_tree_init_filename(&tree, "invalid-filename", 0); + TEST_TREE_DESTROY_ERROR(&tree, mpack_error_io); + + // test successful parse from FILE with auto-close + FILE* file = test_fopen(test_filename, "rb"); + TEST_TRUE(file != NULL); + mpack_tree_init_stdfile(&tree, file, 0, true); + test_file_tree_successful_parse(&tree); + + // test successful parse from FILE with no-close + file = test_fopen(test_filename, "rb"); + TEST_TRUE(file != NULL); + mpack_tree_init_stdfile(&tree, file, 0, false); + test_file_tree_successful_parse(&tree); + test_fclose(file); +} + +typedef struct test_file_stream_t { + size_t length; + char* data; + size_t pos; + size_t step; +} test_file_stream_t; + +// The test stream reader loops over the data in the test file. It returns at +// most the step size so we can test reading from very small to very large +// chunks. +static size_t test_file_stream_read(mpack_tree_t* tree, char* buffer, size_t count) { + test_file_stream_t* stream = (test_file_stream_t*)tree->context; + + if (count > stream->step) + count = stream->step; + + size_t left = count; + while (left > 0) { + size_t n = stream->length - stream->pos; + if (n > left) + n = left; + + mpack_memcpy(buffer, stream->data + stream->pos, n); + + buffer += n; + stream->pos += n; + left -= n; + + if (stream->pos == stream->length) + stream->pos = 0; + } + + return count; +} + +static void test_file_node_stream(void) { + test_file_stream_t stream; + stream.data = test_file_fetch(test_filename, &stream.length); + + size_t steps[] = {11, 23, 32, 127, 369, 4096, SIZE_MAX}; + + size_t i; + for (i = 0; i < sizeof(steps) / sizeof(steps[0]); ++i) { + stream.pos = 0; + stream.step = steps[i]; + + // We use a max_size a bit larger than the file, that way some extra + // data is read from the next tree. + size_t max_size = stream.length * 4 / 3; + + mpack_tree_t tree; + mpack_tree_init_stream(&tree, &test_file_stream_read, &stream, max_size, max_size); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok, "tree initialization failed: %s", + mpack_error_to_string(mpack_tree_error(&tree))); + + // We try parsing the same tree a dozen times repeatedly with this step size. + int j; + for (j = 0; j < 12; ++j) { + mpack_tree_parse(&tree); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok, "tree parsing failed: %s", + mpack_error_to_string(mpack_tree_error(&tree))); + test_file_node_contents(mpack_tree_root(&tree)); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok, "tree contents failed: %s", + mpack_error_to_string(mpack_tree_error(&tree))); + } + + mpack_error_t error = mpack_tree_destroy(&tree); + TEST_TRUE(error == mpack_ok, "tree stream failed with error %s", mpack_error_to_string(error)); + } + + MPACK_FREE(stream.data); +} + +static bool test_file_node_failure(void) { + + // The node failure test may fail with either + // mpack_error_memory or mpack_error_io. + + mpack_tree_t tree; + + #define TEST_POSSIBLE_FAILURE() do { \ + mpack_error_t error = mpack_tree_error(&tree); \ + TEST_TRUE(test_tree_error == error); \ + if (error == mpack_error_memory || error == mpack_error_io) { \ + test_tree_error = mpack_ok; \ + mpack_tree_destroy(&tree); \ + return false; \ + } \ + } while (0) + + mpack_tree_init_filename(&tree, test_filename, 0); + mpack_tree_parse(&tree); + if (mpack_tree_error(&tree) == mpack_error_memory || mpack_tree_error(&tree) == mpack_error_io) { + mpack_tree_destroy(&tree); + return false; + } + mpack_tree_set_error_handler(&tree, test_tree_error_handler); + + + mpack_node_t root = mpack_tree_root(&tree); + + mpack_node_t strings = mpack_node_array_at(root, 0); + size_t length = mpack_node_array_length(strings); + TEST_POSSIBLE_FAILURE(); + TEST_TRUE(6 == length); + + mpack_node_t node = mpack_node_array_at(strings, 0); + char* str = mpack_node_data_alloc(node, 100); + TEST_POSSIBLE_FAILURE(); + TEST_TRUE(str != NULL); + const char* expected = quick_brown_fox; + TEST_TRUE(mpack_node_strlen(node) == strlen(expected)); + if (str) { + TEST_TRUE(memcmp(str, expected, mpack_node_strlen(node)) == 0); + MPACK_FREE(str); + } + + node = mpack_node_array_at(strings, 1); + + str = mpack_node_cstr_alloc(node, 100); + TEST_POSSIBLE_FAILURE(); + TEST_TRUE(str != NULL); + expected = "one"; + if (str) { + TEST_TRUE(strlen(str) == strlen(expected)); + TEST_TRUE(strcmp(str, expected) == 0); + MPACK_FREE(str); + } + + str = mpack_node_utf8_cstr_alloc(node, 100); + TEST_POSSIBLE_FAILURE(); + TEST_TRUE(str != NULL); + if (str) { + TEST_TRUE(strlen(str) == strlen(expected)); + TEST_TRUE(strcmp(str, expected) == 0); + MPACK_FREE(str); + } + + node = mpack_node_array_at(root, 1); + int i; + for (i = 0; i < nesting_depth; ++i) + node = mpack_node_array_at(node, 0); + TEST_TRUE(mpack_ok == mpack_node_error(node)); + mpack_node_nil(node); + + #undef TEST_POSSIBLE_FAILURE + + mpack_error_t error = mpack_tree_destroy(&tree); + if (error == mpack_error_io || error == mpack_error_memory) + return false; + TEST_TRUE(error == mpack_ok, "unexpected error state %i (%s)", (int)error, + mpack_error_to_string(error)); + return true; + +} +#endif + +void test_file(void) { + // write a blank file for test purposes + FILE* blank = fopen(test_blank_filename, "wb"); + fclose(blank); + + #if MPACK_READER && MPACK_DEBUG && MPACK_DOUBLE + test_print(); + #endif + #if MPACK_NODE && MPACK_DEBUG && MPACK_DOUBLE + test_node_print(); + #endif + + #if MPACK_WRITER + test_file_write_failures(); + test_file_write_helper_std_owned(); + test_file_write_helper_std_unowned(); + test_file_write(); + #endif + + #if MPACK_READER + test_file_discard(); + #endif + #if MPACK_EXPECT + test_file_read_missing(); + test_file_read_helper(); + test_file_read_helper_std_owned(); + test_file_read_helper_std_unowned(); + test_file_read_streaming(); + test_file_read_eof(); + #endif + #if MPACK_NODE + test_file_node(); + test_file_node_stream(); + #endif + + #if MPACK_WRITER + test_system_fail_until_ok(&test_file_write_failure); + #endif + #if MPACK_EXPECT + test_system_fail_until_ok(&test_file_expect_failure); + #endif + #if MPACK_NODE + test_system_fail_until_ok(&test_file_node_failure); + #endif + + TEST_TRUE(remove(test_filename) == 0, "failed to delete %s", test_filename); + TEST_TRUE(remove(test_blank_filename) == 0, "failed to delete %s", test_blank_filename); + TEST_TRUE(test_rmdir(test_dir) == 0, "failed to delete %s", test_dir); + + (void)&test_compare_print; +} + +#else + +void test_file(void) { + // if we don't have the writer, nothing to do +} + +#endif +#endif + diff --git a/test/test-file.h b/test/unit/src/test-file.h similarity index 94% rename from test/test-file.h rename to test/unit/src/test-file.h index 4f8ab01..954be46 100644 --- a/test/test-file.h +++ b/test/unit/src/test-file.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR diff --git a/test/test-node.c b/test/unit/src/test-node.c similarity index 73% rename from test/test-node.c rename to test/unit/src/test-node.c index 063a54a..2de2cb6 100644 --- a/test/test-node.c +++ b/test/unit/src/test-node.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -32,16 +32,18 @@ void test_tree_error_handler(mpack_tree_t* tree, mpack_error_t error) { test_tree_error = error; } +static mpack_node_data_t pool[128]; + // tests the example on the messagepack homepage -static void test_example_node() { +static void test_example_node(void) { // add a junk byte at the end to test mpack_tree_size() static const char test[] = "\x82\xA7""compact\xC3\xA6""schema\x00\xC1"; mpack_tree_t tree; // this is a node pool test even if we have malloc. the rest of the // non-simple tests use paging unless malloc is unavailable. - mpack_node_data_t pool[128]; mpack_tree_init_pool(&tree, test, sizeof(test) - 1, pool, sizeof(pool) / sizeof(*pool)); + mpack_tree_parse(&tree); TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); mpack_node_t map = mpack_tree_root(&tree); @@ -52,8 +54,8 @@ static void test_example_node() { TEST_TREE_DESTROY_NOERROR(&tree); } -static void test_node_read_uint_fixnum() { - mpack_node_data_t pool[128]; +static void test_node_read_uint_fixnum(void) { + mpack_tree_t tree; // positive fixnums with u8 TEST_SIMPLE_TREE_READ("\x00", 0 == mpack_node_u8(node)); @@ -97,8 +99,8 @@ static void test_node_read_uint_fixnum() { } -static void test_node_read_uint_signed_fixnum() { - mpack_node_data_t pool[128]; +static void test_node_read_uint_signed_fixnum(void) { + mpack_tree_t tree; // positive fixnums with i8 TEST_SIMPLE_TREE_READ("\x00", 0 == mpack_node_i8(node)); @@ -142,8 +144,8 @@ static void test_node_read_uint_signed_fixnum() { } -static void test_node_read_negative_fixnum() { - mpack_node_data_t pool[128]; +static void test_node_read_negative_fixnum(void) { + mpack_tree_t tree; // negative fixnums with i8 TEST_SIMPLE_TREE_READ("\xff", -1 == mpack_node_i8(node)); @@ -177,8 +179,8 @@ static void test_node_read_negative_fixnum() { } -static void test_node_read_uint() { - mpack_node_data_t pool[128]; +static void test_node_read_uint(void) { + mpack_tree_t tree; // positive signed into u8 TEST_SIMPLE_TREE_READ("\xd0\x7f", 0x7f == mpack_node_u8(node)); @@ -196,7 +198,7 @@ static void test_node_read_uint() { TEST_SIMPLE_TREE_READ("\xd3\x7f\xff\xff\xff\xff\xff\xff\xff", 0x7fffffffffffffff == mpack_node_u64(node)); // positive unsigned into u8 - + TEST_SIMPLE_TREE_READ("\xcc\x80", 0x80 == mpack_node_u8(node)); TEST_SIMPLE_TREE_READ("\xcc\x80", 0x80 == mpack_node_u16(node)); TEST_SIMPLE_TREE_READ("\xcc\x80", 0x80 == mpack_node_u32(node)); @@ -232,8 +234,8 @@ static void test_node_read_uint() { } -static void test_node_read_uint_signed() { - mpack_node_data_t pool[128]; +static void test_node_read_uint_signed(void) { + mpack_tree_t tree; TEST_SIMPLE_TREE_READ("\xcc\x80", 0x80 == mpack_node_i16(node)); TEST_SIMPLE_TREE_READ("\xcc\x80", 0x80 == mpack_node_i32(node)); @@ -252,11 +254,16 @@ static void test_node_read_uint_signed() { TEST_SIMPLE_TREE_READ("\xcd\xff\xff", 0xffff == mpack_node_i32(node)); TEST_SIMPLE_TREE_READ("\xcd\xff\xff", 0xffff == mpack_node_i64(node)); - TEST_SIMPLE_TREE_READ("\xcd\xff\xff", 0xffff == mpack_node_int(node)); + + if (sizeof(int) >= 4) { + TEST_SIMPLE_TREE_READ("\xcd\xff\xff", (int)0xffff == mpack_node_int(node)); + TEST_SIMPLE_TREE_READ("\xce\x00\x01\x00\x00", 0x10000 == mpack_node_int(node)); + } else if (sizeof(int) < 4) { + TEST_SIMPLE_TREE_READ_ERROR("\xcd\xff\xff", 0 == mpack_node_int(node), mpack_error_type); + } TEST_SIMPLE_TREE_READ("\xce\x00\x01\x00\x00", 0x10000 == mpack_node_i32(node)); TEST_SIMPLE_TREE_READ("\xce\x00\x01\x00\x00", 0x10000 == mpack_node_i64(node)); - TEST_SIMPLE_TREE_READ("\xce\x00\x01\x00\x00", 0x10000 == mpack_node_int(node)); TEST_SIMPLE_TREE_READ("\xce\xff\xff\xff\xff", 0xffffffff == mpack_node_i64(node)); @@ -264,8 +271,8 @@ static void test_node_read_uint_signed() { } -static void test_node_read_int() { - mpack_node_data_t pool[128]; +static void test_node_read_int(void) { + mpack_tree_t tree; TEST_SIMPLE_TREE_READ("\xd0\xdf", -33 == mpack_node_i8(node)); TEST_SIMPLE_TREE_READ("\xd0\xdf", -33 == mpack_node_i16(node)); @@ -273,38 +280,40 @@ static void test_node_read_int() { TEST_SIMPLE_TREE_READ("\xd0\xdf", -33 == mpack_node_i64(node)); TEST_SIMPLE_TREE_READ("\xd0\xdf", -33 == mpack_node_int(node)); - TEST_SIMPLE_TREE_READ("\xd0\x80", INT8_MIN == mpack_node_i8(node)); - TEST_SIMPLE_TREE_READ("\xd0\x80", INT8_MIN == mpack_node_i16(node)); - TEST_SIMPLE_TREE_READ("\xd0\x80", INT8_MIN == mpack_node_i32(node)); - TEST_SIMPLE_TREE_READ("\xd0\x80", INT8_MIN == mpack_node_i64(node)); - TEST_SIMPLE_TREE_READ("\xd0\x80", INT8_MIN == mpack_node_int(node)); + TEST_SIMPLE_TREE_READ("\xd0\x80", MPACK_INT8_MIN == mpack_node_i8(node)); + TEST_SIMPLE_TREE_READ("\xd0\x80", MPACK_INT8_MIN == mpack_node_i16(node)); + TEST_SIMPLE_TREE_READ("\xd0\x80", MPACK_INT8_MIN == mpack_node_i32(node)); + TEST_SIMPLE_TREE_READ("\xd0\x80", MPACK_INT8_MIN == mpack_node_i64(node)); + TEST_SIMPLE_TREE_READ("\xd0\x80", MPACK_INT8_MIN == mpack_node_int(node)); - TEST_SIMPLE_TREE_READ("\xd1\xff\x7f", INT8_MIN - 1 == mpack_node_i16(node)); - TEST_SIMPLE_TREE_READ("\xd1\xff\x7f", INT8_MIN - 1 == mpack_node_i32(node)); - TEST_SIMPLE_TREE_READ("\xd1\xff\x7f", INT8_MIN - 1 == mpack_node_i64(node)); - TEST_SIMPLE_TREE_READ("\xd1\xff\x7f", INT8_MIN - 1 == mpack_node_int(node)); + TEST_SIMPLE_TREE_READ("\xd1\xff\x7f", MPACK_INT8_MIN - 1 == mpack_node_i16(node)); + TEST_SIMPLE_TREE_READ("\xd1\xff\x7f", MPACK_INT8_MIN - 1 == mpack_node_i32(node)); + TEST_SIMPLE_TREE_READ("\xd1\xff\x7f", MPACK_INT8_MIN - 1 == mpack_node_i64(node)); + TEST_SIMPLE_TREE_READ("\xd1\xff\x7f", MPACK_INT8_MIN - 1 == mpack_node_int(node)); - TEST_SIMPLE_TREE_READ("\xd1\x80\x00", INT16_MIN == mpack_node_i16(node)); - TEST_SIMPLE_TREE_READ("\xd1\x80\x00", INT16_MIN == mpack_node_i32(node)); - TEST_SIMPLE_TREE_READ("\xd1\x80\x00", INT16_MIN == mpack_node_i64(node)); - TEST_SIMPLE_TREE_READ("\xd1\x80\x00", INT16_MIN == mpack_node_int(node)); + TEST_SIMPLE_TREE_READ("\xd1\x80\x00", MPACK_INT16_MIN == mpack_node_i16(node)); + TEST_SIMPLE_TREE_READ("\xd1\x80\x00", MPACK_INT16_MIN == mpack_node_i32(node)); + TEST_SIMPLE_TREE_READ("\xd1\x80\x00", MPACK_INT16_MIN == mpack_node_i64(node)); + TEST_SIMPLE_TREE_READ("\xd1\x80\x00", MPACK_INT16_MIN == mpack_node_int(node)); - TEST_SIMPLE_TREE_READ("\xd2\xff\xff\x7f\xff", INT16_MIN - 1 == mpack_node_i32(node)); - TEST_SIMPLE_TREE_READ("\xd2\xff\xff\x7f\xff", INT16_MIN - 1 == mpack_node_i64(node)); - TEST_SIMPLE_TREE_READ("\xd2\xff\xff\x7f\xff", INT16_MIN - 1 == mpack_node_int(node)); + TEST_SIMPLE_TREE_READ("\xd2\xff\xff\x7f\xff", (int32_t)MPACK_INT16_MIN - 1 == mpack_node_i32(node)); + TEST_SIMPLE_TREE_READ("\xd2\xff\xff\x7f\xff", (int32_t)MPACK_INT16_MIN - 1 == mpack_node_i64(node)); + if (sizeof(int) >= 4) + TEST_SIMPLE_TREE_READ("\xd2\xff\xff\x7f\xff", (int32_t)MPACK_INT16_MIN - 1 == mpack_node_int(node)); - TEST_SIMPLE_TREE_READ("\xd2\x80\x00\x00\x00", INT32_MIN == mpack_node_i32(node)); - TEST_SIMPLE_TREE_READ("\xd2\x80\x00\x00\x00", INT32_MIN == mpack_node_i64(node)); - TEST_SIMPLE_TREE_READ("\xd2\x80\x00\x00\x00", INT32_MIN == mpack_node_int(node)); + TEST_SIMPLE_TREE_READ("\xd2\x80\x00\x00\x00", MPACK_INT32_MIN == mpack_node_i32(node)); + TEST_SIMPLE_TREE_READ("\xd2\x80\x00\x00\x00", MPACK_INT32_MIN == mpack_node_i64(node)); + if (sizeof(int) >= 4) + TEST_SIMPLE_TREE_READ("\xd2\x80\x00\x00\x00", MPACK_INT32_MIN == mpack_node_int(node)); - TEST_SIMPLE_TREE_READ("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", (int64_t)INT32_MIN - 1 == mpack_node_i64(node)); + TEST_SIMPLE_TREE_READ("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", (int64_t)MPACK_INT32_MIN - 1 == mpack_node_i64(node)); - TEST_SIMPLE_TREE_READ("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", INT64_MIN == mpack_node_i64(node)); + TEST_SIMPLE_TREE_READ("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", MPACK_INT64_MIN == mpack_node_i64(node)); } -static void test_node_read_ints_dynamic_int() { - mpack_node_data_t pool[128]; +static void test_node_read_ints_dynamic_int(void) { + mpack_tree_t tree; // we don't bother to test with different signed/unsigned value // functions; they are tested for equality in test-value.c @@ -330,27 +339,27 @@ static void test_node_read_ints_dynamic_int() { TEST_SIMPLE_TREE_READ("\xcd\xff\xff", mpack_tag_equal(mpack_tag_uint(0xffff), mpack_node_tag(node))); TEST_SIMPLE_TREE_READ("\xce\x00\x01\x00\x00", mpack_tag_equal(mpack_tag_uint(0x10000), mpack_node_tag(node))); TEST_SIMPLE_TREE_READ("\xce\xff\xff\xff\xff", mpack_tag_equal(mpack_tag_uint(0xffffffff), mpack_node_tag(node))); - TEST_SIMPLE_TREE_READ("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_tag_equal(mpack_tag_uint(UINT64_C(0x100000000)), mpack_node_tag(node))); - TEST_SIMPLE_TREE_READ("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_tag_equal(mpack_tag_uint(UINT64_C(0xffffffffffffffff)), mpack_node_tag(node))); + TEST_SIMPLE_TREE_READ("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_tag_equal(mpack_tag_uint(MPACK_UINT64_C(0x100000000)), mpack_node_tag(node))); + TEST_SIMPLE_TREE_READ("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_tag_equal(mpack_tag_uint(MPACK_UINT64_C(0xffffffffffffffff)), mpack_node_tag(node))); // ints TEST_SIMPLE_TREE_READ("\xd0\xdf", mpack_tag_equal(mpack_tag_int(-33), mpack_node_tag(node))); - TEST_SIMPLE_TREE_READ("\xd0\x80", mpack_tag_equal(mpack_tag_int(INT8_MIN), mpack_node_tag(node))); - TEST_SIMPLE_TREE_READ("\xd1\xff\x7f", mpack_tag_equal(mpack_tag_int(INT8_MIN - 1), mpack_node_tag(node))); - TEST_SIMPLE_TREE_READ("\xd1\x80\x00", mpack_tag_equal(mpack_tag_int(INT16_MIN), mpack_node_tag(node))); - TEST_SIMPLE_TREE_READ("\xd2\xff\xff\x7f\xff", mpack_tag_equal(mpack_tag_int(INT16_MIN - 1), mpack_node_tag(node))); + TEST_SIMPLE_TREE_READ("\xd0\x80", mpack_tag_equal(mpack_tag_int(MPACK_INT8_MIN), mpack_node_tag(node))); + TEST_SIMPLE_TREE_READ("\xd1\xff\x7f", mpack_tag_equal(mpack_tag_int(MPACK_INT8_MIN - 1), mpack_node_tag(node))); + TEST_SIMPLE_TREE_READ("\xd1\x80\x00", mpack_tag_equal(mpack_tag_int(MPACK_INT16_MIN), mpack_node_tag(node))); + TEST_SIMPLE_TREE_READ("\xd2\xff\xff\x7f\xff", mpack_tag_equal(mpack_tag_int((int32_t)MPACK_INT16_MIN - 1), mpack_node_tag(node))); - TEST_SIMPLE_TREE_READ("\xd2\x80\x00\x00\x00", mpack_tag_equal(mpack_tag_int(INT32_MIN), mpack_node_tag(node))); - TEST_SIMPLE_TREE_READ("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", mpack_tag_equal(mpack_tag_int((int64_t)INT32_MIN - 1), mpack_node_tag(node))); + TEST_SIMPLE_TREE_READ("\xd2\x80\x00\x00\x00", mpack_tag_equal(mpack_tag_int(MPACK_INT32_MIN), mpack_node_tag(node))); + TEST_SIMPLE_TREE_READ("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", mpack_tag_equal(mpack_tag_int((int64_t)MPACK_INT32_MIN - 1), mpack_node_tag(node))); - TEST_SIMPLE_TREE_READ("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", mpack_tag_equal(mpack_tag_int(INT64_MIN), mpack_node_tag(node))); + TEST_SIMPLE_TREE_READ("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", mpack_tag_equal(mpack_tag_int(MPACK_INT64_MIN), mpack_node_tag(node))); } -static void test_node_read_int_bounds() { - mpack_node_data_t pool[128]; +static void test_node_read_int_bounds(void) { + mpack_tree_t tree; - TEST_SIMPLE_TREE_READ_ERROR("\xd1\xff\x7f", 0 == mpack_node_i8(node), mpack_error_type); + TEST_SIMPLE_TREE_READ_ERROR("\xd1\xff\x7f", 0 == mpack_node_i8(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\xd1\x80\x00", 0 == mpack_node_i8(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\xd2\xff\xff\x7f\xff", 0 == mpack_node_i8(node), mpack_error_type); @@ -369,8 +378,8 @@ static void test_node_read_int_bounds() { } -static void test_node_read_uint_bounds() { - mpack_node_data_t pool[128]; +static void test_node_read_uint_bounds(void) { + mpack_tree_t tree; TEST_SIMPLE_TREE_READ_ERROR("\xcd\x01\x00", 0 == mpack_node_u8(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\xcd\xff\xff", 0 == mpack_node_u8(node), mpack_error_type); @@ -387,8 +396,8 @@ static void test_node_read_uint_bounds() { } -static void test_node_read_misc() { - mpack_node_data_t pool[128]; +static void test_node_read_misc(void) { + mpack_tree_t tree; TEST_SIMPLE_TREE_READ("\xc0", (mpack_node_nil(node), true)); @@ -404,9 +413,21 @@ static void test_node_read_misc() { TEST_SIMPLE_TREE_READ("\xc2", mpack_tag_equal(mpack_tag_false(), mpack_node_tag(node))); TEST_SIMPLE_TREE_READ("\xc3", mpack_tag_equal(mpack_tag_true(), mpack_node_tag(node))); + #if !MPACK_EXTENSIONS + // ext types are unsupported without MPACK_EXTENSIONS + TEST_SIMPLE_TREE_READ_ERROR("\xc7", mpack_type_nil == mpack_node_type(node), mpack_error_unsupported); + TEST_SIMPLE_TREE_READ_ERROR("\xc8", mpack_type_nil == mpack_node_type(node), mpack_error_unsupported); + TEST_SIMPLE_TREE_READ_ERROR("\xc9", mpack_type_nil == mpack_node_type(node), mpack_error_unsupported); + TEST_SIMPLE_TREE_READ_ERROR("\xd4", mpack_type_nil == mpack_node_type(node), mpack_error_unsupported); + TEST_SIMPLE_TREE_READ_ERROR("\xd5", mpack_type_nil == mpack_node_type(node), mpack_error_unsupported); + TEST_SIMPLE_TREE_READ_ERROR("\xd6", mpack_type_nil == mpack_node_type(node), mpack_error_unsupported); + TEST_SIMPLE_TREE_READ_ERROR("\xd7", mpack_type_nil == mpack_node_type(node), mpack_error_unsupported); + TEST_SIMPLE_TREE_READ_ERROR("\xd8", mpack_type_nil == mpack_node_type(node), mpack_error_unsupported); + #endif + // test missing space for cstr null-terminator - mpack_tree_t tree; mpack_tree_init_pool(&tree, "\xa0", 1, pool, sizeof(pool) / sizeof(*pool)); + mpack_tree_parse(&tree); #if MPACK_DEBUG char buf[1]; TEST_ASSERT(mpack_node_copy_cstr(mpack_tree_root(&tree), buf, 0)); @@ -421,50 +442,78 @@ static void test_node_read_misc() { // test pool too small mpack_node_data_t small_pool[1]; mpack_tree_init_pool(&tree, "\x91\xc0", 2, small_pool, 1); + mpack_tree_parse(&tree); TEST_TREE_DESTROY_ERROR(&tree, mpack_error_too_big); TEST_BREAK((mpack_tree_init_pool(&tree, "\xc0", 1, small_pool, 0), true)); TEST_TREE_DESTROY_ERROR(&tree, mpack_error_bug); + + // test forgetting to parse + mpack_tree_init_pool(&tree, "\x91\xc0", 2, small_pool, 1); + TEST_BREAK((mpack_tree_root(&tree), true)); + TEST_TREE_DESTROY_ERROR(&tree, mpack_error_bug); } -static void test_node_read_floats() { - mpack_node_data_t pool[128]; +static void test_node_read_floats(void) { + mpack_tree_t tree; + (void)tree; // these are some very simple floats that don't really test IEEE 742 conformance; // this section could use some improvement + #if MPACK_FLOAT TEST_SIMPLE_TREE_READ("\x00", 0.0f == mpack_node_float(node)); TEST_SIMPLE_TREE_READ("\xd0\x00", 0.0f == mpack_node_float(node)); TEST_SIMPLE_TREE_READ("\xca\x00\x00\x00\x00", 0.0f == mpack_node_float(node)); TEST_SIMPLE_TREE_READ("\xcb\x00\x00\x00\x00\x00\x00\x00\x00", 0.0f == mpack_node_float(node)); + #endif + #if MPACK_DOUBLE TEST_SIMPLE_TREE_READ("\x00", 0.0 == mpack_node_double(node)); TEST_SIMPLE_TREE_READ("\xd0\x00", 0.0 == mpack_node_double(node)); TEST_SIMPLE_TREE_READ("\xca\x00\x00\x00\x00", 0.0 == mpack_node_double(node)); TEST_SIMPLE_TREE_READ("\xcb\x00\x00\x00\x00\x00\x00\x00\x00", 0.0 == mpack_node_double(node)); + #endif - TEST_SIMPLE_TREE_READ("\xca\xff\xff\xff\xff", isnanf(mpack_node_float(node)) != 0); - TEST_SIMPLE_TREE_READ("\xcb\xff\xff\xff\xff\xff\xff\xff\xff", isnanf(mpack_node_float(node)) != 0); - TEST_SIMPLE_TREE_READ("\xca\xff\xff\xff\xff", isnan(mpack_node_double(node)) != 0); - TEST_SIMPLE_TREE_READ("\xcb\xff\xff\xff\xff\xff\xff\xff\xff", isnan(mpack_node_double(node)) != 0); - + #if MPACK_FLOAT TEST_SIMPLE_TREE_READ("\xca\x00\x00\x00\x00", 0.0f == mpack_node_float_strict(node)); TEST_SIMPLE_TREE_READ("\xca\x00\x00\x00\x00", mpack_tag_equal(mpack_tag_float(0.0f), mpack_node_tag(node))); + #endif + #if MPACK_DOUBLE TEST_SIMPLE_TREE_READ("\xca\x00\x00\x00\x00", 0.0 == mpack_node_double_strict(node)); TEST_SIMPLE_TREE_READ("\xcb\x00\x00\x00\x00\x00\x00\x00\x00", mpack_tag_equal(mpack_tag_double(0.0), mpack_node_tag(node))); + #endif + + // when -ffinite-math-only is enabled, isnan() can always return false. + // TODO: we should probably add at least a reader option to + // generate an error on non-finite reals. + #if !MPACK_FINITE_MATH + #if MPACK_FLOAT + TEST_SIMPLE_TREE_READ("\xca\xff\xff\xff\xff", isnanf(mpack_node_float(node)) != 0); TEST_SIMPLE_TREE_READ("\xca\xff\xff\xff\xff", isnanf(mpack_node_float_strict(node)) != 0); + TEST_SIMPLE_TREE_READ("\xcb\xff\xff\xff\xff\xff\xff\xff\xff", isnanf(mpack_node_float(node)) != 0); + #endif + #if MPACK_DOUBLE + TEST_SIMPLE_TREE_READ("\xca\xff\xff\xff\xff", isnan(mpack_node_double(node)) != 0); + TEST_SIMPLE_TREE_READ("\xcb\xff\xff\xff\xff\xff\xff\xff\xff", isnan(mpack_node_double(node)) != 0); TEST_SIMPLE_TREE_READ("\xca\xff\xff\xff\xff", isnan(mpack_node_double_strict(node)) != 0); TEST_SIMPLE_TREE_READ("\xcb\xff\xff\xff\xff\xff\xff\xff\xff", isnan(mpack_node_double_strict(node)) != 0); + #endif + #endif + #if MPACK_FLOAT TEST_SIMPLE_TREE_READ_ERROR("\x00", 0.0f == mpack_node_float_strict(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\xd0\x00", 0.0f == mpack_node_float_strict(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\xcb\x00\x00\x00\x00\x00\x00\x00\x00", 0.0f == mpack_node_float_strict(node), mpack_error_type); + #endif + #if MPACK_DOUBLE TEST_SIMPLE_TREE_READ_ERROR("\x00", 0.0 == mpack_node_double_strict(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\xd0\x00", 0.0 == mpack_node_double_strict(node), mpack_error_type); + #endif } -static void test_node_read_bad_type() { - mpack_node_data_t pool[128]; +static void test_node_read_bad_type(void) { + mpack_tree_t tree; // test that non-compound node functions correctly handle badly typed data TEST_SIMPLE_TREE_READ_ERROR("\xc2", (mpack_node_nil(node), true), mpack_error_type); @@ -479,16 +528,20 @@ static void test_node_read_bad_type() { TEST_SIMPLE_TREE_READ_ERROR("\xc0", 0 == mpack_node_i32(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\xc0", 0 == mpack_node_i64(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\xc0", 0 == mpack_node_int(node), mpack_error_type); + #if MPACK_FLOAT TEST_SIMPLE_TREE_READ_ERROR("\xc0", 0.0f == mpack_node_float(node), mpack_error_type); - TEST_SIMPLE_TREE_READ_ERROR("\xc0", 0.0 == mpack_node_double(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\xc0", 0.0f == mpack_node_float_strict(node), mpack_error_type); + #endif + #if MPACK_DOUBLE + TEST_SIMPLE_TREE_READ_ERROR("\xc0", 0.0 == mpack_node_double(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\xc0", 0.0 == mpack_node_double_strict(node), mpack_error_type); + #endif } -static void test_node_read_possible() { +static void test_node_read_possible(void) { // test early exit for data that contains impossible node numbers - mpack_node_data_t pool[128]; + mpack_tree_t tree; TEST_SIMPLE_TREE_READ_ERROR("\xcc", (MPACK_UNUSED(node), true), mpack_error_invalid); // truncated u8 TEST_SIMPLE_TREE_READ_ERROR("\xcd", (MPACK_UNUSED(node), true), mpack_error_invalid); // truncated u16 TEST_SIMPLE_TREE_READ_ERROR("\xce", (MPACK_UNUSED(node), true), mpack_error_invalid); // truncated u32 @@ -530,16 +583,16 @@ static void test_node_read_possible() { "\xdd\xff\xff\xff\xff\xdd\xff\xff\xff\xff\xdd\xff\xff\xff\xff\xdd\xff\xff\xff\xff" "\xdd\xff\xff\xff\xff\xdd\xff\xff\xff\xff\xdd\xff\xff\xff\xff\xdd\xff\xff\xff\xff"; size_t allocation_count = test_malloc_total_count(); - mpack_tree_t tree; mpack_tree_init(&tree, attack, strlen(attack)); + mpack_tree_parse(&tree); allocation_count = test_malloc_total_count() - allocation_count; TEST_TRUE(allocation_count <= 2, "too many allocations! %i calls to malloc()", (int)allocation_count); TEST_TREE_DESTROY_ERROR(&tree, mpack_error_invalid); #endif } -static void test_node_read_pre_error() { - mpack_node_data_t pool[128]; +static void test_node_read_pre_error(void) { + mpack_tree_t tree; char buf[1]; // test that all node functions correctly handle pre-existing errors @@ -560,10 +613,14 @@ static void test_node_read_pre_error() { TEST_SIMPLE_TREE_READ_ERROR("", 0 == mpack_node_i64(node), mpack_error_invalid); TEST_SIMPLE_TREE_READ_ERROR("", 0 == mpack_node_int(node), mpack_error_invalid); + #if MPACK_FLOAT TEST_SIMPLE_TREE_READ_ERROR("", 0.0f == mpack_node_float(node), mpack_error_invalid); - TEST_SIMPLE_TREE_READ_ERROR("", 0.0 == mpack_node_double(node), mpack_error_invalid); TEST_SIMPLE_TREE_READ_ERROR("", 0.0f == mpack_node_float_strict(node), mpack_error_invalid); + #endif + #if MPACK_DOUBLE + TEST_SIMPLE_TREE_READ_ERROR("", 0.0 == mpack_node_double(node), mpack_error_invalid); TEST_SIMPLE_TREE_READ_ERROR("", 0.0 == mpack_node_double_strict(node), mpack_error_invalid); + #endif TEST_SIMPLE_TREE_READ_ERROR("", 0 == mpack_node_array_length(node), mpack_error_invalid); TEST_SIMPLE_TREE_READ_ERROR("", &tree.nil_node == mpack_node_array_at(node, 0).data, mpack_error_invalid); @@ -579,9 +636,12 @@ static void test_node_read_pre_error() { TEST_SIMPLE_TREE_READ_ERROR("", false == mpack_node_map_contains_str(node, "test", 4), mpack_error_invalid); TEST_SIMPLE_TREE_READ_ERROR("", false == mpack_node_map_contains_cstr(node, "test"), mpack_error_invalid); + #if MPACK_EXTENSIONS TEST_SIMPLE_TREE_READ_ERROR("", 0 == mpack_node_exttype(node), mpack_error_invalid); + #endif TEST_SIMPLE_TREE_READ_ERROR("", 0 == mpack_node_data_len(node), mpack_error_invalid); TEST_SIMPLE_TREE_READ_ERROR("", 0 == mpack_node_strlen(node), mpack_error_invalid); + TEST_SIMPLE_TREE_READ_ERROR("", NULL == mpack_node_str(node), mpack_error_invalid); TEST_SIMPLE_TREE_READ_ERROR("", NULL == mpack_node_data(node), mpack_error_invalid); TEST_SIMPLE_TREE_READ_ERROR("", 0 == mpack_node_copy_data(node, NULL, 0), mpack_error_invalid); buf[0] = 1; @@ -593,9 +653,9 @@ static void test_node_read_pre_error() { #endif } -static void test_node_read_strings() { +static void test_node_read_strings(void) { + mpack_tree_t tree; char buf[256]; - mpack_node_data_t pool[128]; #ifdef MPACK_MALLOC char* test = NULL; #endif @@ -732,10 +792,104 @@ static void test_node_read_strings() { #endif } -static void test_node_read_array() { +static void test_node_read_enum(void) { + mpack_tree_t tree; + + typedef enum { APPLE , BANANA , ORANGE , COUNT} fruit_t; + const char* fruits[] = {"apple", "banana", "orange"}; + + TEST_SIMPLE_TREE_READ("\xa5""apple", APPLE == (fruit_t)mpack_node_enum(node, fruits, COUNT)); + TEST_SIMPLE_TREE_READ("\xa6""banana", BANANA == (fruit_t)mpack_node_enum(node, fruits, COUNT)); + TEST_SIMPLE_TREE_READ("\xa6""orange", ORANGE == (fruit_t)mpack_node_enum(node, fruits, COUNT)); + TEST_SIMPLE_TREE_READ_ERROR("\xa4""kiwi", COUNT == (fruit_t)mpack_node_enum(node, fruits, COUNT), mpack_error_type); + TEST_SIMPLE_TREE_READ_ERROR("\x01", COUNT == (fruit_t)mpack_node_enum(node, fruits, COUNT), mpack_error_type); + + TEST_SIMPLE_TREE_READ("\xa5""apple", APPLE == (fruit_t)mpack_node_enum_optional(node, fruits, COUNT)); + TEST_SIMPLE_TREE_READ("\xa6""banana", BANANA == (fruit_t)mpack_node_enum_optional(node, fruits, COUNT)); + TEST_SIMPLE_TREE_READ("\xa6""orange", ORANGE == (fruit_t)mpack_node_enum_optional(node, fruits, COUNT)); + TEST_SIMPLE_TREE_READ("\xa4""kiwi", COUNT == (fruit_t)mpack_node_enum_optional(node, fruits, COUNT)); + TEST_SIMPLE_TREE_READ("\x01", COUNT == (fruit_t)mpack_node_enum_optional(node, fruits, COUNT)); + + // test pre-existing error + TEST_SIMPLE_TREE_READ_ERROR("\x01", (mpack_node_nil(node), COUNT == (fruit_t)mpack_node_enum(node, fruits, COUNT)), mpack_error_type); +} + +#if MPACK_EXTENSIONS +static void test_node_read_timestamp(void) { + mpack_tree_t tree; + TEST_MPACK_SILENCE_SHADOW_BEGIN + mpack_node_data_t pool[1]; + TEST_MPACK_SILENCE_SHADOW_END + + TEST_SIMPLE_TREE_READ("\xd6\xff\x00\x00\x00\x00", + 0 == mpack_node_timestamp_seconds(node) && + 0 == mpack_node_timestamp_nanoseconds(node)); + TEST_SIMPLE_TREE_READ("\xd6\xff\x00\x00\x01\x00", + 256 == mpack_node_timestamp_seconds(node) && + 0 == mpack_node_timestamp_nanoseconds(node)); + TEST_SIMPLE_TREE_READ("\xd6\xff\xfe\xdc\xba\x98", + 4275878552u == mpack_node_timestamp_seconds(node) && + 0 == mpack_node_timestamp_nanoseconds(node)); + TEST_SIMPLE_TREE_READ("\xd6\xff\xff\xff\xff\xff", + MPACK_UINT32_MAX == mpack_node_timestamp_seconds(node) && + 0 == mpack_node_timestamp_nanoseconds(node)); + + TEST_SIMPLE_TREE_READ("\xd7\xff\x00\x00\x00\x03\x00\x00\x00\x00", + MPACK_INT64_C(12884901888) == mpack_node_timestamp_seconds(node) && + 0 == mpack_node_timestamp_nanoseconds(node)); + TEST_SIMPLE_TREE_READ("\xd7\xff\x00\x00\x00\x00\x00\x00\x00\x00", + 0 == mpack_node_timestamp_seconds(node) && + 0 == mpack_node_timestamp_nanoseconds(node)); + TEST_SIMPLE_TREE_READ("\xd7\xff\xee\x6b\x27\xfc\x00\x00\x00\x00", + 0 == mpack_node_timestamp_seconds(node) && + MPACK_TIMESTAMP_NANOSECONDS_MAX == mpack_node_timestamp_nanoseconds(node)); + TEST_SIMPLE_TREE_READ("\xd7\xff\xee\x6b\x27\xff\xff\xff\xff\xff", + MPACK_INT64_C(17179869183) == mpack_node_timestamp_seconds(node) && + MPACK_TIMESTAMP_NANOSECONDS_MAX == mpack_node_timestamp_nanoseconds(node)); + + TEST_SIMPLE_TREE_READ("\xc7\x0c\xff\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01", + 1 == mpack_node_timestamp_seconds(node) && + 0 == mpack_node_timestamp_nanoseconds(node)); + TEST_SIMPLE_TREE_READ("\xc7\x0c\xff\x3b\x9a\xc9\xff\x00\x00\x00\x00\x00\x00\x00\x00", + 0 == mpack_node_timestamp_seconds(node) && + MPACK_TIMESTAMP_NANOSECONDS_MAX == mpack_node_timestamp_nanoseconds(node)); + TEST_SIMPLE_TREE_READ("\xc7\x0c\xff\x00\x00\x00\x01\xff\xff\xff\xff\xff\xff\xff\xff", + -1 == mpack_node_timestamp_seconds(node) && + 1 == mpack_node_timestamp_nanoseconds(node)); + TEST_SIMPLE_TREE_READ("\xc7\x0c\xff\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff", + MPACK_INT64_MAX == mpack_node_timestamp_seconds(node) && + MPACK_TIMESTAMP_NANOSECONDS_MAX == mpack_node_timestamp_nanoseconds(node)); + TEST_SIMPLE_TREE_READ("\xc7\x0c\xff\x3b\x9a\xc9\xff\x80\x00\x00\x00\x00\x00\x00\x00", + MPACK_INT64_MIN == mpack_node_timestamp_seconds(node) && + MPACK_TIMESTAMP_NANOSECONDS_MAX == mpack_node_timestamp_nanoseconds(node)); + + // invalid timestamps should not flag errors... + TEST_SIMPLE_TREE_READ("\xd7\xff\xff\xff\xff\xff\x00\x00\x00\x00", + (MPACK_UNUSED(node), true)); + TEST_SIMPLE_TREE_READ("\xd7\xff\xee\x6b\x28\x00\xff\xff\xff\xff", + (MPACK_UNUSED(node), true)); + TEST_SIMPLE_TREE_READ("\xc7\x0c\xff\x3b\x9a\xca\x00\x00\x00\x00\x00\x00\x00\x00\x00", + (MPACK_UNUSED(node), true)); + TEST_SIMPLE_TREE_READ("\xc7\x0c\xff\x40\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff", + (MPACK_UNUSED(node), true)); + + // ...unless we ask mpack to interpret them for us. + TEST_SIMPLE_TREE_READ_ERROR("\xd7\xff\xff\xff\xff\xff\x00\x00\x00\x00", + (mpack_node_timestamp(node), true), mpack_error_invalid); + TEST_SIMPLE_TREE_READ_ERROR("\xd7\xff\xee\x6b\x28\x00\xff\xff\xff\xff", + (mpack_node_timestamp(node), true), mpack_error_invalid); + TEST_SIMPLE_TREE_READ_ERROR("\xc7\x0c\xff\x3b\x9a\xca\x00\x00\x00\x00\x00\x00\x00\x00\x00", + (mpack_node_timestamp(node), true), mpack_error_invalid); + TEST_SIMPLE_TREE_READ_ERROR("\xc7\x0c\xff\x40\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff", + (mpack_node_timestamp(node), true), mpack_error_invalid); +} +#endif + +static void test_node_read_array(void) { static const char test[] = "\x93\x90\x91\xc3\x92\xc3\xc3"; mpack_tree_t tree; TEST_TREE_INIT(&tree, test, sizeof(test) - 1); + mpack_tree_parse(&tree); mpack_node_t root = mpack_tree_root(&tree); TEST_TRUE(mpack_type_array == mpack_node_type(root)); @@ -763,11 +917,12 @@ static void test_node_read_array() { TEST_TREE_DESTROY_ERROR(&tree, mpack_error_data); } -static void test_node_read_map() { +static void test_node_read_map(void) { // test map using maps as keys and values static const char test[] = "\x82\x80\x81\x01\x02\x81\x03\x04\xc3"; mpack_tree_t tree; TEST_TREE_INIT(&tree, test, sizeof(test) - 1); + mpack_tree_parse(&tree); mpack_node_t root = mpack_tree_root(&tree); TEST_TRUE(mpack_type_map == mpack_node_type(root)); @@ -796,12 +951,12 @@ static void test_node_read_map() { TEST_TREE_DESTROY_ERROR(&tree, mpack_error_data); } -static void test_node_read_map_search() { +static void test_node_read_map_search(void) { static const char test[] = "\x89\x00\x01\xd0\x7f\x02\xfe\x03\xa5""alice\x04\xa3" "bob\x05\xa4""carl\x06\xa4""carl\x07\x10\x08\x10\x09"; - mpack_node_data_t pool[128]; + mpack_tree_t tree; TEST_SIMPLE_TREE_READ(test, 1 == mpack_node_i32(mpack_node_map_uint(node, 0))); TEST_SIMPLE_TREE_READ(test, 1 == mpack_node_i32(mpack_node_map_int(node, 0))); @@ -829,7 +984,7 @@ static void test_node_read_map_search() { } static void test_node_read_compound_errors(void) { - mpack_node_data_t pool[128]; + mpack_tree_t tree; TEST_SIMPLE_TREE_READ_ERROR("\x00", 0 == mpack_node_array_length(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\x00", 0 == mpack_node_map_count(node), mpack_error_type); @@ -849,9 +1004,12 @@ static void test_node_read_compound_errors(void) { TEST_SIMPLE_TREE_READ_ERROR("\x00", false == mpack_node_map_contains_str(node, "test", 4), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\x00", false == mpack_node_map_contains_cstr(node, "test"), mpack_error_type); + #if MPACK_EXTENSIONS TEST_SIMPLE_TREE_READ_ERROR("\x00", 0 == mpack_node_exttype(node), mpack_error_type); + #endif TEST_SIMPLE_TREE_READ_ERROR("\x00", 0 == mpack_node_data_len(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\x00", 0 == mpack_node_strlen(node), mpack_error_type); + TEST_SIMPLE_TREE_READ_ERROR("\x00", NULL == mpack_node_str(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\x00", NULL == mpack_node_data(node), mpack_error_type); TEST_SIMPLE_TREE_READ_ERROR("\x00", 0 == mpack_node_copy_data(node, NULL, 0), mpack_error_type); @@ -879,9 +1037,18 @@ static void test_node_read_compound_errors(void) { } static void test_node_read_data(void) { - static const char test[] = "\x93\xa5""alice\xc4\x03""bob\xd6\x07""carl"; + static const char test[] = "\x93\xa5""alice\xc4\x03""bob" + // we'll put carl in an ext (of exttype 7) if extensions are supported, + // or in a bin otherwise. + #if MPACK_EXTENSIONS + "\xd6\x07" + #else + "\xc4\x04" + #endif + "carl"; mpack_tree_t tree; TEST_TREE_INIT(&tree, test, sizeof(test) - 1); + mpack_tree_parse(&tree); mpack_node_t root = mpack_tree_root(&tree); mpack_node_t alice = mpack_node_array_at(root, 0); @@ -913,19 +1080,23 @@ static void test_node_read_data(void) { #endif mpack_node_t carl = mpack_node_array_at(root, 2); + #if MPACK_EXTENSIONS TEST_TRUE(7 == mpack_node_exttype(carl)); + #endif TEST_TRUE(4 == mpack_node_data_len(carl)); TEST_TRUE(0 == memcmp("carl", mpack_node_data(carl), 4)); TEST_TREE_DESTROY_NOERROR(&tree); } + static void test_node_read_deep_stack(void) { static const int depth = 1200; - char buf[4096]; + static char buf[4096]; uint8_t* p = (uint8_t*)buf; - for (int i = 0; i < depth; ++i) { + int i; + for (i = 0; i < depth; ++i) { *p++ = 0x81; // one pair map *p++ = 0x04; // key four *p++ = 0x91; // value one element array @@ -934,10 +1105,11 @@ static void test_node_read_deep_stack(void) { mpack_tree_t tree; TEST_TREE_INIT(&tree, buf, (size_t)(p - (uint8_t*)buf)); + mpack_tree_parse(&tree); #ifdef MPACK_MALLOC mpack_node_t node = mpack_tree_root(&tree); - for (int i = 0; i < depth; ++i) { + for (i = 0; i < depth; ++i) { TEST_TRUE(mpack_tree_error(&tree) == mpack_ok, "error at depth %i", i); TEST_TRUE(mpack_node_map_count(node) == 1, "error at depth %i", i); TEST_TRUE(mpack_node_u8(mpack_node_map_key_at(node, 0)) == 4, "error at depth %i", i); @@ -951,7 +1123,193 @@ static void test_node_read_deep_stack(void) { #endif } +static void test_node_multiple_simple(void) { + static const char test[] = "\x00\xa5""hello\xd1\x80\x00\xc0"; + mpack_tree_t tree; + + TEST_MPACK_SILENCE_SHADOW_BEGIN + mpack_node_data_t pool[1]; + TEST_MPACK_SILENCE_SHADOW_END + mpack_tree_init_pool(&tree, test, sizeof(test) - 1, pool, sizeof(pool) / sizeof(*pool)); + + mpack_tree_parse(&tree); + TEST_TRUE(0u == mpack_node_uint(mpack_tree_root(&tree))); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + + mpack_tree_parse(&tree); + char cstr[10]; + mpack_node_copy_cstr(mpack_tree_root(&tree), cstr, sizeof(cstr)); + TEST_TRUE(strcmp(cstr, "hello") == 0); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + + mpack_tree_parse(&tree); + TEST_TRUE(MPACK_INT16_MIN == mpack_node_i16(mpack_tree_root(&tree))); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + + mpack_tree_parse(&tree); + mpack_node_nil(mpack_tree_root(&tree)); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + + TEST_TREE_DESTROY_NOERROR(&tree); +} + +#ifdef MPACK_MALLOC +typedef struct test_node_stream_t { + size_t length; + const char* data; + size_t pos; + size_t step; +} test_node_stream_t; + +static size_t test_node_stream_read(mpack_tree_t* tree, char* buffer, size_t count) { + test_node_stream_t* stream_context = (test_node_stream_t*)tree->context; + + if (count > stream_context->step) + count = stream_context->step; + if (count + stream_context->pos > stream_context->length) + count = stream_context->length - stream_context->pos; + + memcpy(buffer, stream_context->data + stream_context->pos, count); + stream_context->pos += count; + return count; +} + +static bool test_node_multiple_allocs(bool stream, size_t stream_read_size) { + static const char test[] = + "\x82\xa4""true\xc3\xa5""false\xc2" + "\x9a\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a" // larger than config page size + "\x93\xff\xfe\xfd"; + + test_node_stream_t stream_context; + mpack_tree_t tree; + + if (stream) { + stream_context.data = test; + stream_context.length = sizeof(test) - 1; + stream_context.pos = 0; + stream_context.step = stream_read_size; + mpack_tree_init_stream(&tree, &test_node_stream_read, &stream_context, 1000, 1000); + } else { + mpack_tree_init(&tree, test, sizeof(test) - 1); + } + + // we're testing memory errors! other errors are bad. + mpack_tree_parse(&tree); + if (mpack_tree_error(&tree) == mpack_error_memory) { + mpack_tree_destroy(&tree); + return false; + } + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + + TEST_TRUE(false == mpack_node_bool(mpack_node_map_cstr(mpack_tree_root(&tree), "false"))); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + TEST_TRUE(true == mpack_node_bool(mpack_node_map_cstr(mpack_tree_root(&tree), "true"))); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + + // second message... + mpack_tree_parse(&tree); + if (mpack_tree_error(&tree) == mpack_error_memory) { + mpack_tree_destroy(&tree); + return false; + } + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + + TEST_TRUE(10 == mpack_node_array_length(mpack_tree_root(&tree))); + size_t i; + for (i = 0; i < 10; ++i) + TEST_TRUE(i + 1 == mpack_node_uint(mpack_node_array_at(mpack_tree_root(&tree), i))); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + + // third message... + mpack_tree_parse(&tree); + if (mpack_tree_error(&tree) == mpack_error_memory) { + mpack_tree_destroy(&tree); + return false; + } + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + + TEST_TRUE(3 == mpack_node_array_length(mpack_tree_root(&tree))); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + TEST_TRUE(-1 == mpack_node_int(mpack_node_array_at(mpack_tree_root(&tree), 0))); + TEST_TRUE(-2 == mpack_node_int(mpack_node_array_at(mpack_tree_root(&tree), 1))); + TEST_TRUE(-3 == mpack_node_int(mpack_node_array_at(mpack_tree_root(&tree), 2))); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + + // success! + mpack_tree_destroy(&tree); + return true; +} + +static bool test_node_multiple_allocs_memory(void) { + return test_node_multiple_allocs(false, 0); +} + +static bool test_node_multiple_allocs_stream1(void) { + return test_node_multiple_allocs(true, 1); +} + +static bool test_node_multiple_allocs_stream2(void) { + return test_node_multiple_allocs(true, 2); +} + +static bool test_node_multiple_allocs_stream3(void) { + return test_node_multiple_allocs(true, 3); +} + +static bool test_node_multiple_allocs_stream4096(void) { + return test_node_multiple_allocs(true, 4096); +} +#endif + +#if MPACK_DEBUG && MPACK_STDIO +static void test_node_print_buffer(void) { + static const char test[] = "\x82\xA7""compact\xC3\xA6""schema\x00"; + mpack_tree_t tree; + mpack_tree_init(&tree, test, sizeof(test) - 1); + + mpack_tree_parse(&tree); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + mpack_node_t root = mpack_tree_root(&tree); + + char buffer[1024]; + mpack_node_print_to_buffer(root, buffer, sizeof(buffer)); + + static const char* expected = + "{\n" + " \"compact\": true,\n" + " \"schema\": 0\n" + "}"; + TEST_TRUE(buffer[strlen(expected)] == 0); + TEST_TRUE(0 == strcmp(buffer, expected)); + + TEST_TRUE(mpack_ok == mpack_tree_destroy(&tree)); +} + +static void test_node_print_buffer_bounds(void) { + static const char test[] = "\x82\xA7""compact\xC3\xA6""schema\x00"; + mpack_tree_t tree; + mpack_tree_init(&tree, test, sizeof(test) - 1); + + mpack_tree_parse(&tree); + TEST_TRUE(mpack_tree_error(&tree) == mpack_ok); + mpack_node_t root = mpack_tree_root(&tree); + + char buffer[10]; + mpack_node_print_to_buffer(root, buffer, sizeof(buffer)); + TEST_TRUE(mpack_ok == mpack_tree_destroy(&tree)); + + // string should be truncated with a null-terminator + static const char* expected = "{\n \"co"; + TEST_TRUE(buffer[strlen(expected)] == 0); + TEST_TRUE(0 == strcmp(buffer, expected)); +} +#endif + void test_node(void) { + #if MPACK_DEBUG && MPACK_STDIO + test_node_print_buffer(); + test_node_print_buffer_bounds(); + #endif test_example_node(); // int/uint @@ -972,6 +1330,10 @@ void test_node(void) { test_node_read_possible(); test_node_read_pre_error(); test_node_read_strings(); + test_node_read_enum(); + #if MPACK_EXTENSIONS + test_node_read_timestamp(); + #endif // compound types test_node_read_array(); @@ -980,6 +1342,16 @@ void test_node(void) { test_node_read_compound_errors(); test_node_read_data(); test_node_read_deep_stack(); + + // message streams + test_node_multiple_simple(); + #ifdef MPACK_MALLOC + test_system_fail_until_ok(&test_node_multiple_allocs_memory); + test_system_fail_until_ok(&test_node_multiple_allocs_stream1); + test_system_fail_until_ok(&test_node_multiple_allocs_stream2); + test_system_fail_until_ok(&test_node_multiple_allocs_stream3); + test_system_fail_until_ok(&test_node_multiple_allocs_stream4096); + #endif } #endif diff --git a/test/test-node.h b/test/unit/src/test-node.h similarity index 91% rename from test/test-node.h rename to test/unit/src/test-node.h index 70530a4..3d0ba29 100644 --- a/test/test-node.h +++ b/test/unit/src/test-node.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -49,9 +49,9 @@ void test_tree_error_handler(mpack_tree_t* tree, mpack_error_t error); } while (0) #define TEST_SIMPLE_TREE_READ(data, read_expr) do { \ - mpack_tree_t tree; \ mpack_tree_init_pool(&tree, data, sizeof(data) - 1, pool, sizeof(pool) / sizeof(*pool)); \ mpack_tree_set_error_handler(&tree, test_tree_error_handler); \ + mpack_tree_parse(&tree); \ mpack_node_t node = mpack_tree_root(&tree); \ TEST_TRUE((read_expr), "simple tree test did not pass: " #read_expr); \ TEST_TREE_DESTROY_NOERROR(&tree); \ @@ -63,18 +63,17 @@ void test_tree_error_handler(mpack_tree_t* tree, mpack_error_t error); #define TEST_TREE_INIT mpack_tree_init #else #define TEST_TREE_INIT(tree, data, data_size) \ - mpack_node_data_t pool[128]; \ -mpack_tree_init_pool((tree), (data), (data_size), pool, sizeof(pool) / sizeof(*pool)); + mpack_tree_init_pool((tree), (data), (data_size), pool, sizeof(pool) / sizeof(*pool)); #endif // the error handler is only called if the tree is not already in an // error state, so we call it ourselves if the tree init failed. #define TEST_SIMPLE_TREE_READ_ERROR(data, read_expr, error) do { \ - mpack_tree_t tree; \ mpack_tree_init_pool(&tree, data, sizeof(data) - 1, pool, sizeof(pool) / sizeof(*pool)); \ if (mpack_tree_error(&tree) != mpack_ok) \ test_tree_error_handler(&tree, error); \ mpack_tree_set_error_handler(&tree, test_tree_error_handler); \ + mpack_tree_parse(&tree); \ mpack_node_t node = mpack_tree_root(&tree); \ TEST_TRUE((read_expr), "simple read error test did not pass: " #read_expr); \ TEST_TREE_DESTROY_ERROR(&tree, (error)); \ @@ -93,8 +92,11 @@ mpack_tree_init_pool((tree), (data), (data_size), pool, sizeof(pool) / sizeof(*p // (note about volatile, see TEST_ASSERT()) #define TEST_SIMPLE_TREE_READ_ASSERT(data, read_expr) do { \ volatile mpack_tree_t v_tree; \ - mpack_tree_t* tree = (mpack_tree_t*)&v_tree; \ + TEST_MPACK_SILENCE_SHADOW_BEGIN \ + mpack_tree_t* tree = (mpack_tree_t*)(uintptr_t)&v_tree; \ + TEST_MPACK_SILENCE_SHADOW_END \ mpack_tree_init_pool(tree, data, sizeof(data) - 1, pool, sizeof(pool) / sizeof(*pool)); \ + mpack_tree_parse(tree); \ mpack_node_t node = mpack_tree_root(tree); \ TEST_ASSERT(read_expr); \ mpack_tree_destroy(tree); \ @@ -112,8 +114,8 @@ mpack_tree_init_pool((tree), (data), (data_size), pool, sizeof(pool) / sizeof(*p // runs a simple tree test, ensuring it causes a break in // debug mode and flags mpack_error_bug in both debug and release. #define TEST_SIMPLE_TREE_READ_BREAK(data, read_expr) do { \ - mpack_tree_t tree; \ mpack_tree_init_pool(&tree, data, sizeof(data) - 1, pool, sizeof(pool) / sizeof(*pool)); \ + mpack_tree_parse(&tree); \ mpack_node_t node = mpack_tree_root(&tree); \ TEST_BREAK(read_expr); \ TEST_TREE_DESTROY_ERROR(&tree, mpack_error_bug); \ diff --git a/test/unit/src/test-reader.c b/test/unit/src/test-reader.c new file mode 100644 index 0000000..666cf6a --- /dev/null +++ b/test/unit/src/test-reader.c @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "test-expect.h" + +#if MPACK_READER + +mpack_error_t test_read_error = mpack_ok; + +void test_read_error_handler(mpack_reader_t* reader, mpack_error_t error) { + TEST_TRUE(test_read_error == mpack_ok, "error handler was called multiple times"); + TEST_TRUE(error != mpack_ok, "error handler was called with mpack_ok"); + TEST_TRUE(mpack_reader_error(reader) == error, "reader error does not match given error"); + test_read_error = error; +} + +// almost all reader functions are tested by the expect tests. +// minor miscellaneous read tests are added here. + +static void test_reader_should_inplace(void) { + static char buf[4096]; + mpack_reader_t reader; + mpack_reader_init(&reader, buf, sizeof(buf), 0); + + TEST_TRUE(true == mpack_should_read_bytes_inplace(&reader, 0)); + TEST_TRUE(true == mpack_should_read_bytes_inplace(&reader, 1)); + TEST_TRUE(true == mpack_should_read_bytes_inplace(&reader, 20)); + TEST_TRUE(false == mpack_should_read_bytes_inplace(&reader, 500)); + TEST_TRUE(false == mpack_should_read_bytes_inplace(&reader, 10000)); + + mpack_reader_destroy(&reader); +} + +static void test_reader_miscellaneous(void) { + mpack_reader_t reader; + + // 0xc1 is reserved; it should always raise mpack_error_invalid + TEST_SIMPLE_READ_ERROR("\xc1", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); + + #if !MPACK_EXTENSIONS + // ext types are unsupported without MPACK_EXTENSIONS + TEST_SIMPLE_READ_ERROR("\xc7", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); + TEST_SIMPLE_READ_ERROR("\xc8", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); + TEST_SIMPLE_READ_ERROR("\xc9", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); + TEST_SIMPLE_READ_ERROR("\xd4", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); + TEST_SIMPLE_READ_ERROR("\xd5", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); + TEST_SIMPLE_READ_ERROR("\xd6", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); + TEST_SIMPLE_READ_ERROR("\xd7", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); + TEST_SIMPLE_READ_ERROR("\xd8", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_unsupported); + #endif + + // simple truncated tags (testing discard of additional + // temporary data in mpack_parse_tag()) + TEST_SIMPLE_READ_ERROR("\xcc", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); + TEST_SIMPLE_READ_ERROR("\xcd", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); + TEST_SIMPLE_READ_ERROR("\xce", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); + TEST_SIMPLE_READ_ERROR("\xcf", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); + TEST_SIMPLE_READ_ERROR("\xd0", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); + TEST_SIMPLE_READ_ERROR("\xd1", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); + TEST_SIMPLE_READ_ERROR("\xd2", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); + TEST_SIMPLE_READ_ERROR("\xd3", mpack_tag_equal(mpack_read_tag(&reader), mpack_tag_nil()), mpack_error_invalid); + + // truncated discard errors + TEST_SIMPLE_READ_ERROR("\x91", (mpack_discard(&reader), true), mpack_error_invalid); // array + TEST_SIMPLE_READ_ERROR("\x81", (mpack_discard(&reader), true), mpack_error_invalid); // map +} + +#if MPACK_DEBUG && MPACK_STDIO +static void test_print_buffer(void) { + static const char test[] = "\x82\xA7""compact\xC3\xA6""schema\x00"; + + char buffer[1024]; + mpack_print_data_to_buffer(test, sizeof(test) - 1, buffer, sizeof(buffer)); + + static const char* expected = + "{\n" + " \"compact\": true,\n" + " \"schema\": 0\n" + "}"; + TEST_TRUE(buffer[strlen(expected)] == 0); + TEST_TRUE(0 == strcmp(buffer, expected)); +} + +static void test_print_buffer_bounds(void) { + static const char test[] = "\x82\xA7""compact\xC3\xA6""schema\x00"; + + char buffer[10]; + mpack_print_data_to_buffer(test, sizeof(test) - 1, buffer, sizeof(buffer)); + + // string should be truncated with a null-terminator + static const char* expected = "{\n \"co"; + TEST_TRUE(buffer[strlen(expected)] == 0); + TEST_TRUE(0 == strcmp(buffer, expected)); +} + +static void test_print_buffer_hexdump(void) { + static const char test[] = "\xc4\x03""abc"; + + char buffer[64]; + mpack_print_data_to_buffer(test, sizeof(test) - 1, buffer, sizeof(buffer)); + + // string should be truncated with a null-terminator + static const char* expected = ""; + TEST_TRUE(buffer[strlen(expected)] == 0); + TEST_TRUE(0 == strcmp(buffer, expected)); +} + +static void test_print_buffer_no_hexdump(void) { + static const char test[] = "\xc4\x00"; + + char buffer[64]; + mpack_print_data_to_buffer(test, sizeof(test) - 1, buffer, sizeof(buffer)); + + // string should be truncated with a null-terminator + static const char* expected = ""; + TEST_TRUE(buffer[strlen(expected)] == 0); + TEST_TRUE(0 == strcmp(buffer, expected)); +} +#endif + +static bool count_messages(const void* buffer, size_t byte_count, size_t* message_count) { + mpack_reader_t reader; + mpack_reader_init_data(&reader, (const char*)buffer, byte_count); + + *message_count = 0; + while (mpack_reader_remaining(&reader, NULL) > 0) { + ++*message_count; + mpack_discard(&reader); + } + + return mpack_ok == mpack_reader_destroy(&reader); +} + +static void test_count_messages(void) { + static const char test[] = "\x80\x81\xA3""key\xA5""value\x92\xc2\xc3\x90"; + size_t message_count; + TEST_TRUE(count_messages(test, sizeof(test)-1, &message_count)); + TEST_TRUE(message_count == 4); + + static const char test2[] = "\x92\xc0"; // truncated buffer + TEST_TRUE(!count_messages(test2, sizeof(test2)-1, &message_count)); +} + +void test_reader() { + #if MPACK_DEBUG && MPACK_STDIO + test_print_buffer(); + test_print_buffer_bounds(); + test_print_buffer_hexdump(); + test_print_buffer_no_hexdump(); + #endif + test_reader_should_inplace(); + test_reader_miscellaneous(); + test_count_messages(); +} + +#endif + diff --git a/test/test-reader.h b/test/unit/src/test-reader.h similarity index 95% rename from test/test-reader.h rename to test/unit/src/test-reader.h index 19778fd..0a3f596 100644 --- a/test/test-reader.h +++ b/test/unit/src/test-reader.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015 Nicholas Fraser + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in @@ -48,7 +48,6 @@ void test_read_error_handler(mpack_reader_t* reader, mpack_error_t error); (int)error, mpack_error_to_string(error)); \ TEST_TRUE(remaining == 0, \ "reader has %i extra bytes", (int)remaining); \ - mpack_reader_destroy(reader); \ } while (0) // tears down a reader, ensuring it is in the given error state @@ -58,7 +57,6 @@ void test_read_error_handler(mpack_reader_t* reader, mpack_error_t error); TEST_TRUE(actual == expected, "reader is in error state %i (%s) instead of %i (%s)", \ (int)actual, mpack_error_to_string(actual), \ (int)expected, mpack_error_to_string(expected)); \ - mpack_reader_destroy(reader); \ } while (0) @@ -82,7 +80,6 @@ void test_read_error_handler(mpack_reader_t* reader, mpack_error_t error); // runs a simple reader test, ensuring the expression is true and no errors occur #define TEST_SIMPLE_READ(data, read_expr) do { \ - mpack_reader_t reader; \ TEST_READER_INIT_STR(&reader, data); \ mpack_reader_set_error_handler(&reader, test_read_error_handler); \ TEST_TRUE((read_expr), "simple read test did not pass: " #read_expr); \ @@ -93,7 +90,6 @@ void test_read_error_handler(mpack_reader_t* reader, mpack_error_t error); // runs a simple reader test, ensuring the expression is true and no errors occur, cancelling to ignore tracking info #define TEST_SIMPLE_READ_CANCEL(data, read_expr) do { \ - mpack_reader_t reader; \ TEST_READER_INIT_STR(&reader, data); \ TEST_TRUE((read_expr), "simple read test did not pass: " #read_expr); \ mpack_reader_flag_error(&reader, mpack_error_data); \ @@ -102,7 +98,6 @@ void test_read_error_handler(mpack_reader_t* reader, mpack_error_t error); // runs a simple reader test, ensuring the expression is true and the given error is raised #define TEST_SIMPLE_READ_ERROR(data, read_expr, error) do { \ - mpack_reader_t reader; \ TEST_READER_INIT_STR(&reader, data); \ mpack_reader_set_error_handler(&reader, test_read_error_handler); \ TEST_TRUE((read_expr), "simple read error test did not pass: " #read_expr); \ @@ -122,7 +117,9 @@ void test_read_error_handler(mpack_reader_t* reader, mpack_error_t error); // (note about volatile, see TEST_ASSERT()) #define TEST_SIMPLE_READ_ASSERT(data, read_expr) do { \ volatile mpack_reader_t v_reader; \ - mpack_reader_t* reader = (mpack_reader_t*)&v_reader; \ + TEST_MPACK_SILENCE_SHADOW_BEGIN \ + mpack_reader_t* reader = (mpack_reader_t*)(uintptr_t)&v_reader; \ + TEST_MPACK_SILENCE_SHADOW_END \ mpack_reader_init_data(reader, data, sizeof(data) - 1); \ TEST_ASSERT(read_expr); \ mpack_reader_flag_error(reader, mpack_error_data); \ @@ -141,7 +138,6 @@ void test_read_error_handler(mpack_reader_t* reader, mpack_error_t error); // runs a simple reader test, ensuring it causes a break in // debug mode and flags mpack_error_bug in both debug and release. #define TEST_SIMPLE_READ_BREAK(data, read_expr) do { \ - mpack_reader_t reader; \ mpack_reader_init_data(&reader, data, sizeof(data) - 1); \ TEST_BREAK(read_expr); \ TEST_READER_DESTROY_ERROR(&reader, mpack_error_bug); \ diff --git a/test/test-system.c b/test/unit/src/test-system.c similarity index 89% rename from test/test-system.c rename to test/unit/src/test-system.c index 3b18a84..7ec8a71 100644 --- a/test/test-system.c +++ b/test/unit/src/test-system.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -19,6 +19,10 @@ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +#ifdef _WIN32 +#define _CRT_SECURE_NO_WARNINGS +#endif + // We need to include test.h here instead of test-system.h because // test-system.h is included within the mpack-config.h. #include "test.h" @@ -64,8 +68,9 @@ void test_system_fail_until_ok(bool (*test)(void)) { TEST_TRUE(test_files_count() == 0, "files are open before starting failure test"); #endif - for (int i = 0; i < test_system_fail_until_max; ++i) { - test_system_fail_after(i, false); + int i; + for (i = 0; i < test_system_fail_until_max; ++i) { + test_system_fail_after((size_t)i, false); bool ok = test(); #ifdef MPACK_MALLOC @@ -75,7 +80,7 @@ void test_system_fail_until_ok(bool (*test)(void)) { TEST_TRUE(test_files_count() == 0, "test single failure leaked file on iteration %i!", i); #endif - test_system_fail_after(i, true); + test_system_fail_after((size_t)i, true); ok &= test(); #ifdef MPACK_MALLOC @@ -164,6 +169,26 @@ void test_free(void* p) { +#if defined(MPACK_STDLIB) && MPACK_STDLIB +static bool test_system_mock_strlen_enabled = false; +static size_t test_system_mock_strlen_value; + +void test_system_mock_strlen(size_t len) { + test_system_mock_strlen_enabled = true; + test_system_mock_strlen_value = len; +} + +size_t test_strlen(const char* s) { + if (test_system_mock_strlen_enabled) { + test_system_mock_strlen_enabled = false; + return test_system_mock_strlen_value; + } + return strlen(s); +} +#endif + + + #if MPACK_STDIO #undef fopen #undef fclose diff --git a/test/test-system.h b/test/unit/src/test-system.h similarity index 91% rename from test/test-system.h rename to test/unit/src/test-system.h index 70aeb33..9b58330 100644 --- a/test/test-system.h +++ b/test/unit/src/test-system.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -64,6 +64,14 @@ size_t test_malloc_total_count(void); #endif +#if defined(MPACK_STDLIB) && MPACK_STDLIB +// Causes the next call to strlen() to return the given value. +void test_system_mock_strlen(size_t len); + +size_t test_strlen(const char* s); +#endif + + #if defined(MPACK_STDIO) && MPACK_STDIO FILE* test_fopen(const char* path, const char* mode); int test_fclose(FILE* stream); diff --git a/test/test-write.c b/test/unit/src/test-write.c similarity index 58% rename from test/test-write.c rename to test/unit/src/test-write.c index 7e1e50e..96e2655 100644 --- a/test/test-write.c +++ b/test/unit/src/test-write.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -24,6 +24,10 @@ #if MPACK_WRITER +static char buf[4096]; + +static const char* quick_brown_fox = "The quick brown fox jumps over a lazy dog."; + mpack_error_t test_write_error = mpack_ok; void test_write_error_handler(mpack_writer_t* writer, mpack_error_t error) { @@ -34,8 +38,8 @@ void test_write_error_handler(mpack_writer_t* writer, mpack_error_t error) { } // writes ints using the auto int()/uint() functions -static void test_write_simple_auto_int() { - char buf[4096]; +static void test_write_simple_auto_int(void) { + mpack_writer_t writer; // positive fixnums TEST_SIMPLE_WRITE("\x00", mpack_write_uint(&writer, 0)); @@ -69,8 +73,8 @@ static void test_write_simple_auto_int() { TEST_SIMPLE_WRITE("\xcd\xff\xff", mpack_write_uint(&writer, 0xffff)); TEST_SIMPLE_WRITE("\xce\x00\x01\x00\x00", mpack_write_uint(&writer, 0x10000)); TEST_SIMPLE_WRITE("\xce\xff\xff\xff\xff", mpack_write_uint(&writer, 0xffffffff)); - TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_uint(&writer, UINT64_C(0x100000000))); - TEST_SIMPLE_WRITE("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_write_uint(&writer, UINT64_C(0xffffffffffffffff))); + TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_uint(&writer, MPACK_UINT64_C(0x100000000))); + TEST_SIMPLE_WRITE("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_write_uint(&writer, MPACK_UINT64_C(0xffffffffffffffff))); // positive ints with signed value TEST_SIMPLE_WRITE("\xcc\x80", mpack_write_int(&writer, 0x80)); @@ -78,9 +82,9 @@ static void test_write_simple_auto_int() { TEST_SIMPLE_WRITE("\xcd\x01\x00", mpack_write_int(&writer, 0x100)); TEST_SIMPLE_WRITE("\xcd\xff\xff", mpack_write_int(&writer, 0xffff)); TEST_SIMPLE_WRITE("\xce\x00\x01\x00\x00", mpack_write_int(&writer, 0x10000)); - TEST_SIMPLE_WRITE("\xce\xff\xff\xff\xff", mpack_write_int(&writer, INT64_C(0xffffffff))); - TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_int(&writer, INT64_C(0x100000000))); - TEST_SIMPLE_WRITE("\xcf\x7f\xff\xff\xff\xff\xff\xff\xff", mpack_write_int(&writer, INT64_C(0x7fffffffffffffff))); + TEST_SIMPLE_WRITE("\xce\xff\xff\xff\xff", mpack_write_int(&writer, MPACK_INT64_C(0xffffffff))); + TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_int(&writer, MPACK_INT64_C(0x100000000))); + TEST_SIMPLE_WRITE("\xcf\x7f\xff\xff\xff\xff\xff\xff\xff", mpack_write_int(&writer, MPACK_INT64_C(0x7fffffffffffffff))); // ints TEST_SIMPLE_WRITE("\xd0\xdf", mpack_write_int(&writer, -33)); @@ -91,16 +95,16 @@ static void test_write_simple_auto_int() { // when using INT32_C() and compiling the test suite as c++, gcc complains: // error: this decimal constant is unsigned only in ISO C90 [-Werror] - TEST_SIMPLE_WRITE("\xd2\x80\x00\x00\x00", mpack_write_int(&writer, INT64_C(-2147483648))); + TEST_SIMPLE_WRITE("\xd2\x80\x00\x00\x00", mpack_write_int(&writer, MPACK_INT64_C(-2147483648))); - TEST_SIMPLE_WRITE("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", mpack_write_int(&writer, INT64_C(-2147483649))); - TEST_SIMPLE_WRITE("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", mpack_write_int(&writer, INT64_MIN)); + TEST_SIMPLE_WRITE("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", mpack_write_int(&writer, MPACK_INT64_C(-2147483649))); + TEST_SIMPLE_WRITE("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", mpack_write_int(&writer, MPACK_INT64_MIN)); } // writes ints using the sized iXX()/uXX() functions -static void test_write_simple_size_int_fixnums() { - char buf[4096]; +static void test_write_simple_size_int_fixnums(void) { + mpack_writer_t writer; // positive fixnums TEST_SIMPLE_WRITE("\x00", mpack_write_u8(&writer, 0)); @@ -173,8 +177,8 @@ static void test_write_simple_size_int_fixnums() { TEST_SIMPLE_WRITE("\xe0", mpack_write_i64(&writer, -32)); } -static void test_write_simple_size_int() { - char buf[4096]; +static void test_write_simple_size_int(void) { + mpack_writer_t writer; // uints TEST_SIMPLE_WRITE("\xcc\x80", mpack_write_u8(&writer, 0x80)); @@ -195,8 +199,8 @@ static void test_write_simple_size_int() { TEST_SIMPLE_WRITE("\xcd\xff\xff", mpack_write_u64(&writer, 0xffff)); TEST_SIMPLE_WRITE("\xce\x00\x01\x00\x00", mpack_write_u64(&writer, 0x10000)); TEST_SIMPLE_WRITE("\xce\xff\xff\xff\xff", mpack_write_u64(&writer, 0xffffffff)); - TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_u64(&writer, UINT64_C(0x100000000))); - TEST_SIMPLE_WRITE("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_write_u64(&writer, UINT64_C(0xffffffffffffffff))); + TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_u64(&writer, MPACK_UINT64_C(0x100000000))); + TEST_SIMPLE_WRITE("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_write_u64(&writer, MPACK_UINT64_C(0xffffffffffffffff))); // positive ints with signed value TEST_SIMPLE_WRITE("\xcc\x80", mpack_write_i16(&writer, 0x80)); @@ -217,9 +221,9 @@ static void test_write_simple_size_int() { TEST_SIMPLE_WRITE("\xcd\xff\xff", mpack_write_i64(&writer, 0xffff)); TEST_SIMPLE_WRITE("\xce\x00\x01\x00\x00", mpack_write_i64(&writer, 0x10000)); TEST_SIMPLE_WRITE("\xce\x7f\xff\xff\xff", mpack_write_i64(&writer, 0x7fffffff)); - TEST_SIMPLE_WRITE("\xce\xff\xff\xff\xff", mpack_write_i64(&writer, INT64_C(0xffffffff))); - TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_i64(&writer, UINT64_C(0x100000000))); - TEST_SIMPLE_WRITE("\xcf\x7f\xff\xff\xff\xff\xff\xff\xff", mpack_write_i64(&writer, UINT64_C(0x7fffffffffffffff))); + TEST_SIMPLE_WRITE("\xce\xff\xff\xff\xff", mpack_write_i64(&writer, MPACK_INT64_C(0xffffffff))); + TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_i64(&writer, MPACK_UINT64_C(0x100000000))); + TEST_SIMPLE_WRITE("\xcf\x7f\xff\xff\xff\xff\xff\xff\xff", mpack_write_i64(&writer, MPACK_UINT64_C(0x7fffffffffffffff))); // negative ints TEST_SIMPLE_WRITE("\xd0\xdf", mpack_write_i8(&writer, -33)); @@ -236,22 +240,22 @@ static void test_write_simple_size_int() { // when using INT32_C() and compiling the test suite as c++, gcc complains: // error: this decimal constant is unsigned only in ISO C90 [-Werror] - TEST_SIMPLE_WRITE("\xd2\x80\x00\x00\x00", mpack_write_i32(&writer, INT64_C(-2147483648))); - TEST_SIMPLE_WRITE("\xd2\x80\x00\x00\x00", mpack_write_i64(&writer, INT64_C(-2147483648))); + TEST_SIMPLE_WRITE("\xd2\x80\x00\x00\x00", mpack_write_i32(&writer, MPACK_INT64_C(-2147483648))); + TEST_SIMPLE_WRITE("\xd2\x80\x00\x00\x00", mpack_write_i64(&writer, MPACK_INT64_C(-2147483648))); TEST_SIMPLE_WRITE("\xd0\xdf", mpack_write_i64(&writer, -33)); TEST_SIMPLE_WRITE("\xd0\x80", mpack_write_i64(&writer, -128)); TEST_SIMPLE_WRITE("\xd1\xff\x7f", mpack_write_i64(&writer, -129)); TEST_SIMPLE_WRITE("\xd1\x80\x00", mpack_write_i64(&writer, -32768)); TEST_SIMPLE_WRITE("\xd2\xff\xff\x7f\xff", mpack_write_i64(&writer, -32769)); - TEST_SIMPLE_WRITE("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", mpack_write_i64(&writer, INT64_C(-2147483649))); - TEST_SIMPLE_WRITE("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", mpack_write_i64(&writer, INT64_MIN)); + TEST_SIMPLE_WRITE("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", mpack_write_i64(&writer, MPACK_INT64_C(-2147483649))); + TEST_SIMPLE_WRITE("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", mpack_write_i64(&writer, MPACK_INT64_MIN)); } // writes ints using the dynamic tag writer function -static void test_write_simple_tag_int() { - char buf[4096]; +static void test_write_simple_tag_int(void) { + mpack_writer_t writer; // positive fixnums TEST_SIMPLE_WRITE("\x00", mpack_write_tag(&writer, mpack_tag_uint(0))); @@ -282,8 +286,8 @@ static void test_write_simple_tag_int() { TEST_SIMPLE_WRITE("\xcd\xff\xff", mpack_write_tag(&writer, mpack_tag_uint(0xffff))); TEST_SIMPLE_WRITE("\xce\x00\x01\x00\x00", mpack_write_tag(&writer, mpack_tag_uint(0x10000))); TEST_SIMPLE_WRITE("\xce\xff\xff\xff\xff", mpack_write_tag(&writer, mpack_tag_uint(0xffffffff))); - TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_tag(&writer, mpack_tag_uint(UINT64_C(0x100000000)))); - TEST_SIMPLE_WRITE("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_write_tag(&writer, mpack_tag_uint(UINT64_C(0xffffffffffffffff)))); + TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_tag(&writer, mpack_tag_uint(MPACK_UINT64_C(0x100000000)))); + TEST_SIMPLE_WRITE("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_write_tag(&writer, mpack_tag_uint(MPACK_UINT64_C(0xffffffffffffffff)))); // positive ints with signed value TEST_SIMPLE_WRITE("\xcc\x80", mpack_write_tag(&writer, mpack_tag_int(0x80))); @@ -291,9 +295,9 @@ static void test_write_simple_tag_int() { TEST_SIMPLE_WRITE("\xcd\x01\x00", mpack_write_tag(&writer, mpack_tag_int(0x100))); TEST_SIMPLE_WRITE("\xcd\xff\xff", mpack_write_tag(&writer, mpack_tag_int(0xffff))); TEST_SIMPLE_WRITE("\xce\x00\x01\x00\x00", mpack_write_tag(&writer, mpack_tag_int(0x10000))); - TEST_SIMPLE_WRITE("\xce\xff\xff\xff\xff", mpack_write_tag(&writer, mpack_tag_int(INT64_C(0xffffffff)))); - TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_tag(&writer, mpack_tag_int(INT64_C(0x100000000)))); - TEST_SIMPLE_WRITE("\xcf\x7f\xff\xff\xff\xff\xff\xff\xff", mpack_write_tag(&writer, mpack_tag_int(INT64_C(0x7fffffffffffffff)))); + TEST_SIMPLE_WRITE("\xce\xff\xff\xff\xff", mpack_write_tag(&writer, mpack_tag_int(MPACK_INT64_C(0xffffffff)))); + TEST_SIMPLE_WRITE("\xcf\x00\x00\x00\x01\x00\x00\x00\x00", mpack_write_tag(&writer, mpack_tag_int(MPACK_INT64_C(0x100000000)))); + TEST_SIMPLE_WRITE("\xcf\x7f\xff\xff\xff\xff\xff\xff\xff", mpack_write_tag(&writer, mpack_tag_int(MPACK_INT64_C(0x7fffffffffffffff)))); // ints TEST_SIMPLE_WRITE("\xd0\xdf", mpack_write_tag(&writer, mpack_tag_int(-33))); @@ -304,15 +308,15 @@ static void test_write_simple_tag_int() { // when using INT32_C() and compiling the test suite as c++, gcc complains: // error: this decimal constant is unsigned only in ISO C90 [-Werror] - TEST_SIMPLE_WRITE("\xd2\x80\x00\x00\x00", mpack_write_tag(&writer, mpack_tag_int(INT64_C(-2147483648)))); + TEST_SIMPLE_WRITE("\xd2\x80\x00\x00\x00", mpack_write_tag(&writer, mpack_tag_int(MPACK_INT64_C(-2147483648)))); - TEST_SIMPLE_WRITE("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", mpack_write_tag(&writer, mpack_tag_int(INT64_C(-2147483649)))); - TEST_SIMPLE_WRITE("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", mpack_write_tag(&writer, mpack_tag_int(INT64_MIN))); + TEST_SIMPLE_WRITE("\xd3\xff\xff\xff\xff\x7f\xff\xff\xff", mpack_write_tag(&writer, mpack_tag_int(MPACK_INT64_C(-2147483649)))); + TEST_SIMPLE_WRITE("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", mpack_write_tag(&writer, mpack_tag_int(MPACK_INT64_MIN))); } -static void test_write_simple_misc() { - char buf[4096]; +static void test_write_simple_misc(void) { + mpack_writer_t writer; TEST_SIMPLE_WRITE("\xc0", mpack_write_nil(&writer)); TEST_SIMPLE_WRITE("\xc2", mpack_write_bool(&writer, false)); @@ -322,180 +326,264 @@ static void test_write_simple_misc() { // we just test a few floats for now. this could certainly be extended to // test more values like subnormal floats, infinities, etc. + #if MPACK_FLOAT TEST_SIMPLE_WRITE("\xca\x00\x00\x00\x00", mpack_write_float(&writer, 0.0f)); TEST_SIMPLE_WRITE("\xca\x40\x2d\xf3\xb6", mpack_write_float(&writer, 2.718f)); TEST_SIMPLE_WRITE("\xca\xc0\x2d\xf3\xb6", mpack_write_float(&writer, -2.718f)); + #endif + #if MPACK_DOUBLE TEST_SIMPLE_WRITE("\xcb\x00\x00\x00\x00\x00\x00\x00\x00", mpack_write_double(&writer, 0.0)); TEST_SIMPLE_WRITE("\xcb\x40\x09\x21\xfb\x53\xc8\xd4\xf1", mpack_write_double(&writer, 3.14159265)); TEST_SIMPLE_WRITE("\xcb\xc0\x09\x21\xfb\x53\xc8\xd4\xf1", mpack_write_double(&writer, -3.14159265)); + #endif + TEST_SIMPLE_WRITE("\xde\xad\xbe\xef", mpack_write_object_bytes(&writer, "\xde\xad\xbe\xef", 4)); + + #ifdef MPACK_MALLOC + // test writing nothing + char* growable_buf; + size_t size; + mpack_writer_init_growable(&writer, &growable_buf, &size); + TEST_WRITER_DESTROY_NOERROR(&writer); + TEST_TRUE(size == 0); + TEST_TRUE(growable_buf != NULL); + MPACK_FREE(growable_buf); + + // test growing by many steps at once (the initial buffer size during tests + // is 32, and the lipsum string is >700 characters) + mpack_writer_init_growable(&writer, &growable_buf, &size); + mpack_write_cstr(&writer, lipsum); + TEST_WRITER_DESTROY_NOERROR(&writer); + TEST_TRUE(size == MPACK_TAG_SIZE_STR16 + strlen(lipsum)); + TEST_TRUE(growable_buf[0] == '\xda'); + TEST_TRUE(mpack_load_u16(growable_buf + 1) == strlen(lipsum)); + TEST_TRUE(memcmp(growable_buf + MPACK_TAG_SIZE_STR16, lipsum, strlen(lipsum)) == 0); + MPACK_FREE(growable_buf); + #endif } +#if MPACK_EXTENSIONS +static void test_write_timestamp(void) { + mpack_writer_t writer; + + TEST_SIMPLE_WRITE("\xd6\xff\x00\x00\x00\x00", mpack_write_timestamp_seconds(&writer, 0)); + TEST_SIMPLE_WRITE("\xd6\xff\x00\x00\x01\x00", mpack_write_timestamp(&writer, 256, 0)); + TEST_SIMPLE_WRITE("\xd6\xff\xfe\xdc\xba\x98", mpack_write_timestamp_seconds(&writer, 4275878552u)); + TEST_SIMPLE_WRITE("\xd6\xff\xff\xff\xff\xff", mpack_write_timestamp(&writer, MPACK_UINT32_MAX, 0)); + + TEST_SIMPLE_WRITE("\xd7\xff\x00\x00\x00\x03\x00\x00\x00\x00", + mpack_write_timestamp_seconds(&writer, MPACK_INT64_C(12884901888))); + TEST_SIMPLE_WRITE("\xd7\xff\xee\x6b\x27\xfc\x00\x00\x00\x00", + mpack_write_timestamp(&writer, 0, MPACK_TIMESTAMP_NANOSECONDS_MAX)); + TEST_SIMPLE_WRITE("\xd7\xff\xee\x6b\x27\xff\xff\xff\xff\xff", + mpack_write_timestamp(&writer, MPACK_INT64_C(17179869183), MPACK_TIMESTAMP_NANOSECONDS_MAX)); + + TEST_SIMPLE_WRITE("\xc7\x0c\xff\x00\x00\x00\x01\xff\xff\xff\xff\xff\xff\xff\xff", + mpack_write_timestamp(&writer, -1, 1)); + mpack_timestamp_t timestamp = {MPACK_INT64_MAX, MPACK_TIMESTAMP_NANOSECONDS_MAX}; + TEST_SIMPLE_WRITE("\xc7\x0c\xff\x3b\x9a\xc9\xff\x7f\xff\xff\xff\xff\xff\xff\xff", + mpack_write_timestamp_struct(&writer, timestamp)); + TEST_SIMPLE_WRITE("\xc7\x0c\xff\x3b\x9a\xc9\xff\x80\x00\x00\x00\x00\x00\x00\x00", + mpack_write_timestamp(&writer, MPACK_INT64_MIN, MPACK_TIMESTAMP_NANOSECONDS_MAX)); + + mpack_writer_init(&writer, buf, sizeof(buf)); + TEST_BREAK((mpack_write_timestamp(&writer, 0, 1000000000), true)); + TEST_BREAK((mpack_write_timestamp(&writer, 0, MPACK_UINT32_MAX), true)); + TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_bug); +} +#endif + #ifdef MPACK_MALLOC -static void test_write_basic_structures() { - char* buf; +static void test_write_tag_tracking(void) { + char* out; + size_t size; + mpack_writer_t writer; + mpack_writer_init_growable(&writer, &out, &size); + + mpack_start_array(&writer, 8); + mpack_write_tag(&writer, mpack_tag_nil()); + mpack_write_tag(&writer, mpack_tag_bool(true)); + mpack_write_tag(&writer, mpack_tag_bool(false)); + mpack_write_tag(&writer, mpack_tag_uint(4)); + mpack_write_tag(&writer, mpack_tag_int(-3)); + mpack_write_tag(&writer, mpack_tag_str(0)); + mpack_finish_str(&writer); + mpack_write_tag(&writer, mpack_tag_bin(0)); + mpack_finish_bin(&writer); + mpack_write_tag(&writer, mpack_tag_array(1)); + mpack_write_tag(&writer, mpack_tag_nil()); + mpack_finish_array(&writer); + mpack_finish_array(&writer); + + TEST_DESTROY_MATCH(out, "\x98\xC0\xC3\xC2\x04\xFD\xA0\xC4\x00\x91\xC0"); +} + +static void test_write_basic_structures(void) { + char* out; size_t size; mpack_writer_t writer; + int i; // we use a mix of int writers below to test their tracking. // [] - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_array(&writer, 0); mpack_finish_array(&writer); - TEST_DESTROY_MATCH("\x90"); + TEST_DESTROY_MATCH(out, "\x90"); // [nil] - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_array(&writer, 1); mpack_write_nil(&writer); mpack_finish_array(&writer); - TEST_DESTROY_MATCH("\x91\xc0"); + TEST_DESTROY_MATCH(out, "\x91\xc0"); // range(15) - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_array(&writer, 15); - for (int i = 0; i < 15; ++i) + for (i = 0; i < 15; ++i) mpack_write_i32(&writer, i); mpack_finish_array(&writer); - TEST_DESTROY_MATCH( + TEST_DESTROY_MATCH(out, "\x9f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" ); // range(16) (larger than infix) - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_array(&writer, 16); - for (int i = 0; i < 16; ++i) + for (i = 0; i < 16; ++i) mpack_write_u32(&writer, (uint32_t)i); mpack_finish_array(&writer); - TEST_DESTROY_MATCH( + TEST_DESTROY_MATCH(out, "\xdc\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c" "\x0d\x0e\x0f" ); - // UINT16_MAX nils - mpack_writer_init_growable(&writer, &buf, &size); - mpack_start_array(&writer, UINT16_MAX); - for (int i = 0; i < UINT16_MAX; ++i) + // MPACK_UINT16_MAX nils + mpack_writer_init_growable(&writer, &out, &size); + mpack_start_array(&writer, MPACK_UINT16_MAX); + for (i = 0; i < MPACK_UINT16_MAX; ++i) mpack_write_nil(&writer); mpack_finish_array(&writer); { const char prefix[] = "\xdc\xff\xff"; TEST_WRITER_DESTROY_NOERROR(&writer); - TEST_TRUE(memcmp(prefix, buf, sizeof(prefix)-1) == 0, "array prefix is incorrect"); - TEST_TRUE(size == UINT16_MAX + sizeof(prefix)-1); + TEST_TRUE(memcmp(prefix, out, sizeof(prefix)-1) == 0, "array prefix is incorrect"); + TEST_TRUE(size == MPACK_UINT16_MAX + sizeof(prefix)-1); } - if (buf) - MPACK_FREE(buf); + if (out) + MPACK_FREE(out); - // UINT16_MAX+1 nils (largest category) - mpack_writer_init_growable(&writer, &buf, &size); - mpack_start_array(&writer, UINT16_MAX+1); - for (int i = 0; i < UINT16_MAX+1; ++i) + // MPACK_UINT16_MAX+1 nils (largest category) + mpack_writer_init_growable(&writer, &out, &size); + mpack_start_array(&writer, MPACK_UINT16_MAX+1); + for (i = 0; i < MPACK_UINT16_MAX+1; ++i) mpack_write_nil(&writer); mpack_finish_array(&writer); { const char prefix[] = "\xdd\x00\x01\x00\x00"; TEST_WRITER_DESTROY_NOERROR(&writer); - TEST_TRUE(memcmp(prefix, buf, sizeof(prefix)-1) == 0, "array prefix is incorrect"); - TEST_TRUE(size == UINT16_MAX+1 + sizeof(prefix)-1); + TEST_TRUE(memcmp(prefix, out, sizeof(prefix)-1) == 0, "array prefix is incorrect"); + TEST_TRUE(size == MPACK_UINT16_MAX+1 + sizeof(prefix)-1); } - if (buf) - MPACK_FREE(buf); + if (out) + MPACK_FREE(out); // {} - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_map(&writer, 0); mpack_finish_map(&writer); - TEST_DESTROY_MATCH("\x80"); + TEST_DESTROY_MATCH(out, "\x80"); // {nil:nil} - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_map(&writer, 1); mpack_write_nil(&writer); mpack_write_nil(&writer); mpack_finish_map(&writer); - TEST_DESTROY_MATCH("\x81\xc0\xc0"); + TEST_DESTROY_MATCH(out, "\x81\xc0\xc0"); // {0:0,1:1} - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_map(&writer, 2); mpack_write_i8(&writer, 0); mpack_write_i16(&writer, 0); mpack_write_u8(&writer, 1); mpack_write_u16(&writer, 1); mpack_finish_map(&writer); - TEST_DESTROY_MATCH("\x82\x00\x00\x01\x01"); + TEST_DESTROY_MATCH(out, "\x82\x00\x00\x01\x01"); // {0:1, 2:3, ..., 28:29} - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_map(&writer, 15); - for (int i = 0; i < 30; ++i) + for (i = 0; i < 30; ++i) mpack_write_i8(&writer, (int8_t)i); mpack_finish_map(&writer); - TEST_DESTROY_MATCH( + TEST_DESTROY_MATCH(out, "\x8f\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e" "\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d" ); // {0:1, 2:3, ..., 28:29, 30:31} (larger than infix) - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_map(&writer, 16); - for (int i = 0; i < 32; ++i) + for (i = 0; i < 32; ++i) mpack_write_int(&writer, i); mpack_finish_map(&writer); - TEST_DESTROY_MATCH( + TEST_DESTROY_MATCH(out, "\xde\x00\x10\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c" "\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c" "\x1d\x1e\x1f" ); - // UINT16_MAX nil:nils - mpack_writer_init_growable(&writer, &buf, &size); - mpack_start_map(&writer, UINT16_MAX); - for (int i = 0; i < UINT16_MAX*2; ++i) + // MPACK_UINT16_MAX nil:nils + mpack_writer_init_growable(&writer, &out, &size); + mpack_start_map(&writer, MPACK_UINT16_MAX); + for (i = 0; i < MPACK_UINT16_MAX*2; ++i) mpack_write_nil(&writer); mpack_finish_map(&writer); { const char prefix[] = "\xde\xff\xff"; TEST_WRITER_DESTROY_NOERROR(&writer); - TEST_TRUE(memcmp(prefix, buf, sizeof(prefix)-1) == 0, "map prefix is incorrect"); - TEST_TRUE(size == UINT16_MAX*2 + sizeof(prefix)-1); + TEST_TRUE(memcmp(prefix, out, sizeof(prefix)-1) == 0, "map prefix is incorrect"); + TEST_TRUE(size == MPACK_UINT16_MAX*2 + sizeof(prefix)-1); } - if (buf) - MPACK_FREE(buf); + if (out) + MPACK_FREE(out); - // UINT16_MAX+1 nil:nils (largest category) - mpack_writer_init_growable(&writer, &buf, &size); - mpack_start_map(&writer, UINT16_MAX+1); - for (int i = 0; i < (UINT16_MAX+1)*2; ++i) + // MPACK_UINT16_MAX+1 nil:nils (largest category) + mpack_writer_init_growable(&writer, &out, &size); + mpack_start_map(&writer, MPACK_UINT16_MAX+1); + for (i = 0; i < (MPACK_UINT16_MAX+1)*2; ++i) mpack_write_nil(&writer); mpack_finish_map(&writer); { const char prefix[] = "\xdf\x00\x01\x00\x00"; TEST_WRITER_DESTROY_NOERROR(&writer); - TEST_TRUE(memcmp(prefix, buf, sizeof(prefix)-1) == 0, "map prefix is incorrect"); - TEST_TRUE(size == (UINT16_MAX+1)*2 + sizeof(prefix)-1); + TEST_TRUE(memcmp(prefix, out, sizeof(prefix)-1) == 0, "map prefix is incorrect"); + TEST_TRUE(size == (MPACK_UINT16_MAX+1)*2 + sizeof(prefix)-1); } - if (buf) - MPACK_FREE(buf); + if (out) + MPACK_FREE(out); } -static void test_write_small_structure_trees() { - char* buf; +static void test_write_small_structure_trees(void) { + char* out; size_t size; mpack_writer_t writer; + int i; // [[]] - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_array(&writer, 1); mpack_start_array(&writer, 0); mpack_finish_array(&writer); mpack_finish_array(&writer); - TEST_DESTROY_MATCH("\x91\x90"); + TEST_DESTROY_MATCH(out, "\x91\x90"); // [[], [0], [1, 2]] - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_array(&writer, 3); mpack_start_array(&writer, 0); mpack_finish_array(&writer); @@ -507,10 +595,10 @@ static void test_write_small_structure_trees() { mpack_write_int(&writer, 2); mpack_finish_array(&writer); mpack_finish_array(&writer); - TEST_DESTROY_MATCH("\x93\x90\x91\x00\x92\x01\x02"); + TEST_DESTROY_MATCH(out, "\x93\x90\x91\x00\x92\x01\x02"); // miscellaneous tree of arrays of various small sizes - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_array(&writer, 5); mpack_start_array(&writer, 0); @@ -529,18 +617,18 @@ static void test_write_small_structure_trees() { mpack_finish_array(&writer); mpack_start_array(&writer, 15); - for (int i = 0; i < 15; ++i) + for (i = 0; i < 15; ++i) mpack_write_int(&writer, i); mpack_finish_array(&writer); mpack_start_array(&writer, 16); - for (int i = 0; i < 16; ++i) + for (i = 0; i < 16; ++i) mpack_write_int(&writer, i); mpack_finish_array(&writer); mpack_finish_array(&writer); - TEST_DESTROY_MATCH( + TEST_DESTROY_MATCH(out, "\x95\x90\x91\xc0\x92\x90\x91\xc0\x9f\x00\x01\x02\x03\x04\x05\x06" "\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\xdc\x00\x10\x00\x01\x02\x03\x04" "\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" @@ -548,7 +636,7 @@ static void test_write_small_structure_trees() { // miscellaneous tree of maps of various small sizes - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_map(&writer, 5); mpack_write_int(&writer, 0); @@ -575,7 +663,7 @@ static void test_write_small_structure_trees() { mpack_write_int(&writer, 3); mpack_start_map(&writer, 15); - for (int i = 0; i < 15; ++i) { + for (i = 0; i < 15; ++i) { mpack_write_int(&writer, i); mpack_write_int(&writer, i); } @@ -583,7 +671,7 @@ static void test_write_small_structure_trees() { mpack_write_int(&writer, 4); mpack_start_map(&writer, 16); - for (int i = 0; i < 16; ++i) { + for (i = 0; i < 16; ++i) { mpack_write_int(&writer, i); mpack_write_int(&writer, i); } @@ -591,7 +679,7 @@ static void test_write_small_structure_trees() { mpack_finish_map(&writer); - TEST_DESTROY_MATCH( + TEST_DESTROY_MATCH(out, "\x85\x00\x80\x01\x81\x00\xc0\x02\x82\x00\x80\x01\x81\xc0\xc0\x03" "\x8f\x00\x00\x01\x01\x02\x02\x03\x03\x04\x04\x05\x05\x06\x06\x07" "\x07\x08\x08\x09\x09\x0a\x0a\x0b\x0b\x0c\x0c\x0d\x0d\x0e\x0e\x04" @@ -602,7 +690,7 @@ static void test_write_small_structure_trees() { // miscellaneous mix of maps and arrays of various small sizes - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); mpack_start_map(&writer, 5); mpack_write_int(&writer, -47); @@ -640,7 +728,7 @@ static void test_write_small_structure_trees() { mpack_finish_map(&writer); - TEST_DESTROY_MATCH( + TEST_DESTROY_MATCH(out, "\x85\xd0\xd1\x91\xc0\x90\x81\xc0\x00\xc0\x82\xc0\x90\x04\x05\xa5" "\x68\x65\x6c\x6c\x6f\x93\xa7\x62\x6f\x6e\x6a\x6f\x75\x72\xc0\xff" "\x91\x5c\xcd\x01\x5e" @@ -655,7 +743,7 @@ static bool test_write_deep_growth(void) { // stack grow properly. we allow mpack_error_memory as an // error (since it will be simulated by the failure system.) - char* buf; + char* out; size_t size; mpack_writer_t writer; @@ -664,15 +752,15 @@ static bool test_write_deep_growth(void) { TEST_TRUE(test_write_error == mpack_error_memory, "writer error handler was not called?"); \ test_write_error = mpack_ok; \ mpack_writer_destroy(&writer); \ - TEST_TRUE(buf == NULL); \ + TEST_TRUE(out == NULL); \ return false; \ } \ } while (0) - mpack_writer_init_growable(&writer, &buf, &size); + mpack_writer_init_growable(&writer, &out, &size); if (mpack_writer_error(&writer) == mpack_error_memory) { mpack_writer_destroy(&writer); - TEST_TRUE(buf == NULL); + TEST_TRUE(out == NULL); return false; } @@ -682,21 +770,22 @@ static bool test_write_deep_growth(void) { const int depth = 40; const int nums = 1000; - for (int i = 0; i < depth; ++i) { + int i; + for (i = 0; i < depth; ++i) { mpack_start_array(&writer, 1); TEST_POSSIBLE_FAILURE(); } - mpack_start_array(&writer, nums); + mpack_start_array(&writer, (uint32_t)nums); TEST_POSSIBLE_FAILURE(); - for (int i = 0; i < nums; ++i) { - mpack_write_u64(&writer, UINT64_MAX); + for (i = 0; i < nums; ++i) { + mpack_write_u64(&writer, MPACK_UINT64_MAX); TEST_POSSIBLE_FAILURE(); } mpack_finish_array(&writer); TEST_POSSIBLE_FAILURE(); - for (int i = 0; i < depth; ++i) { + for (i = 0; i < depth; ++i) { mpack_finish_array(&writer); TEST_POSSIBLE_FAILURE(); } @@ -705,11 +794,11 @@ static bool test_write_deep_growth(void) { mpack_error_t error = mpack_writer_destroy(&writer); if (error == mpack_ok) { - MPACK_FREE(buf); + MPACK_FREE(out); return true; } if (error == mpack_error_memory) { - TEST_TRUE(buf == NULL); + TEST_TRUE(out == NULL); return false; } TEST_TRUE(false, "unexpected error state %i (%s)", (int)error, mpack_error_to_string(error)); @@ -719,8 +808,7 @@ static bool test_write_deep_growth(void) { #endif #if MPACK_WRITE_TRACKING -static void test_write_tracking() { - char buf[4096]; +static void test_write_tracking(void) { mpack_writer_t writer; // cancel @@ -741,6 +829,12 @@ static void test_write_tracking() { TEST_BREAK((mpack_finish_array(&writer), true)); TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_bug); + // closing wrong type + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_start_array(&writer, 0); + TEST_BREAK((mpack_finish_map(&writer), true)); + TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_bug); + // writing elements in a string mpack_writer_init(&writer, buf, sizeof(buf)); mpack_start_str(&writer, 50); @@ -773,12 +867,145 @@ static void test_write_tracking() { } #endif +#if MPACK_HAS_GENERIC +static void test_write_generic(void) { + mpack_writer_t writer; + + // int8 + TEST_SIMPLE_WRITE("\x7f", mpack_write(&writer, (int8_t)MPACK_INT8_MAX)); + TEST_SIMPLE_WRITE("\x01", mpack_write(&writer, (int8_t)1)); + TEST_SIMPLE_WRITE("\x00", mpack_write(&writer, (int8_t)0)); + TEST_SIMPLE_WRITE("\xd0\x80", mpack_write(&writer, (int8_t)MPACK_INT8_MIN)); + + // int16 + TEST_SIMPLE_WRITE("\xcd\x7f\xff", mpack_write(&writer, (int16_t)MPACK_INT16_MAX)); + TEST_SIMPLE_WRITE("\x7f", mpack_write(&writer, (int16_t)MPACK_INT8_MAX)); + TEST_SIMPLE_WRITE("\x00", mpack_write(&writer, (int16_t)0)); + TEST_SIMPLE_WRITE("\xd0\x80", mpack_write(&writer, (int16_t)MPACK_INT8_MIN)); + TEST_SIMPLE_WRITE("\xd1\x80\x00", mpack_write(&writer, (int16_t)MPACK_INT16_MIN)); + + // int32 + TEST_SIMPLE_WRITE("\xce\x7f\xff\xff\xff", mpack_write(&writer, (int32_t)MPACK_INT32_MAX)); + TEST_SIMPLE_WRITE("\xcd\x7f\xff", mpack_write(&writer, (int32_t)MPACK_INT16_MAX)); + TEST_SIMPLE_WRITE("\x7f", mpack_write(&writer, (int32_t)MPACK_INT8_MAX)); + TEST_SIMPLE_WRITE("\x00", mpack_write(&writer, (int32_t)0)); + TEST_SIMPLE_WRITE("\xd0\x80", mpack_write(&writer, (int32_t)MPACK_INT8_MIN)); + TEST_SIMPLE_WRITE("\xd1\x80\x00", mpack_write(&writer, (int32_t)MPACK_INT16_MIN)); + TEST_SIMPLE_WRITE("\xd2\x80\x00\x00\x00", mpack_write(&writer, (int32_t)MPACK_INT32_MIN)); + + // int64 + TEST_SIMPLE_WRITE("\xcf\x7f\xff\xff\xff\xff\xff\xff\xff", mpack_write(&writer, (int64_t)MPACK_INT64_MAX)); + TEST_SIMPLE_WRITE("\xce\x7f\xff\xff\xff", mpack_write(&writer, (int64_t)MPACK_INT32_MAX)); + TEST_SIMPLE_WRITE("\xcd\x7f\xff", mpack_write(&writer, (int64_t)MPACK_INT16_MAX)); + TEST_SIMPLE_WRITE("\x7f", mpack_write(&writer, (int64_t)MPACK_INT8_MAX)); + TEST_SIMPLE_WRITE("\x00", mpack_write(&writer, (int64_t)0)); + TEST_SIMPLE_WRITE("\xd0\x80", mpack_write(&writer, (int64_t)MPACK_INT8_MIN)); + TEST_SIMPLE_WRITE("\xd1\x80\x00", mpack_write(&writer, (int64_t)MPACK_INT16_MIN)); + TEST_SIMPLE_WRITE("\xd2\x80\x00\x00\x00", mpack_write(&writer, (int64_t)MPACK_INT32_MIN)); + TEST_SIMPLE_WRITE("\xd3\x80\x00\x00\x00\x00\x00\x00\x00", mpack_write(&writer, (int64_t)MPACK_INT64_MIN)); + + // uint8 + TEST_SIMPLE_WRITE("\x00", mpack_write(&writer, (uint8_t)0)); + TEST_SIMPLE_WRITE("\x7f", mpack_write(&writer, (uint8_t)127)); + TEST_SIMPLE_WRITE("\xcc\xff", mpack_write(&writer, (uint8_t)MPACK_UINT8_MAX)); + + // uint16 + TEST_SIMPLE_WRITE("\x00", mpack_write(&writer, (uint16_t)0)); + TEST_SIMPLE_WRITE("\x7f", mpack_write(&writer, (uint16_t)127)); + TEST_SIMPLE_WRITE("\xcc\xff", mpack_write(&writer, (uint16_t)MPACK_UINT8_MAX)); + TEST_SIMPLE_WRITE("\xcd\xff\xff", mpack_write(&writer, (uint16_t)MPACK_UINT16_MAX)); + + // uint32 + TEST_SIMPLE_WRITE("\x00", mpack_write(&writer, (uint32_t)0)); + TEST_SIMPLE_WRITE("\x7f", mpack_write(&writer, (uint32_t)127)); + TEST_SIMPLE_WRITE("\xcc\xff", mpack_write(&writer, (uint32_t)MPACK_UINT8_MAX)); + TEST_SIMPLE_WRITE("\xcd\xff\xff", mpack_write(&writer, (uint32_t)MPACK_UINT16_MAX)); + TEST_SIMPLE_WRITE("\xce\xff\xff\xff\xff", mpack_write(&writer, (uint32_t)MPACK_UINT32_MAX)); + + // uint64 + TEST_SIMPLE_WRITE("\x00", mpack_write(&writer, (uint64_t)0)); + TEST_SIMPLE_WRITE("\x7f", mpack_write(&writer, (uint64_t)127)); + TEST_SIMPLE_WRITE("\xcc\xff", mpack_write(&writer, (uint64_t)MPACK_UINT8_MAX)); + TEST_SIMPLE_WRITE("\xcd\xff\xff", mpack_write(&writer, (uint64_t)MPACK_UINT16_MAX)); + TEST_SIMPLE_WRITE("\xce\xff\xff\xff\xff", mpack_write(&writer, (uint64_t)MPACK_UINT32_MAX)); + TEST_SIMPLE_WRITE("\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_write(&writer, (uint64_t)MPACK_UINT64_MAX)); + + // float and double + // TODO: we just test a few floats for now. this could certainly be extended to + // test more values like subnormal floats, infinities, etc. + #if MPACK_FLOAT + TEST_SIMPLE_WRITE("\xca\x00\x00\x00\x00", mpack_write(&writer, (float)0.0f)); + TEST_SIMPLE_WRITE("\xca\x40\x2d\xf3\xb6", mpack_write(&writer, (float)2.718f)); + TEST_SIMPLE_WRITE("\xca\xc0\x2d\xf3\xb6", mpack_write(&writer, (float)-2.718f)); + #endif + #if MPACK_DOUBLE + TEST_SIMPLE_WRITE("\xcb\x00\x00\x00\x00\x00\x00\x00\x00", mpack_write(&writer, (double)0.0)); + TEST_SIMPLE_WRITE("\xcb\x40\x09\x21\xfb\x53\xc8\xd4\xf1", mpack_write(&writer, (double)3.14159265)); + TEST_SIMPLE_WRITE("\xcb\xc0\x09\x21\xfb\x53\xc8\xd4\xf1", mpack_write(&writer, (double)-3.14159265)); + #endif + + // bool + // TODO: when we pass direct true or false into the _Generic it seems not to emit the correct stream + bool b = false; + TEST_SIMPLE_WRITE("\xc2", mpack_write(&writer, b)); + b = true; + TEST_SIMPLE_WRITE("\xc3", mpack_write(&writer, b)); + + // char * + TEST_SIMPLE_WRITE("\xc0", mpack_write(&writer, (char *)NULL)); + TEST_SIMPLE_WRITE("\xa4""1337", mpack_write(&writer, (char *)"1337")); + + // const char * + TEST_SIMPLE_WRITE("\xc0", mpack_write(&writer, (const char *)NULL)); + TEST_SIMPLE_WRITE("\xa4""1337", mpack_write(&writer, (const char *)"1337")); + + // string literals + TEST_SIMPLE_WRITE("\xa0", mpack_write(&writer, "")); + TEST_SIMPLE_WRITE("\xa4""1337", mpack_write(&writer, "1337")); +} + +static void test_write_generic_kv(void) { + mpack_writer_t writer; + char key[] = "foo"; + char value[] = "bar"; + + // int8, int16, int32, int64 + TEST_SIMPLE_WRITE("\xa3""foo""\x7f", mpack_write_kv(&writer, key, (int8_t)MPACK_INT8_MAX)); + TEST_SIMPLE_WRITE("\xa3""foo""\xcd\x7f\xff", mpack_write_kv(&writer, key, (int16_t)MPACK_INT16_MAX)); + TEST_SIMPLE_WRITE("\xa3""foo""\xce\x7f\xff\xff\xff", mpack_write_kv(&writer, key, (int32_t)MPACK_INT32_MAX)); + TEST_SIMPLE_WRITE("\xa3""foo""\xcf\x7f\xff\xff\xff\xff\xff\xff\xff", mpack_write_kv(&writer, key, (int64_t)MPACK_INT64_MAX)); + + // uint8, uint16, uint32, uint64 + TEST_SIMPLE_WRITE("\xa3""foo""\xcc\xff", mpack_write_kv(&writer, key, (uint8_t)MPACK_UINT8_MAX)); + TEST_SIMPLE_WRITE("\xa3""foo""\xcd\xff\xff", mpack_write_kv(&writer, key, (uint16_t)MPACK_UINT16_MAX)); + TEST_SIMPLE_WRITE("\xa3""foo""\xce\xff\xff\xff\xff", mpack_write_kv(&writer, key, (uint64_t)MPACK_UINT32_MAX)); + TEST_SIMPLE_WRITE("\xa3""foo""\xcf\xff\xff\xff\xff\xff\xff\xff\xff", mpack_write_kv(&writer, key, (uint64_t)MPACK_UINT64_MAX)); + + // float, double and bool + #if MPACK_FLOAT + TEST_SIMPLE_WRITE("\xa3""foo""\xca\xc0\x2d\xf3\xb6", mpack_write_kv(&writer, key, (float)-2.718f)); + #endif + #if MPACK_DOUBLE + TEST_SIMPLE_WRITE("\xa3""foo""\xcb\xc0\x09\x21\xfb\x53\xc8\xd4\xf1", mpack_write_kv(&writer, key, (double)-3.14159265)); + #endif + TEST_SIMPLE_WRITE("\xa3""foo""\xc2", mpack_write_kv(&writer, key, (bool)false)); + + // char *, const char *, literal + TEST_SIMPLE_WRITE("\xa3""foo""\xa3""bar", mpack_write_kv(&writer, key, (char *)value)); + TEST_SIMPLE_WRITE("\xa3""foo""\xa3""bar", mpack_write_kv(&writer, key, (const char *)value)); + TEST_SIMPLE_WRITE("\xa3""foo""\xa3""bar", mpack_write_kv(&writer, key, value)); + TEST_SIMPLE_WRITE("\xa3""foo""\xa3""bar", mpack_write_kv(&writer, key, "bar")); +} + +#endif + static void test_write_utf8(void) { - char buf[4096]; + mpack_writer_t writer; // these test strings are mostly duplicated from test-expect.c, but // without the MessagePack header + const char empty_string[] = ""; const char utf8_null[] = "hello\x00world"; const char utf8_valid[] = " \xCF\x80 \xe4\xb8\xad \xf0\xa0\x80\xb6"; const char utf8_trimmed[] = "\xf0\xa0\x80\xb6"; @@ -790,16 +1017,24 @@ static void test_write_utf8(void) { const char utf8_cesu8[] = " \xED\xA0\x81\xED\xB0\x80 "; const char utf8_wobbly[] = " \xED\xA0\x81 "; + // An empty string should be written successfully + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_null, 0)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, NULL, 0)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, empty_string, 0)); + // all non-UTF-8 writers should write these strings without error - TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_null, sizeof(utf8_null)-1)); - TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_valid, sizeof(utf8_valid)-1)); - TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_trimmed, sizeof(utf8_trimmed)-1)); - TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_invalid, sizeof(utf8_invalid)-1)); - TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_invalid_trimmed, sizeof(utf8_invalid_trimmed)-1)); - TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_truncated, sizeof(utf8_truncated)-1)); - TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_modified, sizeof(utf8_modified)-1)); - TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_cesu8, sizeof(utf8_cesu8)-1)); - TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_wobbly, sizeof(utf8_wobbly)-1)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_null, (uint32_t)sizeof(utf8_null)-1)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_valid, (uint32_t)sizeof(utf8_valid)-1)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_trimmed, (uint32_t)sizeof(utf8_trimmed)-1)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_invalid, (uint32_t)sizeof(utf8_invalid)-1)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_invalid_trimmed, (uint32_t)sizeof(utf8_invalid_trimmed)-1)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_truncated, (uint32_t)sizeof(utf8_truncated)-1)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_modified, (uint32_t)sizeof(utf8_modified)-1)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_cesu8, (uint32_t)sizeof(utf8_cesu8)-1)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_str(&writer, utf8_wobbly, (uint32_t)sizeof(utf8_wobbly)-1)); + + // A c-string with 0 length should be accepted + TEST_SIMPLE_WRITE_NOERROR(mpack_write_cstr(&writer, empty_string)); // as should the non-UTF-8 cstr writers // (the utf8_null test here is writing up to the null-terminator which @@ -815,19 +1050,27 @@ static void test_write_utf8(void) { TEST_SIMPLE_WRITE_NOERROR(mpack_write_cstr(&writer, utf8_cesu8)); TEST_SIMPLE_WRITE_NOERROR(mpack_write_cstr(&writer, utf8_wobbly)); + // A NULL string or string with 0 length should be accepted + TEST_SIMPLE_WRITE_NOERROR(mpack_write_utf8(&writer, utf8_null, 0)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_utf8(&writer, NULL, 0)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_utf8(&writer, empty_string, 0)); + // test UTF-8 cstr writers // NUL is valid in UTF-8, so we allow it by the non-cstr API. (If you're // using it to write, you should also using some non-cstr API to // read, so the NUL will be safely handled.) - TEST_SIMPLE_WRITE_NOERROR(mpack_write_utf8(&writer, utf8_null, sizeof(utf8_null)-1)); - TEST_SIMPLE_WRITE_NOERROR(mpack_write_utf8(&writer, utf8_valid, sizeof(utf8_valid)-1)); - TEST_SIMPLE_WRITE_NOERROR(mpack_write_utf8(&writer, utf8_trimmed, sizeof(utf8_trimmed)-1)); - TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_invalid, sizeof(utf8_invalid)-1), mpack_error_invalid); - TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_invalid_trimmed, sizeof(utf8_invalid_trimmed)-1), mpack_error_invalid); - TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_truncated, sizeof(utf8_truncated)-1), mpack_error_invalid); - TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_modified, sizeof(utf8_modified)-1), mpack_error_invalid); - TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_cesu8, sizeof(utf8_cesu8)-1), mpack_error_invalid); - TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_wobbly, sizeof(utf8_wobbly)-1), mpack_error_invalid); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_utf8(&writer, utf8_null, (uint32_t)sizeof(utf8_null)-1)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_utf8(&writer, utf8_valid, (uint32_t)sizeof(utf8_valid)-1)); + TEST_SIMPLE_WRITE_NOERROR(mpack_write_utf8(&writer, utf8_trimmed, (uint32_t)sizeof(utf8_trimmed)-1)); + TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_invalid, (uint32_t)sizeof(utf8_invalid)-1), mpack_error_invalid); + TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_invalid_trimmed, (uint32_t)sizeof(utf8_invalid_trimmed)-1), mpack_error_invalid); + TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_truncated, (uint32_t)sizeof(utf8_truncated)-1), mpack_error_invalid); + TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_modified, (uint32_t)sizeof(utf8_modified)-1), mpack_error_invalid); + TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_cesu8, (uint32_t)sizeof(utf8_cesu8)-1), mpack_error_invalid); + TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8(&writer, utf8_wobbly, (uint32_t)sizeof(utf8_wobbly)-1), mpack_error_invalid); + + // A c-string with 0 length should be accepted + TEST_SIMPLE_WRITE_NOERROR(mpack_write_utf8_cstr(&writer, empty_string)); // test UTF-8 cstr writers TEST_SIMPLE_WRITE_NOERROR(mpack_write_utf8_cstr(&writer, utf8_null)); // again, up to null-terminator, which is valid... @@ -843,9 +1086,76 @@ static void test_write_utf8(void) { // some basic tests for utf8_cstr_or_nil TEST_SIMPLE_WRITE("\xa5hello", mpack_write_utf8_cstr_or_nil(&writer, "hello")); TEST_SIMPLE_WRITE("\xc0", mpack_write_utf8_cstr_or_nil(&writer, NULL)); + TEST_SIMPLE_WRITE("\xa0", mpack_write_utf8_cstr_or_nil(&writer, empty_string)); TEST_SIMPLE_WRITE_ERROR(mpack_write_utf8_cstr_or_nil(&writer, utf8_invalid), mpack_error_invalid); } +typedef struct test_write_flush_t { + char* out; + size_t capacity; + size_t count; +} test_write_flush_t; + +static void test_write_flush_callback(mpack_writer_t* writer, const char* buffer, size_t count) { + test_write_flush_t* flush = (test_write_flush_t*)writer->context; + if (count > flush->capacity - flush->count) { + mpack_writer_flag_error(writer, mpack_error_io); + return; + } + memcpy(flush->out + flush->count, buffer, count); + flush->count += count; +} + +static void test_write_flush_message(void) { + test_write_flush_t flush = {buf, sizeof(buf), 0}; + + mpack_writer_t writer; + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_writer_set_context(&writer, &flush); + mpack_writer_set_flush(&writer, &test_write_flush_callback); + + TEST_TRUE(flush.count == 0); + mpack_write_cstr(&writer, "hello world!"); + TEST_TRUE(flush.count == 0); + mpack_writer_flush_message(&writer); + TEST_TRUE(flush.count == 13); + + mpack_start_map(&writer, 2); + mpack_write_cstr(&writer, "a"); + mpack_write_int(&writer, 3); + mpack_write_nil(&writer); + mpack_write_true(&writer); + mpack_finish_map(&writer); + TEST_TRUE(flush.count == 13); + mpack_writer_flush_message(&writer); + TEST_TRUE(flush.count == 19); + + TEST_WRITER_DESTROY_NOERROR(&writer); + + // test break due to open message (if tracking is enabled) + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_writer_set_context(&writer, &flush); + mpack_writer_set_flush(&writer, &test_write_flush_callback); + mpack_start_map(&writer, 5); + #if MPACK_WRITE_TRACKING + TEST_BREAK((mpack_writer_flush_message(&writer), true)); + TEST_TRUE(mpack_writer_error(&writer) == mpack_error_bug); + mpack_writer_flush_message(&writer); // no-op + TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_bug); + #else + mpack_writer_flush_message(&writer); + TEST_WRITER_DESTROY_NOERROR(&writer); + #endif + + // test break due to lack of flush + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_write_cstr(&writer, "hello world!"); + TEST_BREAK((mpack_writer_flush_message(&writer), true)); + TEST_TRUE(mpack_writer_error(&writer) == mpack_error_bug); + mpack_writer_flush_message(&writer); // no-op + TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_bug); +} + static void test_misc(void) { // writing too much data without a flush callback @@ -853,10 +1163,86 @@ static void test_misc(void) { char shortbuf[10]; mpack_writer_t writer; mpack_writer_init(&writer, shortbuf, sizeof(shortbuf)); - mpack_write_cstr(&writer, "The quick brown fox jumps over the lazy dog."); + mpack_write_cstr(&writer, quick_brown_fox); TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_too_big); + #if MPACK_STDLIB + // writing strings larger than 32 bits should fail + if (MPACK_UINT32_MAX < SIZE_MAX) { + char single[1]; + + mpack_writer_init(&writer, single, SIZE_MAX); + test_system_mock_strlen((size_t)((uint64_t)MPACK_UINT32_MAX + MPACK_UINT64_C(1))); + mpack_write_cstr(&writer, quick_brown_fox); + TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_invalid); + + mpack_writer_init(&writer, single, SIZE_MAX); + test_system_mock_strlen(SIZE_MAX); + mpack_write_utf8_cstr(&writer, quick_brown_fox); + TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_invalid); + } + #endif + +} + +#if MPACK_COMPATIBILITY +static void test_write_compatibility(void) { + mpack_writer_t writer; + + // test str and bin behavior under all versions + + static const mpack_version_t versions[] = { + mpack_version_v4, + mpack_version_v5, + mpack_version_current, + }; + + size_t i; + for (i = 0; i < sizeof(versions) / sizeof(versions[0]); ++i) { + mpack_version_t version = versions[i]; + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_writer_set_version(&writer, version); + + mpack_start_array(&writer, 2); + mpack_write_cstr(&writer, quick_brown_fox); + mpack_write_bin(&writer, quick_brown_fox, (uint32_t)strlen(quick_brown_fox)); + mpack_finish_array(&writer); + + size_t size = mpack_writer_buffer_used(&writer); + TEST_WRITER_DESTROY_NOERROR(&writer); + + if (version == mpack_version_v4) { + if (size != 1 + 6 + 2 * strlen(quick_brown_fox)) { + TEST_TRUE(false, "incorrect length!"); + } else { + TEST_TRUE(memcmp("\x92", buf, 1) == 0); + TEST_TRUE(memcmp("\xda\x00\x2a", buf + 1, 3) == 0); + TEST_TRUE(memcmp(quick_brown_fox, buf + 1 + 3, strlen(quick_brown_fox)) == 0); + TEST_TRUE(memcmp("\xda\x00\x2a", buf + 1 + 3 + strlen(quick_brown_fox), 3) == 0); + TEST_TRUE(memcmp(quick_brown_fox, buf + 1 + 3 + 3 + strlen(quick_brown_fox), strlen(quick_brown_fox)) == 0); + } + } else { + if (size != 1 + 4 + 2 * strlen(quick_brown_fox)) { + TEST_TRUE(false, "incorrect length!"); + } else { + TEST_TRUE(memcmp("\x92", buf, 1) == 0); + TEST_TRUE(memcmp("\xd9\x2a", buf + 1, 2) == 0); + TEST_TRUE(memcmp(quick_brown_fox, buf + 1 + 2, strlen(quick_brown_fox)) == 0); + TEST_TRUE(memcmp("\xc4\x2a", buf + 1 + 2 + strlen(quick_brown_fox), 2) == 0); + TEST_TRUE(memcmp(quick_brown_fox, buf + 1 + 2 + 2 + strlen(quick_brown_fox), strlen(quick_brown_fox)) == 0); + } + } + } + + #if MPACK_EXTENSIONS + // test ext break in v4 mode + mpack_writer_init(&writer, buf, sizeof(buf)); + mpack_writer_set_version(&writer, mpack_version_v4); + TEST_BREAK(((void)mpack_start_ext(&writer, 1, 1), true)); + TEST_WRITER_DESTROY_ERROR(&writer, mpack_error_bug); + #endif } +#endif void test_writes() { /* @@ -872,10 +1258,22 @@ void test_writes() { test_write_simple_size_int_fixnums(); test_write_simple_size_int(); test_write_simple_tag_int(); + #if MPACK_HAS_GENERIC + test_write_generic(); + test_write_generic_kv(); + #endif test_write_simple_misc(); test_write_utf8(); + #if MPACK_EXTENSIONS + test_write_timestamp(); + #endif + + #if MPACK_COMPATIBILITY + test_write_compatibility(); + #endif #ifdef MPACK_MALLOC + test_write_tag_tracking(); test_write_basic_structures(); test_write_small_structure_trees(); test_system_fail_until_ok(&test_write_deep_growth); @@ -885,6 +1283,7 @@ void test_writes() { test_write_tracking(); #endif + test_write_flush_message(); test_misc(); } diff --git a/test/test-write.h b/test/unit/src/test-write.h similarity index 73% rename from test/test-write.h rename to test/unit/src/test-write.h index 046f74e..a1efeb9 100644 --- a/test/test-write.h +++ b/test/unit/src/test-write.h @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -39,54 +39,55 @@ void test_write_error_handler(mpack_writer_t* writer, mpack_error_t error); // tears down a writer, ensuring it didn't fail -#define TEST_WRITER_DESTROY_NOERROR(writer) \ - TEST_TRUE(mpack_writer_destroy(writer) == mpack_ok, \ - "writer is in error state %i", (int)mpack_writer_error(writer)); \ +#define TEST_WRITER_DESTROY_NOERROR(writer) do {\ + mpack_error_t error = mpack_writer_destroy(writer); \ + TEST_TRUE(error == mpack_ok, "writer is in error state %i (%s)", \ + (int)error, mpack_error_to_string(error)); \ +} while (0) // tears down a writer, ensuring the given error occurred #define TEST_WRITER_DESTROY_ERROR(writer, error) do { \ mpack_error_t expected = (error); \ mpack_error_t actual = mpack_writer_destroy(writer); \ - TEST_TRUE(actual == expected, "writer is in error state %i instead of %i", \ - (int)actual, (int)expected); \ + TEST_TRUE(actual == expected, "writer is in error state %i (%s) instead of %i (%s)", \ + (int)actual, mpack_error_to_string(actual), \ + (int)expected, mpack_error_to_string(expected)); \ } while (0) // performs an operation on a writer, ensuring no error occurs #define TEST_WRITE_NOERROR(writer, write_expr) do { \ (write_expr); \ - TEST_TRUE(mpack_writer_error(writer) == mpack_ok, \ - "writer is in error state %i", (int)mpack_writer_error(writer)); \ + mpack_error_t error = mpack_writer_error(writer); \ + TEST_TRUE(error == mpack_ok, "writer is in error state %i (%s)", \ + (int)error, mpack_error_to_string(error)); \ } while (0) -#define TEST_DESTROY_MATCH_SIZE(expect, size) do { \ +#define TEST_DESTROY_MATCH_IMPL(buf, expect) do { \ static const char data[] = expect; \ + size_t used = mpack_writer_buffer_used(&writer); \ TEST_WRITER_DESTROY_NOERROR(&writer); \ - TEST_TRUE(sizeof(data)-1 == size, \ - "written data length %i does not match length %i of expected", \ - (int)size, (int)(sizeof(data)-1)); \ - TEST_TRUE(memcmp(data, buf, size) == 0, \ - "written data does not match expected"); \ + TEST_TRUE(sizeof(data)-1 == used && memcmp(data, buf, used) == 0, \ + "written data (of length %i) does not match expected (of length %i)", \ + (int)used, (int)(sizeof(data)-1)); \ } while (0) -#define TEST_DESTROY_MATCH(expect) do { \ - TEST_DESTROY_MATCH_SIZE(expect, size); \ +#define TEST_DESTROY_MATCH(buf, expect) do { \ + TEST_DESTROY_MATCH_IMPL(buf, expect); \ if (buf) MPACK_FREE(buf); \ } while (0) // runs a simple writer test, ensuring it matches the given data #define TEST_SIMPLE_WRITE(expect, write_op) do { \ - mpack_writer_t writer; \ mpack_writer_init(&writer, buf, sizeof(buf)); \ mpack_writer_set_error_handler(&writer, test_write_error_handler); \ - (write_op); \ - TEST_DESTROY_MATCH_SIZE(expect, mpack_writer_buffer_used(&writer)); \ + write_op; \ + TEST_DESTROY_MATCH_IMPL(buf, expect); \ TEST_TRUE(test_write_error == mpack_ok); \ test_write_error = mpack_ok; \ } while (0) // runs a simple writer test, ensuring it does not cause an error and ignoring the result #define TEST_SIMPLE_WRITE_NOERROR(write_op) do { \ - mpack_writer_t writer; \ mpack_writer_init(&writer, buf, sizeof(buf)); \ mpack_writer_set_error_handler(&writer, test_write_error_handler); \ (write_op); \ @@ -98,7 +99,6 @@ void test_write_error_handler(mpack_writer_t* writer, mpack_error_t error); // runs a simple writer test, ensuring it does causes the given error #define TEST_SIMPLE_WRITE_ERROR(write_op, error) do { \ mpack_error_t expected2 = (error); \ - mpack_writer_t writer; \ mpack_writer_init(&writer, buf, sizeof(buf)); \ mpack_writer_set_error_handler(&writer, test_write_error_handler); \ (write_op); \ diff --git a/test/test.c b/test/unit/src/test.c similarity index 75% rename from test/test.c rename to test/unit/src/test.c index 58eb05a..7ecc161 100644 --- a/test/test.c +++ b/test/unit/src/test.c @@ -1,16 +1,16 @@ /* - * Copyright (c) 2015 Nicholas Fraser - * + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * * Permission is hereby granted, free of charge, to any person obtaining a copy of * this software and associated documentation files (the "Software"), to deal in * the Software without restriction, including without limitation the rights to * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of * the Software, and to permit persons to whom the Software is furnished to do so, * subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR @@ -21,12 +21,10 @@ #include "test.h" -#include -#include - #include "test-reader.h" #include "test-expect.h" #include "test-write.h" +#include "test-builder.h" #include "test-buffer.h" #include "test-common.h" #include "test-node.h" @@ -37,6 +35,8 @@ mpack_tag_t (*fn_mpack_tag_nil)(void) = &mpack_tag_nil; int passes; int tests; +const char* lipsum = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed nec justo purus. Nunc finibus dolor id lorem sagittis, euismod efficitur arcu aliquam. Nullam a ante eget mi porttitor dignissim vitae at libero. Maecenas in justo massa. Mauris ultricies leo nisl, at ullamcorper erat maximus sit amet. Quisque pharetra sed ligula nec tristique. Mauris consectetur sapien lacus, et pharetra turpis rhoncus a. Sed in eleifend eros. Donec in libero lacus. Sed et finibus ipsum. Etiam eros leo, mollis eget molestie quis, rhoncus ac magna. Donec dolor risus, bibendum et scelerisque at, faucibus in mi. Interdum et malesuada fames ac ante ipsum primis in faucibus. Vestibulum convallis accumsan mollis."; + #if MPACK_CUSTOM_ASSERT bool test_jmp_set = false; jmp_buf test_jmp_buf; @@ -93,6 +93,9 @@ int main(void) { #if MPACK_WRITER test_writes(); #endif + #if MPACK_BUILDER + test_builder(); + #endif #if MPACK_NODE test_node(); #endif diff --git a/test/unit/src/test.h b/test/unit/src/test.h new file mode 100644 index 0000000..6dac522 --- /dev/null +++ b/test/unit/src/test.h @@ -0,0 +1,292 @@ +/* + * Copyright (c) 2015-2021 Nicholas Fraser and the MPack authors + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MPACK_TEST_H +#define MPACK_TEST_H 1 + +#define _DEFAULT_SOURCE 1 +#define _BSD_SOURCE 1 + +#ifdef WIN32 +#define _CRT_SECURE_NO_WARNINGS 1 +#endif + +// mpack poisons float and double when disabled so we give ourselves macros to +// use them manually in tests +#define TEST_FLOAT float +#define TEST_DOUBLE double + +// In the special case where we have float but not double, we have to include +// first since MPack poisons double in the unit test suite to make +// sure it isn't used. +#if defined(MPACK_DOUBLE) && !defined(MPACK_FLOAT) + #if !MPACK_DOUBLE + #include + #endif +#endif + +#include "mpack/mpack.h" + +#if MPACK_CONFORMING + #include + #include + #include + #include + #include + #if MPACK_FLOAT + #include + #endif +#endif + +#if MPACK_STDIO + #ifdef WIN32 + #include + #define mkdir(path, mode) ((void)(mode), _mkdir(path)) + #define rmdir _rmdir + #else + #include + #include + #endif +#endif + +// We silence the same warnings as MPack across the entire unit test suite. +// There's no MPACK_SILENCE_WARNINGS_END to match this. +MPACK_SILENCE_WARNINGS_BEGIN + +// We also silence warnings specifically for unit tests +#ifdef _MSC_VER + #pragma warning(disable:4611) // interaction between '_setjmp' and C++ object destruction is non-portable + #pragma warning(disable:4204) // nonstandard extension used: non-constant aggregate initializer + #pragma warning(disable:4221) // nonstandard extension used: cannot be initialized using address of automatic variable +#endif + +// We silence shadow warnings around variable declarations in some macros +#if defined(MPACK_SILENCE_WARNINGS_PUSH) + #ifdef __GNUC__ + #define TEST_MPACK_SILENCE_SHADOW_BEGIN \ + MPACK_SILENCE_WARNINGS_PUSH \ + _Pragma ("GCC diagnostic ignored \"-Wshadow\"") + #define TEST_MPACK_SILENCE_SHADOW_END \ + MPACK_SILENCE_WARNINGS_POP + #elif defined(_MSC_VER) + #define TEST_MPACK_SILENCE_SHADOW_BEGIN \ + MPACK_SILENCE_WARNINGS_PUSH \ + __pragma(warning(disable:4456)) \ + __pragma(warning(disable:4459)) + #define TEST_MPACK_SILENCE_SHADOW_END \ + MPACK_SILENCE_WARNINGS_POP + #endif +#endif +#ifndef TEST_MPACK_SILENCE_SHADOW_BEGIN + #define TEST_MPACK_SILENCE_SHADOW_BEGIN /*nothing*/ + #define TEST_MPACK_SILENCE_SHADOW_END /*nothing*/ +#endif + +// For some older versions of GCC, we silence shadow warnings globally since +// they don't properly handle push/pop around variable declarations. +#ifdef __GNUC__ + #if __GNUC__ < 9 + #pragma GCC diagnostic ignored "-Wshadow" + #endif +#endif + + + +/** + * Floating point infinities + * + * MSVC is ridiculous with warnings when it comes to infinity. All of this + * wraps various infinities to avoid constant arithmetic overflow warnings. + */ + +#if MPACK_FLOAT + +#ifdef __cplusplus +#include +#define MPACK_FLOAT_POSITIVE_INFINITY (std::numeric_limits::infinity()) +#define MPACK_DOUBLE_POSITIVE_INFINITY (std::numeric_limits::infinity()) +#endif + +#ifdef _MSC_VER + #pragma warning(push) + #pragma warning(disable:4056) // overflow in floating-point constant arithmetic + #pragma warning(disable:4756) // overflow in constant arithmetic + + #ifndef MPACK_FLOAT_POSITIVE_INFINITY + MPACK_STATIC_INLINE float mpack_float_positive_infinity() { + return (float)(INFINITY); + } + #define MPACK_FLOAT_POSITIVE_INFINITY (mpack_float_positive_infinity()) + #endif + + #ifndef MPACK_DOUBLE_POSITIVE_INFINITY + MPACK_STATIC_INLINE double mpack_double_positive_infinity() { + return (double)(INFINITY); + } + #define MPACK_DOUBLE_POSITIVE_INFINITY (mpack_double_positive_infinity()) + #endif + + MPACK_STATIC_INLINE float mpack_float_negative_infinity() { + return -MPACK_FLOAT_POSITIVE_INFINITY; + } + #define MPACK_FLOAT_NEGATIVE_INFINITY (mpack_float_negative_infinity()) + + MPACK_STATIC_INLINE double mpack_double_negative_infinity() { + return -MPACK_DOUBLE_POSITIVE_INFINITY; + } + #define MPACK_DOUBLE_NEGATIVE_INFINITY (mpack_double_negative_infinity()) + + #pragma warning(pop) +#endif + +#if MPACK_FLOAT + #ifndef MPACK_FLOAT_POSITIVE_INFINITY + #define MPACK_FLOAT_POSITIVE_INFINITY ((float)(INFINITY)) + #endif + #ifndef MPACK_FLOAT_NEGATIVE_INFINITY + #define MPACK_FLOAT_NEGATIVE_INFINITY (-MPACK_FLOAT_POSITIVE_INFINITY) + #endif +#endif +#if MPACK_DOUBLE + #ifndef MPACK_DOUBLE_POSITIVE_INFINITY + #define MPACK_DOUBLE_POSITIVE_INFINITY ((double)(INFINITY)) + #endif + #ifndef MPACK_DOUBLE_NEGATIVE_INFINITY + #define MPACK_DOUBLE_NEGATIVE_INFINITY (-MPACK_DOUBLE_POSITIVE_INFINITY) + #endif +#endif + +#endif + + + +#if !MPACK_FINITE_MATH + #if defined(WIN32) + #include + #define isnanf _isnan + #elif defined(__APPLE__) + #define isnanf isnan + #elif defined(__GNUC__) || defined(__clang__) + // On some versions of GCC/Ubuntu (e.g. 4.8.4 on Ubuntu 14.4), + // isnan() incorrectly causes double->float warnings even when + // called on a double + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61501 + #undef isnan + #define isnan(x) __builtin_isnan(x) + #endif + #if !defined(isnanf) && !defined(MPACK_ISNANF_IS_FUNC) + #define isnanf isnan + #endif +#endif + +extern mpack_tag_t (*fn_mpack_tag_nil)(void); + +extern const char* lipsum; + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * This is basically the whole unit testing framework for mpack. + * The reported number of "tests" is the total number of test asserts, + * where each actual test case has several asserts. + */ + +// enable this to exit at the first error +#define TEST_EARLY_EXIT 1 + +// runs the given expression, causing a unit test failure with the +// given printf format string if the expression is not true. +#define TEST_TRUE(...) \ + MPACK_EXPAND(TEST_TRUE_IMPL((MPACK_EXTRACT_ARG0(__VA_ARGS__)), __FILE__, __LINE__, __VA_ARGS__ , "" , NULL)) + +#define TEST_TRUE_IMPL(result, file, line, ignored, ...) \ + MPACK_EXPAND(test_true_impl(result, file, line, __VA_ARGS__)) + +void test_true_impl(bool result, const char* file, int line, const char* format, ...); + +extern int tests; +extern int passes; + +#if MPACK_DEBUG +extern bool test_jmp_set; +extern jmp_buf test_jmp_buf; +extern bool test_break_set; +extern bool test_break_hit; + +// calls setjmp to expect an assert from a unit test. an assertion +// will cause a longjmp to here with a value of 1. +#define TEST_TRUE_SETJMP() \ + (TEST_TRUE(!test_jmp_set, "an assert jmp is already set!"), \ + test_jmp_set = true, \ + setjmp(test_jmp_buf)) + +// clears the expectation of an assert. a subsequent assert will +// cause the unit test suite to abort with error. +#define TEST_TRUE_CLEARJMP() \ + (TEST_TRUE(test_jmp_set, "an assert jmp is not set!"), \ + test_jmp_set = false) + +// runs the given expression, causing a unit test failure if an assertion +// is not triggered. (note that stack variables may need to be marked volatile +// since non-volatile stack variables that are written to after setjmp are +// undefined after longjmp.) +#define TEST_ASSERT(expr) do { \ + volatile bool jumped = false; \ + if (TEST_TRUE_SETJMP()) { \ + jumped = true; \ + } else { \ + (expr); \ + } \ + TEST_TRUE_CLEARJMP(); \ + TEST_TRUE(jumped, "expression should assert, but didn't: " #expr); \ +} while (0) + +#define TEST_BREAK(expr) do { \ + test_break_set = true; \ + test_break_hit = false; \ + TEST_TRUE(expr, "expression is not true: " # expr); \ + TEST_TRUE(test_break_hit, "expression should break, but didn't: " # expr); \ + test_break_set = false; \ +} while (0); + +#else + +// in release mode there are no asserts or break functions, so +// TEST_BREAK() just runs the expr. it is usually used to test +// that something flags mpack_error_bug. +#define TEST_BREAK(expr) do { TEST_TRUE(expr); } while (0) + +// TEST_ASSERT() is not defined because code that causes an assert +// cannot continue to run; it would cause undefined behavior (and +// crash the unit test framework.) it cannot be defined to nothing +// because the tests around it wouldn't make sense (and would cause +// unused warnings); the tests that use it must be disabled entirely. + +#endif + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/tools/afl.sh b/tools/afl.sh new file mode 100755 index 0000000..7887fe3 --- /dev/null +++ b/tools/afl.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Runs mpack-fuzz under american fuzzy lop. + +cd "$(dirname "$0")"/.. + +export AFL_HARDEN=1 +make -f test/fuzz/Makefile || exit 1 + +echo +echo "This will run the first fuzzer as fuzzer01. To run on additional" +echo "cores, run:" +echo +echo " afl-fuzz -i test/messagepack -o .build/fuzz/sync -S fuzzer## -- .build/fuzz/mpack-fuzz" +echo +echo "To watch the overall progress, run:" +echo +echo " watch afl-whatsup .build/fuzz/sync" +echo +echo "Press enter to start..." +read + +afl-fuzz -i test/messagepack -o .build/fuzz/sync -M fuzzer01 -- .build/fuzz/mpack-fuzz diff --git a/tools/amalgamate.sh b/tools/amalgamate.sh index 43e9c1b..2e24e27 100755 --- a/tools/amalgamate.sh +++ b/tools/amalgamate.sh @@ -1,29 +1,51 @@ #!/bin/bash # This script amalgamates the MPack code into a single pair of # source files, mpack.h and mpack.c. The resulting amalgamation -# is in build/amalgamation/ (without documentation.) +# is in .build/amalgamation/ (without documentation.) . "`dirname $0`"/getversion.sh -FILES="\ - mpack-platform \ - mpack-common \ - mpack-writer \ - mpack-reader \ - mpack-expect \ - mpack-node" +HEADERS="\ + mpack/mpack-platform.h \ + mpack/mpack-common.h \ + mpack/mpack-writer.h \ + mpack/mpack-reader.h \ + mpack/mpack-expect.h \ + mpack/mpack-node.h \ + " + +SOURCES="\ + mpack/mpack-platform.c \ + mpack/mpack-common.c \ + mpack/mpack-writer.c \ + mpack/mpack-reader.c \ + mpack/mpack-expect.c \ + mpack/mpack-node.c \ + " TOOLS="\ + tools/afl.sh \ tools/clean.sh \ - tools/gcov.sh \ + tools/coverage.sh \ tools/scan-build.sh \ - tools/valgrind-suppressions" + tools/unit.bat \ + tools/unit.sh \ + tools/valgrind-suppressions \ + " + +FILES="\ + test \ + LICENSE \ + AUTHORS.md \ + README.md \ + CHANGELOG.md \ + " # add top license and comment -rm -rf build/amalgamation -mkdir -p build/amalgamation/src/mpack -HEADER=build/amalgamation/src/mpack/mpack.h -SOURCE=build/amalgamation/src/mpack/mpack.c +rm -rf .build/amalgamation +mkdir -p .build/amalgamation/src/mpack +HEADER=.build/amalgamation/src/mpack/mpack.h +SOURCE=.build/amalgamation/src/mpack/mpack.c echo '/**' > $HEADER sed 's/^/ * /' LICENSE >> $HEADER cat - >> $HEADER <> $HEADER echo -e "#define MPACK_AMALGAMATED 1\n" >> $HEADER -echo -e "#include \"mpack-config.h\"\n" >> $HEADER -for f in $FILES; do +echo -e "#if defined(MPACK_HAS_CONFIG) && MPACK_HAS_CONFIG" >> $HEADER +echo -e "#include \"mpack-config.h\"" >> $HEADER +echo -e "#endif\n" >> $HEADER +for f in $HEADERS; do echo -e "\n/* $f.h */" >> $HEADER - sed -e 's@^#include ".*@/* & */@' -e '0,/^ \*\/$/d' src/mpack/$f.h >> $HEADER + sed -e 's@^#include ".*@/* & */@' -e '0,/^ \*\/$/d' src/$f >> $HEADER done echo -e "#endif\n" >> $HEADER @@ -52,21 +76,15 @@ echo -e "#endif\n" >> $HEADER echo -e "#define MPACK_INTERNAL 1" >> $SOURCE echo -e "#define MPACK_EMIT_INLINE_DEFS 1\n" >> $SOURCE echo -e "#include \"mpack.h\"\n" >> $SOURCE -for f in $FILES; do +for f in $SOURCES; do echo -e "\n/* $f.c */" >> $SOURCE - sed -e 's@^#include ".*@/* & */@' -e '0,/^ \*\/$/d' src/mpack/$f.c >> $SOURCE + sed -e 's@^#include ".*@/* & */@' -e '0,/^ \*\/$/d' src/$f >> $SOURCE done # assemble package contents -CONTENTS="test SConscript SConstruct LICENSE README.md" -cp -ar $CONTENTS build/amalgamation -mkdir -p build/amalgamation/projects/{vs,xcode/MPack.xcodeproj} -cp projects/vs/mpack.{sln,vcxproj,vcxproj.filters} build/amalgamation/projects/vs -cp projects/xcode/MPack.xcodeproj/project.pbxproj build/amalgamation/projects/xcode/MPack.xcodeproj -cp src/mpack-config.h.sample build/amalgamation/src -mkdir -p build/amalgamation/tools -cp $TOOLS build/amalgamation/tools +cp -a $FILES .build/amalgamation +mkdir -p .build/amalgamation/tools +cp -a $TOOLS .build/amalgamation/tools # done! -echo "Done. MPack amalgamation is in build/amalgamation/" - +echo "Done. MPack amalgamation is in .build/amalgamation/" diff --git a/tools/ci.sh b/tools/ci.sh deleted file mode 100755 index 1150158..0000000 --- a/tools/ci.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -# Used to run the test suite under continuous integration. Handles the -# special compiler type "scan-build", which runs the full debug build -# under scan-build; otherwise all variants are built. Also handles -# building the amalgamated package and running code coverage. - -if [[ "$AMALGAMATED" == "1" ]]; then - tools/amalgamate.sh || exit $? - cd build/amalgamation -fi -pwd - -if [[ "$CC" == "scan-build" ]]; then - unset CC - unset CXX - scan-build -o analysis --use-cc=`which clang` --status-bugs scons || exit $? - -elif [[ "$CC" == "gcc" ]] && [[ "$STANDARD" == "1" ]]; then - # We only perform code coverage measurements from the - # GCC non-amalgamated build. - scons gcov=1 all=1 || exit $? - tools/gcov.sh || exit $? - pip install --user cpp-coveralls || exit $? - coveralls --no-gcov --include src || exit $? - -else - scons all=1 || exit $? - -fi diff --git a/tools/clean.bat b/tools/clean.bat new file mode 100644 index 0000000..e21cec3 --- /dev/null +++ b/tools/clean.bat @@ -0,0 +1,3 @@ +if exist test\.build rmdir /s /q test\.build +if exist .ninja_deps del /AH .ninja_deps +if exist .ninja_log del /AH .ninja_log diff --git a/tools/clean.sh b/tools/clean.sh index 166e18f..26812cd 100755 --- a/tools/clean.sh +++ b/tools/clean.sh @@ -1,8 +1,3 @@ -#!/bin/bash -scons -c -rm -rf .sconsign.dblite .sconf_temp config.log -rm -f mpack-*.tar.gz *.gcov *.gcno src/mpack/*.o README.html -rm -f projects/vs/*.{suo,sdf,opensdf} -rm -rf docs/html -rm -rf build analysis -rm -rf projects/vs/{Debug,Release} +#!/bin/sh +rm -rf .build +rm -f vgcore.* mpack-*.tar.gz *.gcov *.gcno src/mpack/*.o README.html diff --git a/tools/coverage.sh b/tools/coverage.sh new file mode 100755 index 0000000..39927e1 --- /dev/null +++ b/tools/coverage.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -e + +# run unit tests with coverage +tools/unit.sh run-coverage + +# run gcov for traditional text-based coverage output +rm -rf coverage +mkdir -p coverage +gcov --object-directory .build/unit/coverage/objs/src/mpack `find src -name '*.c'` || exit $? +mv *.gcov coverage + +# run lcov +lcov --capture --directory .build/unit/coverage/objs/src --output-file coverage/lcov.info + +# generate HTML coverage +genhtml coverage/lcov.info --output-directory coverage/html + +echo +echo "Done. Results written in coverage/" +echo "View HTML results in: file://$(pwd)/coverage/html/index.html" diff --git a/tools/gcov.sh b/tools/gcov.sh deleted file mode 100755 index 9b635bf..0000000 --- a/tools/gcov.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -# tests must be built with "scons gcov=1" -gcov --object-directory build/debug/src/mpack `find src -name '*.c'` || exit $? diff --git a/tools/gendocs.sh b/tools/gendocs.sh index 6dc7149..e16f070 100755 --- a/tools/gendocs.sh +++ b/tools/gendocs.sh @@ -1,20 +1,20 @@ #!/bin/bash -. "`dirname $0`"/getversion.sh +cd "$(dirname "$0")"/.. +source tools/getversion.sh +mkdir -p .build/docs # Write temporary README.md without "Build Status" section cat README.md | \ sed '/^## Build Status/,/^##/{//!d}' | \ sed '/^## Build Status/d' \ - > README.temp.md + > .build/docs/README.temp.md -# Generate docs with edited README.md and correct version number +# Generate docs with correct version number ( - cat docs/doxyfile | sed -e "s/README\.md/README.temp.md/" + cat docs/doxyfile echo "PROJECT_NUMBER = $VERSION" - echo "USE_MDFILE_AS_MAINPAGE = README.temp.md" echo -) | doxygen - +) | doxygen - || exit 1 -RET=$? -rm README.temp.md -(exit $RET) +echo +echo "Docs generated: file://$(pwd)/.build/docs/html/index.html" diff --git a/tools/merge-badges.sh b/tools/merge-badges.sh deleted file mode 100755 index fb7bfc5..0000000 --- a/tools/merge-badges.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -# A wrapper around git merge that updates the -# branch for README.md badges -[[ -z $(git status --porcelain) ]] || { git status --porcelain; echo "Tree is not clean!" ; exit 1; } -git merge --no-ff --no-edit $1 -branch=`git rev-parse --abbrev-ref HEAD` -sed "s@branch\([=/]\)$1@branch\1$branch@g" -i README.md -git add README.md -git commit --amend --no-edit diff --git a/tools/package.sh b/tools/package.sh index 503e5f2..f8d6563 100755 --- a/tools/package.sh +++ b/tools/package.sh @@ -1,27 +1,29 @@ -#!/bin/bash -v +#!/bin/bash # Packages MPack up for amalgamation release. You can run tools/amalgamate.sh -# instead of this script if you just want to generate mpack.h/mpack.c. +# instead of this script if you just want to generate mpack.h and mpack.c. -#[[ -z $(git status --porcelain) ]] || { git status --porcelain; echo "Tree is not clean!" ; exit 1; } +set -v +set -e + +[[ -z $(git status --porcelain) ]] || { git status --porcelain; echo "Tree is not clean!" ; exit 1; } "`dirname $0`"/clean.sh # generate package contents . "`dirname $0`"/amalgamate.sh . "`dirname $0`"/gendocs.sh -cp -ar docs/html build/amalgamation/docs +cp -ar .build/docs/html .build/amalgamation/docs sed -i '/#define MPACK_AMALGAMATED 1/a\ -#define MPACK_RELEASE_VERSION 1' build/amalgamation/src/mpack/mpack.h +#define MPACK_RELEASE_VERSION 1' .build/amalgamation/src/mpack/mpack.h # create package NAME=mpack-amalgamation-$VERSION -tar -C build/amalgamation --transform "s@^@$NAME/@" -czf build/$NAME.UNTESTED.tar.gz `ls build/amalgamation` || exit $? +tar -C .build/amalgamation --transform "s@^@$NAME/@" -czf .build/$NAME.UNTESTED.tar.gz `ls .build/amalgamation` || exit $? # build and run all unit tests -pushd build/amalgamation -scons -j4 all=1 || exit $? +pushd .build/amalgamation +tools/unit.sh all || exit $? popd # done! -mv build/$NAME.UNTESTED.tar.gz $NAME.tar.gz +mv .build/$NAME.UNTESTED.tar.gz $NAME.tar.gz echo Created $NAME.tar.gz - diff --git a/tools/scan-build.sh b/tools/scan-build.sh index 1d8e4df..08270a7 100755 --- a/tools/scan-build.sh +++ b/tools/scan-build.sh @@ -1,2 +1,2 @@ #!/bin/bash -scan-build -o analysis --use-cc=`which clang` --status-bugs --view scons +scan-build -o analysis --status-bugs bash -c 'test/unit/configure.py && ninja -f .build/unit/build.ninja' diff --git a/tools/unit.bat b/tools/unit.bat new file mode 100644 index 0000000..dca3b73 --- /dev/null +++ b/tools/unit.bat @@ -0,0 +1,28 @@ +@echo off + +REM Builds and runs the unit test suite under the Visual Studio C compiler on +REM Windows. +REM +REM Pass a configuration to run or pass "all" to run all configurations. +REM +REM You can run this in a normal command prompt or in a Visual Studio Build +REM Tools command prompt. It will find the build tools automatically if needed +REM but it will run a lot faster if they are already available. + +setlocal enabledelayedexpansion + +REM Find build tools +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -find **\vcvarsall.bat`) do (SET "vcvarsall=%%i") + +REM Enable build tools if needed +where cl 1>NUL 2>NUL +if %errorlevel% neq 0 call "%vcvarsall%" amd64 +if %errorlevel% neq 0 exit /b %errorlevel% + +REM Configure unit tests +python test\unit\configure.py +if %errorlevel% neq 0 exit /b %errorlevel% + +REM Run unit tests +ninja -f .build\unit\build.ninja %* +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/tools/unit.sh b/tools/unit.sh new file mode 100755 index 0000000..50ad59d --- /dev/null +++ b/tools/unit.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Builds and runs the unit test suite. +# Set CC before calling this to use a different compiler. +# Pass a configuration to run or pass "all" to run all configurations. + +set -e +cd "$(dirname $0)/.." +test/unit/configure.py +ninja -f .build/unit/build.ninja $@ diff --git a/tools/update-gh-pages.sh b/tools/update-gh-pages.sh index f1d864d..4ded8a8 100755 --- a/tools/update-gh-pages.sh +++ b/tools/update-gh-pages.sh @@ -1,13 +1,14 @@ #!/bin/bash # updates documentation in gh-pages +set -e + "`dirname $0`"/clean.sh . "`dirname $0`"/gendocs.sh -cp -ar docs/html docs-html +cp -ar .build/docs/html docs-html "`dirname $0`"/clean.sh git checkout gh-pages || exit $? -git pull rm -r *.{html,png,js,css} search cp -r docs-html/* . diff --git a/tools/valgrind-suppressions b/tools/valgrind-suppressions index b34eab9..d27fc6b 100644 --- a/tools/valgrind-suppressions +++ b/tools/valgrind-suppressions @@ -1,9 +1,14 @@ + +# Note: match-leak-kinds is currently commented out because older +# versions of valgrind don't support it + +# C++ GCC 5.1 reachable objects +# https://bugs.kde.org/show_bug.cgi?id=345307 +# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 { - # https://bugs.kde.org/show_bug.cgi?id=345307 - # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65434 GCC-STL-pool-reachable Memcheck:Leak - match-leak-kinds: reachable + #match-leak-kinds: reachable fun:malloc fun:pool fun:__static_initialization_and_destruction_0 @@ -11,3 +16,22 @@ fun:call_init.part.0 fun:_dl_init } + +# musl libc supressions +{ + musl-free + Memcheck:Free + fun:free + obj:/lib/ld-musl-x86_64.so.1 +} +{ + musl-leak + Memcheck:Leak + match-leak-kinds: reachable + fun:calloc + fun:load_library + fun:load_preload + fun:__dls3 + fun:__dls2 + obj:/lib/ld-musl-x86_64.so.1 +} diff --git a/tools/vs2017_x64.bat b/tools/vs2017_x64.bat new file mode 100644 index 0000000..0ec591f --- /dev/null +++ b/tools/vs2017_x64.bat @@ -0,0 +1,12 @@ +@echo off + +REM Sets paths for Visual Studio 2017 build tools for amd64. Visual Studio 2019 +REM creates command prompt shortcuts for 2015 and 2019 but not 2017. For 2017, +REM open a plain command prompt and run this. + +setlocal enabledelayedexpansion + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -find **\vcvarsall.bat`) do (SET "vcvarsall=%%i") +where cl 1>NUL 2>NUL +if %errorlevel% neq 0 call "%vcvarsall%" amd64 -vcvars_ver=14.16 +if %errorlevel% neq 0 exit /b %errorlevel% diff --git a/tools/vs2017_x86.bat b/tools/vs2017_x86.bat new file mode 100644 index 0000000..944309b --- /dev/null +++ b/tools/vs2017_x86.bat @@ -0,0 +1,12 @@ +@echo off + +REM Sets paths for Visual Studio 2017 build tools for x86. Visual Studio 2019 +REM creates command prompt shortcuts for 2015 and 2019 but not 2017. For 2017, +REM open a plain command prompt and run this. + +setlocal enabledelayedexpansion + +for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -find **\vcvarsall.bat`) do (SET "vcvarsall=%%i") +where cl 1>NUL 2>NUL +if %errorlevel% neq 0 call "%vcvarsall%" x86 -vcvars_ver=14.16 +if %errorlevel% neq 0 exit /b %errorlevel%