From: Not Zed Date: Wed, 1 May 2019 09:19:04 +0000 (+0930) Subject: Added XDR-native (de)serialiaser. X-Git-Url: https://code.zedzone.au/cvs?a=commitdiff_plain;h=1d6059b0a8a17836e0789dd54da379d6193ac549;p=libeze Added XDR-native (de)serialiaser. Added a proprietary tagged (de)serialiaser. Prepare for removal of previous serialiser. --- diff --git a/Makefile b/Makefile index 0a828bf..5a3fc12 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ CPPFLAGS=-I. -CFLAGS=-Og -g -fPIC -Wall -mtune=native +CFLAGS=-O0 -g -fPIC -Wall -mtune=native test_LDLIBS=-lpthread -lrt ARFLAGS=rvsUc VERSION=2.1.99 @@ -8,7 +8,12 @@ VERSION=2.1.99 SRCS= \ ez-bitset.c \ ez-blob.c \ + ez-blob.c \ + ez-blob-io.c \ ez-blob-print.c \ + ez-blob-blob.c \ + ez-blob-tagz.c \ + ez-blob-xdrn.c \ ez-port.c \ ez-set.c \ ez-tree.c @@ -16,6 +21,10 @@ SRCS= \ HEADERS = \ ez-bitset.h \ ez-blob.h \ + ez-blobio.h \ + ez-blob-blob.h \ + ez-blob-tagz.h \ + ez-blob-xdrn.h \ ez-list.h \ ez-node.h \ ez-port.h \ @@ -42,7 +51,8 @@ test-%: test-%.o $(CC) $(CFLAGS) -o $@ $< libeze.a $(test_LDLIBS) test-bitset: libeze.a(ez-bitset.o) -test-blob: libeze.a(ez-blob.o) libeze.a(ez-blob-print.o) +test-blob: libeze.a(ez-blob.o) libeze.a(ez-blob-print.o) libeze.a(ez-blob-io.o) \ + libeze.a(ez-blob-xdrn.o) libeze.a(ez-blob-tagz.o) libeze.a(ez-blob-blob.o) test-port: libeze.a(ez-port.o) test-set: libeze.a(ez-set.o) test-tree: libeze.a(ez-tree.o) diff --git a/ez-blob-blob.c b/ez-blob-blob.c new file mode 100644 index 0000000..ecc61a3 --- /dev/null +++ b/ez-blob-blob.c @@ -0,0 +1,367 @@ +/* ez-blob-blob.c: Prototype serialiser. + + Copyright (C) 2019 Michael Zucchi + + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this program. If not, see + . +*/ + +#include +#include +#include + +#include "ez-list.h" + +#include "ez-blob.h" +#include "ez-blob-blob.h" + +/* + This implements a basic 'high performance' serialisation based on the + descriptors. + + Fields are encoded in the same order as the descriptor with no + labelling. Therefore the same descriptor must be used on both ends. + Fields may be at arbitrary alignments. + + primitives + encoded in machine order at native size + + strings + 4-byte length followed by characters. No nul termination. A length + of ~0 indicates a null string pointer. + + arrays + 4-byte length followed by data. ** currently null pointer arrays with zero length are converted to a zero-length array. + + struct + 4-byte length followed by blob-encoded data. + + pointer + 4-byte length followed by blob-encoded data. A length of ~0 + indicates a null pointer. + +*/ + +static const int primitive_size[EZ_BLOB_FLOAT64+1] = { + 1, 2, 4, 8, + 4, 8 +}; + +size_t ez_blob_size(const ez_blob_desc *desc, const void *p) { + size_t size = 0; + + for (int i=0,dlen = desc->bd_length;ibd_type <= EZ_BLOB_FLOAT64) { + size += primitive_size[d->bd_type]; + } else { + const void *v = p + d->bd_offset; + switch (d->bd_type & EZ_BLOB_TYPE) { + case EZ_BLOB_STRING: { + char *s = ((char **)v)[0]; + size += s ? 4 + strlen(s) : 4; + break; + } + case EZ_BLOB_BLOB: + size += 4 + ((const struct ez_blob *)v)->size; + break; + case EZ_BLOB_STRUCT: { + const void *sp = v; + size += 4 + ez_blob_size(d->bd_table, sp); + break; + } + case EZ_BLOB_POINTER: { + const void *s = ((const void * const *)v)[0]; + size += s ? 4 + ez_blob_size(d->bd_table, s) : 4; + break; + } + case EZ_BLOB_LIST: { + ez_list *l = (ez_list *)v; + + size += 4; + for (ez_node *w = ez_list_head(l), *n = ez_node_succ(w); + n; + w = n, n=ez_node_succ(n)) { + size += 4 + ez_blob_size(d->bd_table, w); + } + break; + } + default: + break; + } + } + } + + return size; +} + +int ez_blob_decode_raw(const ez_blob_desc *desc, const ez_blob *blob, void *p) { + const char *b = blob->data; + const char *be = b + blob->size; + + ez_blob_init(desc, p); + + for (int i=0,dlen = desc->bd_length;ibd_offset); + + if (d->bd_type <= EZ_BLOB_FLOAT64) { + int psize = primitive_size[d->bd_type]; + + if (b + psize > be) + goto fail; + memcpy(v, b, psize); + b += psize; + } else if (d->bd_type == EZ_BLOB_LIST) { + uint32_t count; + ez_list *l = v; + + if (b + 4 > be) + goto fail; + memcpy(&count, b, 4); + b += 4; + + ez_list_init(l); + for (int i=0;i be) + goto fail; + memcpy(&ss, b, 4); + b += 4; + + sub.size = ss; + sub.data = (void *)b; + + if (b + ss > be) + goto fail; + if ((s = calloc(d->bd_table->bd_offset, 1)) == NULL) + goto fail; + if (ez_blob_decode_raw(d->bd_table, &sub, s) != 0) { + free(s); + goto fail; + } + ez_list_addtail(l, s); + b+=ss; + } + } else { + uint32_t ss; + ez_blob sub; + + if (b + 4 > be) + goto fail; + memcpy(&ss, b, 4); + b += 4; + + sub.size = ss; + sub.data = (char *)b; + + switch (d->bd_type & EZ_BLOB_TYPE) { + case EZ_BLOB_STRING: { + char *s = NULL; + + if (ss != ~0) { + if (b + ss > be) + goto fail; + if ((s = malloc((size_t)ss + 1)) == NULL) + goto fail; + memcpy(s, b, ss); + s[ss] = 0; + b += ss; + } + ((char **)v)[0] = s; + break; + } + case EZ_BLOB_BLOB: { + struct ez_blob *a = v; + + if (b + ss > be) + goto fail; + a->size = ss; + if ((a->data = malloc(ss)) == NULL) + goto fail; + memcpy(a->data, b, ss); + b += ss; + break; + } + case EZ_BLOB_STRUCT: + if (b + ss > be) + goto fail; + + if (ez_blob_decode_raw(d->bd_table, &sub, v) != 0) + goto fail; + b += ss; + break; + case EZ_BLOB_POINTER: { + void *s = NULL; + + if (ss != ~0) { + if (b + ss > be) + goto fail; + if ((s = calloc(d->bd_table->bd_offset, 1)) == NULL) + goto fail; + if (ez_blob_decode_raw(d->bd_table, &sub, s) != 0) { + free(s); + goto fail; + } + b += ss; + } + ((void **)v)[0] = s; + break; + } + default: + break; + } + } + } + if (b != be) + goto fail; + + return 0; + fail: + ez_blob_free_raw(desc, p); + return 1; +} + +void *ez_blob_decode(const ez_blob_desc *d, const ez_blob *blob) { + void *p = calloc(d->bd_offset, 1); + + if (ez_blob_decode_raw(d, blob, p) != 0) { + free(p); + return NULL; + } + + return p; +} + +int ez_blob_encode_raw(const ez_blob_desc *desc, const void *p, ez_blob *blob) { + char *b = blob->data; + char *be = b+blob->size; + + for (int i=0,dlen = desc->bd_length;ibd_offset); + + if (d->bd_type <= EZ_BLOB_FLOAT64) { + int psize = primitive_size[d->bd_type]; + + assert(b+psize <= be); + memcpy(b, v, psize); + b += psize; + } else if (d->bd_type == EZ_BLOB_LIST) { + ez_list *l = (void *)v; + uint32_t ss = ez_list_size(l); + + memcpy(b, &ss, 4); + b += 4; + for (ez_node *w = ez_list_head(l), *n = ez_node_succ(w); + n; + w = n, n=ez_node_succ(n)) { + ez_blob sub; + + ss = ez_blob_size(d->bd_table, w); + + assert(b+4+ss <= be); + memcpy(b, &ss, 4); + sub.data = b+4; + sub.size = ss; + ez_blob_encode_raw(d->bd_table, w, &sub); + b += 4 + ss; + } + } else { + uint32_t ss; + struct ez_blob sub; + + switch (d->bd_type & EZ_BLOB_TYPE) { + case EZ_BLOB_STRING: { + char *s = ((char **)v)[0]; + if (s) { + ss = strlen(s); + assert(b+4+ss <= be); + memcpy(b, &ss, 4); + memcpy(b+4, s, ss); + b += 4 + ss; + } else { + ss = ~0; + assert(b+4 <= be); + memcpy(b, &ss, 4); + b += 4; + } + break; + } + case EZ_BLOB_BLOB: { + const struct ez_blob *a = v; + + ss = a->size; + assert(b+4+ss <= be); + memcpy(b, &ss, 4); + memcpy(b+4, a->data, ss); + b += 4 + ss; + break; + } + case EZ_BLOB_STRUCT: + ss = ez_blob_size(d->bd_table, v); + + assert(b+4+ss <= be); + memcpy(b, &ss, 4); + sub.data = b+4; + sub.size = ss; + ez_blob_encode_raw(d->bd_table, v, &sub); + b += 4 + ss; + break; + case EZ_BLOB_POINTER: { + const void *s = ((const void **)v)[0]; + + if (s) { + ss = ez_blob_size(d->bd_table, s); + assert(b+4+ss <= be); + memcpy(b, &ss, 4); + sub.data = b+4; + sub.size = ss; + ez_blob_encode_raw(d->bd_table, s, &sub); + b += 4 + ss; + } else { + ss = ~0; + assert(b+4 <= be); + memcpy(b, &ss, 4); + b += 4; + } + break; + } + default: + break; + } + } + } + assert(b == be); + return 0; +} + +int ez_blob_encode(const ez_blob_desc *d, const void *p, ez_blob *blob) { + blob->size = ez_blob_size(d, p); + blob->data = malloc(blob->size); + + if (blob->data) { + if (ez_blob_encode_raw(d, p, blob) == 0) + return 0; + free(blob->data); + blob->data = NULL; + } + blob->size = 0; + + return -1; +} diff --git a/ez-blob-blob.h b/ez-blob-blob.h new file mode 100644 index 0000000..53c9b8c --- /dev/null +++ b/ez-blob-blob.h @@ -0,0 +1,76 @@ +/* ez-blob-blob.h: Prototype serialiser. + + Copyright (C) 2019 Michael Zucchi + + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this program. If not, see + . +*/ + +/* + * The implemented serialisation mechanism only supports + * forward-linked structures and will not perform well when + * marshalling deeply nested structures but other serialisation + * mechanisms are possible. + * + * It's historical and is to be removed. + */ + +/** + * Calculate the serialised blob size for a struct. + * + * @param d descriptor table matching struct. + * @param p struct. + */ +size_t ez_blob_size(const ez_blob_desc *d, const void *p); + +/** + * Marshal a struct to a blob. + * + * @param d descriptor table matching the struct. + * @param p structure. + * @param sizep return pointer for blob size, must not be null. + * @param blob return, the data pointer must be freed with free() when complete. + * @return non-zero on failure. + */ +int ez_blob_encode(const ez_blob_desc *d, const void *p, ez_blob *blob); + +/** + * Marshal a struct to a blob directly + * + * @param d descriptor table matching the struct. + * @param p structure. + * @param blob available target memory, size must be exactly ez_blob_size(). + * @return non-zero on failure. On failure blob.data is invalid. + */ +int ez_blob_encode_raw(const ez_blob_desc *d, const void *p, ez_blob *blob); + +/** + * Unmarshall a blob into a struct. + * + * @param d descriptor table matching blob. + * @param blob raw blob. + * @param size blob size. + * @return the decoded struct. May be freed using ez_blob_free(). + */ +void *ez_blob_decode(const ez_blob_desc *d, const ez_blob *blob); + +/** + * Decode to pre-allocated buffer. + * + * @param b blob data + * @param size blob size + * @return non-zero on failure. Any allocated pointers are freed. + */ +int ez_blob_decode_raw(const ez_blob_desc *d, const ez_blob *blob, void *p); + diff --git a/ez-blob-io.c b/ez-blob-io.c new file mode 100644 index 0000000..9dc6701 --- /dev/null +++ b/ez-blob-io.c @@ -0,0 +1,127 @@ +/* ez-blob-io.c: Common i/o utilities for serialisers + + Copyright (C) 2019 Michael Zucchi + + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this program. If not, see + . +*/ + +#include +#include +#include +#include +#include + +#include "ez-blob.h" +#include "ez-blob-io.h" + +//#define ABORT() abort() +#define ABORT() + +void *blobio_reserve(struct ez_blobio * __restrict io, size_t len) { + size_t size = io->size; + size_t to = io->index + len; + void *v; + + if (io->error) + return NULL; + + switch (io->mode) { + case BLOBIO_WRITE_ALLOC: + if (to > size) { + do { + size = size ? size * 2 : 256; + } while (to > size); + + io->size = size; + io->data = realloc(io->data, size); + + if (!io->data) { + io->error = 1; + ABORT(); + return NULL; + } + } + + v = io->index + io->data; + io->index = to; + return v; + + case BLOBIO_WRITE_FIXED: + if (to > size) { + io->error = 1; + ABORT(); + return NULL; + } + v = io->index + io->data; + io->index = to; + return v; + + case BLOBIO_WRITE_SIZE: + io->index = to; + default: + return NULL; + } +} + +void *blobio_take(struct ez_blobio * __restrict io, size_t len) { + size_t to = io->index + len; + + if (to <= io->size) { + void *v = io->data + io->index; + io->index = to; + return v; + } else { + ABORT(); + io->index = io->size; + io->error = 1; + return NULL; + } +} + +void blobio_write_align(struct ez_blobio *io, unsigned int step) { + int skip = (step-1) & io->index; + uint8_t *v; + + if (skip && (v = blobio_reserve(io, skip))) { + for (int i=0;iindex); +} + +void blobio_dumphex(const char *data, size_t size, const char *prefix) { + for (int i=0;i. +*/ + +// Grow output as necessary +#define BLOBIO_WRITE_ALLOC 0 +// Writing to pre-existing buffer +#define BLOBIO_WRITE_FIXED 1 +// Calculating size only +#define BLOBIO_WRITE_SIZE 2 +// Read mode +#define BLOBIO_READ 3 + +struct ez_blobio { + size_t index; + uint8_t *data; + + // for read: upper limit + // for write: allocation size + size_t size; + + // on read, non-zero means invalid + int error; + // io mode. + int mode; +}; + +/** + * Reserve write-space. + * + * For BLOBIO_WRITE_ALLOC mode this will fail if any allocation fails. + * + * For BLOBIO_WRITE_FIXED this will fail if there is a buffer + * overflow. + * + * For BLOBIO_WRITE_SIZE this will never fail. + * + * @param io + * @param len length required + * @return base of memory, or NULL on error. + */ +void *blobio_reserve(struct ez_blobio *io, size_t len); + +/** + * Take read-space. + * + * For BLOBIO_READ this will fail if there is a buffer underflow. + * + * @param io + * @param len Length required. + * @return base of memory or NULL on error. + */ +void *blobio_take(struct ez_blobio *io, size_t len); + +/** + * Align write output. + * + * Data is padded with 0 bytes. + * + * @param io + * @param step must be power of 2. + */ +void blobio_write_align(struct ez_blobio *io, unsigned int step); + +/** + * Align read input. + * + * @param io + * @param step must be power of 2. + */ +void blobio_read_align(struct ez_blobio *io, unsigned int step); + +/** + * Dump binary data in human-readable hex+ascii format. + * @todo rename ez_blob_dump(prefix, ez_blob)? + */ +void blobio_dumphex(const char *data, size_t size, const char *prefix); + +/* + The read and write functions read or write the type and silently + perform no-operation on error. The error can be checked via + io.error being non-zero. + + @todo I don't think there's any point in making these inline if it just + calls an external function anyway. + */ + +static __inline__ void blobio_write(struct ez_blobio *io, const void *data, size_t len) { + void *v = blobio_reserve(io, len); + if (v) memcpy(v, data, len); +} + +static __inline__ void blobio_writeb(struct ez_blobio *io, uint8_t val) { + uint8_t *v = blobio_reserve(io, 1); + if (v) *v = val; +} + +static __inline__ void blobio_write8(struct ez_blobio *io, uint8_t val) { + uint8_t *v = blobio_reserve(io, 1); + if (v) *v = val; +} + +static __inline__ void blobio_write16(struct ez_blobio *io, uint16_t val) { + uint16_t *v = blobio_reserve(io, 2); + if (v) *v = val; +} + +static __inline__ void blobio_write32(struct ez_blobio *io, uint32_t val) { + uint32_t *v = blobio_reserve(io, 4); + if (v) *v = val; +} + +static __inline__ void blobio_write64(struct ez_blobio *io, uint64_t val) { + uint64_t *v = blobio_reserve(io, 8); + if (v) *v = val; +} + +static __inline__ void blobio_writef(struct ez_blobio *io, float val) { + float *v = blobio_reserve(io, 4); + if (v) *v = val; +} + +static __inline__ void blobio_writed(struct ez_blobio *io, double val) { + double *v = blobio_reserve(io, 8); + if (v) *v = val; +} + +static __inline__ uint8_t blobio_readi8(struct ez_blobio *io) { + uint8_t *v = blobio_take(io, 1); + return v ? *v : 0; +} + +static __inline__ uint16_t blobio_readi16(struct ez_blobio *io) { + uint16_t *v = blobio_take(io, 2); + return v ? *v : 0; +} + +static __inline__ uint32_t blobio_readi32(struct ez_blobio *io) { + uint32_t *v = blobio_take(io, 4); + return v ? *v : 0; +} + +static __inline__ uint64_t blobio_readi64(struct ez_blobio *io) { + uint64_t *v = blobio_take(io, 8); + return v ? *v : 0; +} + +static __inline__ float blobio_readf(struct ez_blobio *io) { + float *v = blobio_take(io, 4); + return v ? *v : 0.0f; +} + +static __inline__ double blobio_readd(struct ez_blobio *io) { + double *v = blobio_take(io, 8); + return v ? *v : 0.0; +} diff --git a/ez-blob-print.c b/ez-blob-print.c index 8fa94dc..1759016 100644 --- a/ez-blob-print.c +++ b/ez-blob-print.c @@ -28,40 +28,21 @@ #include "ez-blob.h" #include "ez-list.h" -static void dumphex(const char *data, size_t size, const char *prefix) { - for (int i=0;ibd_length; memset(x, ' ', depth); x[depth] = 0; - - for (;e && (d->bd_type != EZ_BLOB_END);d++) { + + for (int i=0;i<=len;i++,d++) { const void *u = a + d->bd_offset; printf("%s [type=$%02x, offset=$%04x]", x, d->bd_type, d->bd_offset); - switch (d->bd_type) { + switch (d->bd_type & EZ_BLOB_TYPE) { case EZ_BLOB_INT8: printf(" .%d = %d\n", d->bd_id, *(uint8_t *)u); break; @@ -87,9 +68,10 @@ void ez_blob_print(const ez_blob_desc *d, const void *a, int depth) { const struct ez_blob *ua = u; printf(" .%d %p [%zd] = {\n", d->bd_id, ua->data, ua->size); - dumphex(ua->data, ua->size, x); + blobio_dumphex(ua->data, ua->size, x); printf("%s }\n", x); - break; } + break; + } case EZ_BLOB_POINTER: { void *up = *((void **)u); if (up) { @@ -99,7 +81,8 @@ void ez_blob_print(const ez_blob_desc *d, const void *a, int depth) { } else { printf(" .%d %p\n", d->bd_id, up); } - break; } + break; + } case EZ_BLOB_LIST: { ez_list *l = (void *)u; @@ -112,7 +95,8 @@ void ez_blob_print(const ez_blob_desc *d, const void *a, int depth) { printf("%s},\n", x); } printf("%s }\n", x); - break; } + break; + } default: printf("\n"); } diff --git a/ez-blob-tagz.c b/ez-blob-tagz.c new file mode 100644 index 0000000..d33ae9c --- /dev/null +++ b/ez-blob-tagz.c @@ -0,0 +1,558 @@ +/* ez-blob-tagz.c: A tagged blob serialiser. + + Copyright (C) 2019 Michael Zucchi + + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this program. If not, see + . +*/ + +/* + Tagged blob encoding. + + This is a streamable self-describing byte-oriented binary format. + It is a general purpose format and supports a super-set of the + ez_blob descriptor. It supports primitive and struct types and + sequences thereof and there is room for extension. + + Each item beings with a descriptor byte, then followed by a tag id, + a possible count, and the payload. + + xxxxttcc control byte + + xxxx type code + + 0 uint8 unsigned int, value zero-extended + 1 uint16 + 2 uint32 + 3 uint64 + - reserved + 5 float16 + 6 float32 + 7 float64 + - reserved + f struct + + note that for int/float types, code&3 == log2 of element size in bytes + + tt log2 of tag size in bytes + + 0 1 byte + 1 2 byte + 2 4 byte + 3 reserved, no tag? + + cc log2 of count size in bytes, used to indicate sequence length or non-sequence. + + 0 1 byte + 1 2 byte + 2 4 byte + 3 none, single item follows + + ff is struct-end code + + A header is a control byte followed by an optional 1/2/4 byte-length tag, + followed by an optional 1/2/4 byte-length count. + + A structure payload is a list of tagged fields until a struct-end + code. A structure sequence includes count of a tagged structure. + + Integers can be stored in the smallest number of bytes, i.e. with + all leading $00 bytes removed. + + Possible extension is signed types and ignoring leading $ff bytes. + +ez_blob requirements: + + All fields must have an id. + The id for each field must be unique for any given struct. + The ids must be in incremental order. + Arrays must be stored at their natural size. + +final note: + + This is a bit over-engineered. + +*/ + +#include +#include +#include +#include + +#include "ez-list.h" + +#include "ez-blob.h" +#include "ez-blob-tagz.h" +#include "ez-blob-io.h" + +//#define D(x) do { x; fflush(stdout); } while (0) +#define D(x) + +#define EZT_INT8 0x00 +#define EZT_INT16 0x10 +#define EZT_INT32 0x20 +#define EZT_INT64 0x30 + +#define EZT_FLOAT16 0x50 +#define EZT_FLOAT32 0x60 +#define EZT_FLOAT64 0x70 +#define EZT_STRUCT 0xf0 + +#define EZT_TYPE 0xf0 + +#define EZT_COUNTSHIFT 0 +#define EZT_TAGSHIFT 2 +#define EZT_DATASHIFT 4 + +#define EZT_COUNTSIZE (0x3 << EZT_COUNTSHIFT) +#define EZT_TAGSIZE (0x3 << EZT_TAGSHIFT) +#define EZT_DATASIZE (0x3 << EZT_DATASHIFT) + +#define EZT_NOCOUNT (0x3 << EZT_COUNTSHIFT) + +#define EZT_END 0xff // well, end-struct + +static void blobio_writei(struct ez_blobio *io, int sc, uint64_t val) { + switch (sc) { + case 0: + blobio_write8(io, val); + break; + case 1: + blobio_write16(io, val); + break; + case 2: + blobio_write32(io, val); + break; + case 3: + blobio_write64(io, val); + break; + } +} + +// log2 of significant bytes in value +static int size_code(uint64_t size) { + if (size <= 0xff) + return 0x0; + else if (size <= 0xffff) + return 0x1; + else if (size <= 0xffffffff) + return 0x2; + else + return 0x3; +} + +static int count_code(uint32_t size) { + if (size <= 0xff) + return 0x0; + else if (size <= 0xffff) + return 0x1; + else + return 0x2; +} + +static void blobio_writet(struct ez_blobio *io, uint8_t type, uint32_t tag) { + int tc = size_code(tag); + + blobio_writeb(io, type | (tc << EZT_TAGSHIFT)); + blobio_writei(io, tc, tag); +} + +static void tagz_encode_struct(struct ez_blobio *io, int id, const ez_blob_desc *desc, const void *p) { + blobio_writet(io, EZT_STRUCT | EZT_NOCOUNT, id); + + for (int i=0,dlen=desc->bd_length;ibd_offset); + uint64_t val = 0; + uint32_t count = 0; + const void *ptr = NULL; + + switch (d->bd_type & EZ_BLOB_TYPE) { + case EZ_BLOB_INT8: + val = *(uint8_t *)v; + goto doint; + case EZ_BLOB_INT16: + val = *(uint16_t *)v; + goto doint; + case EZ_BLOB_INT32: + val = *(uint32_t *)v; + goto doint; + case EZ_BLOB_INT64: + val = *(uint64_t *)v; + doint: + if (val) { + int sc = size_code(val); + + blobio_writet(io, EZT_INT8 | EZT_NOCOUNT | (sc << EZT_DATASHIFT), d->bd_id); + blobio_writei(io, sc, val); + } + break; + case EZ_BLOB_FLOAT32: + blobio_writet(io, EZT_FLOAT32 | EZT_NOCOUNT, d->bd_id); + blobio_writef(io, *(const float *)v); + break; + case EZ_BLOB_FLOAT64: + blobio_writet(io, EZT_FLOAT64 | EZT_NOCOUNT, d->bd_id); + blobio_writed(io, *(const double *)v); + break; + case EZ_BLOB_STRING: + ptr = *(const void **)v; + if (ptr) { + count = strlen(ptr); + goto doblob; + } + break; + case EZ_BLOB_BLOB: + count = ((ez_blob *)v)->size; + if (count) { + int cc; + + ptr = ((ez_blob *)v)->data; + doblob: + cc = count_code(count); + blobio_writet(io, EZT_INT8 | (cc << EZT_COUNTSHIFT), d->bd_id); + blobio_writei(io, cc, count); + blobio_write(io, ptr, count); + } + break; + case EZ_BLOB_STRUCT: + tagz_encode_struct(io, d->bd_id, d->bd_table, v); + break; + case EZ_BLOB_POINTER: + ptr = *(const void **)v; + if (ptr) + tagz_encode_struct(io, d->bd_id, d->bd_table, ptr); + break; + case EZ_BLOB_LIST: + count = ez_list_size((ez_list *)v); + if (count) { + int cc = count_code(count); + + blobio_writet(io, EZT_STRUCT | (cc << EZT_COUNTSHIFT), d->bd_id); + blobio_writei(io, cc, count); + + for (ez_node *w = ez_list_head((ez_list *)v), *n = ez_node_succ(w); + n; + w = n, n=ez_node_succ(n)) { + // which id to use? + tagz_encode_struct(io, d->bd_table[0].bd_id, d->bd_table, w); + } + } + break; + } + + } + + blobio_writeb(io, EZT_END); +} + +size_t ez_tagz_size(const ez_blob_desc *d, const void *p) { + struct ez_blobio io = { + .mode = BLOBIO_WRITE_SIZE + }; + tagz_encode_struct(&io, d->bd_id, d, p); + + return io.index; +} + +int ez_tagz_encode_raw(const ez_blob_desc *d, const void *p, ez_blob *blob) { + struct ez_blobio io = { + .data = blob->data, + .size = blob->size, + .mode = BLOBIO_WRITE_FIXED + }; + + tagz_encode_struct(&io, d->bd_id, d, p); + + return io.error; +} + +int ez_tagz_encode(const ez_blob_desc *d, const void *p, ez_blob *blob) { + struct ez_blobio io = { + .mode = BLOBIO_WRITE_ALLOC + }; + + tagz_encode_struct(&io, d->bd_id, d, p); + + if (!io.error) { + blob->size = io.index; + blob->data = io.data; + return 0; + } + free(blob->data); + blob->data = 0; + blob->size = 0; + + return -1; +} + +static int blobio_readb(struct ez_blobio *io) { + uint8_t *v = blobio_take(io, 1); + return v ? *v : EZT_END; +} + +static uint64_t blobio_readi(struct ez_blobio *io, int sc) { + switch (sc) { + case 0: + return blobio_readi8(io); + case 1: + return blobio_readi16(io); + case 2: + return blobio_readi32(io); + case 3: + return blobio_readi64(io); + } + return 0; +} + +// TODO: binary search, length is in first field +// Alternative: if encode/decode is same order, then can just search forward from last spot +static const ez_blob_desc *blob_desc_find(const ez_blob_desc *desc, int id) { + for (int i=0,dlen=desc->bd_length;ibd_id == id) + return d; + } + return NULL; +} + +static void tagz_decode_struct(struct ez_blobio *io, const ez_blob_desc *desc, void *p); +static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, void *p); + +static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, void *p) { + uint8_t h; + + while ((h = blobio_readb(io)) != EZT_END) { + const ez_blob_desc *d; + uint32_t ftag; + uint32_t fcount = 0; + int sc = (h & EZT_DATASIZE) >> EZT_DATASHIFT; /* for primitives */ + + ftag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT); + if ((h & EZT_COUNTSIZE) != EZT_NOCOUNT) + fcount = blobio_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT); + + d = blob_desc_find(desc, ftag); + + if (p && d) { + void *v = p + d->bd_offset; + + switch (h & EZT_TYPE) { + case EZT_INT8: + case EZT_INT16: + case EZT_INT32: + case EZT_INT64: + if (fcount) { + char *mem; + size_t len = fcount * (1<bd_type & EZ_BLOB_TYPE) { + case EZ_BLOB_STRING: + mem = malloc(len+1); + *(char **)v = mem; + memcpy(mem, data, len); + mem[len] = 0; + //*(char **)v = data; + D(printf("string %d '%s'\n", d->bd_id, mem)); + break; + case EZ_BLOB_BLOB: + mem = malloc(len); + ((ez_blob *)v)->data = mem; + ((ez_blob *)v)->size = len; + memcpy(mem, data, len); + //((ez_blob *)v)->data = data; + break; + } + } else { + switch (d->bd_type) { + case EZ_BLOB_INT8: + *(uint8_t *)v = blobio_readi(io, sc); + break; + case EZ_BLOB_INT16: + *(uint16_t *)v = blobio_readi(io, sc); + break; + case EZ_BLOB_INT32: + *(uint32_t *)v = blobio_readi(io, sc); + break; + case EZ_BLOB_INT64: + *(uint64_t *)v = blobio_readi(io, sc); + break; + default: + goto error; + } + } + break; + case EZT_FLOAT32: + if (fcount) { + blobio_take(io, fcount * 4); + } else { + switch (d->bd_type) { + case EZ_BLOB_FLOAT32: + *(float *)v = blobio_readf(io); + break; + case EZ_BLOB_FLOAT64: + *(double *)v = blobio_readf(io); + break; + default: + goto error; + } + } + break; + case EZT_FLOAT64: + if (fcount) { + blobio_take(io, fcount * 8); + } else { + switch (d->bd_type) { + case EZ_BLOB_FLOAT32: + *(float *)v = blobio_readd(io); + break; + case EZ_BLOB_FLOAT64: + *(double *)v = blobio_readd(io); + break; + default: + goto error; + } + } + break; + case EZT_STRUCT: + if (fcount) { + switch (d->bd_type) { + case EZ_BLOB_LIST: + for (int i=0;ibd_table); + + if (!node) + goto error; + + tagz_decode_struct(io, d->bd_table, node); + ez_list_addtail((ez_list *)v, node); + } + break; + default: + goto error; + } + } else { + switch (d->bd_type & EZ_BLOB_TYPE) { + case EZ_BLOB_STRUCT: + D(printf("struct %d\n", d->bd_id)); + ez_blob_init(d->bd_table, v); + tagz_decode_fields(io, d->bd_table, v); + break; + case EZ_BLOB_POINTER: + D(printf("pointer %d\n", d->bd_id)); + *(void **)v = ez_blob_alloc(d->bd_table); + + if (!*(void **)v) + goto error; + + tagz_decode_fields(io, d->bd_table, *(void **)v); + break; + default: + goto error; + } + } + break; + } + } else { + // Just skip fields + switch (h & EZT_TYPE) { + case EZT_INT8: + case EZT_INT16: + case EZT_INT32: + case EZT_INT64: + case EZT_FLOAT32: + case EZT_FLOAT64: + if (fcount) { + blobio_take(io, fcount * (1<bd_table, NULL); + } + } else { + tagz_decode_fields(io, d->bd_table, NULL); + } + break; + default: + goto error; + } + } + } + return; + error: + io->index = io->size; + io->error = 1; +} + +static void tagz_decode_struct(struct ez_blobio *io, const ez_blob_desc *desc, void *p) { + uint8_t h = blobio_readb(io); + uint32_t stag; + //uint32_t scount = 1; + + // must be a struct + // what about a struct list? hmm + if ((h & EZT_TYPE) != EZT_STRUCT) { + //io->error = 1; + return; + } + + stag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT); + stag = stag; + //if (stag != desc->bd_id) { + // ?? + //} + //if (h & EZT_COUNTSIZE) + // count = blobio_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT); + + // read fields + tagz_decode_fields(io, desc, p); +} + +int ez_tagz_decode_raw(const ez_blob_desc *desc, const ez_blob *blob, void *p) { + struct ez_blobio io = { + .size = blob->size, + .data = blob->data, + .mode = BLOBIO_READ + }; + + ez_blob_init(desc, p); + tagz_decode_struct(&io, desc, p); + + if (io.error) { + ez_blob_free_raw(desc, p); + return -1; + } + + return 0; +} + +void *ez_tagz_decode(const ez_blob_desc *desc, const ez_blob *blob) { + void *p = malloc(desc->bd_offset); + + if (p && ez_tagz_decode_raw(desc, blob, p) != 0) { + free(p); + p = NULL; + } + + return p; +} diff --git a/ez-blob-tagz.h b/ez-blob-tagz.h new file mode 100644 index 0000000..df2508c --- /dev/null +++ b/ez-blob-tagz.h @@ -0,0 +1,32 @@ +/* ez-blob-tagz.h: A tagged blob serialiser. + + Copyright (C) 2019 Michael Zucchi + + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this program. If not, see + . +*/ + + +#ifndef _EZ_BLOB_TAGZ_H +#define _EZ_BLOB_TAGZ_H + +size_t ez_tagz_size(const ez_blob_desc *d, const void *p); + +int ez_tagz_encode_raw(const ez_blob_desc *d, const void *p, ez_blob *blob); +int ez_tagz_encode(const ez_blob_desc *d, const void *p, ez_blob *blob); + +int ez_tagz_decode_raw(const ez_blob_desc *desc, const ez_blob *blob, void *p); +void *ez_tagz_decode(const ez_blob_desc *desc, const ez_blob *blob); + +#endif diff --git a/ez-blob-xdrn.c b/ez-blob-xdrn.c new file mode 100644 index 0000000..c96579e --- /dev/null +++ b/ez-blob-xdrn.c @@ -0,0 +1,289 @@ +/* ez-blob-xdrn.c: XDR-like serialiser using native ordering. + + Copyright (C) 2019 Michael Zucchi + + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this program. If not, see + . +*/ + +/* + Implements an XDR serialiser. + + Each end must have identical descriptor tables. + + STRING or POINTER files which may be NULL must be marked as + EZ_BLOB_NULLABLE. They are implemented as optional fields. + +*/ + +#include +#include +#include + +#include "ez-list.h" + +#include "ez-blob.h" +#include "ez-blob-xdrn.h" +#include "ez-blob-io.h" + +static size_t roundup(size_t v) __attribute__ ((always_inline)); +static __inline__ size_t roundup(size_t v) { + return (v+3) & ~3; +} + +static void xdrn_encode_raw(struct ez_blobio *io, const ez_blob_desc *desc, const void *p) { + for (int i=0,dlen=desc->bd_length;ibd_offset); + + switch (d->bd_type) { + case EZ_BLOB_INT8: + blobio_write32(io, *(uint8_t *)v); + break; + case EZ_BLOB_INT16: + blobio_write32(io, *(uint16_t *)v); + break; + case EZ_BLOB_INT32: + blobio_write32(io, *(uint32_t *)v); + break; + case EZ_BLOB_INT64: + blobio_write64(io, *(uint32_t *)v); + break; + case EZ_BLOB_FLOAT32: + blobio_writef(io, *(float *)v); + break; + case EZ_BLOB_FLOAT64: + blobio_writed(io, *(double *)v); + break; + case EZ_BLOB_STRING | EZ_BLOB_NULLABLE: + if (!*(char **)v) { + blobio_write32(io, 0); + break; + } + blobio_write32(io, 1); + // falls through + case EZ_BLOB_STRING: { + int32_t size = strlen(*(char **)v); + + blobio_write32(io, size); + blobio_write(io, *(char **)v, size); + blobio_write_align(io, 4); + break; + } + case EZ_BLOB_BLOB: { + uint32_t size = ((ez_blob *)v)->size; + + blobio_write32(io, size); + blobio_write(io, ((ez_blob *)v)->data, size); + blobio_write_align(io, 4); + break; + } + case EZ_BLOB_STRUCT: + xdrn_encode_raw(io, d->bd_table, v); + break; + case EZ_BLOB_POINTER | EZ_BLOB_NULLABLE: + if (!*(void **)v) { + blobio_write32(io, 0); + break; + } + blobio_write32(io, 1); + // falls through + case EZ_BLOB_POINTER: + xdrn_encode_raw(io, d->bd_table, *(void **)v); + break; + case EZ_BLOB_LIST: + blobio_write32(io, ez_list_size((ez_list *)v)); + for (ez_node *w = ez_list_head((ez_list *)v), *n = ez_node_succ(w); + n; + w = n, n=ez_node_succ(n)) { + xdrn_encode_raw(io, d->bd_table, w); + } + break; + default: + io->error = 1; + return; + } + } +} + +size_t ez_xdrn_size(const ez_blob_desc *d, const void *p) { + struct ez_blobio io = { + .mode = BLOBIO_WRITE_SIZE + }; + xdrn_encode_raw(&io, d, p); + + return io.index; +} + +int ez_xdrn_encode_raw(const ez_blob_desc *d, const void *p, ez_blob *blob) { + struct ez_blobio io = { + .data = blob->data, + .size = blob->size, + .mode = BLOBIO_WRITE_FIXED + }; + + xdrn_encode_raw(&io, d, p); + + return io.error; +} + +int ez_xdrn_encode(const ez_blob_desc *d, const void *p, ez_blob *blob) { + struct ez_blobio io = { + .mode = BLOBIO_WRITE_ALLOC + }; + + xdrn_encode_raw(&io, d, p); + + if (!io.error) { + blob->size = io.index; + blob->data = io.data; + return 0; + } + + free(blob->data); + + return -1; +} + +static void xdrn_decode_raw(struct ez_blobio *io, const ez_blob_desc *desc, void *p) { + ez_blob_init(desc, p); + + for (int i=0,dlen=desc->bd_length;ibd_offset); + + switch (d->bd_type) { + case EZ_BLOB_INT8: + *(uint8_t *)v = blobio_readi32(io); + break; + case EZ_BLOB_INT16: + *(uint16_t *)v = blobio_readi32(io); + break; + case EZ_BLOB_INT32: + *(uint32_t *)v = blobio_readi32(io); + break; + case EZ_BLOB_INT64: + *(uint64_t *)v = blobio_readi64(io); + break; + case EZ_BLOB_FLOAT32: + *(float *)v = blobio_readf(io); + break; + case EZ_BLOB_FLOAT64: + *(double *)v = blobio_readd(io); + break; + case EZ_BLOB_STRING | EZ_BLOB_NULLABLE: + if (blobio_readi32(io) == 0) + break; + // falls through + case EZ_BLOB_STRING: { + uint32_t size = blobio_readi32(io); + void *src = blobio_take(io, size); + + if (src) { + char *mem = malloc(size); + + if (!mem) + goto error; + + memcpy(mem, src, size); + mem[size] = 0; + *(char **)v = mem; + blobio_read_align(io, 4); + } + break; + } + case EZ_BLOB_BLOB: { + uint32_t size = blobio_readi32(io); + void *src = blobio_take(io, size); + + if (src) { + char *mem = malloc(size); + + if (!mem) + goto error; + + memcpy(mem, src, size); + ((ez_blob *)v)->data = mem; + ((ez_blob *)v)->size = size; + blobio_read_align(io, 4); + } + break; + } + case EZ_BLOB_STRUCT: + xdrn_decode_raw(io, d->bd_table, v); + break; + case EZ_BLOB_POINTER | EZ_BLOB_NULLABLE: + if (blobio_readi32(io) == 0) + break; + case EZ_BLOB_POINTER: { + void *mem = malloc(d->bd_table->bd_offset); + + if (!mem) + goto error; + + *(void **)v = mem; + xdrn_decode_raw(io, d->bd_table, mem); + break; + } + case EZ_BLOB_LIST: { + uint32_t count = blobio_readi32(io); + + for (int j=0;jbd_table->bd_offset); + + if (!node) + goto error; + + xdrn_decode_raw(io, d->bd_table, node); + ez_list_addtail((ez_list *)v, node); + } + break; + } + default: + goto error; + } + } + return; + error: + io->index = io->size; + io->error = 1; +} + +int ez_xdrn_decode_raw(const ez_blob_desc *desc, const ez_blob *blob, void *p) { + struct ez_blobio io = { + .size = blob->size, + .data = blob->data, + .mode = BLOBIO_READ + }; + + xdrn_decode_raw(&io, desc, p); + + if (io.error) { + ez_blob_free_raw(desc, p); + return -1; + } + + return 0; +} + +void *ez_xdrn_decode(const ez_blob_desc *desc, const ez_blob *blob) { + void *p = malloc(desc->bd_offset); + + if (p && ez_xdrn_decode_raw(desc, blob, p) != 0) { + free(p); + p = NULL; + } + + return p; +} diff --git a/ez-blob-xdrn.h b/ez-blob-xdrn.h new file mode 100644 index 0000000..90e7af5 --- /dev/null +++ b/ez-blob-xdrn.h @@ -0,0 +1,30 @@ +/* ez-blob-xdrn.c: XDR-like serialiser using native ordering. + + Copyright (C) 2019 Michael Zucchi + + This program is free software: you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public License + as published by the Free Software Foundation, either version 3 of + the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this program. If not, see + . +*/ + +#ifndef _EZ_BLOB_XDRN_H +#define _EZ_BLOB_XDRN_H + +size_t ez_xdrn_size(const ez_blob_desc *d, const void *p); +int ez_xdrn_encode_raw(const ez_blob_desc *d, const void *p, ez_blob *blob); +int ez_xdrn_encode(const ez_blob_desc *d, const void *p, ez_blob *blob); + +int ez_xdrn_decode_raw(const ez_blob_desc *desc, const ez_blob *blob, void *p); +void *ez_xdrn_decode(const ez_blob_desc *desc, const ez_blob *blob); + +#endif diff --git a/ez-blob.c b/ez-blob.c index be6ddc0..14271cb 100644 --- a/ez-blob.c +++ b/ez-blob.c @@ -1,4 +1,4 @@ -/* ez-blob.h: Serialising description and serialiser. +/* ez-blob.h: Serialising description and utils. Copyright (C) 2019 Michael Zucchi @@ -26,83 +26,9 @@ #include "ez-blob.h" #include "ez-list.h" -/* - This implements a basic 'high performance' serialisation based on the - descriptors. - - Fields are encoded in the same order as the descriptor with no - labelling. Therefore the same descriptor must be used on both ends. - Fields may be at arbitrary alignments. - - primitives - encoded in machine order at native size - - strings - 4-byte length followed by characters. No nul termination. A length - of ~0 indicates a null string pointer. - - arrays - 4-byte length followed by data. ** currently null pointer arrays with zero length are converted to a zero-length array. - - struct - 4-byte length followed by blob-encoded data. - - pointer - 4-byte length followed by blob-encoded data. A length of ~0 - indicates a null pointer. - -*/ - -static const int primitive_size[EZ_BLOB_FLOAT64+1] = { - 1, 2, 4, 8, - 4, 8 -}; - -size_t ez_blob_size(const ez_blob_desc *d, const void *p) { - size_t size = 0; - - for (;(d->bd_type != EZ_BLOB_END);d++) { - if (d->bd_type <= EZ_BLOB_FLOAT64) { - size += primitive_size[d->bd_type]; - } else { - const void *v = p + d->bd_offset; - switch (d->bd_type) { - case EZ_BLOB_STRING: { - char *s = ((char **)v)[0]; - size += s ? 4 + strlen(s) : 4; - break; } - case EZ_BLOB_BLOB: - size += 4 + ((const struct ez_blob *)v)->size; - break; - case EZ_BLOB_STRUCT: { - const void *sp = v; - size += 4 + ez_blob_size(d->bd_table, sp); - break; } - case EZ_BLOB_POINTER: { - const void *s = ((const void * const *)v)[0]; - size += s ? 4 + ez_blob_size(d->bd_table, s) : 4; - break; } - case EZ_BLOB_LIST: { - ez_list *l = (ez_list *)v; - - size += 4; - for (ez_node *w = ez_list_head(l), *n = ez_node_succ(w); - n; - w = n, n=ez_node_succ(n)) { - size += 4 + ez_blob_size(d->bd_table, w); - } - break; } - default: - break; - } - } - } - - return size; -} - -void ez_blob_free_raw(const ez_blob_desc *d, void *p) { - for (;(d->bd_type != EZ_BLOB_END);d++) { +void ez_blob_free_raw(const ez_blob_desc *desc, void *p) { + for (int i=0,dlen = desc->bd_length;ibd_offset; switch (d->bd_type) { @@ -125,7 +51,8 @@ void ez_blob_free_raw(const ez_blob_desc *d, void *p) { w = n, n=ez_node_succ(n)) { ez_blob_free(d->bd_table, w); } - break; } + break; + } default: break; } @@ -139,250 +66,20 @@ void ez_blob_free(const ez_blob_desc *d, void *p) { } } -void *ez_blob_decode_raw(const ez_blob_desc *d, const ez_blob *blob, void *p) { - const char *b = blob->data; - const char *be = b + blob->size; - - memset(p, 0, d->bd_offset); - - for (;(d->bd_type != EZ_BLOB_END);d++) { - void *v = (p + d->bd_offset); - - if (d->bd_type <= EZ_BLOB_FLOAT64) { - int psize = primitive_size[d->bd_type]; - - if (b + psize > be) - goto fail; - memcpy(v, b, psize); - b += psize; - } else if (d->bd_type == EZ_BLOB_LIST) { - uint32_t count; - ez_list *l = v; - - if (b + 4 > be) - goto fail; - memcpy(&count, b, 4); - b += 4; - - ez_list_init(l); - for (int i=0;i be) - goto fail; - memcpy(&ss, b, 4); - b += 4; - - sub.size = ss; - sub.data = (void *)b; - - if (b + ss > be) - goto fail; - if ((s = calloc(d->bd_table->bd_offset, 1)) == NULL) - goto fail; - if (ez_blob_decode_raw(d->bd_table, &sub, s) == NULL) { - free(s); - goto fail; - } - ez_list_addtail(l, s); - b+=ss; - } - } else if (d->bd_type <= EZ_BLOB_LAST) { - uint32_t ss; - ez_blob sub; - - if (b + 4 > be) - goto fail; - memcpy(&ss, b, 4); - b += 4; +void ez_blob_init(const ez_blob_desc *desc, void *p) { + memset(p, 0, desc->bd_offset); - sub.size = ss; - sub.data = (char *)b; + for (int i=0,dlen = desc->bd_length;ibd_type) { - case EZ_BLOB_STRING: { - char *s = NULL; - - if (ss != ~0) { - if (b + ss > be) - goto fail; - if ((s = malloc((size_t)ss + 1)) == NULL) - goto fail; - memcpy(s, b, ss); - s[ss] = 0; - b += ss; - } - ((char **)v)[0] = s; - break; } - case EZ_BLOB_BLOB: { - struct ez_blob *a = v; - - if (b + ss > be) - goto fail; - a->size = ss; - if ((a->data = malloc(ss)) == NULL) - goto fail; - memcpy(a->data, b, ss); - b += ss; - break; } - case EZ_BLOB_STRUCT: - if (b + ss > be) - goto fail; - - if (ez_blob_decode_raw(d->bd_table, &sub, v) == NULL) - goto fail; - b += ss; - break; - case EZ_BLOB_POINTER: { - void *s = NULL; - - if (ss != ~0) { - if (b + ss > be) - goto fail; - if ((s = calloc(d->bd_table->bd_offset, 1)) == NULL) - goto fail; - if (ez_blob_decode_raw(d->bd_table, &sub, s) == NULL) { - free(s); - goto fail; - } - b += ss; - } - ((void **)v)[0] = s; - break; } - default: - break; - } - } + if (d->bd_type == EZ_BLOB_LIST) + ez_list_init((ez_list *)(p + d->bd_offset)); } - if (b != be) - goto fail; - - return p; - fail: - ez_blob_free_raw(d, p); - return NULL; } -void *ez_blob_decode(const ez_blob_desc *d, const ez_blob *blob) { - void *p = calloc(d->bd_offset, 1); - - if (ez_blob_decode_raw(d, blob, p) == NULL) { - free(p); - return NULL; - } - +void *ez_blob_alloc(const ez_blob_desc *d) { + void *p = malloc(d->bd_offset); + if (p) + ez_blob_init(d, p); return p; } - -void ez_blob_encode_raw(const ez_blob_desc *d, const void *p, ez_blob *blob) { - char *b = blob->data; - char *be = b+blob->size; - - for (;(d->bd_type != EZ_BLOB_END);d++) { - const void *v = (p + d->bd_offset); - - if (d->bd_type <= EZ_BLOB_FLOAT64) { - int psize = primitive_size[d->bd_type]; - - assert(b+psize <= be); - memcpy(b, v, psize); - b += psize; - } else if (d->bd_type == EZ_BLOB_LIST) { - ez_list *l = (void *)v; - uint32_t ss = ez_list_size(l); - - memcpy(b, &ss, 4); - b += 4; - for (ez_node *w = ez_list_head(l), *n = ez_node_succ(w); - n; - w = n, n=ez_node_succ(n)) { - ez_blob sub; - - ss = ez_blob_size(d->bd_table, w); - - assert(b+4+ss <= be); - memcpy(b, &ss, 4); - sub.data = b+4; - sub.size = ss; - ez_blob_encode_raw(d->bd_table, w, &sub); - b += 4 + ss; - } - } else if (d->bd_type <= EZ_BLOB_LAST) { - uint32_t ss; - struct ez_blob sub; - - switch (d->bd_type) { - case EZ_BLOB_STRING: { - char *s = ((char **)v)[0]; - if (s) { - ss = strlen(s); - assert(b+4+ss <= be); - memcpy(b, &ss, 4); - memcpy(b+4, s, ss); - b += 4 + ss; - } else { - ss = ~0; - assert(b+4 <= be); - memcpy(b, &ss, 4); - b += 4; - } - break; } - case EZ_BLOB_BLOB: { - const struct ez_blob *a = v; - - ss = a->size; - assert(b+4+ss <= be); - memcpy(b, &ss, 4); - memcpy(b+4, a->data, ss); - b += 4 + ss; - break; } - case EZ_BLOB_STRUCT: - ss = ez_blob_size(d->bd_table, v); - - assert(b+4+ss <= be); - memcpy(b, &ss, 4); - sub.data = b+4; - sub.size = ss; - ez_blob_encode_raw(d->bd_table, v, &sub); - b += 4 + ss; - break; - case EZ_BLOB_POINTER: { - const void *s = ((const void **)v)[0]; - - if (s) { - ss = ez_blob_size(d->bd_table, s); - assert(b+4+ss <= be); - memcpy(b, &ss, 4); - sub.data = b+4; - sub.size = ss; - ez_blob_encode_raw(d->bd_table, s, &sub); - b += 4 + ss; - } else { - ss = ~0; - assert(b+4 <= be); - memcpy(b, &ss, 4); - b += 4; - } - break; } - default: - break; - } - } - } - assert(b == be); -} - -int ez_blob_encode(const ez_blob_desc *d, const void *p, ez_blob *blob) { - blob->size = ez_blob_size(d, p); - blob->data = malloc(blob->size); - - if (blob->data) { - ez_blob_encode_raw(d, p, blob); - return 0; - } - - return -1; -} - diff --git a/ez-blob.h b/ez-blob.h index 9dfad2f..6ee3f05 100644 --- a/ez-blob.h +++ b/ez-blob.h @@ -1,4 +1,4 @@ -/* ez-blob.h: Serialising description and serialiser. +/* ez-blob.h: Serialising description. Copyright (C) 2019 Michael Zucchi @@ -23,23 +23,17 @@ #include /** - * This is a C structure serialiser. + * This is a C structure annotation for implementing a serialiser. * * It is table driven and can describe most common C data structures, * including nested and linked structures. * - * The implemented serialisation mechanism only supports - * forward-linked structures and will not perform well when - * marshalling deeply nested structures but other serialisation - * mechanisms are possible. - */ - -/** - * An array of ez_blob_desc is passed to ez_blob_encode/decode. The first - * entry must be of type EZ_BLOB_PK and it's 'offset' must be the size - * of the structure. The final entry must be of type EZ_BLOB_END. + * A structure is described by an array of ez_blob_desc. The first + * entry must be of type EZ_BLOB_PK, it's 'offset' must be the size + * of the structure, and .u.count must be the number of fields + * following. * - * As an example: + * As an example: * * typedef struct dbdisk { * int id; @@ -48,19 +42,16 @@ * char *type; * } dbdisk; * - * Has a table such as this, using the init macros. + * Has a table such as this, using the init macros. * * static ez_blob_desc DISK_ENC[] = { - * EZ_BLOB_START(dbdisk), + * EZ_BLOB_START(dbdisk, 0, 3), * EZ_BLOB_STRING(dbdisk, 1, uuid), * EZ_BLOB_STRING(dbdisk, 2, label), * EZ_BLOB_STRING(dbdisk, 3, type), - * EZ_BLOB_END(dbdisk) * }; * - * The 'id' field (of ez_blob_desc) is currently unused by the blob - * serialiser but could be implemented for data versioning or partial - * serialisation. + * The 'id' field and it's semantics are serialiser dependent. * */ typedef struct ez_blob_desc { @@ -69,8 +60,10 @@ typedef struct ez_blob_desc { unsigned int bd_offset; // offset into structure, if type == EZ_BLOB_PK then this is sizeof(struct) union { const struct ez_blob_desc *table;// embedded structure or pointer + unsigned int length; } u; #define bd_table u.table +#define bd_length u.length } ez_blob_desc; /** @@ -83,13 +76,15 @@ typedef enum ez_blob_desc_type { EZ_BLOB_INT64, EZ_BLOB_FLOAT32, EZ_BLOB_FLOAT64, - EZ_BLOB_STRING, // normall c strring + EZ_BLOB_STRING, // normal c strring EZ_BLOB_BLOB, // embedded ez_blob for arbitrary bytes. EZ_BLOB_STRUCT, // bd_table points to another descriptor list, object embedded EZ_BLOB_POINTER, // bd_table points to another descriptor list, object pointer EZ_BLOB_LIST, // ez_list nodes. EZ_BLOB_LAST = EZ_BLOB_LIST, EZ_BLOB_PK = 16, // MUST always be first element in descriptor and occur only once + EZ_BLOB_NULLABLE = 0x8000, // Pointer may be NULL for STRING, POINTER + EZ_BLOB_TYPE = 0xff, // Type mask EZ_BLOB_END } ez_blob_desc_type; @@ -101,7 +96,7 @@ typedef struct ez_blob { void *data; } ez_blob; -#define EZ_BLOB_START(s) { EZ_BLOB_PK, 0, sizeof(s) } +#define EZ_BLOB_START(s, id, len) { EZ_BLOB_PK, 0, sizeof(s), .u.length = len } #define EZ_BLOB_INT8(s, id, f) { EZ_BLOB_INT8, id, offsetof(s, f) } #define EZ_BLOB_INT16(s, id, f) { EZ_BLOB_INT16, id, offsetof(s, f) } #define EZ_BLOB_INT32(s, id, f) { EZ_BLOB_INT32, id, offsetof(s, f) } @@ -109,58 +104,31 @@ typedef struct ez_blob { #define EZ_BLOB_FLOAT32(s, id, f) { EZ_BLOB_FLOAT32, id, offsetof(s, f) } #define EZ_BLOB_FLOAT64(s, id, f) { EZ_BLOB_FLOAT64, id, offsetof(s, f) } #define EZ_BLOB_STRING(s, id, f) { EZ_BLOB_STRING, id, offsetof(s, f) } +#define EZ_BLOB_STRING_NULL(s, id, f) { EZ_BLOB_STRING | EZ_BLOB_NULLABLE, id, offsetof(s, f) } #define EZ_BLOB_BLOB(s, id, f) { EZ_BLOB_BLOB, id, offsetof(s, f) } #define EZ_BLOB_STRUCT(s, id, f, other) { EZ_BLOB_STRUCT, id, offsetof(s, f), .u.table = other } #define EZ_BLOB_POINTER(s, id, f, other) { EZ_BLOB_POINTER, id, offsetof(s, f), .u.table = other } +#define EZ_BLOB_POINTER_NULL(s, id, f, other) { EZ_BLOB_POINTER | EZ_BLOB_NULLABLE, id, offsetof(s, f), .u.table = other } #define EZ_BLOB_LIST(s, id, f, other) { EZ_BLOB_LIST, id, offsetof(s, f), .u.table = other } -#define EZ_BLOB_END(s) { EZ_BLOB_END } /** - * Calculate the serialised blob size for a struct. + * Allocate a new structure to hold a blob. * - * @param d descriptor table matching struct. - * @param p struct. - */ -size_t ez_blob_size(const ez_blob_desc *d, const void *p); - -/** - * Marshal a struct to a blob. - * - * @param d descriptor table matching the struct. - * @param p structure. - * @param sizep return pointer for blob size, must not be null. - * @param blob return, the data pointer must be freed with free() when complete. - * @return non-zero on failure. - */ -int ez_blob_encode(const ez_blob_desc *d, const void *p, ez_blob *blob); - -/** - * Marshal a struct to a blob directly + * The structure is initialised by ez_blob_init(). * - * @param d descriptor table matching the struct. - * @param p structure. - * @param blob available target memory, size must be exactly ez_blob_size(). + * @param d the descriptor table for the blob. */ -void ez_blob_encode_raw(const ez_blob_desc *d, const void *p, ez_blob *blob); +void *ez_blob_alloc(const ez_blob_desc *d); /** - * Unmarshall a blob into a struct. + * Initialise a struct to the default state. * - * @param d descriptor table matching blob. - * @param blob raw blob. - * @param size blob size. - * @return the decoded struct. May be freed using ez_blob_free(). - */ -void *ez_blob_decode(const ez_blob_desc *d, const ez_blob *blob); - -/** - * Decode to pre-allocated buffer. + * This sets all values to zero and lists to empty. * - * @param b blob data - * @param size blob size - * @return p is returned on success. On failure NULL is returned and any internal pointers are freed. + * @param d descriptor table matching struct. + * @param p pointer to struct base. */ -void *ez_blob_decode_raw(const ez_blob_desc *d, const ez_blob *blob, void *p); +void ez_blob_init(const ez_blob_desc *d, void *p); /** * Free struct unmarshalled by ez_blob_decode(). @@ -173,7 +141,8 @@ void ez_blob_free(const ez_blob_desc *d, void *p); /** * Free the contents of a structure, but don't free the container. * - * @todo should this zero the structure too? + * Use ez_blob_init() if you want to re-use the object. + * * @param p structure to free. p will not be freed. */ void ez_blob_free_raw(const ez_blob_desc *d, void *p); diff --git a/test-blob.c b/test-blob.c index bce25d4..0a05817 100644 --- a/test-blob.c +++ b/test-blob.c @@ -10,6 +10,10 @@ #include "ez-blob.h" #include "ez-list.h" +#include "ez-blob-blob.h" +#include "ez-blob-tagz.h" +#include "ez-blob-xdrn.h" + struct simple { char b; short s; @@ -19,13 +23,12 @@ struct simple { }; static const ez_blob_desc simple_DESC[] = { - EZ_BLOB_START(struct simple), + EZ_BLOB_START(struct simple, 0, 5), EZ_BLOB_INT8(struct simple, 1, b), EZ_BLOB_INT16(struct simple, 2, s), EZ_BLOB_INT32(struct simple, 3, a), EZ_BLOB_FLOAT32(struct simple, 4, c), EZ_BLOB_FLOAT64(struct simple, 5, d), - EZ_BLOB_END(struct simple) }; struct arrays { @@ -35,10 +38,9 @@ struct arrays { }; static const ez_blob_desc array_DESC[] = { - EZ_BLOB_START(struct arrays), + EZ_BLOB_START(struct arrays, 0, 2), EZ_BLOB_STRING(struct arrays, 1, string), EZ_BLOB_BLOB(struct arrays, 2, array), - EZ_BLOB_END(struct arrays) }; struct embedded { @@ -47,10 +49,9 @@ struct embedded { }; static const ez_blob_desc embed_DESC[] = { - EZ_BLOB_START(struct embedded), + EZ_BLOB_START(struct embedded, 0, 2), EZ_BLOB_STRUCT(struct embedded, 1, s, simple_DESC), EZ_BLOB_STRUCT(struct embedded, 2, a, array_DESC), - EZ_BLOB_END(struct embedded) }; struct tree { @@ -61,11 +62,10 @@ struct tree { }; static const ez_blob_desc tree_DESC[] = { - EZ_BLOB_START(struct tree), + EZ_BLOB_START(struct tree, 0, 3), EZ_BLOB_STRING(struct tree, 1, s), - EZ_BLOB_POINTER(struct tree, 2, left, tree_DESC), - EZ_BLOB_POINTER(struct tree, 3, right, tree_DESC), - EZ_BLOB_END(struct tree) + EZ_BLOB_POINTER_NULL(struct tree, 2, left, tree_DESC), + EZ_BLOB_POINTER_NULL(struct tree, 3, right, tree_DESC), }; struct listnode { @@ -75,10 +75,9 @@ struct listnode { }; static const ez_blob_desc listnode_DESC[] = { - EZ_BLOB_START(struct listnode), + EZ_BLOB_START(struct listnode, 0, 2), EZ_BLOB_STRING(struct listnode, 1, name), EZ_BLOB_BLOB(struct listnode, 2, value), - EZ_BLOB_END(struct listnode) }; struct list { @@ -87,10 +86,9 @@ struct list { }; static const ez_blob_desc list_DESC[] = { - EZ_BLOB_START(struct list), + EZ_BLOB_START(struct list, 0, 2), EZ_BLOB_STRING(struct list, 1, name), EZ_BLOB_LIST(struct list, 2, list, listnode_DESC), - EZ_BLOB_END(struct list) }; static void dumphex(const char *data, size_t size, const char *prefix) { @@ -115,9 +113,10 @@ static void blob_print(const ez_blob_desc *d, const void *a, int depth) { ez_blob_print(d, a, depth); } -static int blob_equals(const ez_blob_desc *d, const void *a, const void *b) { +static int blob_equals(const ez_blob_desc *desc, const void *a, const void *b) { int e = 1; - for (;e && (d->bd_type != EZ_BLOB_END);d++) { + for (int i=0, dlen=desc->bd_length; e && i < dlen; i++) { + const ez_blob_desc *d = &desc[i+1]; const void *u = a + d->bd_offset; const void *v = b + d->bd_offset; @@ -140,8 +139,19 @@ static int blob_equals(const ez_blob_desc *d, const void *a, const void *b) { case EZ_BLOB_FLOAT64: e &= (*(double *)u) == (*(double *)v); break; + case EZ_BLOB_STRING | EZ_BLOB_NULLABLE: + if (!*(char **)u || !*(char **)v) { + e &= (*(char **)u) == (*(char **)v); + break; + } + // falls through case EZ_BLOB_STRING: - e &= strcmp((*(char **)u), (*(char **)v)) == 0; + if (!*(char **)u || !*(char **)v) { + printf("strings can't be null\n"); + e = 0; + } else { + e &= strcmp(*(char **)u, *(char **)v) == 0; + } break; case EZ_BLOB_BLOB: { const struct ez_blob *ua = u; @@ -149,7 +159,17 @@ static int blob_equals(const ez_blob_desc *d, const void *a, const void *b) { e &= ua->size == va->size && memcmp(ua->data, va->data, ua->size) == 0; - break; } + break; + } + case EZ_BLOB_POINTER | EZ_BLOB_NULLABLE: + if (!*(void **)u || !*(void **)v) { + e &= *(void **)u == *(void **)v; + break; + } + // falls through + case EZ_BLOB_POINTER: + e &= blob_equals(d->bd_table, *(void **)u, *(void **)v); + break; case EZ_BLOB_LIST: { ez_list *ul = (void *)u; ez_list *vl = (void *)v; @@ -163,27 +183,45 @@ static int blob_equals(const ez_blob_desc *d, const void *a, const void *b) { e &= blob_equals(d->bd_table, uw, vw); } - break; } + break; + } + default: + printf("cmp unknown field type=%d id=%d\n", d->bd_type, d->bd_id); + e = 0; + break; + } + if (!e) { + printf("cmp failed in struct %d at field %d type %04x\n", desc->bd_id, d->bd_id, d->bd_type); + abort(); } } return e; } -static void test_basics(const ez_blob_desc *d, void *src) { +struct test_funcs { + const char *name; + int (*blob_encode)(const ez_blob_desc *d, const void *p, ez_blob *blob); + void *(*blob_decode)(const ez_blob_desc *desc, const ez_blob *blob); + int (*blob_decode_raw)(const ez_blob_desc *desc, const ez_blob *blob, void *p); +}; + +static void test_basics(const ez_blob_desc *d, void *src, struct test_funcs *funcs) { ez_blob blob, blob2; void *t; size_t dsize = d->bd_offset; char tmp[dsize + 64] __attribute__ ((aligned(64))); + int res; printf("source\n"); blob_print(d, src, 4); - ez_blob_encode(d, src, &blob); + res = funcs->blob_encode(d, src, &blob); + assert(res == 0); dumphex(blob.data, blob.size, "serial: "); // check decode equals - t = ez_blob_decode(d, &blob); + t = funcs->blob_decode(d, &blob); assert(t != NULL); assert(blob_equals(d, t, src)); @@ -191,7 +229,7 @@ static void test_basics(const ez_blob_desc *d, void *src) { blob_print(d, t, 4); // check encode-decoded equals source - ez_blob_encode(d, t, &blob2); + funcs->blob_encode(d, t, &blob2); assert(blob.size == blob2.size); assert(memcmp(blob.data, blob2.data, blob2.size) == 0); free(blob2.data); @@ -200,7 +238,7 @@ static void test_basics(const ez_blob_desc *d, void *src) { // check raw decode stays within bounds of struct memset(tmp, 0xbe, sizeof(tmp)); - t = ez_blob_decode_raw(d, &blob, &tmp[32]); + funcs->blob_decode_raw(d, &blob, &tmp[32]); //assert(t != NULL); //assert(blob_equals(d, tmp+32, src)); dumphex(tmp, sizeof(tmp), "object: "); @@ -213,16 +251,17 @@ static void test_basics(const ez_blob_desc *d, void *src) { free(blob.data); } -static void test_simple(void) { +static void test_simple(struct test_funcs *funcs) { struct simple src = { 'z', 0xff0f, 1, 3.14159f, 5644941221133 }; const ez_blob_desc *d = simple_DESC; - test_basics(d, &src); + printf("test simple\n"); + test_basics(d, &src, funcs); } -static void test_arrays(void) { +static void test_arrays(struct test_funcs *funcs) { char data[7] = { 1, 2, 3, 4, 5, 6, 7 }; struct arrays src = { 32, @@ -232,10 +271,11 @@ static void test_arrays(void) { }; const ez_blob_desc *d = array_DESC; - test_basics(d, &src); + printf("test arrays\n"); + test_basics(d, &src, funcs); } -static void test_rawtree(void) { +static void test_rawtree(struct test_funcs *funcs) { struct tree src[6] = { { &src[1], &src[3], "root" }, { NULL, &src[2], "left" }, @@ -246,10 +286,11 @@ static void test_rawtree(void) { }; const ez_blob_desc *d = tree_DESC; - test_basics(d, &src); + printf("test raw tree\n"); + test_basics(d, &src, funcs); } -static void test_list(void) { +static void test_list(struct test_funcs *funcs) { struct list list = { .name = "a list", .list = EZ_INIT_LIST(list.list) @@ -267,12 +308,41 @@ static void test_list(void) { const ez_blob_desc *d = list_DESC; - test_basics(d, &list); + printf("test list\n"); + test_basics(d, &list, funcs); } +struct test_funcs backends[] = { + { + "blob legacy", + ez_blob_encode, + ez_blob_decode, + ez_blob_decode_raw + }, + { + "tagz", + ez_tagz_encode, + ez_tagz_decode, + ez_tagz_decode_raw + }, + { + "xdrn", + ez_xdrn_encode, + ez_xdrn_decode, + ez_xdrn_decode_raw + }, +}; + int main(int argc, char **argv) { - test_simple(); - test_arrays(); - test_rawtree(); - test_list(); + for (int i=0;i<3;i++) { + struct test_funcs *f = &backends[i]; + printf("* **********************************************************************\n"); + printf("test: %s\n", f->name); + fflush(stdout); + + test_simple(f); + test_arrays(f); + test_rawtree(f); + test_list(f); + } }