From: Not Zed Date: Thu, 2 May 2019 01:33:27 +0000 (+0930) Subject: Support 64-bit sequence lengths with some bit fiddling. X-Git-Url: https://code.zedzone.au/cvs?a=commitdiff_plain;h=802c70067e39ddde84d886a1b86eabb931cbcfcc;p=libeze Support 64-bit sequence lengths with some bit fiddling. Fix incorrect use of realloc() Fix incorrect free on failure. Added a 64-bit array length test. --- diff --git a/ez-blob-io.c b/ez-blob-io.c index 0a8310b..c943ced 100644 --- a/ez-blob-io.c +++ b/ez-blob-io.c @@ -40,18 +40,23 @@ void *blobio_reserve(struct ez_blobio * __restrict io, size_t len) { switch (io->mode) { case BLOBIO_WRITE_ALLOC: if (to > size) { + void *alloc; + do { size = size ? size * 2 : 256; } while (to > size); io->size = size; - io->data = realloc(io->data, size); + alloc = realloc(io->data, size); - if (!io->data) { + if (!alloc) { io->error = 1; + printf("realloc failed\n"); ABORT(); return NULL; } + + io->data = alloc; } v = io->index + io->data; diff --git a/ez-blob-tagz.c b/ez-blob-tagz.c index fe88304..1c6b7f1 100644 --- a/ez-blob-tagz.c +++ b/ez-blob-tagz.c @@ -50,19 +50,19 @@ 0 1 byte 1 2 byte 2 4 byte - 3 reserved, no tag? + 3 use the count size for tt instead, and there is no count byte cc log2 of count size in bytes, used to indicate sequence length or non-sequence. 0 1 byte 1 2 byte 2 4 byte - 3 none, single item follows + 3 8 byte ff is struct-end code A header is a control byte followed by an optional 1/2/4 byte-length tag, - followed by an optional 1/2/4 byte-length count. + followed by an optional 1/2/4/8 byte-length count. A structure payload is a list of tagged fields until a struct-end code. A structure sequence is a list of count struct-encoded blocks. @@ -119,7 +119,7 @@ final note: #define EZT_TAGSIZE (0x3 << EZT_TAGSHIFT) #define EZT_DATASIZE (0x3 << EZT_DATASHIFT) -#define EZT_NOCOUNT (0x3 << EZT_COUNTSHIFT) +#define EZT_NOCOUNT (0x3 << EZT_TAGSHIFT) #define EZT_END 0xff // well, end-struct @@ -152,7 +152,7 @@ static int size_code(uint64_t size) { return 0x3; } -static int count_code(uint32_t size) { +static int tag_code(uint32_t size) { if (size <= 0xff) return 0x0; else if (size <= 0xffff) @@ -161,21 +161,32 @@ static int count_code(uint32_t size) { return 0x2; } +// tag single item, tt=11 and cc=tag static void blobio_writet(struct ez_blobio *io, uint8_t type, uint32_t tag) { - int tc = size_code(tag); + int tc = tag_code(tag); - blobio_writeb(io, type | (tc << EZT_TAGSHIFT)); + blobio_writeb(io, type | EZT_NOCOUNT | (tc << EZT_COUNTSHIFT)); blobio_writei(io, tc, tag); } +// tag counted item, tt=tag, cc=count +static void blobio_writetc(struct ez_blobio *io, uint8_t type, uint32_t tag, uint64_t count) { + int tc = tag_code(tag); + int cc = size_code(count); + + blobio_writeb(io, type | (tc << EZT_TAGSHIFT) | (cc << EZT_COUNTSHIFT)); + blobio_writei(io, tc, tag); + blobio_writei(io, cc, count); +} + static void tagz_encode_struct(struct ez_blobio *io, int id, const ez_blob_desc *desc, const void *p) { - blobio_writet(io, EZT_STRUCT | EZT_NOCOUNT, id); + blobio_writet(io, EZT_STRUCT, id); for (int i=0,dlen=desc->bd_length;ibd_offset); uint64_t val = 0; - uint32_t count = 0; + uint64_t count = 0; const void *ptr = NULL; switch (d->bd_type & EZ_BLOB_TYPE) { @@ -194,16 +205,16 @@ static void tagz_encode_struct(struct ez_blobio *io, int id, const ez_blob_desc if (val) { int sc = size_code(val); - blobio_writet(io, EZT_INT8 | EZT_NOCOUNT | (sc << EZT_DATASHIFT), d->bd_id); + blobio_writet(io, EZT_INT8 | (sc << EZT_DATASHIFT), d->bd_id); blobio_writei(io, sc, val); } break; case EZ_BLOB_FLOAT32: - blobio_writet(io, EZT_FLOAT32 | EZT_NOCOUNT, d->bd_id); + blobio_writet(io, EZT_FLOAT32, d->bd_id); blobio_writef(io, *(const float *)v); break; case EZ_BLOB_FLOAT64: - blobio_writet(io, EZT_FLOAT64 | EZT_NOCOUNT, d->bd_id); + blobio_writet(io, EZT_FLOAT64, d->bd_id); blobio_writed(io, *(const double *)v); break; case EZ_BLOB_STRING: @@ -218,13 +229,9 @@ static void tagz_encode_struct(struct ez_blobio *io, int id, const ez_blob_desc // empty blobs are the default, so implement same, empty are not encoded count = ((ez_blob *)v)->size; if (count) { - int cc; - ptr = ((ez_blob *)v)->data; doblob: - cc = count_code(count); - blobio_writet(io, EZT_INT8 | (cc << EZT_COUNTSHIFT), d->bd_id); - blobio_writei(io, cc, count); + blobio_writetc(io, EZT_INT8, d->bd_id, count); blobio_write(io, ptr, count); } break; @@ -239,10 +246,7 @@ static void tagz_encode_struct(struct ez_blobio *io, int id, const ez_blob_desc case EZ_BLOB_LIST: count = ez_list_size((ez_list *)v); if (count) { - int cc = count_code(count); - - blobio_writet(io, EZT_STRUCT | (cc << EZT_COUNTSHIFT), d->bd_id); - blobio_writei(io, cc, count); + blobio_writetc(io, EZT_STRUCT, d->bd_id, count); for (ez_node *w = ez_list_head((ez_list *)v), *n = ez_node_succ(w); n; @@ -292,7 +296,7 @@ int ez_tagz_encode(const ez_blob_desc *d, const void *p, ez_blob *blob) { blob->data = io.data; return 0; } - free(blob->data); + free(io.data); blob->data = 0; blob->size = 0; @@ -328,15 +332,15 @@ static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, v while ((h = blobio_readb(io)) != EZT_END) { uint32_t ftag; - uint32_t fcount = 0; - int sc = (h & EZT_DATASIZE) >> EZT_DATASHIFT; /* for primitives */ - int use_count = 0; - - ftag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT); - if ((h & EZT_COUNTSIZE) != EZT_NOCOUNT) { - fcount = blobio_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT); - use_count = 1; - } + uint64_t fcount; + int sc = (h >> EZT_DATASHIFT) & 3; + int tc = (h >> EZT_TAGSHIFT) & 3; + int cc = (h >> EZT_COUNTSHIFT) & 3; + int use_count = tc != 3; + + tc = tc != 3 ? tc : cc; + ftag = blobio_readi(io, tc); + fcount = use_count ? blobio_readi(io, cc) : 1; // This forces the tags to be in order // It ensures each field can only be visited once @@ -474,6 +478,8 @@ static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, v } } break; + default: + goto error; } } else { // Just skip fields @@ -513,25 +519,24 @@ static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, v static void tagz_decode_struct(struct ez_blobio *io, const ez_blob_desc *desc, void *p) { uint8_t h = blobio_readb(io); uint32_t stag; - //uint32_t scount = 1; - - // must be a struct - // what about a struct list? hmm - if (io->error || (h & EZT_TYPE) != EZT_STRUCT) { - //io->error = 1; - return; - } + int tc = (h >> EZT_TAGSHIFT) & 3; + int cc = (h >> EZT_COUNTSHIFT) & 3; + + // must be a struct with tc=11 and cc!=11 + if ((h & EZT_TYPE) != EZT_STRUCT) + goto error; + if (tc != 3 || cc == 3) + goto error; - stag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT); + // unused, but could check against desc->bd_id + stag = blobio_readi(io, cc); stag = stag; - //if (stag != desc->bd_id) { - // ?? - //} - //if (h & EZT_COUNTSIZE) - // count = blobio_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT); - // read fields tagz_decode_fields(io, desc, p); + + return; + error: + io->error = 1; } int ez_tagz_decode_raw(const ez_blob_desc *desc, const ez_blob *blob, void *p) { @@ -565,26 +570,35 @@ void *ez_tagz_decode(const ez_blob_desc *desc, const ez_blob *blob) { /* ********************************************************************** */ // TODO: move to separate .o +// TODO: copy fail behaviour of decode() static void tagz_dump_struct(struct ez_blobio *io, int depth); static void tagz_dump_fields(struct ez_blobio *io, int depth); static void tagz_dump_struct(struct ez_blobio *io, int depth) { uint8_t h = blobio_readb(io); uint32_t stag; - //uint32_t scount = 1; + int tc = (h >> EZT_TAGSHIFT) & 3; + int cc = (h >> EZT_COUNTSHIFT) & 3; char s[depth+1]; memset(s, ' ', depth); s[depth] = 0; - // must be a struct - // what about a struct list? hmm - if ((h & EZT_TYPE) != EZT_STRUCT) + // must be a struct with tc=11 and cc!=11 + if ((h & EZT_TYPE) != EZT_STRUCT) { + printf(" not a struct\n"); + return; + } + if (tc != 3) { + printf(" not a unitary object tc=%d\n", tc); + return; + } + if (cc == 3) { + printf(" invalid tag size %d\n", cc); return; + } - stag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT); - //if (h & EZT_COUNTSIZE) - // count = blob_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT); + stag = blobio_readi(io, cc); printf("%s[%02x] %d = {\n", s, h, stag); @@ -601,15 +615,15 @@ static void tagz_dump_fields(struct ez_blobio *io, int depth) { // read fields while ((h = blobio_readb(io)) != EZT_END) { uint32_t ftag; - uint32_t fcount = 0; - int sc = (h & EZT_DATASIZE) >> EZT_DATASHIFT; - int use_count = 0; - - ftag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT); - if ((h & EZT_COUNTSIZE) != EZT_NOCOUNT) { - fcount = blobio_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT); - use_count = 1; - } + uint64_t fcount; + int sc = (h >> EZT_DATASHIFT) & 3; + int tc = (h >> EZT_TAGSHIFT) & 3; + int cc = (h >> EZT_COUNTSHIFT) & 3; + int use_count = tc != 3; + + tc = tc != 3 ? tc : cc; + ftag = blobio_readi(io, tc); + fcount = use_count ? blobio_readi(io, cc) : 1; switch (h & EZT_TYPE) { case EZT_INT8: @@ -619,7 +633,7 @@ static void tagz_dump_fields(struct ez_blobio *io, int depth) { if (use_count) { void *v = blobio_take(io, fcount * (1<index-1); - printf("%s}\n", s); } + printf("%s[%02x] END $%04lx\n", s, h, io->index-1); + printf("%s}\n", s); } /** diff --git a/ez-blob-tagz.h b/ez-blob-tagz.h index df2508c..3a257ed 100644 --- a/ez-blob-tagz.h +++ b/ez-blob-tagz.h @@ -29,4 +29,6 @@ int ez_tagz_encode(const ez_blob_desc *d, const void *p, ez_blob *blob); int ez_tagz_decode_raw(const ez_blob_desc *desc, const ez_blob *blob, void *p); void *ez_tagz_decode(const ez_blob_desc *desc, const ez_blob *blob); +void ez_tagz_dump(ez_blob *blob); + #endif diff --git a/test-blob.c b/test-blob.c index 273b7d4..67f93c9 100644 --- a/test-blob.c +++ b/test-blob.c @@ -254,10 +254,15 @@ static void test_basics(const ez_blob_desc *d, void *src, struct test_funcs *fun } res = funcs->blob_encode(d, src, &blob); - assert(res == 0); + if (res != 0) { + printf(" encode failed: probably out of memory, skipped testing this\n"); + return; + } if (doprint) dumphex(blob.data, blob.size, "serial: "); - + else + dumphex(blob.data, blob.size < 256 ? blob.size : 256, "header: "); + // check decode equals t = funcs->blob_decode(d, &blob); assert(t != NULL); @@ -269,11 +274,12 @@ static void test_basics(const ez_blob_desc *d, void *src, struct test_funcs *fun assert(blob_equals(d, t, src)); // check encode-decoded equals source - funcs->blob_encode(d, t, &blob2); + res = funcs->blob_encode(d, t, &blob2); + assert(res == 0); assert(blob.size == blob2.size); assert(memcmp(blob.data, blob2.data, blob2.size) == 0); free(blob2.data); - + ez_blob_free(d, t); // check raw decode stays within bounds of struct @@ -389,20 +395,32 @@ static void test_intsize(struct test_funcs *funcs) { } static void test_stringsize(struct test_funcs *funcs) { - int sizes[6] = { 0, 1, 255, 256, 65535, 65536 }; + size_t sizes[7] = { 0, 1, 255, 256, 65535, 65536, 0x100000000ULL }; const ez_blob_desc *d = stringsize_DESC; struct stringsize src; int sprint = doprint; - for (int i=0;i<6;i++) { - int size = sizes[i]; + for (int i=0;i<7;i++) { + size_t size = sizes[i]; + + // only tagz supports >32 bit counts + if (size >= 0x100000000ULL + && strcmp(funcs->name, "tagz") != 0) { + printf("skip string size %zd\n", size); + continue; + } + src.value = malloc(size+1); - memset(src.value, ' ', size); - src.value[size] = 0; - printf("test string size %d\n", size); - doprint = size <= 256; - test_basics(d, &src, funcs); - free(src.value); + if (src.value) { + memset(src.value, ' ', size); + src.value[size] = 0; + printf("test string size %zd\n", size); + doprint = size <= 256; + test_basics(d, &src, funcs); + free(src.value); + } else { + printf("skip string size %zd: memory allocation failed\n", size); + } } doprint = sprint; }