From 1b6c3fb16e0d3f8711dcf64fcb546bf11c12249c Mon Sep 17 00:00:00 2001 From: Not Zed Date: Wed, 1 May 2019 23:46:16 +0930 Subject: [PATCH] More blob test cases. Fixed some errors and potential leaks on bad data. --- ez-blob-io.c | 10 +- ez-blob-print.c | 8 +- ez-blob-tagz.c | 177 ++++++++++++++++++++++++----- ez-blob-xdrn.c | 4 +- ez-blob.c | 11 +- test-blob.c | 290 +++++++++++++++++++++++++++++++++++++++++------- 6 files changed, 419 insertions(+), 81 deletions(-) diff --git a/ez-blob-io.c b/ez-blob-io.c index 9dc6701..0a8310b 100644 --- a/ez-blob-io.c +++ b/ez-blob-io.c @@ -91,17 +91,19 @@ void *blobio_take(struct ez_blobio * __restrict io, size_t len) { } void blobio_write_align(struct ez_blobio *io, unsigned int step) { - int skip = (step-1) & io->index; + int skip = (step - io->index) & (step-1); uint8_t *v; - + if (skip && (v = blobio_reserve(io, skip))) { for (int i=0;iindex); + int skip = (step - io->index) & (step-1); + + blobio_take(io, skip); } void blobio_dumphex(const char *data, size_t size, const char *prefix) { diff --git a/ez-blob-print.c b/ez-blob-print.c index 1759016..6cea3bd 100644 --- a/ez-blob-print.c +++ b/ez-blob-print.c @@ -44,16 +44,16 @@ void ez_blob_print(const ez_blob_desc *d, const void *a, int depth) { switch (d->bd_type & EZ_BLOB_TYPE) { case EZ_BLOB_INT8: - printf(" .%d = %d\n", d->bd_id, *(uint8_t *)u); + printf(" .%d = %02x\n", d->bd_id, *(uint8_t *)u); break; case EZ_BLOB_INT16: - printf(" .%d = %d\n", d->bd_id, *(uint16_t *)u); + printf(" .%d = %04x\n", d->bd_id, *(uint16_t *)u); break; case EZ_BLOB_INT32: - printf(" .%d = %d\n", d->bd_id, *(uint32_t *)u); + printf(" .%d = %08x\n", d->bd_id, *(uint32_t *)u); break; case EZ_BLOB_INT64: - printf(" .%d = %ld\n", d->bd_id, *(uint64_t *)u); + printf(" .%d = %016lx\n", d->bd_id, *(uint64_t *)u); break; case EZ_BLOB_FLOAT32: printf(" .%d = %f\n", d->bd_id, *(float *)u); diff --git a/ez-blob-tagz.c b/ez-blob-tagz.c index d33ae9c..fe88304 100644 --- a/ez-blob-tagz.c +++ b/ez-blob-tagz.c @@ -65,7 +65,7 @@ followed by an optional 1/2/4 byte-length count. A structure payload is a list of tagged fields until a struct-end - code. A structure sequence includes count of a tagged structure. + code. A structure sequence is a list of count struct-encoded blocks. Integers can be stored in the smallest number of bytes, i.e. with all leading $00 bytes removed. @@ -207,6 +207,7 @@ static void tagz_encode_struct(struct ez_blobio *io, int id, const ez_blob_desc blobio_writed(io, *(const double *)v); break; case EZ_BLOB_STRING: + // NULL strings are the default, so implement same but allow for empty strings ptr = *(const void **)v; if (ptr) { count = strlen(ptr); @@ -214,6 +215,7 @@ static void tagz_encode_struct(struct ez_blobio *io, int id, const ez_blob_desc } break; case EZ_BLOB_BLOB: + // empty blobs are the default, so implement same, empty are not encoded count = ((ez_blob *)v)->size; if (count) { int cc; @@ -316,35 +318,35 @@ static uint64_t blobio_readi(struct ez_blobio *io, int sc) { return 0; } -// TODO: binary search, length is in first field -// Alternative: if encode/decode is same order, then can just search forward from last spot -static const ez_blob_desc *blob_desc_find(const ez_blob_desc *desc, int id) { - for (int i=0,dlen=desc->bd_length;ibd_id == id) - return d; - } - return NULL; -} - static void tagz_decode_struct(struct ez_blobio *io, const ez_blob_desc *desc, void *p); static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, void *p); static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, void *p) { uint8_t h; + const ez_blob_desc *dscan = desc ? desc + 1 : desc; + const ez_blob_desc *dend = desc ? desc + desc->bd_length + 1 : desc; while ((h = blobio_readb(io)) != EZT_END) { - const ez_blob_desc *d; uint32_t ftag; uint32_t fcount = 0; int sc = (h & EZT_DATASIZE) >> EZT_DATASHIFT; /* for primitives */ + int use_count = 0; ftag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT); - if ((h & EZT_COUNTSIZE) != EZT_NOCOUNT) + if ((h & EZT_COUNTSIZE) != EZT_NOCOUNT) { fcount = blobio_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT); + use_count = 1; + } - d = blob_desc_find(desc, ftag); + // This forces the tags to be in order + // It ensures each field can only be visited once + const ez_blob_desc *d = NULL; + { + while (dscan < dend && dscan->bd_id < ftag) + dscan++; + if (dscan < dend && dscan->bd_id == ftag) + d = dscan++; + } if (p && d) { void *v = p + d->bd_offset; @@ -354,7 +356,7 @@ static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, v case EZT_INT16: case EZT_INT32: case EZT_INT64: - if (fcount) { + if (use_count) { char *mem; size_t len = fcount * (1<bd_type & EZ_BLOB_TYPE) { case EZ_BLOB_STRING: mem = malloc(len+1); + if (!mem) + goto error; *(char **)v = mem; memcpy(mem, data, len); mem[len] = 0; @@ -373,6 +377,8 @@ static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, v break; case EZ_BLOB_BLOB: mem = malloc(len); + if (!mem) + goto error; ((ez_blob *)v)->data = mem; ((ez_blob *)v)->size = len; memcpy(mem, data, len); @@ -399,7 +405,7 @@ static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, v } break; case EZT_FLOAT32: - if (fcount) { + if (use_count) { blobio_take(io, fcount * 4); } else { switch (d->bd_type) { @@ -415,7 +421,7 @@ static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, v } break; case EZT_FLOAT64: - if (fcount) { + if (use_count) { blobio_take(io, fcount * 8); } else { switch (d->bd_type) { @@ -431,10 +437,10 @@ static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, v } break; case EZT_STRUCT: - if (fcount) { + if (use_count) { switch (d->bd_type) { case EZ_BLOB_LIST: - for (int i=0;ierror && i < fcount; i++) { void *node = ez_blob_alloc(d->bd_table); if (!node) @@ -451,7 +457,7 @@ static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, v switch (d->bd_type & EZ_BLOB_TYPE) { case EZ_BLOB_STRUCT: D(printf("struct %d\n", d->bd_id)); - ez_blob_init(d->bd_table, v); + //ez_blob_init(d->bd_table, v); tagz_decode_fields(io, d->bd_table, v); break; case EZ_BLOB_POINTER: @@ -478,7 +484,7 @@ static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, v case EZT_INT64: case EZT_FLOAT32: case EZT_FLOAT64: - if (fcount) { + if (use_count) { blobio_take(io, fcount * (1<bd_table, NULL); + for (int i=0; !io->error && i < fcount; i++) { + tagz_decode_struct(io, NULL, NULL); } } else { - tagz_decode_fields(io, d->bd_table, NULL); + tagz_decode_fields(io, NULL, NULL); } break; default: @@ -511,7 +517,7 @@ static void tagz_decode_struct(struct ez_blobio *io, const ez_blob_desc *desc, v // must be a struct // what about a struct list? hmm - if ((h & EZT_TYPE) != EZT_STRUCT) { + if (io->error || (h & EZT_TYPE) != EZT_STRUCT) { //io->error = 1; return; } @@ -556,3 +562,120 @@ void *ez_tagz_decode(const ez_blob_desc *desc, const ez_blob *blob) { return p; } + +/* ********************************************************************** */ +// TODO: move to separate .o +static void tagz_dump_struct(struct ez_blobio *io, int depth); +static void tagz_dump_fields(struct ez_blobio *io, int depth); + +static void tagz_dump_struct(struct ez_blobio *io, int depth) { + uint8_t h = blobio_readb(io); + uint32_t stag; + //uint32_t scount = 1; + + char s[depth+1]; + memset(s, ' ', depth); + s[depth] = 0; + + // must be a struct + // what about a struct list? hmm + if ((h & EZT_TYPE) != EZT_STRUCT) + return; + + stag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT); + //if (h & EZT_COUNTSIZE) + // count = blob_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT); + + printf("%s[%02x] %d = {\n", s, h, stag); + + tagz_dump_fields(io, depth+4); +} + +static void tagz_dump_fields(struct ez_blobio *io, int depth) { + int h; + + char s[depth+1]; + memset(s, ' ', depth); + s[depth] = 0; + + // read fields + while ((h = blobio_readb(io)) != EZT_END) { + uint32_t ftag; + uint32_t fcount = 0; + int sc = (h & EZT_DATASIZE) >> EZT_DATASHIFT; + int use_count = 0; + + ftag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT); + if ((h & EZT_COUNTSIZE) != EZT_NOCOUNT) { + fcount = blobio_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT); + use_count = 1; + } + + switch (h & EZT_TYPE) { + case EZT_INT8: + case EZT_INT16: + case EZT_INT32: + case EZT_INT64: + if (use_count) { + void *v = blobio_take(io, fcount * (1<index-1); + printf("%s}\n", s); + } +} + +/** + * ezt format is self-describing so can be dumped directly. + */ +void ez_tagz_dump(ez_blob *blob) { + struct ez_blobio io = { + .data = blob->data, + .size = blob->size, + .mode = BLOBIO_READ + }; + + tagz_dump_struct(&io, 4); +} diff --git a/ez-blob-xdrn.c b/ez-blob-xdrn.c index c96579e..28a94d2 100644 --- a/ez-blob-xdrn.c +++ b/ez-blob-xdrn.c @@ -58,7 +58,7 @@ static void xdrn_encode_raw(struct ez_blobio *io, const ez_blob_desc *desc, cons blobio_write32(io, *(uint32_t *)v); break; case EZ_BLOB_INT64: - blobio_write64(io, *(uint32_t *)v); + blobio_write64(io, *(uint64_t *)v); break; case EZ_BLOB_FLOAT32: blobio_writef(io, *(float *)v); @@ -191,7 +191,7 @@ static void xdrn_decode_raw(struct ez_blobio *io, const ez_blob_desc *desc, void void *src = blobio_take(io, size); if (src) { - char *mem = malloc(size); + char *mem = malloc(size+1); if (!mem) goto error; diff --git a/ez-blob.c b/ez-blob.c index 14271cb..66b7856 100644 --- a/ez-blob.c +++ b/ez-blob.c @@ -27,11 +27,11 @@ #include "ez-list.h" void ez_blob_free_raw(const ez_blob_desc *desc, void *p) { - for (int i=0,dlen = desc->bd_length;ibd_length; i < dlen; i++) { const ez_blob_desc *d = &desc[i+1]; void *v = p + d->bd_offset; - switch (d->bd_type) { + switch (d->bd_type & EZ_BLOB_TYPE) { case EZ_BLOB_STRING: free(((char **)v)[0]); break; @@ -69,11 +69,14 @@ void ez_blob_free(const ez_blob_desc *d, void *p) { void ez_blob_init(const ez_blob_desc *desc, void *p) { memset(p, 0, desc->bd_offset); - for (int i=0,dlen = desc->bd_length;ibd_length; i < dlen; i++) { const ez_blob_desc *d = &desc[i+1]; + void *v = p + d->bd_offset; if (d->bd_type == EZ_BLOB_LIST) - ez_list_init((ez_list *)(p + d->bd_offset)); + ez_list_init((ez_list *)v); + else if (d->bd_type == EZ_BLOB_STRUCT) + ez_blob_init(d->bd_table, v); } } diff --git a/test-blob.c b/test-blob.c index 0a05817..273b7d4 100644 --- a/test-blob.c +++ b/test-blob.c @@ -14,21 +14,42 @@ #include "ez-blob-tagz.h" #include "ez-blob-xdrn.h" +#include "ez-blob-io.h" + +/* + TODO: some bad data tests. + + This depends a lot on the structure and the format. + + * lengths outside of data + * counts that will exhaust memory + * field id's out of range + * data corruption (wrong output) + * meta-data corruption (wrong/invalid structure) + * truncated data + * too much data + + Primarily it shouldn't crash or corrupt memory. + With explicit knowledge of format, it should fail on corrupt meta-data. + */ + struct simple { char b; short s; int a; + int64_t l; float c; double d; }; static const ez_blob_desc simple_DESC[] = { - EZ_BLOB_START(struct simple, 0, 5), + EZ_BLOB_START(struct simple, 0, 6), EZ_BLOB_INT8(struct simple, 1, b), EZ_BLOB_INT16(struct simple, 2, s), EZ_BLOB_INT32(struct simple, 3, a), - EZ_BLOB_FLOAT32(struct simple, 4, c), - EZ_BLOB_FLOAT64(struct simple, 5, d), + EZ_BLOB_INT64(struct simple, 4, l), + EZ_BLOB_FLOAT32(struct simple, 5, c), + EZ_BLOB_FLOAT64(struct simple, 6, d), }; struct arrays { @@ -38,9 +59,10 @@ struct arrays { }; static const ez_blob_desc array_DESC[] = { - EZ_BLOB_START(struct arrays, 0, 2), - EZ_BLOB_STRING(struct arrays, 1, string), - EZ_BLOB_BLOB(struct arrays, 2, array), + EZ_BLOB_START(struct arrays, 0, 3), + EZ_BLOB_INT32(struct arrays, 1, count), + EZ_BLOB_STRING(struct arrays, 2, string), + EZ_BLOB_BLOB(struct arrays, 3, array), }; struct embedded { @@ -91,22 +113,30 @@ static const ez_blob_desc list_DESC[] = { EZ_BLOB_LIST(struct list, 2, list, listnode_DESC), }; +struct intsize { + int16_t value16; + int32_t value32; + int64_t value64; +}; + +static const ez_blob_desc intsize_DESC[] = { + EZ_BLOB_START(struct intsize, 7, 3), + EZ_BLOB_INT16(struct intsize, 1, value16), + EZ_BLOB_INT32(struct intsize, 2, value32), + EZ_BLOB_INT64(struct intsize, 3, value64), +}; + +struct stringsize { + char *value; +}; + +static const ez_blob_desc stringsize_DESC[] = { + EZ_BLOB_START(struct stringsize, 8, 1), + EZ_BLOB_STRING(struct stringsize, 1, value), +}; + static void dumphex(const char *data, size_t size, const char *prefix) { - for (int i=0;idata, va->data, ua->size) == 0; break; } + case EZ_BLOB_STRUCT: + e &= blob_equals(d->bd_table, u, v); + break; case EZ_BLOB_POINTER | EZ_BLOB_NULLABLE: if (!*(void **)u || !*(void **)v) { e &= *(void **)u == *(void **)v; @@ -206,27 +239,34 @@ struct test_funcs { int (*blob_decode_raw)(const ez_blob_desc *desc, const ez_blob *blob, void *p); }; +static int doprint = 1; + static void test_basics(const ez_blob_desc *d, void *src, struct test_funcs *funcs) { ez_blob blob, blob2; void *t; size_t dsize = d->bd_offset; char tmp[dsize + 64] __attribute__ ((aligned(64))); int res; - - printf("source\n"); - blob_print(d, src, 4); + + if (doprint) { + printf("source\n"); + blob_print(d, src, 4); + } res = funcs->blob_encode(d, src, &blob); assert(res == 0); - dumphex(blob.data, blob.size, "serial: "); + if (doprint) + dumphex(blob.data, blob.size, "serial: "); // check decode equals t = funcs->blob_decode(d, &blob); assert(t != NULL); - assert(blob_equals(d, t, src)); - printf("decoded\n"); - blob_print(d, t, 4); + if (doprint) { + printf("decoded\n"); + blob_print(d, t, 4); + } + assert(blob_equals(d, t, src)); // check encode-decoded equals source funcs->blob_encode(d, t, &blob2); @@ -241,7 +281,8 @@ static void test_basics(const ez_blob_desc *d, void *src, struct test_funcs *fun funcs->blob_decode_raw(d, &blob, &tmp[32]); //assert(t != NULL); //assert(blob_equals(d, tmp+32, src)); - dumphex(tmp, sizeof(tmp), "object: "); + if (doprint) + dumphex(tmp, sizeof(tmp), "object: "); for (int i=0;i<32;i++) { assert((tmp[i] & 0xff) == 0xbe); @@ -253,7 +294,7 @@ static void test_basics(const ez_blob_desc *d, void *src, struct test_funcs *fun static void test_simple(struct test_funcs *funcs) { struct simple src = { - 'z', 0xff0f, 1, 3.14159f, 5644941221133 + 'z', 0xff0f, 1, 0xf01040a04010dULL, 3.14159f, 5644941221133 }; const ez_blob_desc *d = simple_DESC; @@ -275,6 +316,22 @@ static void test_arrays(struct test_funcs *funcs) { test_basics(d, &src, funcs); } +static void test_struct(struct test_funcs *funcs) { + char data[10] = { 1, 2, 3, 4, 5, 6, 7, 1, 2, 3 }; + struct embedded src = { + { + -1, -1, -1, -1, 1337.4004, 6.022E23, + }, + { + 55, "Jane was there.", .array.size = 10, .array.data = data + } + }; + const ez_blob_desc *d = embed_DESC; + + printf("test embedded structures\n"); + test_basics(d, &src, funcs); +} + static void test_rawtree(struct test_funcs *funcs) { struct tree src[6] = { { &src[1], &src[3], "root" }, @@ -312,9 +369,143 @@ static void test_list(struct test_funcs *funcs) { test_basics(d, &list, funcs); } +static void test_intsize(struct test_funcs *funcs) { + uint64_t sizes[8] = { 0x00, 0xff, 0x101, 0xffff, 0x10002, 0xffffffff, 0x100000003ULL, ~0ULL }; + const ez_blob_desc *d = intsize_DESC; + struct intsize src; + + // Checks that the sizes reconstruct to their original values + + for (int i=0;i<8;i++) { + uint64_t size = sizes[i]; + + src.value16 = size; + src.value32 = size; + src.value64 = size; + + printf("test int value %016lx\n", size); + test_basics(d, &src, funcs); + } +} + +static void test_stringsize(struct test_funcs *funcs) { + int sizes[6] = { 0, 1, 255, 256, 65535, 65536 }; + const ez_blob_desc *d = stringsize_DESC; + struct stringsize src; + int sprint = doprint; + + for (int i=0;i<6;i++) { + int size = sizes[i]; + src.value = malloc(size+1); + memset(src.value, ' ', size); + src.value[size] = 0; + printf("test string size %d\n", size); + doprint = size <= 256; + test_basics(d, &src, funcs); + free(src.value); + } + doprint = sprint; +} + +// the corruption tests don't know the format details so can't tell what a given +// change should cause. +// however they should not crash, cause leaks, or valgrind oddness. + +static void test_corrupt_bit(struct test_funcs *funcs) { + // this is definitely not robust + if (strcmp(funcs->name, "legacy") == 0) { + printf("Not supposed to be safe\n"); + return; + } + + char data[10] = { 1, 2, 3, 4, 5, 6, 7, 1, 2, 3 }; + struct embedded src = { + { + -1, -1, -1, -1, 1337.4004, 6.022E23, + }, + { + 55, "Jane was there.", .array.size = 10, .array.data = data + } + }; + const ez_blob_desc *d = embed_DESC; + struct embedded *dst; + int res; + ez_blob blob; + + // test 1 corrupt bit in every position + + // we don't know the internals so which ones would fail, but + // can check for bad memory accesses with valgrind or crash + + int good = 0, bad = 0; + res = funcs->blob_encode(d, &src, &blob); + assert(res == 0); + for (int i=0;iblob_decode(d, &blob); + data[i] ^= (1<name, "legacy") == 0) { + printf("Not supposed to be safe\n"); + return; + } + + char data[10] = { 1, 2, 3, 4, 5, 6, 7, 1, 2, 3 }; + struct embedded src = { + { + -1, -1, -1, -1, 1337.4004, 6.022E23, + }, + { + 55, "Jane was there.", .array.size = 10, .array.data = data + } + }; + const ez_blob_desc *d = embed_DESC; + struct embedded *dst; + int res; + ez_blob blob; + + int good = 0, bad = 0; + res = funcs->blob_encode(d, &src, &blob); + assert(res == 0); + for (int i=0;iblob_decode(d, &blob); + if (dst) { + good++; + ez_blob_free(d, dst); + } else { + bad++; + } + } + data[i] = save; + } + free(blob.data); + printf(" info: %d failed, %d decoded - but not important\n", bad, good); +} + struct test_funcs backends[] = { { - "blob legacy", + "legacy", ez_blob_encode, ez_blob_decode, ez_blob_decode_raw @@ -333,16 +524,35 @@ struct test_funcs backends[] = { }, }; +struct { + const char *name; + void (*test)(struct test_funcs *); +} tests[] = { + { "primitives", test_simple }, + { "arrays", test_arrays }, + { "structs", test_struct }, + { "tree as links", test_rawtree }, + { "lists", test_list }, + { "string sizes", test_stringsize }, + { "int sizes", test_intsize }, + { "corrupted bit", test_corrupt_bit }, + { "corrupted byte", test_corrupt_byte }, +}; + int main(int argc, char **argv) { - for (int i=0;i<3;i++) { + int nfunc = sizeof(backends)/sizeof(backends[0]); + int ntest = sizeof(tests)/sizeof(tests[0]); + + for (int i=0;iname); - fflush(stdout); - - test_simple(f); - test_arrays(f); - test_rawtree(f); - test_list(f); + + for (int j=0;jname, tests[j].name); + fflush(stdout); + + tests[j].test(f); + } } + return 0; } -- 2.39.2