followed by an optional 1/2/4 byte-length count.
A structure payload is a list of tagged fields until a struct-end
- code. A structure sequence includes count of a tagged structure.
+ code. A structure sequence is a list of count struct-encoded blocks.
Integers can be stored in the smallest number of bytes, i.e. with
all leading $00 bytes removed.
blobio_writed(io, *(const double *)v);
break;
case EZ_BLOB_STRING:
+ // NULL strings are the default, so implement same but allow for empty strings
ptr = *(const void **)v;
if (ptr) {
count = strlen(ptr);
}
break;
case EZ_BLOB_BLOB:
+ // empty blobs are the default, so implement same, empty are not encoded
count = ((ez_blob *)v)->size;
if (count) {
int cc;
return 0;
}
-// TODO: binary search, length is in first field
-// Alternative: if encode/decode is same order, then can just search forward from last spot
-static const ez_blob_desc *blob_desc_find(const ez_blob_desc *desc, int id) {
- for (int i=0,dlen=desc->bd_length;i<dlen;i++) {
- const ez_blob_desc *d = &desc[i+1];
-
- if (d->bd_id == id)
- return d;
- }
- return NULL;
-}
-
static void tagz_decode_struct(struct ez_blobio *io, const ez_blob_desc *desc, void *p);
static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, void *p);
static void tagz_decode_fields(struct ez_blobio *io, const ez_blob_desc *desc, void *p) {
uint8_t h;
+ const ez_blob_desc *dscan = desc ? desc + 1 : desc;
+ const ez_blob_desc *dend = desc ? desc + desc->bd_length + 1 : desc;
while ((h = blobio_readb(io)) != EZT_END) {
- const ez_blob_desc *d;
uint32_t ftag;
uint32_t fcount = 0;
int sc = (h & EZT_DATASIZE) >> EZT_DATASHIFT; /* for primitives */
+ int use_count = 0;
ftag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT);
- if ((h & EZT_COUNTSIZE) != EZT_NOCOUNT)
+ if ((h & EZT_COUNTSIZE) != EZT_NOCOUNT) {
fcount = blobio_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT);
+ use_count = 1;
+ }
- d = blob_desc_find(desc, ftag);
+ // This forces the tags to be in order
+ // It ensures each field can only be visited once
+ const ez_blob_desc *d = NULL;
+ {
+ while (dscan < dend && dscan->bd_id < ftag)
+ dscan++;
+ if (dscan < dend && dscan->bd_id == ftag)
+ d = dscan++;
+ }
if (p && d) {
void *v = p + d->bd_offset;
case EZT_INT16:
case EZT_INT32:
case EZT_INT64:
- if (fcount) {
+ if (use_count) {
char *mem;
size_t len = fcount * (1<<sc);
void *data = blobio_take(io, len);
switch (d->bd_type & EZ_BLOB_TYPE) {
case EZ_BLOB_STRING:
mem = malloc(len+1);
+ if (!mem)
+ goto error;
*(char **)v = mem;
memcpy(mem, data, len);
mem[len] = 0;
break;
case EZ_BLOB_BLOB:
mem = malloc(len);
+ if (!mem)
+ goto error;
((ez_blob *)v)->data = mem;
((ez_blob *)v)->size = len;
memcpy(mem, data, len);
}
break;
case EZT_FLOAT32:
- if (fcount) {
+ if (use_count) {
blobio_take(io, fcount * 4);
} else {
switch (d->bd_type) {
}
break;
case EZT_FLOAT64:
- if (fcount) {
+ if (use_count) {
blobio_take(io, fcount * 8);
} else {
switch (d->bd_type) {
}
break;
case EZT_STRUCT:
- if (fcount) {
+ if (use_count) {
switch (d->bd_type) {
case EZ_BLOB_LIST:
- for (int i=0;i<fcount;i++) {
+ for (int i=0; !io->error && i < fcount; i++) {
void *node = ez_blob_alloc(d->bd_table);
if (!node)
switch (d->bd_type & EZ_BLOB_TYPE) {
case EZ_BLOB_STRUCT:
D(printf("struct %d\n", d->bd_id));
- ez_blob_init(d->bd_table, v);
+ //ez_blob_init(d->bd_table, v);
tagz_decode_fields(io, d->bd_table, v);
break;
case EZ_BLOB_POINTER:
case EZT_INT64:
case EZT_FLOAT32:
case EZT_FLOAT64:
- if (fcount) {
+ if (use_count) {
blobio_take(io, fcount * (1<<sc));
} else {
blobio_take(io, (1<<sc));
break;
case EZT_STRUCT:
if (fcount) {
- for (int i=0;i<fcount;i++) {
- tagz_decode_struct(io, d->bd_table, NULL);
+ for (int i=0; !io->error && i < fcount; i++) {
+ tagz_decode_struct(io, NULL, NULL);
}
} else {
- tagz_decode_fields(io, d->bd_table, NULL);
+ tagz_decode_fields(io, NULL, NULL);
}
break;
default:
// must be a struct
// what about a struct list? hmm
- if ((h & EZT_TYPE) != EZT_STRUCT) {
+ if (io->error || (h & EZT_TYPE) != EZT_STRUCT) {
//io->error = 1;
return;
}
return p;
}
+
+/* ********************************************************************** */
+// TODO: move to separate .o
+static void tagz_dump_struct(struct ez_blobio *io, int depth);
+static void tagz_dump_fields(struct ez_blobio *io, int depth);
+
+static void tagz_dump_struct(struct ez_blobio *io, int depth) {
+ uint8_t h = blobio_readb(io);
+ uint32_t stag;
+ //uint32_t scount = 1;
+
+ char s[depth+1];
+ memset(s, ' ', depth);
+ s[depth] = 0;
+
+ // must be a struct
+ // what about a struct list? hmm
+ if ((h & EZT_TYPE) != EZT_STRUCT)
+ return;
+
+ stag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT);
+ //if (h & EZT_COUNTSIZE)
+ // count = blob_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT);
+
+ printf("%s[%02x] %d = {\n", s, h, stag);
+
+ tagz_dump_fields(io, depth+4);
+}
+
+static void tagz_dump_fields(struct ez_blobio *io, int depth) {
+ int h;
+
+ char s[depth+1];
+ memset(s, ' ', depth);
+ s[depth] = 0;
+
+ // read fields
+ while ((h = blobio_readb(io)) != EZT_END) {
+ uint32_t ftag;
+ uint32_t fcount = 0;
+ int sc = (h & EZT_DATASIZE) >> EZT_DATASHIFT;
+ int use_count = 0;
+
+ ftag = blobio_readi(io, (h & EZT_TAGSIZE) >> EZT_TAGSHIFT);
+ if ((h & EZT_COUNTSIZE) != EZT_NOCOUNT) {
+ fcount = blobio_readi(io, (h & EZT_COUNTSIZE) >> EZT_COUNTSHIFT);
+ use_count = 1;
+ }
+
+ switch (h & EZT_TYPE) {
+ case EZT_INT8:
+ case EZT_INT16:
+ case EZT_INT32:
+ case EZT_INT64:
+ if (use_count) {
+ void *v = blobio_take(io, fcount * (1<<sc));
+ if (v) {
+ printf("%s[%02x] integer-%-2d %d [%d] = {\n", s, h, (1<<sc)*8, ftag, fcount);
+ blobio_dumphex(v, fcount * (1<<sc), s);
+ printf("%s}\n", s);
+ }
+ } else {
+ printf("%s[%02x] integer-%-2d %d = %02lx\n", s, h, (1<<sc)*8, ftag, blobio_readi(io, sc));
+ }
+ break;
+ case EZT_FLOAT32:
+ if (use_count) {
+ printf("%sfloat %d [%d] = {\n", s, ftag, fcount);
+ for (int i=0;i<fcount;i++)
+ printf(" %f\n", blobio_readf(io));
+ printf("%s}\n", s);
+ } else {
+ printf("%sfloat %d = %f\n", s, ftag, blobio_readf(io));
+ }
+ break;
+ case EZT_FLOAT64:
+ if (use_count) {
+ printf("%s%d double[%d] = {\n", s, ftag, fcount);
+ for (int i=0;i<fcount;i++)
+ printf("%s %f\n", s, blobio_readd(io));
+ printf("%s}\n", s);
+ } else {
+ printf("%s%d double = %f\n", s, ftag, blobio_readd(io));
+ }
+ break;
+ case EZT_STRUCT:
+ if (use_count) {
+ printf("%sstruct %d [%d] = {\n", s, ftag, fcount);
+ for (int i=0;i<fcount;i++) {
+ tagz_dump_struct(io, depth+4);
+ }
+ printf("%s}\n", s);
+ } else {
+ tagz_dump_fields(io, depth+4);
+ }
+ break;
+ default:
+ printf("%s[%02x] ??\n", s, h);
+ break;
+ }
+ printf("%s[%02x] END $%04lx\n", s, h, io->index-1);
+ printf("%s}\n", s);
+ }
+}
+
+/**
+ * ezt format is self-describing so can be dumped directly.
+ */
+void ez_tagz_dump(ez_blob *blob) {
+ struct ez_blobio io = {
+ .data = blob->data,
+ .size = blob->size,
+ .mode = BLOBIO_READ
+ };
+
+ tagz_dump_struct(&io, 4);
+}
#include "ez-blob-tagz.h"
#include "ez-blob-xdrn.h"
+#include "ez-blob-io.h"
+
+/*
+ TODO: some bad data tests.
+
+ This depends a lot on the structure and the format.
+
+ * lengths outside of data
+ * counts that will exhaust memory
+ * field id's out of range
+ * data corruption (wrong output)
+ * meta-data corruption (wrong/invalid structure)
+ * truncated data
+ * too much data
+
+ Primarily it shouldn't crash or corrupt memory.
+ With explicit knowledge of format, it should fail on corrupt meta-data.
+ */
+
struct simple {
char b;
short s;
int a;
+ int64_t l;
float c;
double d;
};
static const ez_blob_desc simple_DESC[] = {
- EZ_BLOB_START(struct simple, 0, 5),
+ EZ_BLOB_START(struct simple, 0, 6),
EZ_BLOB_INT8(struct simple, 1, b),
EZ_BLOB_INT16(struct simple, 2, s),
EZ_BLOB_INT32(struct simple, 3, a),
- EZ_BLOB_FLOAT32(struct simple, 4, c),
- EZ_BLOB_FLOAT64(struct simple, 5, d),
+ EZ_BLOB_INT64(struct simple, 4, l),
+ EZ_BLOB_FLOAT32(struct simple, 5, c),
+ EZ_BLOB_FLOAT64(struct simple, 6, d),
};
struct arrays {
};
static const ez_blob_desc array_DESC[] = {
- EZ_BLOB_START(struct arrays, 0, 2),
- EZ_BLOB_STRING(struct arrays, 1, string),
- EZ_BLOB_BLOB(struct arrays, 2, array),
+ EZ_BLOB_START(struct arrays, 0, 3),
+ EZ_BLOB_INT32(struct arrays, 1, count),
+ EZ_BLOB_STRING(struct arrays, 2, string),
+ EZ_BLOB_BLOB(struct arrays, 3, array),
};
struct embedded {
EZ_BLOB_LIST(struct list, 2, list, listnode_DESC),
};
+struct intsize {
+ int16_t value16;
+ int32_t value32;
+ int64_t value64;
+};
+
+static const ez_blob_desc intsize_DESC[] = {
+ EZ_BLOB_START(struct intsize, 7, 3),
+ EZ_BLOB_INT16(struct intsize, 1, value16),
+ EZ_BLOB_INT32(struct intsize, 2, value32),
+ EZ_BLOB_INT64(struct intsize, 3, value64),
+};
+
+struct stringsize {
+ char *value;
+};
+
+static const ez_blob_desc stringsize_DESC[] = {
+ EZ_BLOB_START(struct stringsize, 8, 1),
+ EZ_BLOB_STRING(struct stringsize, 1, value),
+};
+
static void dumphex(const char *data, size_t size, const char *prefix) {
- for (int i=0;i<size;i+=16) {
- fputs(prefix, stdout);
- printf(" %04x:", i);
- for (int j=0;j<16;j++) {
- if (i+j<size)
- printf(" %02x", data[i+j] & 0xff);
- else
- fputs(" ", stdout);
- }
- fputs(" ", stdout);
- for (int j=0;j<16;j++) {
- putchar(i+j<size && isprint(data[i+j]) ? data[i+j] : '.');
- }
- putchar('\n');
- }
+ blobio_dumphex(data, size, prefix);
}
static void blob_print(const ez_blob_desc *d, const void *a, int depth) {
&& memcmp(ua->data, va->data, ua->size) == 0;
break;
}
+ case EZ_BLOB_STRUCT:
+ e &= blob_equals(d->bd_table, u, v);
+ break;
case EZ_BLOB_POINTER | EZ_BLOB_NULLABLE:
if (!*(void **)u || !*(void **)v) {
e &= *(void **)u == *(void **)v;
int (*blob_decode_raw)(const ez_blob_desc *desc, const ez_blob *blob, void *p);
};
+static int doprint = 1;
+
static void test_basics(const ez_blob_desc *d, void *src, struct test_funcs *funcs) {
ez_blob blob, blob2;
void *t;
size_t dsize = d->bd_offset;
char tmp[dsize + 64] __attribute__ ((aligned(64)));
int res;
-
- printf("source\n");
- blob_print(d, src, 4);
+
+ if (doprint) {
+ printf("source\n");
+ blob_print(d, src, 4);
+ }
res = funcs->blob_encode(d, src, &blob);
assert(res == 0);
- dumphex(blob.data, blob.size, "serial: ");
+ if (doprint)
+ dumphex(blob.data, blob.size, "serial: ");
// check decode equals
t = funcs->blob_decode(d, &blob);
assert(t != NULL);
- assert(blob_equals(d, t, src));
- printf("decoded\n");
- blob_print(d, t, 4);
+ if (doprint) {
+ printf("decoded\n");
+ blob_print(d, t, 4);
+ }
+ assert(blob_equals(d, t, src));
// check encode-decoded equals source
funcs->blob_encode(d, t, &blob2);
funcs->blob_decode_raw(d, &blob, &tmp[32]);
//assert(t != NULL);
//assert(blob_equals(d, tmp+32, src));
- dumphex(tmp, sizeof(tmp), "object: ");
+ if (doprint)
+ dumphex(tmp, sizeof(tmp), "object: ");
for (int i=0;i<32;i++) {
assert((tmp[i] & 0xff) == 0xbe);
static void test_simple(struct test_funcs *funcs) {
struct simple src = {
- 'z', 0xff0f, 1, 3.14159f, 5644941221133
+ 'z', 0xff0f, 1, 0xf01040a04010dULL, 3.14159f, 5644941221133
};
const ez_blob_desc *d = simple_DESC;
test_basics(d, &src, funcs);
}
+static void test_struct(struct test_funcs *funcs) {
+ char data[10] = { 1, 2, 3, 4, 5, 6, 7, 1, 2, 3 };
+ struct embedded src = {
+ {
+ -1, -1, -1, -1, 1337.4004, 6.022E23,
+ },
+ {
+ 55, "Jane was there.", .array.size = 10, .array.data = data
+ }
+ };
+ const ez_blob_desc *d = embed_DESC;
+
+ printf("test embedded structures\n");
+ test_basics(d, &src, funcs);
+}
+
static void test_rawtree(struct test_funcs *funcs) {
struct tree src[6] = {
{ &src[1], &src[3], "root" },
test_basics(d, &list, funcs);
}
+static void test_intsize(struct test_funcs *funcs) {
+ uint64_t sizes[8] = { 0x00, 0xff, 0x101, 0xffff, 0x10002, 0xffffffff, 0x100000003ULL, ~0ULL };
+ const ez_blob_desc *d = intsize_DESC;
+ struct intsize src;
+
+ // Checks that the sizes reconstruct to their original values
+
+ for (int i=0;i<8;i++) {
+ uint64_t size = sizes[i];
+
+ src.value16 = size;
+ src.value32 = size;
+ src.value64 = size;
+
+ printf("test int value %016lx\n", size);
+ test_basics(d, &src, funcs);
+ }
+}
+
+static void test_stringsize(struct test_funcs *funcs) {
+ int sizes[6] = { 0, 1, 255, 256, 65535, 65536 };
+ const ez_blob_desc *d = stringsize_DESC;
+ struct stringsize src;
+ int sprint = doprint;
+
+ for (int i=0;i<6;i++) {
+ int size = sizes[i];
+ src.value = malloc(size+1);
+ memset(src.value, ' ', size);
+ src.value[size] = 0;
+ printf("test string size %d\n", size);
+ doprint = size <= 256;
+ test_basics(d, &src, funcs);
+ free(src.value);
+ }
+ doprint = sprint;
+}
+
+// the corruption tests don't know the format details so can't tell what a given
+// change should cause.
+// however they should not crash, cause leaks, or valgrind oddness.
+
+static void test_corrupt_bit(struct test_funcs *funcs) {
+ // this is definitely not robust
+ if (strcmp(funcs->name, "legacy") == 0) {
+ printf("Not supposed to be safe\n");
+ return;
+ }
+
+ char data[10] = { 1, 2, 3, 4, 5, 6, 7, 1, 2, 3 };
+ struct embedded src = {
+ {
+ -1, -1, -1, -1, 1337.4004, 6.022E23,
+ },
+ {
+ 55, "Jane was there.", .array.size = 10, .array.data = data
+ }
+ };
+ const ez_blob_desc *d = embed_DESC;
+ struct embedded *dst;
+ int res;
+ ez_blob blob;
+
+ // test 1 corrupt bit in every position
+
+ // we don't know the internals so which ones would fail, but
+ // can check for bad memory accesses with valgrind or crash
+
+ int good = 0, bad = 0;
+ res = funcs->blob_encode(d, &src, &blob);
+ assert(res == 0);
+ for (int i=0;i<blob.size;i++) {
+ uint8_t *data = blob.data;
+ for (int j=0;j<8;j++) {
+ data[i] ^= (1<<j);
+ dst = funcs->blob_decode(d, &blob);
+ data[i] ^= (1<<j);
+
+ if (dst) {
+ good++;
+ ez_blob_free(d, dst);
+ } else {
+ bad++;
+ }
+ }
+ }
+ free(blob.data);
+ printf(" info: %d failed, %d decoded - but not important\n", bad, good);
+}
+
+static void test_corrupt_byte(struct test_funcs *funcs) {
+ // this is definitely not robust
+ if (strcmp(funcs->name, "legacy") == 0) {
+ printf("Not supposed to be safe\n");
+ return;
+ }
+
+ char data[10] = { 1, 2, 3, 4, 5, 6, 7, 1, 2, 3 };
+ struct embedded src = {
+ {
+ -1, -1, -1, -1, 1337.4004, 6.022E23,
+ },
+ {
+ 55, "Jane was there.", .array.size = 10, .array.data = data
+ }
+ };
+ const ez_blob_desc *d = embed_DESC;
+ struct embedded *dst;
+ int res;
+ ez_blob blob;
+
+ int good = 0, bad = 0;
+ res = funcs->blob_encode(d, &src, &blob);
+ assert(res == 0);
+ for (int i=0;i<blob.size;i++) {
+ uint8_t *data = blob.data;
+ uint8_t save = data[i];
+
+ for (int j=0;j<256;j++) {
+ data[i] = j;
+ dst = funcs->blob_decode(d, &blob);
+ if (dst) {
+ good++;
+ ez_blob_free(d, dst);
+ } else {
+ bad++;
+ }
+ }
+ data[i] = save;
+ }
+ free(blob.data);
+ printf(" info: %d failed, %d decoded - but not important\n", bad, good);
+}
+
struct test_funcs backends[] = {
{
- "blob legacy",
+ "legacy",
ez_blob_encode,
ez_blob_decode,
ez_blob_decode_raw
},
};
+struct {
+ const char *name;
+ void (*test)(struct test_funcs *);
+} tests[] = {
+ { "primitives", test_simple },
+ { "arrays", test_arrays },
+ { "structs", test_struct },
+ { "tree as links", test_rawtree },
+ { "lists", test_list },
+ { "string sizes", test_stringsize },
+ { "int sizes", test_intsize },
+ { "corrupted bit", test_corrupt_bit },
+ { "corrupted byte", test_corrupt_byte },
+};
+
int main(int argc, char **argv) {
- for (int i=0;i<3;i++) {
+ int nfunc = sizeof(backends)/sizeof(backends[0]);
+ int ntest = sizeof(tests)/sizeof(tests[0]);
+
+ for (int i=0;i<nfunc;i++) {
struct test_funcs *f = &backends[i];
- printf("* **********************************************************************\n");
- printf("test: %s\n", f->name);
- fflush(stdout);
-
- test_simple(f);
- test_arrays(f);
- test_rawtree(f);
- test_list(f);
+
+ for (int j=0;j<ntest;j++) {
+ printf("* **********************************************************************\n");
+ printf("* test: %s / %s\n", f->name, tests[j].name);
+ fflush(stdout);
+
+ tests[j].test(f);
+ }
}
+ return 0;
}