--- /dev/null
+/*
+ articledb.c Article Database.
+
+ Copyright (C) 2019 Michael Zucchi
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as
+ published by the Free Software Foundation, either version 3 of the
+ License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+*/
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <errno.h>
+
+#include "ez-blob.h"
+#include "ez-blob-tagz.h"
+
+#include "ez-list.h"
+
+typedef uint64_t articleid_t;
+typedef uint64_t dataid_t;
+typedef uint64_t revisionid_t;
+typedef uint64_t authorid_t;
+
+struct property {
+ ez_node ln;
+ char *name;
+ ez_blob value;
+};
+
+struct article {
+ articleid_t id;
+ revisionid_t revisionid;
+ dataid_t dataid;
+ articleid_t previd,upid,nextid;
+
+ char *title;
+ char *type;
+ char *path;
+ char *keyword;
+
+ // move votes to a property?
+ uint32_t upvote, downvote;
+ ez_list prop;
+};
+
+struct revision {
+ revisionid_t id;
+ authorid_t authorid;
+ uint64_t ctime;
+ ez_list prop;
+};
+
+struct author {
+ authorid_t id;
+ uint64_t ctime;
+ uint64_t mtime;
+ char *nick;
+ char *email;
+ char *pass;
+ ez_list prop;
+};
+
+/*
+ tagged format
+
+ type:4
+ tagsize:2
+ lengthsize:2
+
+ tag
+ length (or data)
+ data[length]
+
+ */
+
+static const ez_blob_desc property_DESC[] = {
+ EZ_BLOB_START(struct property, 1, 2),
+ EZ_BLOB_STRING(struct property, 1, name),
+ EZ_BLOB_INT8V(struct property, 2, value),
+};
+
+static ez_blob_desc article_DESC[] = {
+ EZ_BLOB_START(struct article, 2, 12),
+
+ EZ_BLOB_INT64(struct article, 1, revisionid),
+ EZ_BLOB_INT64(struct article, 2, dataid),
+ EZ_BLOB_INT64(struct article, 3, previd),
+ EZ_BLOB_INT64(struct article, 4, upid),
+ EZ_BLOB_INT64(struct article, 5, nextid),
+
+ EZ_BLOB_STRING(struct article, 6, title),
+ EZ_BLOB_STRING(struct article, 7, type),
+ EZ_BLOB_STRING(struct article, 8, path),
+ EZ_BLOB_STRING(struct article, 9, keyword),
+
+ EZ_BLOB_INT32(struct article, 10, upvote),
+ EZ_BLOB_INT32(struct article, 11, downvote),
+
+ EZ_BLOB_LIST(struct article, 12, prop, property_DESC),
+};
+
+static const ez_blob_desc revision_DESC[] = {
+ EZ_BLOB_START(struct revision, 3, 3),
+ EZ_BLOB_INT64(struct revision, 1, authorid),
+ EZ_BLOB_INT64(struct revision, 2, ctime),
+ EZ_BLOB_LIST(struct revision, 3, prop, property_DESC),
+};
+
+static ez_blob_desc author_DESC[] = {
+ EZ_BLOB_START(struct author, 4, 6),
+ EZ_BLOB_INT64(struct author, 1, ctime),
+ EZ_BLOB_INT64(struct author, 2, mtime),
+ EZ_BLOB_STRING(struct author, 3, nick),
+ EZ_BLOB_STRING(struct author, 4, email),
+ EZ_BLOB_STRING(struct author, 5, pass),
+ EZ_BLOB_LIST(struct author, 6, prop, property_DESC),
+};
+
+#include <lmdb.h>
+
+struct articledb {
+ int res;
+
+ MDB_env *env;
+
+ MDB_dbi article;
+ //MDB_dbi article_by_revision;
+ //MDB_dbi article_by_root;
+ MDB_dbi article_by_type;
+ MDB_dbi article_by_title;
+ MDB_dbi article_by_path;
+ MDB_dbi article_by_keyword;
+ //MDB_dbi article_by_prev;
+ //MDB_dbi article_by_up;
+ //MDB_dbi article_by_next;
+
+ MDB_dbi revision;
+ //MDB_dbi revison_by_author;
+
+ MDB_dbi author;
+ MDB_dbi author_by_email;
+};
+
+struct articledb *db_open(const char *path) {
+ struct articledb *db = calloc(sizeof(*db), 1);
+ int res;
+ MDB_txn *tx;
+
+ res = mdb_env_create(&db->env);
+ if (res)
+ goto fail;
+ res = mdb_env_set_maxdbs(db->env, 8);
+ if (res)
+ goto fail;
+ res = mdb_env_set_mapsize(db->env, 1<<28); // 256MB
+ if (res)
+ goto fail;
+ res = mdb_env_open(db->env, path, 0, 0664);
+ if (res)
+ goto fail;
+
+ res = mdb_txn_begin(db->env, NULL, 0, &tx);
+ if (res)
+ goto fail;
+
+ // Main store
+ res |= mdb_dbi_open(tx, "article", MDB_CREATE | MDB_INTEGERKEY, &db->article);
+ res |= mdb_dbi_open(tx, "article#title", MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED | MDB_INTEGERDUP, &db->article_by_title);
+ res |= mdb_dbi_open(tx, "article#path", MDB_CREATE, &db->article_by_path);
+ res |= mdb_dbi_open(tx, "article#keyword", MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED | MDB_INTEGERDUP, &db->article_by_keyword);
+ res |= mdb_dbi_open(tx, "article#type", MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED | MDB_INTEGERDUP, &db->article_by_type);
+
+ // Revision
+ res |= mdb_dbi_open(tx, "revision", MDB_CREATE | MDB_INTEGERKEY, &db->revision);
+
+ // Author
+ res |= mdb_dbi_open(tx, "author", MDB_CREATE | MDB_INTEGERKEY, &db->author);
+ res |= mdb_dbi_open(tx, "author#email", MDB_CREATE, &db->author_by_email);
+
+ res |= mdb_txn_commit(tx);
+
+ if (res)
+ goto fail;
+
+ printf("db open\n");
+
+ return db;
+ fail:
+ printf("db setup fail: %s\n", mdb_strerror(res));
+ // shutdown
+ if (db->env)
+ mdb_env_close(db->env);
+ free(db);
+ return NULL;
+}
+
+void db_close(struct articledb *db) {
+ if (db) {
+ mdb_env_close(db->env);
+ free(db);
+ }
+}
+
+static int object_put(struct articledb *db, MDB_txn *tx, MDB_dbi index, uint64_t id, void *p, const ez_blob_desc *desc, int flag) {
+ MDB_val key, data;
+ ez_blob blob;
+ int res;
+
+ key.mv_data = &id;
+ key.mv_size = sizeof(id);
+
+ ez_tagz_encode(desc, p, &blob);
+
+ data.mv_size = blob.eb_size;
+ data.mv_data = blob.eb_data;
+
+ res = mdb_put(tx, index, &key, &data, flag);
+
+ free(blob.eb_data);
+
+ return res;
+}
+
+void author_init(struct author *a) {
+ memset(a, 0, sizeof(*a));
+ ez_list_init(&a->prop);
+}
+
+int author_create(struct articledb *db, MDB_txn *txn, struct author *a) {
+ MDB_txn *tx;
+ MDB_val key, dat;
+ int res = -1;
+
+ mdb_txn_begin(db->env, txn, 0, &tx);
+
+ if (!*a->email || !a->id)
+ goto fail;
+
+ key.mv_data = a->email;
+ key.mv_size = strlen(a->email);
+ dat.mv_data = &a->id;
+ dat.mv_size = sizeof(a->id);
+
+ if ((res = mdb_put(tx, db->author_by_email, &key, &dat, MDB_NOOVERWRITE)) != 0)
+ goto fail;
+
+ if ((res = object_put(db, tx, db->author, a->id, a, author_DESC, MDB_NOOVERWRITE)) != 0)
+ goto fail;
+
+ return mdb_txn_commit(tx);
+ fail:
+ mdb_txn_abort(tx);
+ return res;
+}
+
+int author_update(struct articledb *db, MDB_txn *txn, struct author *a) {
+ MDB_txn *tx;
+ MDB_val key, data;
+ int res = -1;
+
+ mdb_txn_begin(db->env, txn, 0, &tx);
+
+ // Check email hasn't changed
+ key.mv_data = a->email;
+ key.mv_size = strlen(a->email);
+
+ if ((res = mdb_get(tx, db->author_by_email, &key, &data)) != 0)
+ goto fail;
+
+ if (data.mv_size != sizeof(a->id)
+ || memcmp(data.mv_data, &a->id, sizeof(a->id)) != 0)
+ goto fail;
+
+ // Write
+ res = object_put(db, tx, db->author, a->id, a, author_DESC, 0);
+ if (res != 0)
+ goto fail;
+
+ return mdb_txn_commit(tx);
+ fail:
+ mdb_txn_abort(tx);
+ return res;
+}
+
+int author_exists(struct articledb *db, MDB_txn *tx, authorid_t aid) {
+ MDB_val key, data;
+ int res;
+
+ key.mv_data = &aid;
+ key.mv_size = sizeof(aid);
+
+ res = mdb_get(tx, db->author, &key, &data);
+ if (res == 0)
+ return 1;
+ else if (res == MDB_NOTFOUND)
+ return 0;
+ else
+ // set error state? abort txn?
+ return 0;
+}
+
+void revision_init(struct revision *a) {
+ memset(a, 0, sizeof(*a));
+ ez_list_init(&a->prop);
+}
+
+int revision_create(struct articledb *db, MDB_txn *txn, struct revision *r) {
+ MDB_txn *tx;
+ int res = -1;
+
+ mdb_txn_begin(db->env, txn, 0, &tx);
+
+ if (!author_exists(db, tx, r->authorid))
+ goto fail;
+
+ res = object_put(db, tx, db->revision, r->id, r, revision_DESC, MDB_NOOVERWRITE);
+ if (res != 0)
+ goto fail;
+
+ return mdb_txn_commit(tx);
+ fail:
+ mdb_txn_abort(tx);
+ return res;
+}
+
+#include <ctype.h>
+// first call, val.mv_data = keyword, val.mv_size = 0
+int keyword_next(MDB_val *val) {
+ unsigned char *k = val->mv_data + val->mv_size;
+ unsigned char *start, *end;
+
+ while (isspace(*k) || *k == ',')
+ k++;
+
+ start = k;
+ while (*k && !isspace(*k) && *k != ',')
+ k++;
+ end = k;
+ if (start == end)
+ return 0;
+ val->mv_data = start;
+ val->mv_size = end-start;
+ return 1;
+}
+
+void article_init(struct article *a) {
+ memset(a, 0, sizeof(*a));
+ ez_list_init(&a->prop);
+}
+
+// What could we update?
+// could update keywords at least, maybe properties
+// or force a new revision?
+// hrmmmmm.
+int article_create(struct articledb *db, MDB_txn *txn, struct article *a) {
+ MDB_txn *tx;
+ MDB_val key, data;
+ int res = -1;
+
+ mdb_txn_begin(db->env, txn, 0, &tx);
+
+ data.mv_data = &a->id;
+ data.mv_size = sizeof(a->id);
+
+ if (a->type == NULL || *a->type == 0) {
+ printf("no article.type\n");
+ res = EINVAL;
+ goto fail;
+ }
+
+ key.mv_data = a->type;
+ key.mv_size = strlen(a->type);
+ if ((res = mdb_put(tx, db->article_by_type, &key, &data, 0)) != 0)
+ goto fail;
+
+ if (a->title && *a->title) {
+ key.mv_data = a->title;
+ key.mv_size = strlen(a->title);
+ if ((res = mdb_put(tx, db->article_by_title, &key, &data, 0)) != 0)
+ goto fail;
+ }
+
+ if (a->path && *a->path) {
+ key.mv_data = a->path;
+ key.mv_size = strlen(a->path);
+ if ((res = mdb_put(tx, db->article_by_path, &key, &data, MDB_NOOVERWRITE)) != 0)
+ goto fail;
+ }
+
+ if (a->keyword && *a->keyword) {
+ key.mv_data = a->keyword;
+ key.mv_size = 0;
+
+ while (keyword_next(&key)) {
+ if ((res = mdb_put(tx, db->article_by_keyword, &key, &data, 0)) != 0)
+ goto fail;
+ }
+ }
+
+ if ((res = object_put(db, tx, db->article, a->id, a, article_DESC, MDB_NOOVERWRITE)) != 0)
+ goto fail;
+
+ return mdb_txn_commit(tx);
+ fail:
+ mdb_txn_abort(tx);
+ return res;
+}
+
+static int strequal(const char *a, const char *b) {
+ return a ? (b ? strcmp(a, b) == 0 : 0) : (b ? 0 : 1);
+}
+
+int article_update(struct articledb *db, MDB_txn *txn, struct article *a) {
+ MDB_txn *tx;
+ MDB_val key, data;
+ int res = -1;
+
+ mdb_txn_begin(db->env, txn, 0, &tx);
+
+ key.mv_data = &a->id;
+ key.mv_size = sizeof(a->id);
+ if ((res = mdb_get(tx, db->article, &key, &data)) != 0)
+ goto fail;
+
+ ez_blob blob = { .eb_data = data.mv_data, .eb_size = data.mv_size };
+ struct article o;
+
+ if (ez_tagz_decode_raw(article_DESC, &blob, &o) != 0)
+ goto fail;
+
+ // check things that can't change
+ if (!strequal(o.type, a->type))
+ goto fail_invalid;
+ if (o.revisionid != a->revisionid)
+ goto fail_invalid;
+ if (o.dataid != a->dataid)
+ goto fail_invalid;
+
+ // find things that changed, need to copy as data is only valid until the next update
+ char *title = NULL;
+ char *path = NULL;
+ char *keyword = NULL;
+
+ if (!strequal(o.title, a->title))
+ title = strdup(o.title);
+ if (!strequal(o.path, a->path))
+ path = strdup(o.path);
+ if (!strequal(o.keyword, a->keyword))
+ keyword = strdup(o.keyword);
+
+ ez_blob_free_raw(article_DESC, &o);
+
+ data.mv_data = &a->id;
+ data.mv_size = sizeof(a->id);
+
+ if (title) {
+ key.mv_data = title;
+ key.mv_size = strlen(title);
+ if ((res = mdb_del(tx, db->article_by_title, &key, &data)) != 0)
+ goto fail;
+ if (a->title && *a->title) {
+ key.mv_data = a->title;
+ key.mv_size = strlen(a->title);
+ if ((res = mdb_put(tx, db->article_by_title, &key, &data, 0)) != 0)
+ goto fail;
+ }
+ }
+
+ if (path) {
+ if ((res = mdb_del(tx, db->article_by_path, &key, &data)) != 0)
+ goto fail;
+ if (a->path && *a->path) {
+ key.mv_data = a->path;
+ key.mv_size = strlen(a->path);
+ if ((res = mdb_put(tx, db->article_by_path, &key, &data, MDB_NOOVERWRITE)) != 0)
+ goto fail;
+ }
+ }
+
+ if (keyword) {
+ key.mv_data = keyword;
+ key.mv_size = 0;
+ while (keyword_next(&key)) {
+ if ((res = mdb_del(tx, db->article_by_keyword, &key, &data)) != 0)
+ goto fail;
+ }
+
+ key.mv_data = a->keyword;
+ key.mv_size = 0;
+ while (keyword_next(&key)) {
+ if ((res = mdb_put(tx, db->article_by_keyword, &key, &data, 0)) != 0)
+ goto fail;
+ }
+ }
+
+ if ((res = object_put(db, tx, db->article, a->id, a, article_DESC, 0)) != 0)
+ goto fail;
+
+ return mdb_txn_commit(tx);
+
+ fail_invalid:
+ res = -1;
+ fail:
+ mdb_txn_abort(tx);
+ return res;
+}
+
+// find articles for a page from a given data, in reverse date order
+// secondary: secondary index used
+// type: key in secondary index
+int article_page(struct articledb *db, MDB_txn *tx, MDB_dbi secondary, char *type, articleid_t from, articleid_t *ids, int count, articleid_t *newer, articleid_t *older) {
+ MDB_val key, dat;
+ MDB_cursor *cursor;
+ int res = -1;
+ int got = 0;
+
+ //res = mdb_txn_begin(db->env, txn, MDB_RDONLY, &tx);
+ //if (res != 0)
+ // return -1;
+
+ res = mdb_cursor_open(tx, secondary, &cursor);
+ if (res != 0)
+ goto fail_noclose;
+
+ key.mv_data = type;
+ key.mv_size = strlen(type);
+ dat.mv_data = &from;
+ dat.mv_size = sizeof(from);
+
+ if (from != ~0)
+ res = mdb_cursor_get(cursor, &key, &dat, MDB_GET_BOTH_RANGE);
+ else {
+ res = mdb_cursor_get(cursor, &key, &dat, MDB_SET_RANGE);
+ res = mdb_cursor_get(cursor, &key, &dat, MDB_LAST_DUP);
+ }
+ if (res != 0)
+ goto fail;
+
+ // Copy up to count older posts than this one
+ for (;got<count && res == 0;got++) {
+ memcpy(&ids[got], dat.mv_data, sizeof(articleid_t));
+ res = mdb_cursor_get(cursor, &key, &dat, MDB_PREV_DUP);
+ }
+
+ // Next oldest is older page start
+ if (res == 0)
+ memcpy(older, dat.mv_data, sizeof(*older));
+ else
+ *older = 0;
+
+ // Skip up to count newer posts than this one, that is the newer page start
+ *newer = 0;
+ if (from != ~0) {
+ dat.mv_data = &from;
+ dat.mv_size = sizeof(from);
+ res = mdb_cursor_get(cursor, &key, &dat, MDB_GET_BOTH_RANGE);
+ for (int i=0;i<count && (res = mdb_cursor_get(cursor, &key, &dat, MDB_NEXT_DUP)) == 0;i++) {
+ memcpy(newer, dat.mv_data, sizeof(*newer));
+ }
+ }
+ mdb_cursor_close(cursor);
+ //mdb_txn_commit(tx);
+ return got;
+ fail:
+ mdb_cursor_close(cursor);
+ fail_noclose:
+ //mdb_txn_abort(tx);
+ return 0;
+}
+
+/*
+
+54321
+
+
+43, older=2, newer=5
+
+21, older=0, newer=4
+
+12345
+
+
+
+ */
+#include <time.h>
+#include <sys/time.h>
+
+typedef uint64_t dbid_t;
+
+dbid_t next_id(void) {
+ struct timespec tv;
+ clock_gettime(CLOCK_REALTIME, &tv);
+
+ return tv.tv_sec * (1000000000ULL) + tv.tv_nsec;
+}
+
+static void object_list(struct articledb *db, MDB_dbi index, const ez_blob_desc *desc) {
+ MDB_txn *tx;
+ MDB_val key, data;
+ MDB_cursor *cursor;
+ int res = -1;
+
+ mdb_txn_begin(db->env, NULL, MDB_RDONLY, &tx);
+
+ res = mdb_cursor_open(tx, index, &cursor);
+ res = mdb_cursor_get(cursor, &key, &data, MDB_FIRST);
+ while (res == 0) {
+ void *p = ez_tagz_decode(desc, (ez_blob *)&data);
+ if (p) {
+ printf(" id: %016lx\n", *(uint64_t *)key.mv_data);
+ //ez_blob_print(desc, p, 4);
+ ez_blob_free(desc, p);
+ }
+ res = mdb_cursor_get(cursor, &key, &data, MDB_NEXT);
+ }
+ mdb_txn_commit(tx);
+}
+
+// list a secondary index where the keys are strings
+static void string_list(struct articledb *db, MDB_dbi primary, MDB_dbi secondary, const ez_blob_desc *desc) {
+ MDB_txn *tx;
+ MDB_val key, data, blob;
+ MDB_cursor *cursor;
+ int res = -1;
+
+ mdb_txn_begin(db->env, NULL, MDB_RDONLY, &tx);
+
+ res = mdb_cursor_open(tx, secondary, &cursor);
+ res = mdb_cursor_get(cursor, &key, &data, MDB_FIRST);
+ while (res == 0) {
+ printf("key '");
+ fwrite(key.mv_data, key.mv_size, 1, stdout);
+ printf("'\n");
+
+ res = mdb_get(tx, primary, &data, &blob);
+ if (res == 0) {
+ void *p = ez_tagz_decode(desc, (ez_blob *)&blob);
+ if (p) {
+ printf(" id: %016lx\n", *(uint64_t *)data.mv_data);
+ ez_blob_print(desc, p, 4);
+ ez_blob_free(desc, p);
+ }
+
+ //res = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_NODUP);
+ res = mdb_cursor_get(cursor, &key, &data, MDB_NEXT);
+ }
+ }
+ mdb_txn_commit(tx);
+}
+
+#include <sys/types.h>
+#include <dirent.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+typedef struct ez_scan {
+ ez_blob data;
+ size_t ptr;
+} ez_scan;
+
+int prop_next(ez_scan *b, char **name, char **value) {
+ unsigned char *p = b->data.eb_data + b->ptr;
+ unsigned char *e = b->data.eb_data + b->data.eb_size;
+ int state = 0;
+
+ *name = NULL;
+ *value = NULL;
+
+ while (p < e) {
+ unsigned char c = *p++;
+
+ switch (state) {
+ case 0:
+ if (c == '#')
+ state = 4;
+ else {
+ *name = (char *)p-1;
+ state = 1;
+ }
+ break;
+ case 1:
+ if (c == '=') {
+ p[-1] = 0;
+ *value = (char *)p;
+ state = 2;
+ }
+ break;
+ case 2:
+ if (c == '\r')
+ p[-1] = 0;
+ if (c == '\n') {
+ p[-1] = 0;
+ b->ptr = (p-(unsigned char *)b->data.eb_data);
+ return 1;
+ }
+ case 4:
+ if (c == '\n')
+ state = 0;
+ break;
+ }
+ }
+ b->ptr = b->data.eb_size;
+ return 0;
+}
+
+void import(struct articledb *db, const char *postdir) {
+ DIR *dir = opendir(postdir);
+ struct dirent *d;
+ int ok = 1;
+ MDB_txn *tx;
+ int count = 0;
+ int res;
+
+ mdb_txn_begin(db->env, NULL, 0, &tx);
+
+ struct author author ={
+ .id = 2,
+ .email = "notzed@gmail.com",
+ .nick = "Not Zed",
+ .prop = EZ_INIT_LIST(author.prop)
+ };
+ res = author_create(db, tx, &author);
+ if (res != 0) {
+ ok = 0;
+ printf("author: %s\n", mdb_strerror(res));
+ }
+
+ while ((d = readdir(dir))) {
+ size_t len = strlen(d->d_name);
+ char buffer[1024];
+
+ if (len > 5 && strcmp(d->d_name + len - 5, ".meta") == 0) {
+ char key[len];
+ int fd = openat(dirfd(dir), d->d_name, O_RDONLY);
+
+ memcpy(key, d->d_name, len-5);
+ key[len-5] = 0;
+
+ if (fd != -1) {
+ len = read(fd, buffer, sizeof(buffer)-1);
+ if (len >= 0) {
+ ez_scan scan = {
+ .data.eb_size = len,
+ .data.eb_data = buffer
+ };
+ struct article post = {
+ .prop = EZ_INIT_LIST(post.prop),
+ .type = "post"
+ };
+ struct revision rev = {
+ .prop = EZ_INIT_LIST(post.prop),
+ .authorid = author.id
+ };
+
+ rev.id = post.revisionid = post.id = strtoull(key, NULL, 16),
+
+ buffer[len] = '\n';
+ char *name, *value;
+ char *orig = NULL, *path = NULL;
+ while (prop_next(&scan, &name, &value)) {
+ if (strcmp(name, "title") == 0)
+ post.title = value;
+ else if (strcmp(name, "keywords") == 0)
+ post.keyword = value;
+ else if (strcmp(name, "original") == 0)
+ orig = value;
+ else if (strcmp(name, "path") == 0)
+ path = value;
+ else if (strcmp(name, "ctime") == 0) // do i care?
+ rev.ctime = atoll(value);
+ }
+ if (orig) {
+ char *p = strstr(orig, ".com/");
+ if (p) {
+ char *e = strstr(p, ".html");
+ if (e)
+ *e = 0;
+ post.path = alloca(strlen(p+5) + strlen("/post/"));
+ sprintf(post.path, "/post/%s", p+5);
+ }
+ }
+ if (!post.path)
+ post.path = path;
+
+ //printf("Post %s\n", d->d_name);
+ //printf(" rev %012lx\n", rev.id);
+ //ez_blob_print(revision_DESC, &rev, 4);
+ //printf(" post %012lx\n", post.id);
+ //ez_blob_print(article_DESC, &post, 4);
+ ez_blob rblob, pblob;
+ ez_tagz_encode(revision_DESC, &rev, &rblob);
+ ez_tagz_encode(article_DESC, &post, &pblob);
+
+ res = revision_create(db, tx, &rev);
+ if (res != 0) {
+ printf("revision: %s\n", mdb_strerror(res));
+ ok = 0;
+ }
+ res = article_create(db, tx, &post);
+ if (res != 0) {
+ printf("post %s\n", mdb_strerror(res));
+ ok = 0;
+ }
+ count++;
+
+ free(pblob.eb_data);
+ free(rblob.eb_data);
+ }
+ close(fd);
+ }
+ }
+
+ }
+ closedir(dir);
+
+ if (ok) {
+ printf("%d posts ready\n", count);
+ //mdb_txn_abort(tx);
+ mdb_txn_commit(tx);
+ } else
+ mdb_txn_abort(tx);
+}
+
+/*
+ Some meta-data one might want to cache
+
+ all keywords, although really why bother
+
+ */
+
+int main(int argc, char **argv) {
+ struct articledb *db = db_open("/tmp/post-db");
+ int res;
+
+ if (0) {
+ articleid_t ids[8];
+ articleid_t older, newer;
+ int count;
+ articleid_t from;
+ MDB_txn *tx;
+
+ from = 0x0166de2bae4fULL;
+ from = 0x0166a3cf8de6ULL;
+ from = ~0;
+ //from = 0x014eb53eab22ULL;
+
+ mdb_txn_begin(db->env, NULL, MDB_RDONLY, &tx);
+
+ //count = article_page(db, tx, db->article_by_keyword, "philosophy", from, ids, 8, &newer, &older);
+ count = article_page(db, tx, db->article_by_type, "post", from, ids, 8, &newer, &older);
+ printf("found %d:\n", count);
+ for (int i=0;i<count;i++) {
+ MDB_val key, dat;
+
+ key.mv_data = &ids[i];
+ key.mv_size = sizeof(ids[i]);
+
+ res = mdb_get(tx, db->article, &key, &dat);
+ if (res == 0) {
+ struct article *a = ez_tagz_decode(article_DESC, (ez_blob *)&dat);
+ //ez_blob_print(article_DESC, p);
+ printf(" %012lx '%s'\n", ids[i], a->title);
+
+ ez_blob_free(article_DESC, a);
+ }
+
+ }
+ printf(" older %016lx\n", older);
+ printf(" newer %016lx\n", newer);
+
+ mdb_txn_commit(tx);
+ db_close(db);
+ return 0;
+
+ }
+
+ if (0) {
+ import(db, "post");
+ db_close(db);
+ return 0;
+ }
+
+ if (0) {
+ articleid_t ids[3];
+ articleid_t older, newer;
+ int count;
+ articleid_t from;
+
+ printf("articles\n");
+ object_list(db, db->article, article_DESC);
+
+ from = 0x1599e4164192bb36; // last
+ from = 0x1599e4164192842a; // las-1
+ from = 0x1599e416418abb25; // firsr
+ from = 0x1599e416418e0d4e; //middle
+ count = article_page(db, NULL, db->article_by_type, "post", from, ids, 3, &newer, &older);
+ if (count >= 0) {
+ articleid_t save = newer;
+ printf("articles %d\n", count);
+ for (int i=0;i<count;i++)
+ printf(" %016lx\n", ids[i]);
+ printf("older %016lx\n", older);
+ printf("newer %016lx\n", newer);
+
+ printf("from older\n");
+ count = article_page(db, NULL, db->article_by_type, "post", older, ids, 3, &newer, &older);
+ printf(" older %016lx\n", older);
+ printf(" newer %016lx\n", newer);
+
+ printf("from newer\n");
+ count = article_page(db, NULL, db->article_by_type, "post", save, ids, 3, &newer, &older);
+ printf(" older %016lx\n", older);
+ printf(" newer %016lx\n", newer);
+ }
+
+ db_close(db);
+ return 0;
+ }
+
+ if (1) {
+ printf("authors\n");
+ object_list(db, db->author, author_DESC);
+ printf("revisions\n");
+ object_list(db, db->revision, revision_DESC);
+ printf("articles\n");
+ object_list(db, db->article, article_DESC);
+ if (1) {
+ printf("articles by path\n");
+ string_list(db, db->article, db->article_by_path, article_DESC);
+ printf("articles by keyword\n");
+ string_list(db, db->article, db->article_by_keyword, article_DESC);
+ printf("articles by type\n");
+ string_list(db, db->article, db->article_by_type, article_DESC);
+ }
+
+
+ db_close(db);
+ return 0;
+ }
+
+ struct author author ={
+ .id = 1,
+ .email = "guest",
+ .nick = "guest",
+ .prop = EZ_INIT_LIST(author.prop)
+ };
+
+ MDB_txn *tx;
+ mdb_txn_begin(db->env, NULL, 0, &tx);
+
+ res = author_create(db, tx, &author);
+ if (res != 0)
+ printf("author create failed: %s\n", mdb_strerror(res));
+
+ char *keywords[4] = {
+ "zedzone",
+ "rants",
+ "jjmpeg,java",
+ "java"
+ };
+ for (int i=0;i<30;i++) {
+ char path[64];
+
+ struct revision r = {
+ .id = next_id(),
+ .authorid = author.id,
+ .prop = EZ_INIT_LIST(r.prop)
+ };
+
+ res = revision_create(db, tx, &r);
+ if (res != 0) {
+ printf("revision create failed: %s\n", mdb_strerror(res));
+ goto fail;
+ }
+
+ struct article post = {
+ .id = r.id,
+ .revisionid = r.id,
+ .title = "A post",
+ .type = "post",
+ .path = path,
+ .keyword = keywords[i%4],
+ .upvote = i+1,
+ .downvote = 7,
+ .prop = EZ_INIT_LIST(post.prop)
+ };
+ sprintf(path, "/post/%012lx", post.id);
+ res = article_create(db, tx, &post);
+ if (res != 0) {
+ printf("article create failed: %s\n", mdb_strerror(res));
+ goto fail;
+ }
+ }
+ mdb_txn_commit(tx);
+ db_close(db);
+ return 0;
+ fail:
+ mdb_txn_abort(tx);
+ db_close(db);
+ return 1;
+}