From: Not Zed Date: Sat, 29 Apr 2023 22:11:46 +0000 (+0930) Subject: Add flags to dbdisk to support non-recursive scans. X-Git-Url: https://code.zedzone.au/cvs?a=commitdiff_plain;h=6a48ccfed7ed5d3e72384466c66c0b09142f714f;p=playerz Add flags to dbdisk to support non-recursive scans. Remove redundant copies of serialisation descriptors. Fix a double-free with dbfile. Add update command to disk-indexer. Add usage to disk-util. Sanitize utf8 in json strings. Minor cleanups. --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..31dc307 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +*.d +*.o diff --git a/Makefile b/Makefile index 13cec01..cc9486e 100644 --- a/Makefile +++ b/Makefile @@ -102,13 +102,15 @@ SOURCES= \ music-player.c \ notify.c -disk-monitor: disk-monitor.o dbindex.o dbmarshal.o notify.o -disk-indexer: disk-indexer.o dbindex.o dbmarshal.o notify.o analyse.o +disk-monitor: disk-monitor.o dbindex.o dbmarshal.o notify.o blobs.o +disk-indexer: disk-indexer.o dbindex.o dbmarshal.o notify.o analyse.o blobs.o audio-cmd: audio-cmd.o notify.o blobs.o -music-player: music-player.o notify.o dbindex.o dbmarshal.o +music-player: music-player.o notify.o dbindex.o dbmarshal.o blobs.o input-monitor: input-monitor.o notify.o blobs.o -disk-util: disk-util.o dbindex.o dbmarshal.o -http-monitor: http-monitor.o dbindex.o notify.o dbmarshal.o ../libeze/libeze.a +disk-util: disk-util.o dbindex.o dbmarshal.o blobs.o +http-monitor: http-monitor.o dbindex.o notify.o dbmarshal.o blobs.o ../libeze/libeze.a + +dbindex.c: dbmarshal.h %.d: %.c @rm -rf $@ diff --git a/blobs.c b/blobs.c index 0f47034..d98bc5a 100644 --- a/blobs.c +++ b/blobs.c @@ -27,9 +27,9 @@ ez_blob_desc DBDISK_DESC[] = { EZ_BLOB_START(dbdisk, 1, 4), - EZ_BLOB_STRING(dbdisk, 1, uuid), - EZ_BLOB_STRING(dbdisk, 2, label), - EZ_BLOB_STRING(dbdisk, 3, type), + EZ_BLOB_INT32(dbdisk, 1, flags), + EZ_BLOB_STRING(dbdisk, 2, uuid), + EZ_BLOB_STRING(dbdisk, 3, label), EZ_BLOB_STRING(dbdisk, 4, mount), }; diff --git a/dbindex.c b/dbindex.c index 2b17ac4..89eeaf1 100644 --- a/dbindex.c +++ b/dbindex.c @@ -37,26 +37,6 @@ // prototype void dblist_dump(dbtxn *txn, dbindex *db); -ez_blob_desc DBDISK_DESC[] = { - EZ_BLOB_START(dbdisk, 1, 4), - EZ_BLOB_STRING(dbdisk, 1, uuid), - EZ_BLOB_STRING(dbdisk, 2, label), - EZ_BLOB_STRING(dbdisk, 3, type), - EZ_BLOB_STRING(dbdisk, 4, mount), -}; - -ez_blob_desc DBFILE_DESC[] = { - EZ_BLOB_START(dbfile, 2, 7), - EZ_BLOB_INT32(dbfile, 1, diskid), - EZ_BLOB_INT64(dbfile, 2, size), - EZ_BLOB_INT64(dbfile, 3, mtime), - EZ_BLOB_INT64(dbfile, 4, duration), - EZ_BLOB_STRING(dbfile, 5, path), - EZ_BLOB_STRING(dbfile, 6, title), - EZ_BLOB_STRING(dbfile, 7, artist), - EZ_BLOB_TRANSIENTP(dbfile, 8, full_path), -}; - /* TODO: playlist should be linked list @@ -105,13 +85,6 @@ struct dblistfile { uint32_t seq; }; -ez_blob_desc DBLIST_DESC[] = { - EZ_BLOB_START(dblist, 3, 3), - EZ_BLOB_INT32(dblist, 1, size), - EZ_BLOB_STRING(dblist, 2, name), - EZ_BLOB_STRING(dblist, 3, comment), -}; - struct dbindex { int res; // last result @@ -166,16 +139,12 @@ static uint32_t find_next_id(MDB_txn *tx, MDB_dbi db) { int r; uint32_t id = 1; - printf("find last value of db\n"); mdb_cursor_open(tx, db, &cursor); r = mdb_cursor_get(cursor, &key, &data, MDB_LAST); if (r == 0) { assert(key.mv_size == sizeof(id)); memcpy(&id, key.mv_data, sizeof(id)); - printf("found, was %d\n", id); id += 1; - } else { - printf("not found (%d), using %d\n", r, id); } mdb_cursor_close(cursor); @@ -244,28 +213,6 @@ dbindex *dbindex_open(const char *path) { db->listid = find_next_id(tx, db->list); db->fileid = find_next_id(tx, db->file); - { - MDB_cursor *cursor; - MDB_val key = { 0 }, data = { 0 }; - int r; - - printf("Known disks:\n"); - mdb_cursor_open(tx, db->disk, &cursor); - r = mdb_cursor_get(cursor, &key, &data, MDB_FIRST); - while (r == 0) { - dbdisk *p = ez_basic_decode(DBDISK_DESC, (ez_blob *)&data); - p->id = *(int *)key.mv_data; - printf("id=%d\n", p->id); - printf(" uuid=%s\n", p->uuid); - printf(" label=%s\n", p->label); - printf(" type=%s\n", p->type); - printf(" mount=%s\n", p->mount); - ez_blob_free(DBDISK_DESC, p); - r = mdb_cursor_get(cursor, &key, &data, MDB_NEXT); - } - mdb_cursor_close(cursor); - } - res |= mdb_txn_commit(tx); if (res) { @@ -275,7 +222,7 @@ dbindex *dbindex_open(const char *path) { db = NULL; } - printf("dbindex open, disk.id=%d list.id=%d file.id=%d\n", db->diskid, db->listid, db->fileid); + printf("dbindex open, path=`%s' disk.id=%d list.id=%d file.id=%d\n", path, db->diskid, db->listid, db->fileid); return db; fail: @@ -571,10 +518,7 @@ char *dbfile_full_path(dbtxn *tx, dbindex *db, dbfile *file) { } void dbfile_free(dbfile *f) { - if (f) { - free(f->full_path); - ez_blob_free(DBFILE_DESC, f); - } + ez_blob_free(DBFILE_DESC, f); } #include "dbmarshal.h" @@ -2086,9 +2030,9 @@ void dbindex_dump(dbindex *db) { dbdisk *p = ez_basic_decode(DBDISK_DESC, (ez_blob *)&data); p->id = *(int *)key.mv_data; printf("id=%d\n", p->id); + printf(" flags=%08x\n", p->flags); printf(" uuid=%s\n", p->uuid); printf(" label=%s\n", p->label); - printf(" type=%s\n", p->type); printf(" mount=%s\n", p->mount); ez_blob_free(DBDISK_DESC, p); r = mdb_cursor_get(cursor, &key, &data, MDB_NEXT); diff --git a/dbindex.h b/dbindex.h index a87f293..5de544c 100644 --- a/dbindex.h +++ b/dbindex.h @@ -25,12 +25,16 @@ typedef uint32_t dbid_t; struct dbdisk { dbid_t id; + uint32_t flags; char *uuid; char *label; - char *type; char *mount; // last mount point }; +// dbdisk flags +#define DBDISKF_RECURSIVE 1 +#define DBDISKF_MOUNT 2 + typedef struct dblist dblist; struct dblist { @@ -95,7 +99,8 @@ int dbstate_put(dbtxn *tx, dbindex *db, dbstate *s); dbdisk *dbdisk_get(dbtxn *tx, dbindex *db, int diskid); dbdisk *dbdisk_get_uuid(dbtxn *tx, dbindex *db, const char *uuid); void dbdisk_free(dbdisk *d); -int dbdisk_add(dbtxn *txn, dbindex *db, dbdisk *d); +int dbdisk_add(dbtxn *txn, dbindex *db, dbdisk *d); +int dbdisk_del(dbtxn *txn, dbindex *db, dbdisk *disk); dbfile *dbfile_get(dbtxn *tx, dbindex *db, int fileid); dbfile *dbfile_get_path(dbtxn *tx, dbindex *db, int diskid, const char *path); @@ -115,6 +120,7 @@ void dbfile_scan_close(dbscan *scan); extern ez_blob_desc DBDISK_DESC[]; extern ez_blob_desc DBFILE_DESC[]; +extern ez_blob_desc DBLIST_DESC[]; /* Player support */ int dbfile_next(dbindex *db, dbfile **f, char **fpath); diff --git a/disk-indexer.c b/disk-indexer.c index c9dcd08..f03c098 100644 --- a/disk-indexer.c +++ b/disk-indexer.c @@ -49,8 +49,6 @@ struct indexer { dbtxn *tx; // global transaction for insert 1 disk - int isnew; - ez_list queue; struct dirent *entry; @@ -185,50 +183,24 @@ static void dir_free(struct dir_node *d) { void indexer_destroy(struct indexer *ix); -int indexer_init(struct indexer *ix, dbindex *db, const char *path, const char *uuid) { +int indexer_init(struct indexer *ix, dbindex *db, const dbdisk *info) { int res; memset(ix, 0, sizeof(*ix)); ez_list_init(&ix->queue); - ez_list_addtail(&ix->queue, dir_new(path)); - - // find out how big dirent needs to be - int name_max = pathconf(path, _PC_NAME_MAX); - - name_max = name_max == -1 ? 256 : name_max; - ix->entry = malloc(sizeof(struct dirent) - sizeof(ix->entry->d_name) + name_max + 1); - - ix->root = strdup(path); - - res = regcomp(&ix->match, "\\.(mp3|avi|mpeg|mp2|mpg|mp4|mov)$", REG_ICASE | REG_NOSUB | REG_EXTENDED); + res = regcomp(&ix->match, "\\.(mp3|avi|mpeg|mp2|mpg|mp4|mov|aac)$", REG_ICASE | REG_NOSUB | REG_EXTENDED); if (res != 0) { perror("regcomp"); exit(1); } - // Lookup this disk + // init disk and perhaps list of existing fids ix->db = db; ix->tx = dbindex_begin(db, NULL, 0); + ix->disk = dbdisk_get_uuid(ix->tx, db, info->uuid); - ix->disk = dbdisk_get_uuid(ix->tx, db, uuid); - if (!ix->disk) { - dbdisk *disk; - - disk = malloc(sizeof(*ix->disk)); - - disk->id = 0; - disk->uuid = strdup(uuid); - disk->label = strdup("unknown"); - disk->type = strdup("jfs"); - disk->mount = strdup(path); - - res = dbdisk_add(ix->tx, db, disk); - if (res != 0) - goto fail; - printf("%8d : add new disk %s\n", disk->id, uuid); - ix->disk = disk; - } else { + if (ix->disk) { int count = 0; dbtxn *tx = dbindex_begin(db, NULL, 1); @@ -249,10 +221,34 @@ int indexer_init(struct indexer *ix, dbindex *db, const char *path, const char * printf("bitset count %d actual count %d\n", ez_bitset_card(ix->existing), count); } - printf("%8d : add old disk %s (%d existing files)\n", ix->disk->id, uuid, count); + printf("%8d : add old disk %s (%d existing files)\n", ix->disk->id, ix->disk->uuid, count); + } else if (info->mount[0] == '/') { + dbdisk *disk = malloc(sizeof(*ix->disk)); + + disk->id = 0; + disk->flags = info->flags; + disk->uuid = strdup(info->uuid); + disk->label = strdup(info->label); + disk->mount = strdup(info->mount); + + res = dbdisk_add(ix->tx, db, disk); + if (res != 0) + goto fail; + printf("%8d : add new disk %s\n", disk->id, disk->uuid); + ix->disk = disk; + } else { + printf("cannot add non-absolute path: `%s'\n", info->mount); + res = -1; + goto fail; } - // FIXME: error handling + ix->root = strdup(ix->disk->mount); + ez_list_addtail(&ix->queue, dir_new(ix->root)); + + // find out how big dirent needs to be + int name_max = pathconf(ix->root, _PC_NAME_MAX); + name_max = name_max == -1 ? 256 : name_max; + ix->entry = malloc(sizeof(struct dirent) - sizeof(ix->entry->d_name) + name_max + 1); return 0; @@ -415,7 +411,7 @@ int indexer_scan(struct indexer *ix) { if ((count % 1000) == 0) indexer_info(ix); } - } else if (S_ISDIR(st.st_mode)) { + } else if (S_ISDIR(st.st_mode) && (ix->disk->flags & DBDISKF_RECURSIVE)) { ez_list_addtail(&ix->queue, dir_new(name)); } else if (S_ISLNK(st.st_mode)) { // softlinks are ignored for simplicity reasons @@ -498,7 +494,7 @@ static void indexer(const char *path) { printf("Indexer: scanning '%s'.\n", disk->mount); - res = indexer_init(&ix, db, disk->mount, disk->uuid); + res = indexer_init(&ix, db, disk); if (res == 0) { res = indexer_scan(&ix); @@ -685,6 +681,8 @@ void search_suffix(const char *path) { #include int main(int argc, char **argv) { + char *self = argv[0]; + av_log_set_level(AV_LOG_ERROR); //avcodec_register_all(); @@ -694,8 +692,11 @@ int main(int argc, char **argv) { setlocale(LC_ALL, "en_AU.UTF-8"); - if (argc > 2 && strcmp(argv[1], "-d") == 0) { - dbdir = argv[2]; + argv += 1; + argc -= 1; + + if (argc > 1 && strcmp(argv[0], "-d") == 0) { + dbdir = argv[1]; argv += 2; argc -= 2; } @@ -725,87 +726,51 @@ int main(int argc, char **argv) { } #endif - if (argc > 1) { - if (strcmp(argv[1], "check") == 0) + if (argc > 0) { + if (strcmp(argv[0], "start") == 0) + indexer(dbdir); + else if (strcmp(argv[0], "check") == 0) check(dbdir); else { notify_t q = notify_writer_new(NOTIFY_INDEXER); if (q) { - if (strcmp(argv[1], "quit") == 0) { + if (strcmp(argv[0], "quit") == 0) { notify_msg_send(q, NOTIFY_QUIT, 0, 0); - } else if (strcmp(argv[1], "shuffle") == 0) { + } else if (strcmp(argv[0], "shuffle") == 0) { notify_msg_send(q, NOTIFY_SHUFFLE, 0, 0); - } else if (strcmp(argv[1], "add") == 0 && argc == 4) { - dbdisk disk = { - .uuid = argv[2], - .label = "system", - .type = "system", - .mount = argv[3] - }; - notify_msg_send(q, NOTIFY_DISK_ADD, 0, &disk); + } else if (strcmp(argv[0], "add") == 0) { + dbdisk disk = { 0 }; + + argv++; argc--; + if (argc > 0 && strcmp(argv[0], "-r") == 0) { + disk.flags |= DBDISKF_RECURSIVE; + argv++; argc--; + } + + if (argc == 2) { + disk.uuid = argv[0]; + disk.mount = argv[1]; + disk.label = "system"; + notify_msg_send(q, NOTIFY_DISK_ADD, 0, &disk); + } else { + printf("usage: %s add [-r] uuid mount\n", self); + } + } else if (strcmp(argv[0], "update") == 0) { + dbdisk disk = { 0 }; + if (argc == 2) { + disk.uuid = argv[1]; + disk.mount = ""; + notify_msg_send(q, NOTIFY_DISK_ADD, 0, &disk); + } } notify_close(q); } } - } else - indexer(dbdir); - - return 0; - - - char *uuid = argc > 1 ? argv[1] : "some-disk"; - - - dbindex *db = dbindex_open("/home/notzed/playerz.db"); - int res; - - if (!db) { - return 1; - } - - if (0) { - dbtxn *tx = dbindex_begin(db, NULL, 1); - int diskid = 4; - dbscan *scan = dbfile_scan_disk(tx, db, diskid); - int count = 0; - uint32_t fid; - - while ((fid = dbfile_scan_next(scan)) != ~0) { - count++; - } - printf(" %d files on disk %d\n", count, diskid); - - dbindex_commit(tx); - dbindex_close(db); - return 0; - } - - struct indexer ix; - - res = indexer_init(&ix, db, "/data/hd4/Music", uuid); - if (res == 0) { - - //for (int i=1;idisk.uuid); free(m->disk.label); - free(m->disk.type); free(m->disk.mount); free(m); @@ -237,7 +236,7 @@ static void partition_add(struct monitor *m, const char *dev) { md->dev = strdup(dev); md->disk.uuid = strdup(uuid); - md->disk.type = strdup(type); + md->disk.flags = DBDISKF_RECURSIVE | DBDISKF_MOUNT; md->disk.label = plabel ? strdup(label) : NULL; md->disk.mount = mountp; diff --git a/disk-util.c b/disk-util.c index d149a46..8f2a44d 100644 --- a/disk-util.c +++ b/disk-util.c @@ -52,6 +52,32 @@ int main(int argc, char **argv) { setlocale(LC_ALL, "en_AU.UTF-8"); + if (argc == 1) { + printf("usage: %s [-d database] options\n" + " [ -f fileid ]\n" + " [ -s seq ]\n" + " [ -d diskid ] selectors\n" + " --shuffle did run dbshuffle 2 for disk [did]\n" + " --file fid show file [fid] info\n" + " --files list files or [fileid]\n" + " --file-del delete [fileid]\n" + " --lists show playlists\n" + " --lists-reset clear playlists\n" + " --list-add name add list [name]\n" + " --list-del lid delete list\n" + " --list-add-file lid fid\n" + " add file [fid] to list [lid]\n" + " --list-del-file lid seq\n" + " delete file [seq] of list [lid]\n" + " --list-dump lid dump lists or list entry [seq]\n" + " --disks dump disks\n" + " --disk-del did delete disk [did]\n" + " --validate db check\n" + " --search query run search\n", + argv[0]); + return 0; + } + if (argc > 2 && strcmp(argv[1], "-d") == 0) { dbdir = argv[2]; argv += 2; @@ -72,11 +98,32 @@ int main(int argc, char **argv) { seq = atoi(argv[++i]); } else if (strcmp(cmd, "-d") == 0) { diskid = atoi(argv[++i]); + } else if (strcmp(cmd, "-l") == 0) { + listid = atoi(argv[++i]); } else if (strcmp(cmd, "--shuffle") == 0) { int did = atoi(argv[++i]); dbshuffle_init2(db, did); - } else if (strcmp(cmd, "--file-dump") == 0) { + } else if (strcmp(cmd, "--file") == 0) { + dbtxn *tx = dbindex_begin(db, NULL, 1); + int fid = atoi(argv[++i]); + dbfile *file = dbfile_get(tx, db, fid); + + if (file) { + printf("id: %u\n", file->id); + printf("diskid: %u\n", file->diskid); + printf("size: %lu\n", file->size); + printf("mtime: %lu\n", file->mtime); + printf("duration: %lu\n", file->duration); + printf("path: %s\n", file->path); + printf("title: %s\n", file->title); + printf("artist: %s\n", file->artist); + printf("full_path: %s\n", file->full_path); + } + + dbfile_free(file); + dbindex_abort(tx); + // dump file info // dump lists it's in } else if (strcmp(cmd, "--files") == 0) { @@ -167,9 +214,9 @@ int main(int argc, char **argv) { for (dbdisk *disk = dbscan_disk(tx, &scan, db, 0); disk; disk = dbscan_disk_next(&scan)) { printf("id: %d\n", disk->id); + printf("flags: %08x\n", disk->flags); printf("uuid: %s\n", disk->uuid); printf("label: %s\n", disk->label); - printf("type: %s\n", disk->type); printf("mount: %s\n", disk->mount); printf("\n"); dbdisk_free(disk); diff --git a/http-monitor.c b/http-monitor.c index e14cf62..1c47834 100644 --- a/http-monitor.c +++ b/http-monitor.c @@ -124,13 +124,35 @@ static void duration_value(struct obstack *os, uint64_t ms) { obstack_printf(os, "\"%02d:%02d:%02d\"", (int)h, (int)m, (int)s); } +/* cleanup utf8, FIXME: this should be done at index time */ +static void json_print(struct obstack *os, uint8_t *s) { + uint8_t c; + + while ((c = *s++)) { + if (c < 32) { + obstack_1grow(os, '?'); + } else if (c == '\\') { + obstack_1grow(os, c); + obstack_1grow(os, c); + } else { + obstack_1grow(os, c); + } + } +} + static void write_file_json(struct obstack *os, dbfile *file) { obstack_printf(os, "\"id\":\"%d\"", file->id); - if (file->title) - obstack_printf(os, ",\"title\":\"%s\"", file->title); - if (file->artist) - obstack_printf(os, ",\"artist\":\"%s\"", file->artist); + if (file->title) { + obstack_printf(os, ",\"title\":\""); + json_print(os, file->title); + obstack_printf(os, "\""); + } + if (file->artist) { + obstack_printf(os, ",\"artist\":\""); + json_print(os, file->artist); + obstack_printf(os, "\""); + } obstack_sgrow(os, ",\"length\":"); duration_value(os, file->duration / 1000); @@ -154,7 +176,7 @@ static void write_state_json(dbindex *db, struct obstack *os) { obstack_1grow(os, '{'); write_file_json(os, file); - obstack_printf(os, ",\"path\":\"%s\"", file->path); + obstack_printf(os, ",\"path\":\"%s\"", dbfile_full_path(tx, db, file)); obstack_printf(os, ",\"status\":\"%s\"", (state.state & 2) ? "paused" : "playing"); obstack_printf(os, ",\"listid\":\"%u\"", state.listid);