From 05baba9c03f9df72d65de1623b432f28eb428566 Mon Sep 17 00:00:00 2001 From: William Casarin Date: Sun, 15 Dec 2024 11:23:09 -0800 Subject: [PATCH] nostrdb: flags: make some indexes optional Make fulltext indices and note blocks optional. This will be useful for quickly building databases when testing, since more stuff in the write queue when writing can slow things down. Signed-off-by: William Casarin --- nostrdb/src/nostrdb.c | 28 ++++++++++++++++++---------- nostrdb/src/nostrdb.h | 5 ++++- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/nostrdb/src/nostrdb.c b/nostrdb/src/nostrdb.c index 0a92d322..704f114f 100644 --- a/nostrdb/src/nostrdb.c +++ b/nostrdb/src/nostrdb.c @@ -162,6 +162,7 @@ struct ndb_writer { struct ndb_lmdb *lmdb; struct ndb_monitor *monitor; + uint32_t ndb_flags; void *queue_buf; int queue_buflen; pthread_t thread_id; @@ -4171,7 +4172,8 @@ static int ndb_write_new_blocks(struct ndb_txn *txn, struct ndb_note *note, static uint64_t ndb_write_note(struct ndb_txn *txn, struct ndb_writer_note *note, - unsigned char *scratch, size_t scratch_size) + unsigned char *scratch, size_t scratch_size, + uint32_t ndb_flags) { int rc; uint64_t note_key; @@ -4207,15 +4209,18 @@ static uint64_t ndb_write_note(struct ndb_txn *txn, // only parse content and do fulltext index on text and longform notes if (note->note->kind == 1 || note->note->kind == 30023) { - if (!ndb_write_note_fulltext_index(txn, note->note, note_key)) - return 0; + if (!ndb_flag_set(ndb_flags, NDB_FLAG_NO_FULLTEXT)) { + if (!ndb_write_note_fulltext_index(txn, note->note, note_key)) + return 0; + } // write note blocks - ndb_write_new_blocks(txn, note->note, note_key, scratch, - scratch_size); + if (!ndb_flag_set(ndb_flags, NDB_FLAG_NO_NOTE_BLOCKS)) { + ndb_write_new_blocks(txn, note->note, note_key, scratch, scratch_size); + } } - if (note->note->kind == 7) { + if (note->note->kind == 7 && !ndb_flag_set(ndb_flags, NDB_FLAG_NO_STATS)) { ndb_write_reaction_stats(txn, note->note); } @@ -4365,7 +4370,8 @@ static void *ndb_writer_thread(void *data) case NDB_WRITER_PROFILE: note_nkey = ndb_write_note(&txn, &msg->note, - scratch, scratch_size); + scratch, scratch_size, + writer->ndb_flags); if (note_nkey > 0) { written_notes[num_notes++] = (struct written_note){ @@ -4384,7 +4390,8 @@ static void *ndb_writer_thread(void *data) case NDB_WRITER_NOTE: note_nkey = ndb_write_note(&txn, &msg->note, scratch, - scratch_size); + scratch_size, + writer->ndb_flags); if (note_nkey > 0) { written_notes[num_notes++] = (struct written_note){ @@ -4514,10 +4521,11 @@ static void *ndb_ingester_thread(void *data) static int ndb_writer_init(struct ndb_writer *writer, struct ndb_lmdb *lmdb, - struct ndb_monitor *monitor) + struct ndb_monitor *monitor, uint32_t ndb_flags) { writer->lmdb = lmdb; writer->monitor = monitor; + writer->ndb_flags = ndb_flags; writer->queue_buflen = sizeof(struct ndb_writer_msg) * DEFAULT_QUEUE_SIZE; writer->queue_buf = malloc(writer->queue_buflen); if (writer->queue_buf == NULL) { @@ -4842,7 +4850,7 @@ int ndb_init(struct ndb **pndb, const char *filename, const struct ndb_config *c ndb_monitor_init(&ndb->monitor, config->sub_cb, config->sub_cb_ctx); - if (!ndb_writer_init(&ndb->writer, &ndb->lmdb, &ndb->monitor)) { + if (!ndb_writer_init(&ndb->writer, &ndb->lmdb, &ndb->monitor, ndb->flags)) { fprintf(stderr, "ndb_writer_init failed\n"); return 0; } diff --git a/nostrdb/src/nostrdb.h b/nostrdb/src/nostrdb.h index 0c092e4f..4c9423f6 100644 --- a/nostrdb/src/nostrdb.h +++ b/nostrdb/src/nostrdb.h @@ -9,8 +9,11 @@ #define NDB_PACKED_STR 0x1 #define NDB_PACKED_ID 0x2 -#define NDB_FLAG_NOMIGRATE (1 << 0) +#define NDB_FLAG_NOMIGRATE (1 << 0) #define NDB_FLAG_SKIP_NOTE_VERIFY (1 << 1) +#define NDB_FLAG_NO_FULLTEXT (1 << 2) +#define NDB_FLAG_NO_NOTE_BLOCKS (1 << 3) +#define NDB_FLAG_NO_STATS (1 << 4) //#define DEBUG 1