tyiu/damus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ndb: switch profile queries to use transactions

Browse Source 
this should ensure no crashing occurs when querying profiles
This commit is contained in:
William Casarin
2023-09-10 14:51:55 -07:00
parent 622a436589
commit fc9b9f2940
51 changed files with 435 additions and 252 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
nostrdb/Ndb.swift
View File

@@ -49,44 +49,130 @@ class Ndb {
self.ndb = ndb
}
func lookup_note_by_key(_ key: UInt64) -> NdbNote? {
guard let note_p = ndb_get_note_by_key(ndb.ndb, key, nil) else {
func lookup_note_by_key_with_txn<Y>(_ key: NoteKey, txn: NdbTxn<Y>) -> NdbNote? {
guard let note_p = ndb_get_note_by_key(&txn.txn, key, nil) else {
return nil
}
return NdbNote(note: note_p, owned_size: nil)
return NdbNote(note: note_p, owned_size: nil, key: key)
}
func lookup_note(_ id: NoteId) -> NdbNote? {
id.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NdbNote? in
guard let baseAddress = ptr.baseAddress,
let note_p = ndb_get_note_by_id(ndb.ndb, baseAddress, nil) else {
return nil
}
return NdbNote(note: note_p, owned_size: nil)
func lookup_note_by_key(_ key: NoteKey) -> NdbTxn<NdbNote?> {
return NdbTxn(ndb: self) { txn in
lookup_note_by_key_with_txn(key, txn: txn)
}
}
func lookup_profile(_ pubkey: Pubkey) -> ProfileRecord? {
private func lookup_profile_by_key_inner<Y>(_ key: ProfileKey, txn: NdbTxn<Y>) -> ProfileRecord? {
var size: Int = 0
guard let profile_p = ndb_get_profile_by_key(&txn.txn, key, &size) else {
return nil
}
return profile_flatbuf_to_record(ptr: profile_p, size: size, key: key)
}
private func profile_flatbuf_to_record(ptr: UnsafeMutableRawPointer, size: Int, key: UInt64) -> ProfileRecord? {
do {
var buf = ByteBuffer(assumingMemoryBound: ptr, capacity: size)
let rec: NdbProfileRecord = try getDebugCheckedRoot(byteBuffer: &buf)
return ProfileRecord(data: rec, key: key)
} catch {
// Handle error appropriately
print("UNUSUAL: \(error)")
return nil
}
}
private func lookup_note_with_txn_inner<Y>(id: NoteId, txn: NdbTxn<Y>) -> NdbNote? {
return id.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NdbNote? in
var key: UInt64 = 0
guard let baseAddress = ptr.baseAddress,
let note_p = ndb_get_note_by_id(&txn.txn, baseAddress, nil, &key) else {
return nil
}
return NdbNote(note: note_p, owned_size: nil, key: key)
}
}
private func lookup_profile_with_txn_inner<Y>(pubkey: Pubkey, txn: NdbTxn<Y>) -> ProfileRecord? {
return pubkey.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> ProfileRecord? in
var size: Int = 0
var key: UInt64 = 0
guard let baseAddress = ptr.baseAddress,
let profile_p = ndb_get_profile_by_pubkey(ndb.ndb, baseAddress, &size)
let profile_p = ndb_get_profile_by_pubkey(&txn.txn, baseAddress, &size, &key)
else {
return nil
}
do {
var buf = ByteBuffer(assumingMemoryBound: profile_p, capacity: size)
let rec: NdbProfileRecord = try getDebugCheckedRoot(byteBuffer: &buf)
return ProfileRecord(data: rec)
} catch {
// Handle error appropriately
print("UNUSUAL: \(error)")
return nil
}
return profile_flatbuf_to_record(ptr: profile_p, size: size, key: key)
}
}
func lookup_profile_by_key_with_txn<Y>(key: ProfileKey, txn: NdbTxn<Y>) -> ProfileRecord? {
lookup_profile_by_key_inner(key, txn: txn)
}
func lookup_profile_by_key(key: ProfileKey) -> NdbTxn<ProfileRecord?> {
return NdbTxn(ndb: self) { txn in
lookup_profile_by_key_inner(key, txn: txn)
}
}
func lookup_note_with_txn<Y>(id: NoteId, txn: NdbTxn<Y>) -> NdbNote? {
lookup_note_with_txn_inner(id: id, txn: txn)
}
func lookup_profile_key(_ pubkey: Pubkey) -> ProfileKey? {
return NdbTxn(ndb: self) { txn in
lookup_profile_key_with_txn(pubkey, txn: txn)
}.value
}
func lookup_profile_key_with_txn<Y>(_ pubkey: Pubkey, txn: NdbTxn<Y>) -> ProfileKey? {
return pubkey.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NoteKey? in
guard let p = ptr.baseAddress else { return nil }
let r = ndb_get_profilekey_by_pubkey(&txn.txn, p)
if r == 0 {
return nil
}
return r
}
}
func lookup_note_key_with_txn<Y>(_ id: NoteId, txn: NdbTxn<Y>) -> NoteKey? {
return id.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NoteKey? in
guard let p = ptr.baseAddress else {
return nil
}
let r = ndb_get_notekey_by_id(&txn.txn, p)
if r == 0 {
return nil
}
return r
}
}
func lookup_note_key(_ id: NoteId) -> NoteKey? {
NdbTxn(ndb: self, with: { txn in lookup_note_key_with_txn(id, txn: txn) }).value
}
func lookup_note(_ id: NoteId) -> NdbTxn<NdbNote?> {
return NdbTxn(ndb: self) { txn in
lookup_note_with_txn_inner(id: id, txn: txn)
}
}
func lookup_profile(_ pubkey: Pubkey) -> NdbTxn<ProfileRecord?> {
return NdbTxn(ndb: self) { txn in
lookup_profile_with_txn_inner(pubkey: pubkey, txn: txn)
}
}
func lookup_profile_with_txn<Y>(_ pubkey: Pubkey, txn: NdbTxn<Y>) -> ProfileRecord? {
lookup_profile_with_txn_inner(pubkey: pubkey, txn: txn)
}
func process_event(_ str: String) -> Bool {
return str.withCString { cstr in
return ndb_process_event(ndb.ndb, cstr, Int32(str.utf8.count)) != 0
@@ -106,7 +192,7 @@ class Ndb {
#if DEBUG
func getDebugCheckedRoot<T: FlatBufferObject & Verifiable>(byteBuffer: inout ByteBuffer) throws -> T {
return try getCheckedRoot(byteBuffer: &byteBuffer)
return try getRoot(byteBuffer: &byteBuffer)
}
#else
func getDebugCheckedRoot<T: FlatBufferObject>(byteBuffer: inout ByteBuffer) throws -> T {

7
nostrdb/NdbNote.swift
View File

@@ -38,6 +38,7 @@ class NdbNote: Encodable, Equatable, Hashable {
// we can have owned notes, but we can also have lmdb virtual-memory mapped notes so its optional
private let owned: Bool
let count: Int
let key: NoteKey?
let note: UnsafeMutablePointer<ndb_note>
// cached stuff (TODO: remove these)
@@ -48,10 +49,11 @@ class NdbNote: Encodable, Equatable, Hashable {
return NdbNote.owned_from_json_cstr(json: content_raw, json_len: content_len)
}()
init(note: UnsafeMutablePointer<ndb_note>, owned_size: Int?) {
init(note: UnsafeMutablePointer<ndb_note>, owned_size: Int?, key: NoteKey?) {
self.note = note
self.owned = owned_size != nil
self.count = owned_size ?? 0
self.key = key
#if DEBUG_NOTE_SIZE
if let owned_size {
@@ -218,6 +220,7 @@ class NdbNote: Encodable, Equatable, Hashable {
}
self.note = r.assumingMemoryBound(to: ndb_note.self)
self.key = nil
}
static func owned_from_json(json: String, bufsize: Int = 2 << 18) -> NdbNote? {
@@ -245,7 +248,7 @@ class NdbNote: Encodable, Equatable, Hashable {
guard let note_data = realloc(data, Int(len)) else { return nil }
let new_note = note_data.assumingMemoryBound(to: ndb_note.self)
return NdbNote(note: new_note, owned_size: Int(len))
return NdbNote(note: new_note, owned_size: Int(len), key: nil)
}
}

2
nostrdb/NdbTagsIterator.swift
View File

@@ -70,7 +70,7 @@ struct TagsSequence: Encodable, Sequence {
}
precondition(false, "sequence subscript oob")
// it seems like the compiler needs this or it gets bitchy
return .init(note: .init(note: .allocate(capacity: 1), owned_size: nil), tag: .allocate(capacity: 1))
return .init(note: .init(note: .allocate(capacity: 1), owned_size: nil, key: nil), tag: .allocate(capacity: 1))
}
func makeIterator() -> TagsIterator {

2
nostrdb/NdbTxn.swift
View File

@@ -17,7 +17,7 @@ class NdbTxn<T> {
private var val: T!
var moved: Bool
init(ndb: Ndb, with: (NdbTxn<T>) -> T) {
init(ndb: Ndb, with: (NdbTxn<T>) -> T = { _ in () }) {
self.txn = ndb_txn()
#if TXNDEBUG
txn_count += 1

26
nostrdb/nostrdb.c
View File

@@ -264,7 +264,9 @@ int ndb_get_tsid(MDB_txn *txn, struct ndb_lmdb *lmdb, enum ndb_dbs db,
int success = 0;
struct ndb_tsid tsid;
// position at the most recent
ndb_tsid_high(&tsid, id);
k.mv_data = &tsid;
k.mv_size = sizeof(tsid);
@@ -341,32 +343,44 @@ static void *ndb_lookup_tsid(struct ndb_txn *txn, enum ndb_dbs ind,
return res;
}
void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pk, size_t *len, uint32_t *key)
void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pk, size_t *len, uint64_t *key)
{
return ndb_lookup_tsid(txn, NDB_DB_PROFILE_PK, NDB_DB_PROFILE, pk, len, key);
}
struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint32_t *key)
struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint64_t *key)
{
return ndb_lookup_tsid(txn, NDB_DB_NOTE_ID, NDB_DB_NOTE, id, len, key);
}
uint32_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id)
static inline uint64_t ndb_get_indexkey_by_id(struct ndb_txn *txn,
enum ndb_dbs db,
const unsigned char *id)
{
MDB_val k;
if (!ndb_get_tsid(txn->mdb_txn, &txn->ndb->lmdb, NDB_DB_NOTE_ID, id, &k))
if (!ndb_get_tsid(txn->mdb_txn, &txn->ndb->lmdb, db, id, &k))
return 0;
return *(uint32_t*)k.mv_data;
}
struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint32_t key, size_t *len)
uint64_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id)
{
return ndb_get_indexkey_by_id(txn, NDB_DB_NOTE_ID, id);
}
uint64_t ndb_get_profilekey_by_pubkey(struct ndb_txn *txn, const unsigned char *id)
{
return ndb_get_indexkey_by_id(txn, NDB_DB_PROFILE_PK, id);
}
struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint64_t key, size_t *len)
{
return ndb_lookup_by_key(txn, key, NDB_DB_NOTE, len);
}
void *ndb_get_profile_by_key(struct ndb_txn *txn, uint32_t key, size_t *len)
void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len)
{
return ndb_lookup_by_key(txn, key, NDB_DB_PROFILE, len);
}

1
nostrdb/nostrdb.h
View File

@@ -166,6 +166,7 @@ void ndb_end_query(struct ndb_txn *);
void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pubkey, size_t *len, uint64_t *primkey);
void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len);
uint64_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id);
uint64_t ndb_get_profilekey_by_pubkey(struct ndb_txn *txn, const unsigned char *id);
struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint64_t *primkey);
struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint64_t key, size_t *len);
void ndb_destroy(struct ndb *);