nostrdb: ccan: sync with normal versions.

This is the version of CCAN which CLN was using at the time these
were taken.  Unfortunately lots of whitespace has been changed,
but AFAICT no source changes.

Here's the command I ran (with ../ccan checked out to 1ae4c432):

```
make update-ccan CCAN_NEW="alignof array_size build_assert check_type container_of cppmagic likely list mem short_types str structeq take tal tal/str typesafe_cb utf8 endian crypto/sha256"
```

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Signed-off-by: William Casarin <jb55@jb55.com>

nostrdb/ccan/ccan/htable/LICENSE

@@ -0,0 +1 @@
+../../licenses/LGPL-2.1

nostrdb/ccan/ccan/htable/_info

@@ -0,0 +1,118 @@
+#include "config.h"
+#include <string.h>
+#include <stdio.h>
+
+/**
+ * htable - hash table routines
+ *
+ * A hash table is an efficient structure for looking up keys.  This version
+ * grows with usage and allows efficient deletion.
+ *
+ * Example:
+ *	#include <ccan/htable/htable.h>
+ *	#include <ccan/hash/hash.h>
+ *	#include <stdio.h>
+ *	#include <err.h>
+ *	#include <string.h>
+ *
+ *	struct name_to_digit {
+ *		const char *name;
+ *		unsigned int val;
+ *	};
+ *
+ *	static struct name_to_digit map[] = {
+ *		{ "zero", 0},
+ *		{ "one", 1 },
+ *		{ "two", 2 },
+ *		{ "three", 3 },
+ *		{ "four", 4 },
+ *		{ "five", 5 },
+ *		{ "six", 6 },
+ *		{ "seven", 7 },
+ *		{ "eight", 8 },
+ *		{ "nine", 9 }
+ *	};
+ *
+ *	// Wrapper for rehash function pointer.
+ *	static size_t rehash(const void *e, void *unused)
+ *	{
+ *		(void)unused;
+ *		return hash_string(((struct name_to_digit *)e)->name);
+ *	}
+ *
+ *	// Comparison function.
+ *	static bool nameeq(const void *e, void *string)
+ *	{
+ *		return strcmp(((struct name_to_digit *)e)->name, string) == 0;
+ *	}
+ *
+ *	// We let them add their own aliases, eg. --alias=v=5
+ *	static void add_alias(struct htable *ht, const char *alias)
+ *	{
+ *		char *eq, *name;
+ *		struct name_to_digit *n;
+ *
+ *		n = malloc(sizeof(*n));
+ *		n->name = name = strdup(alias);
+ *
+ *		eq = strchr(name, '=');
+ *		if (!eq || ((n->val = atoi(eq+1)) == 0 && !strcmp(eq+1, "0")))
+ *			errx(1, "Usage: --alias=<name>=<value>");
+ *		*eq = '\0';
+ *		htable_add(ht, hash_string(n->name), n);
+ *	}
+ *
+ *	int main(int argc, char *argv[])
+ *	{
+ *		struct htable ht;
+ *		int i;
+ *		unsigned long val;
+ *
+ *		if (argc < 2)
+ *			errx(1, "Usage: %s [--alias=<name>=<val>]... <str>...",
+ *			     argv[0]);
+ *
+ *		// Create and populate hash table.
+ *		htable_init(&ht, rehash, NULL);
+ *		for (i = 0; i < (int)(sizeof(map)/sizeof(map[0])); i++)
+ *			htable_add(&ht, hash_string(map[i].name), &map[i]);
+ *
+ *		// Add any aliases to the hash table.
+ *		for (i = 1; i < argc; i++) {
+ *			if (!strncmp(argv[i], "--alias=", strlen("--alias=")))
+ *				add_alias(&ht, argv[i] + strlen("--alias="));
+ *			else
+ *				break;
+ *		}
+ *
+ *		// Find the other args in the hash table.
+ *		for (val = 0; i < argc; i++) {
+ *			struct name_to_digit *n;
+ *			n = htable_get(&ht, hash_string(argv[i]),
+ *				       nameeq, argv[i]);
+ *			if (!n)
+ *				errx(1, "Invalid digit name %s", argv[i]);
+ *			// Append it to the value we are building up.
+ *			val *= 10;
+ *			val += n->val;
+ *		}
+ *		printf("%lu\n", val);
+ *		return 0;
+ *	}
+ *
+ * License: LGPL (v2.1 or any later version)
+ * Author: Rusty Russell <rusty@rustcorp.com.au>
+ */
+int main(int argc, char *argv[])
+{
+	if (argc != 2)
+		return 1;
+
+	if (strcmp(argv[1], "depends") == 0) {
+		printf("ccan/compiler\n");
+		printf("ccan/str\n");
+		return 0;
+	}
+
+	return 1;
+}

nostrdb/ccan/ccan/htable/htable.c

@@ -0,0 +1,491 @@
+/* Licensed under LGPLv2+ - see LICENSE file for details */
+#include <ccan/htable/htable.h>
+#include <ccan/compiler/compiler.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <assert.h>
+#include <string.h>
+
+/* We use 0x1 as deleted marker. */
+#define HTABLE_DELETED (0x1)
+
+/* perfect_bitnum 63 means there's no perfect bitnum */
+#define NO_PERFECT_BIT (sizeof(uintptr_t) * CHAR_BIT - 1)
+
+static void *htable_default_alloc(struct htable *ht, size_t len)
+{
+	return calloc(len, 1);
+}
+
+static void htable_default_free(struct htable *ht, void *p)
+{
+	free(p);
+}
+
+static void *(*htable_alloc)(struct htable *, size_t) = htable_default_alloc;
+static void (*htable_free)(struct htable *, void *) = htable_default_free;
+
+void htable_set_allocator(void *(*alloc)(struct htable *, size_t len),
+			  void (*free)(struct htable *, void *p))
+{
+	if (!alloc)
+		alloc = htable_default_alloc;
+	if (!free)
+		free = htable_default_free;
+	htable_alloc = alloc;
+	htable_free = free;
+}
+
+/* We clear out the bits which are always the same, and put metadata there. */
+static inline uintptr_t get_extra_ptr_bits(const struct htable *ht,
+					   uintptr_t e)
+{
+	return e & ht->common_mask;
+}
+
+static inline void *get_raw_ptr(const struct htable *ht, uintptr_t e)
+{
+	return (void *)((e & ~ht->common_mask) | ht->common_bits);
+}
+
+static inline uintptr_t make_hval(const struct htable *ht,
+				  const void *p, uintptr_t bits)
+{
+	return ((uintptr_t)p & ~ht->common_mask) | bits;
+}
+
+static inline bool entry_is_valid(uintptr_t e)
+{
+	return e > HTABLE_DELETED;
+}
+
+static inline uintptr_t ht_perfect_mask(const struct htable *ht)
+{
+	return (uintptr_t)2 << ht->perfect_bitnum;
+}
+
+static inline uintptr_t get_hash_ptr_bits(const struct htable *ht,
+					  size_t hash)
+{
+	/* Shuffling the extra bits (as specified in mask) down the
+	 * end is quite expensive.  But the lower bits are redundant, so
+	 * we fold the value first. */
+	return (hash ^ (hash >> ht->bits))
+		& ht->common_mask & ~ht_perfect_mask(ht);
+}
+
+void htable_init(struct htable *ht,
+		 size_t (*rehash)(const void *elem, void *priv), void *priv)
+{
+	struct htable empty = HTABLE_INITIALIZER(empty, NULL, NULL);
+	*ht = empty;
+	ht->rehash = rehash;
+	ht->priv = priv;
+	ht->table = &ht->common_bits;
+}
+
+/* Fill to 87.5% */
+static inline size_t ht_max(const struct htable *ht)
+{
+	return ((size_t)7 << ht->bits) / 8;
+}
+
+/* Clean deleted if we're full, and more than 12.5% deleted */
+static inline size_t ht_max_deleted(const struct htable *ht)
+{
+	return ((size_t)1 << ht->bits) / 8;
+}
+
+bool htable_init_sized(struct htable *ht,
+		       size_t (*rehash)(const void *, void *),
+		       void *priv, size_t expect)
+{
+	htable_init(ht, rehash, priv);
+
+	/* Don't go insane with sizing. */
+	for (ht->bits = 1; ht_max(ht) < expect; ht->bits++) {
+		if (ht->bits == 30)
+			break;
+	}
+
+	ht->table = htable_alloc(ht, sizeof(size_t) << ht->bits);
+	if (!ht->table) {
+		ht->table = &ht->common_bits;
+		return false;
+	}
+	(void)htable_debug(ht, HTABLE_LOC);
+	return true;
+}
+	
+void htable_clear(struct htable *ht)
+{
+	if (ht->table != &ht->common_bits)
+		htable_free(ht, (void *)ht->table);
+	htable_init(ht, ht->rehash, ht->priv);
+}
+
+bool htable_copy_(struct htable *dst, const struct htable *src)
+{
+	uintptr_t *htable = htable_alloc(dst, sizeof(size_t) << src->bits);
+
+	if (!htable)
+		return false;
+
+	*dst = *src;
+	dst->table = htable;
+	memcpy(dst->table, src->table, sizeof(size_t) << src->bits);
+	return true;
+}
+
+static size_t hash_bucket(const struct htable *ht, size_t h)
+{
+	return h & ((1 << ht->bits)-1);
+}
+
+static void *htable_val(const struct htable *ht,
+			struct htable_iter *i, size_t hash, uintptr_t perfect)
+{
+	uintptr_t h2 = get_hash_ptr_bits(ht, hash) | perfect;
+
+	while (ht->table[i->off]) {
+		if (ht->table[i->off] != HTABLE_DELETED) {
+			if (get_extra_ptr_bits(ht, ht->table[i->off]) == h2)
+				return get_raw_ptr(ht, ht->table[i->off]);
+		}
+		i->off = (i->off + 1) & ((1 << ht->bits)-1);
+		h2 &= ~perfect;
+	}
+	return NULL;
+}
+
+void *htable_firstval_(const struct htable *ht,
+		       struct htable_iter *i, size_t hash)
+{
+	i->off = hash_bucket(ht, hash);
+	return htable_val(ht, i, hash, ht_perfect_mask(ht));
+}
+
+void *htable_nextval_(const struct htable *ht,
+		      struct htable_iter *i, size_t hash)
+{
+	i->off = (i->off + 1) & ((1 << ht->bits)-1);
+	return htable_val(ht, i, hash, 0);
+}
+
+void *htable_first_(const struct htable *ht, struct htable_iter *i)
+{
+	for (i->off = 0; i->off < (size_t)1 << ht->bits; i->off++) {
+		if (entry_is_valid(ht->table[i->off]))
+			return get_raw_ptr(ht, ht->table[i->off]);
+	}
+	return NULL;
+}
+
+void *htable_next_(const struct htable *ht, struct htable_iter *i)
+{
+	for (i->off++; i->off < (size_t)1 << ht->bits; i->off++) {
+		if (entry_is_valid(ht->table[i->off]))
+			return get_raw_ptr(ht, ht->table[i->off]);
+	}
+	return NULL;
+}
+
+void *htable_prev_(const struct htable *ht, struct htable_iter *i)
+{
+	for (;;) {
+		if (!i->off)
+			return NULL;
+		i->off--;
+		if (entry_is_valid(ht->table[i->off]))
+			return get_raw_ptr(ht, ht->table[i->off]);
+	}
+}
+
+/* Another bit currently in mask needs to be exposed, so that a bucket with p in
+ * it won't appear invalid */
+static COLD void unset_another_common_bit(struct htable *ht,
+					  uintptr_t *maskdiff,
+					  const void *p)
+{
+	size_t i;
+
+	for (i = sizeof(uintptr_t) * CHAR_BIT - 1; i > 0; i--) {
+		if (((uintptr_t)p & ((uintptr_t)1 << i))
+		    && ht->common_mask & ~*maskdiff & ((uintptr_t)1 << i))
+			break;
+	}
+	/* There must have been one, right? */
+	assert(i > 0);
+
+	*maskdiff |= ((uintptr_t)1 << i);
+}
+
+/* We want to change the common mask: this fixes up the table */
+static COLD void fixup_table_common(struct htable *ht, uintptr_t maskdiff)
+{
+	size_t i;
+	uintptr_t bitsdiff;
+
+again:
+	bitsdiff = ht->common_bits & maskdiff;
+
+	for (i = 0; i < (size_t)1 << ht->bits; i++) {
+		uintptr_t e;
+		if (!entry_is_valid(e = ht->table[i]))
+			continue;
+
+		/* Clear the bits no longer in the mask, set them as
+		 * expected. */
+		e &= ~maskdiff;
+		e |= bitsdiff;
+		/* If this made it invalid, restart with more exposed */
+		if (!entry_is_valid(e)) {
+			unset_another_common_bit(ht, &maskdiff, get_raw_ptr(ht, e));
+			goto again;
+		}
+		ht->table[i] = e;
+	}
+
+	/* Take away those bits from our mask, bits and perfect bit. */
+	ht->common_mask &= ~maskdiff;
+	ht->common_bits &= ~maskdiff;
+	if (ht_perfect_mask(ht) & maskdiff)
+		ht->perfect_bitnum = NO_PERFECT_BIT;
+}
+
+/* Limited recursion */
+static void ht_add(struct htable *ht, const void *new, size_t h);
+
+/* We tried to add this entry, but it looked invalid!  We need to
+ * let another pointer bit through mask */
+static COLD void update_common_fix_invalid(struct htable *ht, const void *p, size_t h)
+{
+	uintptr_t maskdiff;
+
+	assert(ht->elems != 0);
+
+	maskdiff = 0;
+	unset_another_common_bit(ht, &maskdiff, p);
+	fixup_table_common(ht, maskdiff);
+
+	/* Now won't recurse */
+	ht_add(ht, p, h);
+}
+
+/* This does not expand the hash table, that's up to caller. */
+static void ht_add(struct htable *ht, const void *new, size_t h)
+{
+	size_t i;
+	uintptr_t perfect = ht_perfect_mask(ht);
+
+	i = hash_bucket(ht, h);
+
+	while (entry_is_valid(ht->table[i])) {
+		perfect = 0;
+		i = (i + 1) & ((1 << ht->bits)-1);
+	}
+	ht->table[i] = make_hval(ht, new, get_hash_ptr_bits(ht, h)|perfect);
+	if (!entry_is_valid(ht->table[i]))
+		update_common_fix_invalid(ht, new, h);
+}
+
+static COLD bool double_table(struct htable *ht)
+{
+	unsigned int i;
+	size_t oldnum = (size_t)1 << ht->bits;
+	uintptr_t *oldtable, e;
+
+	oldtable = ht->table;
+	ht->table = htable_alloc(ht, sizeof(size_t) << (ht->bits+1));
+	if (!ht->table) {
+		ht->table = oldtable;
+		return false;
+	}
+	ht->bits++;
+
+	/* If we lost our "perfect bit", get it back now. */
+	if (ht->perfect_bitnum == NO_PERFECT_BIT && ht->common_mask) {
+		for (i = 0; i < sizeof(ht->common_mask) * CHAR_BIT; i++) {
+			if (ht->common_mask & ((size_t)2 << i)) {
+				ht->perfect_bitnum = i;
+				break;
+			}
+		}
+	}
+
+	if (oldtable != &ht->common_bits) {
+		for (i = 0; i < oldnum; i++) {
+			if (entry_is_valid(e = oldtable[i])) {
+				void *p = get_raw_ptr(ht, e);
+				ht_add(ht, p, ht->rehash(p, ht->priv));
+			}
+		}
+		htable_free(ht, oldtable);
+	}
+	ht->deleted = 0;
+
+	(void)htable_debug(ht, HTABLE_LOC);
+	return true;
+}
+
+static COLD void rehash_table(struct htable *ht)
+{
+	size_t start, i;
+	uintptr_t e, perfect = ht_perfect_mask(ht);
+
+	/* Beware wrap cases: we need to start from first empty bucket. */
+	for (start = 0; ht->table[start]; start++);
+
+	for (i = 0; i < (size_t)1 << ht->bits; i++) {
+		size_t h = (i + start) & ((1 << ht->bits)-1);
+		e = ht->table[h];
+		if (!e)
+			continue;
+		if (e == HTABLE_DELETED)
+			ht->table[h] = 0;
+		else if (!(e & perfect)) {
+			void *p = get_raw_ptr(ht, e);
+			ht->table[h] = 0;
+			ht_add(ht, p, ht->rehash(p, ht->priv));
+		}
+	}
+	ht->deleted = 0;
+	(void)htable_debug(ht, HTABLE_LOC);
+}
+
+/* We stole some bits, now we need to put them back... */
+static COLD void update_common(struct htable *ht, const void *p)
+{
+	uintptr_t maskdiff;
+
+	if (ht->elems == 0) {
+		ht->common_mask = -1;
+		ht->common_bits = ((uintptr_t)p & ht->common_mask);
+		ht->perfect_bitnum = 0;
+		(void)htable_debug(ht, HTABLE_LOC);
+		return;
+	}
+
+	/* Find bits which are unequal to old common set. */
+	maskdiff = ht->common_bits ^ ((uintptr_t)p & ht->common_mask);
+
+	fixup_table_common(ht, maskdiff);
+	(void)htable_debug(ht, HTABLE_LOC);
+}
+
+bool htable_add_(struct htable *ht, size_t hash, const void *p)
+{
+	/* Cannot insert NULL, or (void *)1. */
+	assert(p);
+	assert(entry_is_valid((uintptr_t)p));
+
+	/* Getting too full? */
+	if (ht->elems+1 + ht->deleted > ht_max(ht)) {
+		/* If we're more than 1/8 deleted, clean those,
+		 * otherwise double table size. */
+		if (ht->deleted > ht_max_deleted(ht))
+			rehash_table(ht);
+		else if (!double_table(ht))
+			return false;
+	}
+	if (((uintptr_t)p & ht->common_mask) != ht->common_bits)
+		update_common(ht, p);
+
+	ht_add(ht, p, hash);
+	ht->elems++;
+	return true;
+}
+
+bool htable_del_(struct htable *ht, size_t h, const void *p)
+{
+	struct htable_iter i;
+	void *c;
+
+	for (c = htable_firstval(ht,&i,h); c; c = htable_nextval(ht,&i,h)) {
+		if (c == p) {
+			htable_delval(ht, &i);
+			return true;
+		}
+	}
+	return false;
+}
+
+void htable_delval_(struct htable *ht, struct htable_iter *i)
+{
+	assert(i->off < (size_t)1 << ht->bits);
+	assert(entry_is_valid(ht->table[i->off]));
+
+	ht->elems--;
+	/* Cheap test: if the next bucket is empty, don't need delete marker */
+	if (ht->table[hash_bucket(ht, i->off+1)] != 0) {
+		ht->table[i->off] = HTABLE_DELETED;
+		ht->deleted++;
+	} else
+		ht->table[i->off] = 0;
+}
+
+void *htable_pick_(const struct htable *ht, size_t seed, struct htable_iter *i)
+{
+	void *e;
+	struct htable_iter unwanted;
+
+	if (!i)
+		i = &unwanted;
+	i->off = seed % ((size_t)1 << ht->bits);
+	e = htable_next(ht, i);
+	if (!e)
+		e = htable_first(ht, i);
+	return e;
+}
+
+struct htable *htable_check(const struct htable *ht, const char *abortstr)
+{
+	void *p;
+	struct htable_iter i;
+	size_t n = 0;
+
+	/* Use non-DEBUG versions here, to avoid infinite recursion with
+	 * CCAN_HTABLE_DEBUG! */
+	for (p = htable_first_(ht, &i); p; p = htable_next_(ht, &i)) {
+		struct htable_iter i2;
+		void *c;
+		size_t h = ht->rehash(p, ht->priv);
+		bool found = false;
+
+		n++;
+
+		/* Open-code htable_get to avoid CCAN_HTABLE_DEBUG */
+		for (c = htable_firstval_(ht, &i2, h);
+		     c;
+		     c = htable_nextval_(ht, &i2, h)) {
+			if (c == p) {
+				found = true;
+				break;
+			}
+		}
+
+		if (!found) {
+			if (abortstr) {
+				fprintf(stderr,
+					"%s: element %p in position %zu"
+					" cannot find itself\n",
+					abortstr, p, i.off);
+				abort();
+			}
+			return NULL;
+		}
+	}
+	if (n != ht->elems) {
+		if (abortstr) {
+			fprintf(stderr,
+				"%s: found %zu elems, expected %zu\n",
+				abortstr, n, ht->elems);
+			abort();
+		}
+		return NULL;
+	}
+
+	return (struct htable *)ht;
+}

nostrdb/ccan/ccan/htable/htable.h

@@ -0,0 +1,290 @@
+/* Licensed under LGPLv2+ - see LICENSE file for details */
+#ifndef CCAN_HTABLE_H
+#define CCAN_HTABLE_H
+#include "config.h"
+#include <ccan/str/str.h>
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+
+/* Define CCAN_HTABLE_DEBUG for expensive debugging checks on each call. */
+#define HTABLE_LOC __FILE__ ":" stringify(__LINE__)
+#ifdef CCAN_HTABLE_DEBUG
+#define htable_debug(h, loc) htable_check((h), loc)
+#else
+#define htable_debug(h, loc) ((void)loc, h)
+#endif
+
+/**
+ * struct htable - private definition of a htable.
+ *
+ * It's exposed here so you can put it in your structures and so we can
+ * supply inline functions.
+ */
+struct htable {
+	size_t (*rehash)(const void *elem, void *priv);
+	void *priv;
+	unsigned int bits, perfect_bitnum;
+	size_t elems, deleted;
+	/* These are the bits which are the same in all pointers. */
+	uintptr_t common_mask, common_bits;
+	uintptr_t *table;
+};
+
+/**
+ * HTABLE_INITIALIZER - static initialization for a hash table.
+ * @name: name of this htable.
+ * @rehash: hash function to use for rehashing.
+ * @priv: private argument to @rehash function.
+ *
+ * This is useful for setting up static and global hash tables.
+ *
+ * Example:
+ *	// For simplicity's sake, say hash value is contents of elem.
+ *	static size_t rehash(const void *elem, void *unused)
+ *	{
+ *		(void)unused;
+ *		return *(size_t *)elem;
+ *	}
+ *	static struct htable ht = HTABLE_INITIALIZER(ht, rehash, NULL);
+ */
+#define HTABLE_INITIALIZER(name, rehash, priv)				\
+	{ rehash, priv, 0, 0, 0, 0, -1, 0, &name.common_bits }
+
+/**
+ * htable_init - initialize an empty hash table.
+ * @ht: the hash table to initialize
+ * @rehash: hash function to use for rehashing.
+ * @priv: private argument to @rehash function.
+ */
+void htable_init(struct htable *ht,
+		 size_t (*rehash)(const void *elem, void *priv), void *priv);
+
+/**
+ * htable_init_sized - initialize an empty hash table of given size.
+ * @ht: the hash table to initialize
+ * @rehash: hash function to use for rehashing.
+ * @priv: private argument to @rehash function.
+ * @size: the number of element.
+ *
+ * If this returns false, @ht is still usable, but may need to do reallocation
+ * upon an add.  If this returns true, it will not need to reallocate within
+ * @size htable_adds.
+ */
+bool htable_init_sized(struct htable *ht,
+		       size_t (*rehash)(const void *elem, void *priv),
+		       void *priv, size_t size);
+
+/**
+ * htable_count - count number of entries in a hash table.
+ * @ht: the hash table
+ */
+static inline size_t htable_count(const struct htable *ht)
+{
+	return ht->elems;
+}
+
+/**
+ * htable_clear - empty a hash table.
+ * @ht: the hash table to clear
+ *
+ * This doesn't do anything to any pointers left in it.
+ */
+void htable_clear(struct htable *ht);
+
+
+/**
+ * htable_check - check hash table for consistency
+ * @ht: the htable
+ * @abortstr: the location to print on aborting, or NULL.
+ *
+ * Because hash tables have redundant information, consistency checking that
+ * each element is in the correct location can be done.  This is useful as a
+ * debugging check.  If @abortstr is non-NULL, that will be printed in a
+ * diagnostic if the htable is inconsistent, and the function will abort.
+ *
+ * Returns the htable if it is consistent, NULL if not (it can never return
+ * NULL if @abortstr is set).
+ */
+struct htable *htable_check(const struct htable *ht, const char *abortstr);
+
+/**
+ * htable_copy - duplicate a hash table.
+ * @dst: the hash table to overwrite
+ * @src: the hash table to copy
+ *
+ * Only fails on out-of-memory.
+ *
+ * Equivalent to (but faster than):
+ *    if (!htable_init_sized(dst, src->rehash, src->priv, 1U << src->bits))
+ *	   return false;
+ *    v = htable_first(src, &i);
+ *    while (v) {
+ *		htable_add(dst, v);
+ *		v = htable_next(src, i);
+ *    }
+ *    return true;
+ */
+#define htable_copy(dst, src) htable_copy_(dst, htable_debug(src, HTABLE_LOC))
+bool htable_copy_(struct htable *dst, const struct htable *src);
+
+/**
+ * htable_add - add a pointer into a hash table.
+ * @ht: the htable
+ * @hash: the hash value of the object
+ * @p: the non-NULL pointer (also cannot be (void *)1).
+ *
+ * Also note that this can only fail due to allocation failure.  Otherwise, it
+ * returns true.
+ */
+#define htable_add(ht, hash, p) \
+	htable_add_(htable_debug(ht, HTABLE_LOC), hash, p)
+bool htable_add_(struct htable *ht, size_t hash, const void *p);
+
+/**
+ * htable_del - remove a pointer from a hash table
+ * @ht: the htable
+ * @hash: the hash value of the object
+ * @p: the pointer
+ *
+ * Returns true if the pointer was found (and deleted).
+ */
+#define htable_del(ht, hash, p) \
+	htable_del_(htable_debug(ht, HTABLE_LOC), hash, p)
+bool htable_del_(struct htable *ht, size_t hash, const void *p);
+
+/**
+ * struct htable_iter - iterator or htable_first or htable_firstval etc.
+ *
+ * This refers to a location inside the hashtable.
+ */
+struct htable_iter {
+	size_t off;
+};
+
+/**
+ * htable_firstval - find a candidate for a given hash value
+ * @htable: the hashtable
+ * @i: the struct htable_iter to initialize
+ * @hash: the hash value
+ *
+ * You'll need to check the value is what you want; returns NULL if none.
+ * See Also:
+ *	htable_delval()
+ */
+#define htable_firstval(htable, i, hash) \
+	htable_firstval_(htable_debug(htable, HTABLE_LOC), i, hash)
+
+void *htable_firstval_(const struct htable *htable,
+		       struct htable_iter *i, size_t hash);
+
+/**
+ * htable_nextval - find another candidate for a given hash value
+ * @htable: the hashtable
+ * @i: the struct htable_iter to initialize
+ * @hash: the hash value
+ *
+ * You'll need to check the value is what you want; returns NULL if no more.
+ */
+#define htable_nextval(htable, i, hash) \
+	htable_nextval_(htable_debug(htable, HTABLE_LOC), i, hash)
+void *htable_nextval_(const struct htable *htable,
+		      struct htable_iter *i, size_t hash);
+
+/**
+ * htable_get - find an entry in the hash table
+ * @ht: the hashtable
+ * @h: the hash value of the entry
+ * @cmp: the comparison function
+ * @ptr: the pointer to hand to the comparison function.
+ *
+ * Convenient inline wrapper for htable_firstval/htable_nextval loop.
+ */
+static inline void *htable_get(const struct htable *ht,
+			       size_t h,
+			       bool (*cmp)(const void *candidate, void *ptr),
+			       const void *ptr)
+{
+	struct htable_iter i;
+	void *c;
+
+	for (c = htable_firstval(ht,&i,h); c; c = htable_nextval(ht,&i,h)) {
+		if (cmp(c, (void *)ptr))
+			return c;
+	}
+	return NULL;
+}
+
+/**
+ * htable_first - find an entry in the hash table
+ * @ht: the hashtable
+ * @i: the struct htable_iter to initialize
+ *
+ * Get an entry in the hashtable; NULL if empty.
+ */
+#define htable_first(htable, i) \
+	htable_first_(htable_debug(htable, HTABLE_LOC), i)
+void *htable_first_(const struct htable *htable, struct htable_iter *i);
+
+/**
+ * htable_next - find another entry in the hash table
+ * @ht: the hashtable
+ * @i: the struct htable_iter to use
+ *
+ * Get another entry in the hashtable; NULL if all done.
+ * This is usually used after htable_first or prior non-NULL htable_next.
+ */
+#define htable_next(htable, i) \
+	htable_next_(htable_debug(htable, HTABLE_LOC), i)
+void *htable_next_(const struct htable *htable, struct htable_iter *i);
+
+/**
+ * htable_prev - find the previous entry in the hash table
+ * @ht: the hashtable
+ * @i: the struct htable_iter to use
+ *
+ * Get previous entry in the hashtable; NULL if all done.
+ *
+ * "previous" here only means the item that would have been returned by
+ * htable_next() before the item it returned most recently.
+ *
+ * This is usually used in the middle of (or after) a htable_next iteration and
+ * to "unwind" actions taken.
+ */
+#define htable_prev(htable, i) \
+	htable_prev_(htable_debug(htable, HTABLE_LOC), i)
+void *htable_prev_(const struct htable *htable, struct htable_iter *i);
+
+/**
+ * htable_delval - remove an iterated pointer from a hash table
+ * @ht: the htable
+ * @i: the htable_iter
+ *
+ * Usually used to delete a hash entry after it has been found with
+ * htable_firstval etc.
+ */
+#define htable_delval(htable, i) \
+	htable_delval_(htable_debug(htable, HTABLE_LOC), i)
+void htable_delval_(struct htable *ht, struct htable_iter *i);
+
+/**
+ * htable_pick - set iterator to a random valid entry.
+ * @ht: the htable
+ * @seed: a random number to use.
+ * @i: the htable_iter which is output (or NULL).
+ *
+ * Usually used with htable_delval to delete a random entry.  Returns
+ * NULL iff the table is empty, otherwise a random entry.
+ */
+#define htable_pick(htable, seed, i)					\
+	htable_pick_(htable_debug(htable, HTABLE_LOC), seed, i)
+void *htable_pick_(const struct htable *ht, size_t seed, struct htable_iter *i);
+
+/**
+ * htable_set_allocator - set calloc/free functions.
+ * @alloc: allocator to use, must zero memory!
+ * @free: unallocator to use (@p is NULL or a return from @alloc)
+ */
+void htable_set_allocator(void *(*alloc)(struct htable *, size_t len),
+			  void (*free)(struct htable *, void *p));
+#endif /* CCAN_HTABLE_H */

nostrdb/ccan/ccan/htable/htable_type.h

@@ -0,0 +1,188 @@
+/* Licensed under LGPLv2+ - see LICENSE file for details */
+#ifndef CCAN_HTABLE_TYPE_H
+#define CCAN_HTABLE_TYPE_H
+#include <ccan/htable/htable.h>
+#include <ccan/compiler/compiler.h>
+#include "config.h"
+
+/**
+ * HTABLE_DEFINE_TYPE - create a set of htable ops for a type
+ * @type: a type whose pointers will be values in the hash.
+ * @keyof: a function/macro to extract a key: <keytype> @keyof(const type *elem)
+ * @hashfn: a hash function for a @key: size_t @hashfn(const <keytype> *)
+ * @eqfn: an equality function keys: bool @eqfn(const type *, const <keytype> *)
+ * @prefix: a prefix for all the functions to define (of form <name>_*)
+ *
+ * NULL values may not be placed into the hash table.
+ *
+ * This defines the type hashtable type and an iterator type:
+ *	struct <name>;
+ *	struct <name>_iter;
+ *
+ * It also defines initialization and freeing functions:
+ *	void <name>_init(struct <name> *);
+ *	bool <name>_init_sized(struct <name> *, size_t);
+ *	void <name>_clear(struct <name> *);
+ *	bool <name>_copy(struct <name> *dst, const struct <name> *src);
+ *
+ * Count entries:
+ *	size_t <name>_count(const struct <name> *ht);
+ *
+ * Add function only fails if we run out of memory:
+ *	bool <name>_add(struct <name> *ht, const <type> *e);
+ *
+ * Delete and delete-by key return true if it was in the set:
+ *	bool <name>_del(struct <name> *ht, const <type> *e);
+ *	bool <name>_delkey(struct <name> *ht, const <keytype> *k);
+ *
+ * Delete by iterator:
+ *	bool <name>_delval(struct <name> *ht, struct <name>_iter *i);
+ *
+ * Find and return the (first) matching element, or NULL:
+ *	type *<name>_get(const struct @name *ht, const <keytype> *k);
+ *
+ * Find and return all matching elements, or NULL:
+ *	type *<name>_getfirst(const struct @name *ht, const <keytype> *k,
+ *			      struct <name>_iter *i);
+ *	type *<name>_getnext(const struct @name *ht, const <keytype> *k,
+ *			     struct <name>_iter *i);
+ *
+ * Iteration over hashtable is also supported:
+ *	type *<name>_first(const struct <name> *ht, struct <name>_iter *i);
+ *	type *<name>_next(const struct <name> *ht, struct <name>_iter *i);
+ *	type *<name>_prev(const struct <name> *ht, struct <name>_iter *i);
+ *      type *<name>_pick(const struct <name> *ht, size_t seed,
+ *                        struct <name>_iter *i);
+ * It's currently safe to iterate over a changing hashtable, but you might
+ * miss an element.  Iteration isn't very efficient, either.
+ *
+ * You can use HTABLE_INITIALIZER like so:
+ *	struct <name> ht = { HTABLE_INITIALIZER(ht.raw, <name>_hash, NULL) };
+ */
+#define HTABLE_DEFINE_TYPE(type, keyof, hashfn, eqfn, name)		\
+	struct name { struct htable raw; };				\
+	struct name##_iter { struct htable_iter i; };			\
+	static inline size_t name##_hash(const void *elem, void *priv)	\
+	{								\
+		(void)priv;						\
+		return hashfn(keyof((const type *)elem));		\
+	}								\
+	static inline UNNEEDED void name##_init(struct name *ht)	\
+	{								\
+		htable_init(&ht->raw, name##_hash, NULL);		\
+	}								\
+	static inline UNNEEDED bool name##_init_sized(struct name *ht,	\
+						      size_t s)		\
+	{								\
+		return htable_init_sized(&ht->raw, name##_hash, NULL, s); \
+	}								\
+	static inline UNNEEDED size_t name##_count(const struct name *ht) \
+	{								\
+		return htable_count(&ht->raw);				\
+	}								\
+	static inline UNNEEDED void name##_clear(struct name *ht)	\
+	{								\
+		htable_clear(&ht->raw);					\
+	}								\
+	static inline UNNEEDED bool name##_copy(struct name *dst,	\
+						const struct name *src)	\
+	{								\
+		return htable_copy(&dst->raw, &src->raw);		\
+	}								\
+	static inline bool name##_add(struct name *ht, const type *elem) \
+	{								\
+		return htable_add(&ht->raw, hashfn(keyof(elem)), elem);	\
+	}								\
+	static inline UNNEEDED bool name##_del(struct name *ht,		\
+					       const type *elem)	\
+	{								\
+		return htable_del(&ht->raw, hashfn(keyof(elem)), elem);	\
+	}								\
+	static inline UNNEEDED type *name##_get(const struct name *ht,	\
+				       const HTABLE_KTYPE(keyof, type) k) \
+	{								\
+		struct htable_iter i;					\
+		size_t h = hashfn(k);					\
+		void *c;						\
+									\
+		for (c = htable_firstval(&ht->raw,&i,h);		\
+		     c;							\
+		     c = htable_nextval(&ht->raw,&i,h)) {		\
+			if (eqfn(c, k))					\
+				return c;				\
+		}							\
+		return NULL;						\
+	}								\
+	static inline UNNEEDED type *name##_getmatch_(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+				         size_t h,			\
+				         type *v,			\
+					 struct name##_iter *iter)	\
+	{								\
+		while (v) {						\
+			if (eqfn(v, k))					\
+				break;					\
+			v = htable_nextval(&ht->raw, &iter->i, h);	\
+		}							\
+		return v;						\
+	}								\
+	static inline UNNEEDED type *name##_getfirst(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+					 struct name##_iter *iter)	\
+	{								\
+		size_t h = hashfn(k);					\
+		type *v = htable_firstval(&ht->raw, &iter->i, h);	\
+		return name##_getmatch_(ht, k, h, v, iter);			\
+	}								\
+	static inline UNNEEDED type *name##_getnext(const struct name *ht, \
+				         const HTABLE_KTYPE(keyof, type) k, \
+					 struct name##_iter *iter)	\
+	{								\
+		size_t h = hashfn(k);					\
+		type *v = htable_nextval(&ht->raw, &iter->i, h);	\
+		return name##_getmatch_(ht, k, h, v, iter);		\
+	}								\
+	static inline UNNEEDED bool name##_delkey(struct name *ht,	\
+					 const HTABLE_KTYPE(keyof, type) k) \
+	{								\
+		type *elem = name##_get(ht, k);				\
+		if (elem)						\
+			return name##_del(ht, elem);			\
+		return false;						\
+	}								\
+	static inline UNNEEDED void name##_delval(struct name *ht,	\
+						  struct name##_iter *iter) \
+	{								\
+		htable_delval(&ht->raw, &iter->i);			\
+	}								\
+	static inline UNNEEDED type *name##_pick(const struct name *ht,	\
+						size_t seed,		\
+						struct name##_iter *iter) \
+	{								\
+		return htable_pick(&ht->raw, seed, iter ? &iter->i : NULL); \
+	}								\
+	static inline UNNEEDED type *name##_first(const struct name *ht, \
+					 struct name##_iter *iter)	\
+	{								\
+		return htable_first(&ht->raw, &iter->i);		\
+	}								\
+	static inline UNNEEDED type *name##_next(const struct name *ht,	\
+					struct name##_iter *iter)	\
+	{								\
+		return htable_next(&ht->raw, &iter->i);			\
+	}								\
+	static inline UNNEEDED type *name##_prev(const struct name *ht,	\
+					struct name##_iter *iter)	\
+	{								\
+		return htable_prev(&ht->raw, &iter->i);			\
+	}
+
+#if HAVE_TYPEOF
+#define HTABLE_KTYPE(keyof, type) typeof(keyof((const type *)NULL))
+#else
+/* Assumes keys are a pointer: if not, override. */
+#ifndef HTABLE_KTYPE
+#define HTABLE_KTYPE(keyof, type) void *
+#endif
+#endif
+#endif /* CCAN_HTABLE_TYPE_H */

nostrdb/ccan/ccan/htable/tools/Makefile

@@ -0,0 +1,41 @@
+CCANDIR=../../..
+CFLAGS=-Wall -Werror -O3 -I$(CCANDIR)
+#CFLAGS=-Wall -Werror -g -I$(CCANDIR)
+
+CCAN_OBJS:=ccan-tal.o ccan-tal-str.o ccan-tal-grab_file.o ccan-take.o ccan-time.o ccan-str.o ccan-noerr.o ccan-list.o
+
+all: speed stringspeed hsearchspeed density
+
+speed: speed.o hash.o $(CCAN_OBJS)
+density: density.o hash.o $(CCAN_OBJS)
+
+speed.o: speed.c ../htable.h ../htable.c
+
+hash.o: ../../hash/hash.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+
+stringspeed: stringspeed.o hash.o $(CCAN_OBJS)
+
+stringspeed.o: speed.c ../htable.h ../htable.c
+
+hsearchspeed: hsearchspeed.o $(CCAN_OBJS)
+
+clean:
+	rm -f stringspeed speed hsearchspeed *.o
+
+ccan-tal.o: $(CCANDIR)/ccan/tal/tal.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-tal-str.o: $(CCANDIR)/ccan/tal/str/str.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-take.o: $(CCANDIR)/ccan/take/take.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-tal-grab_file.o: $(CCANDIR)/ccan/tal/grab_file/grab_file.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-time.o: $(CCANDIR)/ccan/time/time.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-list.o: $(CCANDIR)/ccan/list/list.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-str.o: $(CCANDIR)/ccan/str/str.c
+	$(CC) $(CFLAGS) -c -o $@ $<
+ccan-noerr.o: $(CCANDIR)/ccan/noerr/noerr.c
+	$(CC) $(CFLAGS) -c -o $@ $<

nostrdb/ccan/ccan/htable/tools/density.c

@@ -0,0 +1,107 @@
+/* Density measurements for hashtables. */
+#include <ccan/err/err.h>
+#include <ccan/htable/htable_type.h>
+#include <ccan/htable/htable.c>
+#include <ccan/hash/hash.h>
+#include <ccan/ptrint/ptrint.h>
+#include <ccan/time/time.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+/* We don't actually hash objects: we put values in as if they were ptrs */
+static uintptr_t key(const ptrint_t *obj)
+{
+	return ptr2int(obj);
+}
+
+static size_t hash_uintptr(uintptr_t key)
+{
+	return hashl(&key, 1, 0);
+}
+
+static bool cmp(const ptrint_t *p, uintptr_t k)
+{
+	return key(p) == k;
+}
+
+HTABLE_DEFINE_TYPE(ptrint_t, key, hash_uintptr, cmp, htable_ptrint);
+
+/* Nanoseconds per operation */
+static size_t normalize(const struct timeabs *start,
+			const struct timeabs *stop,
+			unsigned int num)
+{
+	return time_to_nsec(time_divide(time_between(*stop, *start), num));
+}
+
+static size_t average_run(const struct htable_ptrint *ht, size_t count, size_t *longest)
+{
+	size_t i, total = 0;
+
+	*longest = 0;
+	for (i = 0; i < count; i++) {
+		size_t h = hash_uintptr(i + 2);
+		size_t run = 1;
+
+		while (get_raw_ptr(&ht->raw, ht->raw.table[h % ((size_t)1 << ht->raw.bits)]) != int2ptr(i + 2)) {
+			h++;
+			run++;
+		}
+		total += run;
+		if (run > *longest)
+			*longest = run;
+	}
+	return total / count;
+}
+
+int main(int argc, char *argv[])
+{
+	unsigned int i;
+	size_t num;
+	struct timeabs start, stop;
+	struct htable_ptrint ht;
+
+	if (argc != 2)
+		errx(1, "Usage: density <power-of-2-tablesize>");
+
+	num = atoi(argv[1]);
+
+	printf("Total buckets, buckets used, nanoseconds search time per element, avg run, longest run\n");
+	for (i = 1; i <= 99; i++) {
+		uintptr_t j;
+		struct htable_ptrint_iter it;
+		size_t count, avg_run, longest_run;
+		ptrint_t *p;
+
+		htable_ptrint_init_sized(&ht, num * 3 / 4);
+		assert((1 << ht.raw.bits) == num);
+
+		/* Can't put 0 or 1 in the hash table: multiply by a prime. */
+		for (j = 0; j < num * i / 100; j++) {
+			htable_ptrint_add(&ht, int2ptr(j + 2));
+			/* stop it from doubling! */
+			ht.raw.elems = num / 2;
+		}
+		/* Must not have changed! */
+		assert((1 << ht.raw.bits) == num);
+
+		/* Clean cache */
+		count = 0;
+		for (p = htable_ptrint_first(&ht, &it); p; p = htable_ptrint_next(&ht, &it))
+			count++;
+		assert(count == num * i / 100);
+		start = time_now();
+		for (j = 0; j < count; j++)
+			assert(htable_ptrint_get(&ht, j + 2));
+		stop = time_now();
+		avg_run = average_run(&ht, count, &longest_run);
+		printf("%zu,%zu,%zu,%zu,%zu\n",
+		       num, count, normalize(&start, &stop, count), avg_run, longest_run);
+		fflush(stdout);
+		htable_ptrint_clear(&ht);
+	}
+
+	return 0;
+}

nostrdb/ccan/ccan/htable/tools/hsearchspeed.c

@@ -0,0 +1,95 @@
+/* Simple speed tests for a hash of strings using hsearch */
+#include <ccan/htable/htable_type.h>
+#include <ccan/htable/htable.c>
+#include <ccan/tal/str/str.h>
+#include <ccan/tal/grab_file/grab_file.h>
+#include <ccan/tal/tal.h>
+#include <ccan/hash/hash.h>
+#include <ccan/time/time.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/time.h>
+#include <search.h>
+
+/* Nanoseconds per operation */
+static size_t normalize(const struct timeabs *start,
+			const struct timeabs *stop,
+			unsigned int num)
+{
+	return time_to_nsec(time_divide(time_between(*stop, *start), num));
+}
+
+int main(int argc, char *argv[])
+{
+	size_t i, j, num;
+	struct timeabs start, stop;
+	char **w;
+	ENTRY *words, *misswords;
+
+	w = tal_strsplit(NULL, grab_file(NULL,
+					 argv[1] ? argv[1] : "/usr/share/dict/words"), "\n", STR_NO_EMPTY);
+	num = tal_count(w) - 1;
+	printf("%zu words\n", num);
+
+	hcreate(num+num/3);
+
+	words = tal_arr(w, ENTRY, num);
+	for (i = 0; i < num; i++) {
+		words[i].key = w[i];
+		words[i].data = words[i].key;
+	}
+
+	/* Append and prepend last char for miss testing. */
+	misswords = tal_arr(w, ENTRY, num);
+	for (i = 0; i < num; i++) {
+		char lastc;
+		if (strlen(w[i]))
+			lastc = w[i][strlen(w[i])-1];
+		else
+			lastc = 'z';
+		misswords[i].key = tal_fmt(misswords, "%c%s%c%c",
+					   lastc, w[i], lastc, lastc);
+	}
+
+	printf("#01: Initial insert: ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++)
+		hsearch(words[i], ENTER);
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("#02: Initial lookup (match): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++)
+		if (hsearch(words[i], FIND)->data != words[i].data)
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("#03: Initial lookup (miss): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++) {
+		if (hsearch(misswords[i], FIND))
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	/* Lookups in order are very cache-friendly for judy; try random */
+	printf("#04: Initial lookup (random): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0, j = 0; i < num; i++, j = (j + 10007) % num)
+		if (hsearch(words[i], FIND)->data != words[i].data)
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	return 0;
+}

nostrdb/ccan/ccan/htable/tools/speed.c

@@ -0,0 +1,370 @@
+/* Simple speed tests for hashtables. */
+#include <ccan/htable/htable_type.h>
+#include <ccan/htable/htable.c>
+#include <ccan/hash/hash.h>
+#include <ccan/time/time.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+static size_t hashcount;
+struct object {
+	/* The key. */
+	unsigned int key;
+
+	/* Some contents. Doubles as consistency check. */
+	struct object *self;
+};
+
+static const unsigned int *objkey(const struct object *obj)
+{
+	return &obj->key;
+}
+
+static size_t hash_obj(const unsigned int *key)
+{
+	hashcount++;
+	return hashl(key, 1, 0);
+}
+
+static bool cmp(const struct object *object, const unsigned int *key)
+{
+	return object->key == *key;
+}
+
+HTABLE_DEFINE_TYPE(struct object, objkey, hash_obj, cmp, htable_obj);
+
+static unsigned int popcount(unsigned long val)
+{
+#if HAVE_BUILTIN_POPCOUNTL
+	return __builtin_popcountl(val);
+#else
+	if (sizeof(long) == sizeof(u64)) {
+		u64 v = val;
+		v = (v & 0x5555555555555555ULL)
+			+ ((v >> 1) & 0x5555555555555555ULL);
+		v = (v & 0x3333333333333333ULL)
+			+ ((v >> 1) & 0x3333333333333333ULL);
+		v = (v & 0x0F0F0F0F0F0F0F0FULL)
+			+ ((v >> 1) & 0x0F0F0F0F0F0F0F0FULL);
+		v = (v & 0x00FF00FF00FF00FFULL)
+			+ ((v >> 1) & 0x00FF00FF00FF00FFULL);
+		v = (v & 0x0000FFFF0000FFFFULL)
+			+ ((v >> 1) & 0x0000FFFF0000FFFFULL);
+		v = (v & 0x00000000FFFFFFFFULL)
+			+ ((v >> 1) & 0x00000000FFFFFFFFULL);
+		return v;
+	}
+	val = (val & 0x55555555ULL) + ((val >> 1) & 0x55555555ULL);
+	val = (val & 0x33333333ULL) + ((val >> 1) & 0x33333333ULL);
+	val = (val & 0x0F0F0F0FULL) + ((val >> 1) & 0x0F0F0F0FULL);
+	val = (val & 0x00FF00FFULL) + ((val >> 1) & 0x00FF00FFULL);
+	val = (val & 0x0000FFFFULL) + ((val >> 1) & 0x0000FFFFULL);
+	return val;
+#endif
+}
+
+static size_t perfect(const struct htable *ht)
+{
+	size_t i, placed_perfect = 0;
+
+	for (i = 0; i < ((size_t)1 << ht->bits); i++) {
+		if (!entry_is_valid(ht->table[i]))
+			continue;
+		if (hash_bucket(ht, ht->rehash(get_raw_ptr(ht, ht->table[i]),
+					       ht->priv)) == i) {
+			assert((ht->table[i] & ht_perfect_mask(ht))
+			       == ht_perfect_mask(ht));
+			placed_perfect++;
+		}
+	}
+	return placed_perfect;
+}
+
+static size_t count_deleted(const struct htable *ht)
+{
+	size_t i, delete_markers = 0;
+
+	for (i = 0; i < ((size_t)1 << ht->bits); i++) {
+		if (ht->table[i] == HTABLE_DELETED)
+			delete_markers++;
+	}
+	return delete_markers;
+}
+
+/* Nanoseconds per operation */
+static size_t normalize(const struct timeabs *start,
+			const struct timeabs *stop,
+			unsigned int num)
+{
+	return time_to_nsec(time_divide(time_between(*stop, *start), num));
+}
+
+static size_t worst_run(struct htable *ht, size_t *deleted)
+{
+	size_t longest = 0, len = 0, this_del = 0, i;
+
+	*deleted = 0;
+	/* This doesn't take into account end-wrap, but gives an idea. */
+	for (i = 0; i < ((size_t)1 << ht->bits); i++) {
+		if (ht->table[i]) {
+			len++;
+			if (ht->table[i] == HTABLE_DELETED)
+				this_del++;
+		} else {
+			if (len > longest) {
+				longest = len;
+				*deleted = this_del;
+			}
+			len = 0;
+			this_del = 0;
+		}
+	}
+	return longest;
+}
+
+int main(int argc, char *argv[])
+{
+	struct object *objs;
+	unsigned int i, j;
+	size_t num, deleted;
+	struct timeabs start, stop;
+	struct htable_obj ht;
+	bool make_dumb = false;
+
+	if (argv[1] && strcmp(argv[1], "--dumb") == 0) {
+		argv++;
+		make_dumb = true;
+	}
+	num = argv[1] ? atoi(argv[1]) : 1000000;
+	objs = calloc(num, sizeof(objs[0]));
+
+	for (i = 0; i < num; i++) {
+		objs[i].key = i;
+		objs[i].self = &objs[i];
+	}
+
+	htable_obj_init(&ht);
+
+	printf("Initial insert: ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++)
+		htable_obj_add(&ht, objs[i].self);
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+	printf("Details: hash size %u, mask bits %u, perfect %.0f%%\n",
+	       1U << ht.raw.bits, popcount(ht.raw.common_mask),
+	       perfect(&ht.raw) * 100.0 / ht.raw.elems);
+
+	if (make_dumb) {
+		/* Screw with mask, to hobble us. */
+		update_common(&ht.raw, (void *)~ht.raw.common_bits);
+		printf("Details: DUMB MODE: mask bits %u\n",
+		       popcount(ht.raw.common_mask));
+	}
+
+	printf("Initial lookup (match): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++)
+		if (htable_obj_get(&ht, &i)->self != objs[i].self)
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Initial lookup (miss): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++) {
+		unsigned int n = i + num;
+		if (htable_obj_get(&ht, &n))
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	/* Lookups in order are very cache-friendly for judy; try random */
+	printf("Initial lookup (random): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0, j = 0; i < num; i++, j = (j + 10007) % num)
+		if (htable_obj_get(&ht, &j)->self != &objs[j])
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	hashcount = 0;
+	printf("Initial delete all: ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++)
+		if (!htable_obj_del(&ht, objs[i].self))
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+	printf("Details: rehashes %zu\n", hashcount);
+
+	printf("Initial re-inserting: ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++)
+		htable_obj_add(&ht, objs[i].self);
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	hashcount = 0;
+	printf("Deleting first half: ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i+=2)
+		if (!htable_obj_del(&ht, objs[i].self))
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Details: rehashes %zu, delete markers %zu\n",
+	       hashcount, count_deleted(&ht.raw));
+
+	printf("Adding (a different) half: ");
+	fflush(stdout);
+
+	for (i = 0; i < num; i+=2)
+		objs[i].key = num+i;
+
+	start = time_now();
+	for (i = 0; i < num; i+=2)
+		htable_obj_add(&ht, objs[i].self);
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Details: delete markers %zu, perfect %.0f%%\n",
+	       count_deleted(&ht.raw), perfect(&ht.raw) * 100.0 / ht.raw.elems);
+
+	printf("Lookup after half-change (match): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 1; i < num; i+=2)
+		if (htable_obj_get(&ht, &i)->self != objs[i].self)
+			abort();
+	for (i = 0; i < num; i+=2) {
+		unsigned int n = i + num;
+		if (htable_obj_get(&ht, &n)->self != objs[i].self)
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Lookup after half-change (miss): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++) {
+		unsigned int n = i + num * 2;
+		if (htable_obj_get(&ht, &n))
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	/* Hashtables with delete markers can fill with markers over time.
+	 * so do some changes to see how it operates in long-term. */
+	for (i = 0; i < 5; i++) {
+		if (i == 0) {
+			/* We don't measure this: jmap is different. */
+			printf("Details: initial churn\n");
+		} else {
+			printf("Churning %s time: ",
+			       i == 1 ? "second"
+			       : i == 2 ? "third"
+			       : i == 3 ? "fourth"
+			       : "fifth");
+			fflush(stdout);
+		}
+		start = time_now();
+		for (j = 0; j < num; j++) {
+			if (!htable_obj_del(&ht, &objs[j]))
+				abort();
+			objs[j].key = num*i+j;
+			if (!htable_obj_add(&ht, &objs[j]))
+				abort();
+		}
+		stop = time_now();
+		if (i != 0)
+			printf(" %zu ns\n", normalize(&start, &stop, num));
+	}
+
+	/* Spread out the keys more to try to make it harder. */
+	printf("Details: reinserting with spread\n");
+	for (i = 0; i < num; i++) {
+		if (!htable_obj_del(&ht, objs[i].self))
+			abort();
+		objs[i].key = num * 5 + i * 9;
+		if (!htable_obj_add(&ht, objs[i].self))
+			abort();
+	}
+	printf("Details: delete markers %zu, perfect %.0f%%\n",
+	       count_deleted(&ht.raw), perfect(&ht.raw) * 100.0 / ht.raw.elems);
+	i = worst_run(&ht.raw, &deleted);
+	printf("Details: worst run %u (%zu deleted)\n", i, deleted);
+
+	printf("Lookup after churn & spread (match): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++) {
+		unsigned int n = num * 5 + i * 9;
+		if (htable_obj_get(&ht, &n)->self != objs[i].self)
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Lookup after churn & spread (miss): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++) {
+		unsigned int n = num * (5 + 9) + i * 9;
+		if (htable_obj_get(&ht, &n))
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Lookup after churn & spread (random): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0, j = 0; i < num; i++, j = (j + 10007) % num) {
+		unsigned int n = num * 5 + j * 9;
+		if (htable_obj_get(&ht, &n)->self != &objs[j])
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	hashcount = 0;
+	printf("Deleting half after churn & spread: ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i+=2)
+		if (!htable_obj_del(&ht, objs[i].self))
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Adding (a different) half after churn & spread: ");
+	fflush(stdout);
+
+	for (i = 0; i < num; i+=2)
+		objs[i].key = num*6+i*9;
+
+	start = time_now();
+	for (i = 0; i < num; i+=2)
+		htable_obj_add(&ht, objs[i].self);
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Details: delete markers %zu, perfect %.0f%%\n",
+	       count_deleted(&ht.raw), perfect(&ht.raw) * 100.0 / ht.raw.elems);
+
+	return 0;
+}

nostrdb/ccan/ccan/htable/tools/stringspeed.c

@@ -0,0 +1,240 @@
+/* Simple speed tests for a hash of strings. */
+#include <ccan/htable/htable_type.h>
+#include <ccan/htable/htable.c>
+#include <ccan/tal/str/str.h>
+#include <ccan/tal/grab_file/grab_file.h>
+#include <ccan/tal/tal.h>
+#include <ccan/hash/hash.h>
+#include <ccan/time/time.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include <unistd.h>
+#include <sys/time.h>
+
+static size_t hashcount;
+
+static const char *strkey(const char *str)
+{
+	return str;
+}
+
+static size_t hash_str(const char *key)
+{
+	hashcount++;
+	return hash(key, strlen(key), 0);
+}
+
+static bool cmp(const char *obj, const char *key)
+{
+	return strcmp(obj, key) == 0;
+}
+
+HTABLE_DEFINE_TYPE(char, strkey, hash_str, cmp, htable_str);
+
+/* Nanoseconds per operation */
+static size_t normalize(const struct timeabs *start,
+			const struct timeabs *stop,
+			unsigned int num)
+{
+	return time_to_nsec(time_divide(time_between(*stop, *start), num));
+}
+
+int main(int argc, char *argv[])
+{
+	size_t i, j, num;
+	struct timeabs start, stop;
+	struct htable_str ht;
+	char **words, **misswords;
+
+	words = tal_strsplit(NULL, grab_file(NULL,
+					     argv[1] ? argv[1] : "/usr/share/dict/words"), "\n",
+			     STR_NO_EMPTY);
+	htable_str_init(&ht);
+	num = tal_count(words) - 1;
+	/* Note that on my system, num is just > 98304, where we double! */
+	printf("%zu words\n", num);
+
+	/* Append and prepend last char for miss testing. */
+	misswords = tal_arr(words, char *, num);
+	for (i = 0; i < num; i++) {
+		char lastc;
+		if (strlen(words[i]))
+			lastc = words[i][strlen(words[i])-1];
+		else
+			lastc = 'z';
+		misswords[i] = tal_fmt(misswords, "%c%s%c%c",
+				       lastc, words[i], lastc, lastc);
+	}
+
+	printf("#01: Initial insert: ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++)
+		htable_str_add(&ht, words[i]);
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("Bytes allocated: %zu\n",
+	       sizeof(ht.raw.table[0]) << ht.raw.bits);
+
+	printf("#02: Initial lookup (match): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++)
+		if (htable_str_get(&ht, words[i]) != words[i])
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("#03: Initial lookup (miss): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++) {
+		if (htable_str_get(&ht, misswords[i]))
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	/* Lookups in order are very cache-friendly for judy; try random */
+	printf("#04: Initial lookup (random): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0, j = 0; i < num; i++, j = (j + 10007) % num)
+		if (htable_str_get(&ht, words[j]) != words[j])
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	hashcount = 0;
+	printf("#05: Initial delete all: ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++)
+		if (!htable_str_del(&ht, words[i]))
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("#06: Initial re-inserting: ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++)
+		htable_str_add(&ht, words[i]);
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	hashcount = 0;
+	printf("#07: Deleting first half: ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i+=2)
+		if (!htable_str_del(&ht, words[i]))
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("#08: Adding (a different) half: ");
+	fflush(stdout);
+
+	start = time_now();
+	for (i = 0; i < num; i+=2)
+		htable_str_add(&ht, misswords[i]);
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("#09: Lookup after half-change (match): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 1; i < num; i+=2)
+		if (htable_str_get(&ht, words[i]) != words[i])
+			abort();
+	for (i = 0; i < num; i+=2) {
+		if (htable_str_get(&ht, misswords[i]) != misswords[i])
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("#10: Lookup after half-change (miss): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i+=2)
+		if (htable_str_get(&ht, words[i]))
+			abort();
+	for (i = 1; i < num; i+=2) {
+		if (htable_str_get(&ht, misswords[i]))
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	/* Hashtables with delete markers can fill with markers over time.
+	 * so do some changes to see how it operates in long-term. */
+	printf("#11: Churn 1: ");
+	start = time_now();
+	for (j = 0; j < num; j+=2) {
+		if (!htable_str_del(&ht, misswords[j]))
+			abort();
+		if (!htable_str_add(&ht, words[j]))
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("#12: Churn 2: ");
+	start = time_now();
+	for (j = 1; j < num; j+=2) {
+		if (!htable_str_del(&ht, words[j]))
+			abort();
+		if (!htable_str_add(&ht, misswords[j]))
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("#13: Churn 3: ");
+	start = time_now();
+	for (j = 1; j < num; j+=2) {
+		if (!htable_str_del(&ht, misswords[j]))
+			abort();
+		if (!htable_str_add(&ht, words[j]))
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	/* Now it's back to normal... */
+	printf("#14: Post-Churn lookup (match): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++)
+		if (htable_str_get(&ht, words[i]) != words[i])
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	printf("#15: Post-Churn lookup (miss): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0; i < num; i++) {
+		if (htable_str_get(&ht, misswords[i]))
+			abort();
+	}
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	/* Lookups in order are very cache-friendly for judy; try random */
+	printf("#16: Post-Churn lookup (random): ");
+	fflush(stdout);
+	start = time_now();
+	for (i = 0, j = 0; i < num; i++, j = (j + 10007) % num)
+		if (htable_str_get(&ht, words[j]) != words[j])
+			abort();
+	stop = time_now();
+	printf(" %zu ns\n", normalize(&start, &stop, num));
+
+	return 0;
+}