From 0df59049fe13ef89d362fa7f109f289b297441dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ond=C5=99ej=20Sur=C3=BD?= <ondrej@isc.org>
Date: Tue, 22 Feb 2022 23:40:39 +0100
Subject: [PATCH] Provide alternative isc_queue implementation based on locked
 list

The current implementation of isc_queue uses Michael-Scott lock-free
queue that in turn uses hazard pointers.  It was discovered that the way
we use the isc_queue, such complicated mechanism isn't really needed,
because most of the time, we either execute the work directly when on
nmthread (in case of UDP) or schedule the work from the matching
nmthreads.

Provide alternative implementation for the isc_queue based on locked
ISC_LIST.
---
 bin/named/main.c            |   1 -
 configure.ac                |  12 ++++
 lib/isc/include/isc/queue.h |   3 +-
 lib/isc/queue.c             | 121 ++++++++++++++++++++++++++++++++++++
 4 files changed, 134 insertions(+), 3 deletions(-)

diff --git a/bin/named/main.c b/bin/named/main.c
index 9ad2d0e..8870933 100644
--- a/bin/named/main.c
+++ b/bin/named/main.c
@@ -34,7 +34,6 @@
 #include <isc/dir.h>
 #include <isc/file.h>
 #include <isc/hash.h>
-#include <isc/hp.h>
 #include <isc/httpd.h>
 #include <isc/managers.h>
 #include <isc/netmgr.h>
diff --git a/configure.ac b/configure.ac
index 79d33d1..26241a0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2263,8 +2263,20 @@ AS_CASE([$with_cmocka],
 AC_SUBST([CMOCKA_CFLAGS])
 AC_SUBST([CMOCKA_LIBS])
 
+#
+# Use lock-free Michael-Scott's queue implementation or locked-list queue
+#
+# [pairwise: --enable-lock-free-queue, --disable-lock-free-queue]
+AC_ARG_ENABLE([lock-free-queue],
+	    [AS_HELP_STRING([--enable-lock-free-queue],[enable lock-free queue implementation (default is enabled)])],
+	    [],[enable_lock_free_queue=yes])
+AS_CASE([$enable_lock_free_queue],
+	[no],[],
+	[yes],[AC_DEFINE([USE_LOCK_FREE_QUEUE],[1],[Define to 1 to enable lock-free queue])])
+
 AC_DEFINE([SKIPPED_TEST_EXIT_CODE], [0], [Exit code for skipped tests])
 
+
 #
 # Check for kyua execution engine if CMocka was requested
 # and bail out if execution engine was not found
diff --git a/lib/isc/include/isc/queue.h b/lib/isc/include/isc/queue.h
index 0927075..568bf18 100644
--- a/lib/isc/include/isc/queue.h
+++ b/lib/isc/include/isc/queue.h
@@ -39,8 +39,7 @@ uintptr_t
 isc_queue_dequeue(isc_queue_t *queue);
 /*%<
  * Remove an object pointer from the head of the queue and return the
- * pointer. If the queue is empty, return `nulluintptr` (the uintptr_t
- * representation of NULL).
+ * pointer. If the queue is empty, return `NULL`.
  *
  * Requires:
  * \li	'queue' is not null.
diff --git a/lib/isc/queue.c b/lib/isc/queue.c
index d7ea824..c4cb404 100644
--- a/lib/isc/queue.c
+++ b/lib/isc/queue.c
@@ -28,6 +28,10 @@
 
 static uintptr_t nulluintptr = (uintptr_t)NULL;
 
+#if USE_LOCK_FREE_QUEUE
+
+#define BUFFER_SIZE 1024
+
 typedef struct node {
 	atomic_uint_fast32_t deqidx;
 	atomic_uintptr_t items[BUFFER_SIZE];
@@ -232,3 +236,120 @@ isc_queue_destroy(isc_queue_t *queue) {
 	alloced = queue->alloced_ptr;
 	isc_mem_putanddetach(&queue->mctx, alloced, sizeof(*queue) + ALIGNMENT);
 }
+
+#else /* USE_LOCK_FREE_QUEUE */
+
+typedef struct node node_t;
+
+struct node {
+	uintptr_t item;
+	ISC_LINK(node_t) link;
+};
+
+struct isc_queue {
+	isc_mem_t *mctx;
+	isc_mutex_t lock;
+	int max_threads;
+	ISC_LIST(node_t) nodes;
+	void *alloced_ptr;
+};
+
+static node_t *
+node_new(isc_mem_t *mctx, uintptr_t item) {
+	node_t *node = isc_mem_get(mctx, sizeof(*node));
+	*node = (node_t){
+		.item = item,
+	};
+
+	ISC_LINK_INIT(node, link);
+
+	return (node);
+}
+
+static void
+node_destroy(isc_mem_t *mctx, node_t *node) {
+	isc_mem_put(mctx, node, sizeof(*node));
+}
+
+isc_queue_t *
+isc_queue_new(isc_mem_t *mctx, int max_threads) {
+	isc_queue_t *queue = NULL;
+	void *qbuf = NULL;
+	uintptr_t qptr;
+
+	qbuf = isc_mem_get(mctx, sizeof(*queue) + ALIGNMENT);
+	qptr = (uintptr_t)qbuf;
+	queue = (isc_queue_t *)(qptr + (ALIGNMENT - (qptr % ALIGNMENT)));
+
+	if (max_threads == 0) {
+		max_threads = MAX_THREADS;
+	}
+
+	*queue = (isc_queue_t){
+		.max_threads = max_threads,
+		.alloced_ptr = qbuf,
+	};
+
+	ISC_LIST_INIT(queue->nodes);
+
+	isc_mutex_init(&queue->lock);
+	isc_mem_attach(mctx, &queue->mctx);
+
+	return (queue);
+}
+
+void
+isc_queue_enqueue(isc_queue_t *queue, uintptr_t item) {
+	node_t *node = node_new(queue->mctx, item);
+	REQUIRE(item != nulluintptr);
+
+	LOCK(&queue->lock);
+	ISC_LIST_ENQUEUE(queue->nodes, node, link);
+	UNLOCK(&queue->lock);
+}
+
+uintptr_t
+isc_queue_dequeue(isc_queue_t *queue) {
+	node_t *node = NULL;
+	uintptr_t item = nulluintptr;
+	REQUIRE(queue != NULL);
+
+	LOCK(&queue->lock);
+	node = ISC_LIST_HEAD(queue->nodes);
+	if (node != NULL) {
+		ISC_LIST_DEQUEUE(queue->nodes, node, link);
+		item = node->item;
+	}
+	UNLOCK(&queue->lock);
+
+	if (node != NULL) {
+		node_destroy(queue->mctx, node);
+	}
+
+	return (item);
+}
+
+void
+isc_queue_destroy(isc_queue_t *queue) {
+	node_t *node = NULL;
+	void *alloced = NULL;
+
+	REQUIRE(queue != NULL);
+
+	LOCK(&queue->lock);
+	node = ISC_LIST_HEAD(queue->nodes);
+	while (node != NULL) {
+		node_t *next = ISC_LIST_NEXT(node, link);
+		ISC_LIST_DEQUEUE(queue->nodes, node, link);
+		node_destroy(queue->mctx, node);
+		node = next;
+	}
+	UNLOCK(&queue->lock);
+
+	isc_mutex_destroy(&queue->lock);
+
+	alloced = queue->alloced_ptr;
+	isc_mem_putanddetach(&queue->mctx, alloced, sizeof(*queue) + ALIGNMENT);
+}
+
+#endif /* USE_LOCK_FREE_QUEUE */
-- 
2.34.1