[PATCH 03 of 23 cubicweb V2] [database/pool] switch to a Lifo queue while closing less used connections first

Laurent Peuch cortex at worlddomination.be
Fri Jan 17 14:37:00 CET 2020


# HG changeset patch
# User Laurent Peuch <cortex at worlddomination.be>
# Date 1579267492 -3600
#      Fri Jan 17 14:24:52 2020 +0100
# Node ID 8881f32c71d3732eaffb782e94021d708e94d0c5
# Parent  cc81dd2c317bcc289fdd03337ada54eacdb977cc
# Available At https://hg.logilab.org/users/lpeuch/cubicweb
#              hg pull https://hg.logilab.org/users/lpeuch/cubicweb -r 8881f32c71d3
[database/pool] switch to a Lifo queue while closing less used connections first

This is to be closest to pgbouncer algorithm which probably makes more sens in
regards to caching etc...

diff --git a/cubicweb/server/repository.py b/cubicweb/server/repository.py
--- a/cubicweb/server/repository.py
+++ b/cubicweb/server/repository.py
@@ -160,6 +160,31 @@ class FastReadCounter(object):
             self.value += 1
 
 
+# python STL doens't provide any concurrent deque implementation with blocking .get
+class LifoDeque(queue.LifoQueue):
+    def get_oldest(self, block=True, timeout=None):
+        'Like queue.get except it gets the oldest one'
+        with self.not_empty:
+            if not block:
+                if not self._qsize():
+                    raise queue.Empty
+            elif timeout is None:
+                while not self._qsize():
+                    self.not_empty.wait()
+            elif timeout < 0:
+                raise ValueError("'timeout' must be a non-negative number")
+            else:
+                endtime = time() + timeout
+                while not self._qsize():
+                    remaining = endtime - time()
+                    if remaining <= 0.0:
+                        raise queue.Empty
+                    self.not_empty.wait(remaining)
+            item = self.queue.popleft()  # <- that's the only code difference from super().get
+            self.not_full.notify()
+        return item
+
+
 class _CnxSetPool:
 
     def __init__(self, source, size, min_size=3, min_timeout=0.1, max_timeout=5,
@@ -176,7 +201,7 @@ class _CnxSetPool:
         self._last_time_we_waited = None
 
         if self.size is not None:
-            self._queue = queue.Queue()
+            self._queue = LifoDeque()
 
             if self.min_size > self.size:
                 self.min_size = max(1, int(self.size / 3))
@@ -287,7 +312,7 @@ class _CnxSetPool:
 
         # let's try to close a connection
         try:
-            connection = self._queue.get(block=False)
+            connection = self._queue.get_oldest(block=False)
         except queue.Empty:
             pass  # probably for some race condition we missed it
         else:



More information about the cubicweb-devel mailing list