pool.py 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187
  1. """Connection pooling for psycopg2
  2. This module implements thread-safe (and not) connection pools.
  3. """
  4. # psycopg/pool.py - pooling code for psycopg
  5. #
  6. # Copyright (C) 2003-2019 Federico Di Gregorio <fog@debian.org>
  7. # Copyright (C) 2020-2021 The Psycopg Team
  8. #
  9. # psycopg2 is free software: you can redistribute it and/or modify it
  10. # under the terms of the GNU Lesser General Public License as published
  11. # by the Free Software Foundation, either version 3 of the License, or
  12. # (at your option) any later version.
  13. #
  14. # In addition, as a special exception, the copyright holders give
  15. # permission to link this program with the OpenSSL library (or with
  16. # modified versions of OpenSSL that use the same license as OpenSSL),
  17. # and distribute linked combinations including the two.
  18. #
  19. # You must obey the GNU Lesser General Public License in all respects for
  20. # all of the code used other than OpenSSL.
  21. #
  22. # psycopg2 is distributed in the hope that it will be useful, but WITHOUT
  23. # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  24. # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
  25. # License for more details.
  26. import psycopg2
  27. from psycopg2 import extensions as _ext
  28. class PoolError(psycopg2.Error):
  29. pass
  30. class AbstractConnectionPool:
  31. """Generic key-based pooling code."""
  32. def __init__(self, minconn, maxconn, *args, **kwargs):
  33. """Initialize the connection pool.
  34. New 'minconn' connections are created immediately calling 'connfunc'
  35. with given parameters. The connection pool will support a maximum of
  36. about 'maxconn' connections.
  37. """
  38. self.minconn = int(minconn)
  39. self.maxconn = int(maxconn)
  40. self.closed = False
  41. self._args = args
  42. self._kwargs = kwargs
  43. self._pool = []
  44. self._used = {}
  45. self._rused = {} # id(conn) -> key map
  46. self._keys = 0
  47. for i in range(self.minconn):
  48. self._connect()
  49. def _connect(self, key=None):
  50. """Create a new connection and assign it to 'key' if not None."""
  51. conn = psycopg2.connect(*self._args, **self._kwargs)
  52. if key is not None:
  53. self._used[key] = conn
  54. self._rused[id(conn)] = key
  55. else:
  56. self._pool.append(conn)
  57. return conn
  58. def _getkey(self):
  59. """Return a new unique key."""
  60. self._keys += 1
  61. return self._keys
  62. def _getconn(self, key=None):
  63. """Get a free connection and assign it to 'key' if not None."""
  64. if self.closed:
  65. raise PoolError("connection pool is closed")
  66. if key is None:
  67. key = self._getkey()
  68. if key in self._used:
  69. return self._used[key]
  70. if self._pool:
  71. self._used[key] = conn = self._pool.pop()
  72. self._rused[id(conn)] = key
  73. return conn
  74. else:
  75. if len(self._used) == self.maxconn:
  76. raise PoolError("connection pool exhausted")
  77. return self._connect(key)
  78. def _putconn(self, conn, key=None, close=False):
  79. """Put away a connection."""
  80. if self.closed:
  81. raise PoolError("connection pool is closed")
  82. if key is None:
  83. key = self._rused.get(id(conn))
  84. if key is None:
  85. raise PoolError("trying to put unkeyed connection")
  86. if len(self._pool) < self.minconn and not close:
  87. # Return the connection into a consistent state before putting
  88. # it back into the pool
  89. if not conn.closed:
  90. status = conn.info.transaction_status
  91. if status == _ext.TRANSACTION_STATUS_UNKNOWN:
  92. # server connection lost
  93. conn.close()
  94. elif status != _ext.TRANSACTION_STATUS_IDLE:
  95. # connection in error or in transaction
  96. conn.rollback()
  97. self._pool.append(conn)
  98. else:
  99. # regular idle connection
  100. self._pool.append(conn)
  101. # If the connection is closed, we just discard it.
  102. else:
  103. conn.close()
  104. # here we check for the presence of key because it can happen that a
  105. # thread tries to put back a connection after a call to close
  106. if not self.closed or key in self._used:
  107. del self._used[key]
  108. del self._rused[id(conn)]
  109. def _closeall(self):
  110. """Close all connections.
  111. Note that this can lead to some code fail badly when trying to use
  112. an already closed connection. If you call .closeall() make sure
  113. your code can deal with it.
  114. """
  115. if self.closed:
  116. raise PoolError("connection pool is closed")
  117. for conn in self._pool + list(self._used.values()):
  118. try:
  119. conn.close()
  120. except Exception:
  121. pass
  122. self.closed = True
  123. class SimpleConnectionPool(AbstractConnectionPool):
  124. """A connection pool that can't be shared across different threads."""
  125. getconn = AbstractConnectionPool._getconn
  126. putconn = AbstractConnectionPool._putconn
  127. closeall = AbstractConnectionPool._closeall
  128. class ThreadedConnectionPool(AbstractConnectionPool):
  129. """A connection pool that works with the threading module."""
  130. def __init__(self, minconn, maxconn, *args, **kwargs):
  131. """Initialize the threading lock."""
  132. import threading
  133. AbstractConnectionPool.__init__(
  134. self, minconn, maxconn, *args, **kwargs)
  135. self._lock = threading.Lock()
  136. def getconn(self, key=None):
  137. """Get a free connection and assign it to 'key' if not None."""
  138. self._lock.acquire()
  139. try:
  140. return self._getconn(key)
  141. finally:
  142. self._lock.release()
  143. def putconn(self, conn=None, key=None, close=False):
  144. """Put away an unused connection."""
  145. self._lock.acquire()
  146. try:
  147. self._putconn(conn, key, close)
  148. finally:
  149. self._lock.release()
  150. def closeall(self):
  151. """Close all connections (even the one currently in use.)"""
  152. self._lock.acquire()
  153. try:
  154. self._closeall()
  155. finally:
  156. self._lock.release()