From 5c0d5499b6c30290098cef505ceadcfbe9e00636 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Thu, 23 Jan 2025 20:41:51 +0100 Subject: [PATCH 1/9] add ldappool module --- Lib/ldappool/README.md | 65 ++++++ Lib/ldappool/__init__.py | 422 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 487 insertions(+) create mode 100644 Lib/ldappool/README.md create mode 100644 Lib/ldappool/__init__.py diff --git a/Lib/ldappool/README.md b/Lib/ldappool/README.md new file mode 100644 index 00000000..c926097a --- /dev/null +++ b/Lib/ldappool/README.md @@ -0,0 +1,65 @@ +# LDAP Pooling example + +## entries as dict + +```python +from ldappool import ConnectionPool +pool = ConnectionPool( + params={"keep": True, "autoBind": True, "retries": 2}, + max=5) +pool.set_uri("ldaps://ldap.example.com:636/dc=example,dc=com?uid,mail?sub?(|(uid=test)(mail=test@example.com))") +pool.set_credentials("binddn", "bindpw") +with pool.get() as conn: + for entry in conn.search_s(pool.basedn, + pool.scope, + pool.filter, + pool.attributes): + print(f"{entry[0]}: {entry[1].get('uid')} {entry[1].get('mail')}") + for member in entry[1].get("memberOf", []): + print(member) +``` + +## entry to dataclass example +```python +from ldappool import ConnectionPool +from ldappool import e2c +pool = ConnectionPool( + params={"keep": True, "autoBind": True, "retries": 2}, + max=5) +pool.set_uri("ldaps://ldap.example.com:636/dc=example,dc=com?uid,mail?sub?(|(uid=test)(mail=test@example.com))") +pool.set_credentials("binddn", "bindpw") +with pool.get() as conn: + for entry in map(e2c, conn.search_s(pool.basedn, + pool.scope, + pool.filter, + pool.attributes)): + print(f"{entry.dn}: {entry.uid} {entry.mail}") + for member in entry.memberOf: + print(member) +``` + +## changing the connection or credentials for the pool + +```python +from ldappool import ConnectionPool +from ldappool import e2c +pool = ConnectionPool( + params={"keep": True, "autoBind": True, "retries": 2}, + max=5) +pool.set_uri("ldaps://ldap.example.com:636/dc=example,dc=com?uid,mail?sub?(|(uid=test)(mail=test@example.com))") +pool.set_credentials("binddn", "bindpw") +with pool.get() as conn: + for entry in map(e2c, conn.search_s(pool.basedn, + pool.scope, + pool.filter, + pool.attributes)): + print(f"{entry.dn}: {entry.uid} {entry.mail}") + +pool.set_credentials(entry.dn, "changeme") +with pool.get() as conn: + for entry in map(e2c, conn.search_s(pool.basedn, + pool.scope, + pool.filter, + pool.attributes)): + print(f"{entry.dn}: {entry.uid} {entry.mail}") +``` diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py new file mode 100644 index 00000000..60aad335 --- /dev/null +++ b/Lib/ldappool/__init__.py @@ -0,0 +1,422 @@ +import dataclasses +import logging +import sys +import threading +import time +from urllib.parse import urlparse + +import ldap +from ldapurl import LDAPUrl + +# nano seconds to ensure we know the locked time +ns = 1_000_000_000 +ns_locktimeout = 15.0 + +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + + +class LDAPPoolExhausted(Exception): + pass + + +class LDAPPoolDown(Exception): + pass + + +class LDAPLockTimeout(Exception): + pass + + +def e2c(entry): + cls = dataclasses.make_dataclass("", ["dn"] + list(entry[1].keys()), frozen=True) + return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) + + +class Connection(object): + def __init__( + self, + uri: LDAPUrl, + binddn: str, + bindpw: str, + params: dict = {}, + ): + self.uri = uri + self.binddn = binddn + self.bindpw = bindpw + self.params = params + self.established = False + self.inUse = False + self._whoami = None + self._conn = False + self._lock = threading.Lock() + self._pool = None + self._health = 0.0 + (f"ConnectionPool new Connection {self}") + if self.params.get("prewarm", False): + self.__enter__() + + def __locktime(self): + if self._health == 0.0: + self._health = time.perf_counter_ns() + return True + if (time.perf_counter_ns() - self._health) / ns < ns_locktimeout: + return False + return True + + @property + def whoami(self): + return self._whoami + + def __whoami(self): + # do not stress the connection too often + if not self.__locktime(): + return + for r in range(self.params.get("retries", 3)): + try: + self._whoami = self._conn.whoami_s() + return + except ldap.SERVER_DOWN as ldaperr: + logging.error(f"__whoami ConnectionPool {ldaperr}") + self.established = False + # just catch that error until we finished iterating + try: + self.__enter__() + except: + pass + raise ldap.SERVER_DOWN( + f"max retries {self.params.get('retries', 3)} reached" + ) + + @property + def conn(self): + if self._conn == False: + self.__enter__() + try: + if self.established: + self.__whoami() + except ldap.SERVER_DOWN as ldaperr: + self.established = False + raise LDAPPoolDown( + f"could not establish connection with {self.uri.initializeUrl()}" + + f" with max retries of {self.params.get('retries', 3)}" + ) + return self._conn + + def __lock_acquire(self): + try: + if self._lock.acquire(blocking=True, timeout=1): + return True + else: + raise LDAPLockTimeout() + except Exception as lockerr: + return False + + def __lock_release(self): + try: + self._lock.release() + return True + except Exception as lockerr: + return False + + def authenticate( + self, + binddn: str, + bindpw: str, + ): + + if not self.__lock_acquire(): + raise LDAPLockTimeout() + + try: + self.conn.simple_bind_s( + binddn, + bindpw, + ) + if not self.__lock_release(): + raise LDAPLockTimeout() + except ldap.INVALID_CREDENTIALS as ldaperr: + # rollback auth anyway + self.__lock_release() + self.__authenticate__() + raise ldap.INVALID_CREDENTIALS + # rollback auth anyway + self.__authenticate__() + return True + + def __authenticate__(self): + if not self.__lock_acquire(): + raise LDAPLockTimeout() + try: + self.conn.simple_bind_s( + self.binddn, + self.bindpw, + ) + logging.debug("__whoami from __authenticate__") + self.__whoami() + self.__lock_release() + except ldap.INVALID_CREDENTIALS as ldaperr: + self.__lock_release() + logging.info(ldaperr) + raise ldap.INVALID_CREDENTIALS + + def __set_connection_parameters__(self): + self._conn.set_option(ldap.OPT_REFERRALS, self.params.get("referrals", False)) + self._conn.set_option( + ldap.OPT_NETWORK_TIMEOUT, self.params.get("network_timeout", 10.0) + ) + self._conn.set_option(ldap.OPT_TIMEOUT, self.params.get("timeout", 10.0)) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_IDLE, self.params.get("keepalive_idle", 10) + ) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_INTERVAL, self.params.get("keepalive_interval", 5) + ) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_PROBES, self.params.get("keepalive_probes", 3) + ) + self._conn.set_option(ldap.OPT_RESTART, ldap.OPT_ON) + if self.params.get("allow_tls_fallback", False): + logging.debug("TLS Fallback enabled in LDAP") + self._conn.set_option(ldap.OPT_X_TLS_TRY, 1) + self._conn.set_option(ldap.OPT_X_TLS_NEWCTX, ldap.OPT_OFF) + + def __enter__(self): + self.inUse = True + if not self.established: + logging.debug( + f"ConnectionPool {self} initializin LDAP {self.uri.initializeUrl()}" + ) + try: + self._conn = ldap.initialize(self.uri.initializeUrl()) + if self.params.get("autoBind", False): + ( + f"ConnectionPool {self} autoBind with {self.binddn} password {'x'*len(self.bindpw)}" + ) + self.__authenticate__() + except Exception as ldaperr: + (ldaperr) + raise ldaperr + self.established = True + return self.conn + + def giveback(self): + try: + if self.params.get("autoBind", False): + if not self.params.get("keep", False): + logging.debug(f"ConnectionPool unbind connection {self}") + try: + self._conn.unbind_s() + except Exception as ldaperr: + logging.error( + "ConnectionPool unbind connection" + + f"{self} exception {ldaperr}" + ) + self.inUse = False + except AttributeError: + self.inUse = False + + def __del__(self): + self.giveback() + if all([self._pool is not None, not self.params.get("keep", False)]): + logging.debug(f"ConnectionPool deleteing connection {self} from Pool") + self._pool.delete(self) + + def __exit__(self, type, value, traceback): + self.giveback() + if all([self._pool is not None, not self.params.get("keep", False)]): + self._pool.delete(self) + + def __cmp__(self, other): + if isinstance(other, LDAPUrl): + return self.uri.initializeUrl() == other.uri.initializeUrl() + return False + + def set_uri(self, uri: LDAPUrl): + self.uri = uri + return True + + def set_binddn(self, binddn: str): + self.binddn = binddn + return True + + def set_bindpw(self, bindpw: str): + self.bindpw = bindpw + return True + + def set_credentials(self, binddn: str, bindpw: str): + self.set_binddn(binddn) + self.set_bindpw(bindpw) + return True + + +class ConnectionPool(object): + def __init__( + self, + uri: LDAPUrl = LDAPUrl("ldap:///"), + binddn: str = "", + bindpw: str = "", + params: dict = {}, + max: int = 10, + ): + self.uri = uri + self.binddn = binddn + self.bindpw = bindpw + self.params = params + self.max = int(max) + self._lock = threading.Lock() + self._pool = [] + logging.debug(f"ConnectionPool {self} starting with {self.max} connections") + if self.params.get("prewarm", False): + self.scale + + @property + def basedn(self): + return self.uri.dn + + @property + def scope(self): + return self.uri.scope + + @property + def filter(self): + return self.uri.filterstr + + @property + def attributes(self): + return self.uri.attrs + + @property + def extensions(self): + return self.uri.extensions + + def set_uri(self, uri: LDAPUrl): + if not isinstance(uri, LDAPUrl): + uri = LDAPUrl(uri) + if len(self._pool) > 0: + map( + lambda c: (c.set_uri(uri), c.giveback(force=True)), + filter(lambda cp: cp.uri != uri, self._pool), + ) + self.uri = uri + return True + + def set_binddn(self, binddn: str): + if len(self._pool) > 0: + map( + lambda c: (c.set_binddn(binddn), c.giveback(force=True)), + filter(lambda cp: cp.binddn != binddn, self._pool), + ) + self.binddn = binddn + return True + + def set_bindpw(self, bindpw: str): + if len(self._pool) > 0: + map( + lambda c: (c.set_bindpw(bindpw), c.giveback(force=True)), + filter(lambda cp: cp.bindpw != bindpw, self._pool), + ) + self.bindpw = bindpw + return True + + def set_credentials(self, binddn: str, bindpw: str): + self.set_binddn(binddn) + self.set_bindpw(bindpw) + return True + + @property + def scale(self): + for _ in range(self.max - len(self._pool)): + self.put( + Connection( + uri=self.uri, + binddn=self.binddn, + bindpw=self.bindpw, + params=self.params, + ) + ) + + def __enter__(self): + if len(self._pool) == 0: + self.scale + with self.get() as conn: + yield conn + self.put(conn) + + @property + def ping(self): + with self.get() as conn: + try: + return True + except Exception as ldaperr: + try: + if conn.search_s("cn=config", ldap.SCOPE_ONELEVEL) != []: + return True + else: + # we might have ACI's in place + return True + except Exception as ldaperr: # collect with parent exception + pass + logging.error( + f"LDAP exception pinging server {self.uri.initializeUrl()} {ldaperr}" + ) + raise ldaperr + return True + + def get(self, binddn: str = "", bindpw: str = ""): + if len(self._pool) == 0: + self.scale + self._lock.acquire() + if len(self._pool) == 0: + self._lock.release() + logging.warning( + f"max connections {self.max} reached, consider increasing pool size" + ) + raise LDAPPoolExhausted( + f"max connections {self.max} reached, consider increasing pool size" + ) + try: + con = list(filter(lambda x: not x.inUse, self._pool))[0] + except IndexError: + self._lock.release() + logging.warning( + f"all connections {self.max} in use, consider increasing pool size" + ) + raise LDAPPoolExhausted( + f"all connections {self.max} in use, consider increasing pool size" + ) + con.inUse = True + self._lock.release() + if all([binddn != "", bindpw != ""]): + try: + con.authenticate(binddn, bindpw) + except ldap.INVALID_CREDENTIALS: + self.put(con) + raise ldap.INVALID_CREDENTIALS + return con + + def put(self, connection): + self._lock.acquire() + if connection.inUse: + connection.giveback() + if not connection in self._pool: + self._pool.append(connection) + connection._pool = self + self._lock.release() + return True + + def status(self): + self._lock.acquire() + for p in self._pool: + if p.inUse: + if sys.getrefcount(p) < 4: + p.giveback() + logging.info(f"Id {p} inUse {p.inUse} {p.established} {p.whoami}") + self._lock.release() + + def delete(self, connection, force=True): + return + self._lock.acquire() + if connection in self._pool: + if any([not self.params.get("keep", False), force]): + self._pool.remove(connection) + self._lock.release() From 665dc810fe2206fd40680de4159599309269aced Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Mon, 27 Jan 2025 13:19:06 +0100 Subject: [PATCH 2/9] rebased for easier integration --- Lib/ldappool/__init__.py | 59 +++-- Tests/__init__.py | 21 +- Tests/t_ldappool.py | 513 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 556 insertions(+), 37 deletions(-) create mode 100644 Tests/t_ldappool.py diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index 60aad335..0dffa137 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -82,10 +82,8 @@ def __whoami(self): try: self.__enter__() except: - pass - raise ldap.SERVER_DOWN( - f"max retries {self.params.get('retries', 3)} reached" - ) + continue + raise ldap.SERVER_DOWN(f"max retries {self.params.get('retries', 3)} reached") @property def conn(self): @@ -188,6 +186,7 @@ def __enter__(self): ) try: self._conn = ldap.initialize(self.uri.initializeUrl()) + self.__set_connection_parameters__() if self.params.get("autoBind", False): ( f"ConnectionPool {self} autoBind with {self.binddn} password {'x'*len(self.bindpw)}" @@ -199,8 +198,19 @@ def __enter__(self): self.established = True return self.conn - def giveback(self): + def giveback(self, force=False): try: + if force: + try: + self._conn.unbind_s() + except Exception as ldaperr: + logging.error( + "ConnectionPool unbind connection" + + f"{self} exception {ldaperr}" + ) + self.inUse = False + return + if self.params.get("autoBind", False): if not self.params.get("keep", False): logging.debug(f"ConnectionPool unbind connection {self}") @@ -227,7 +237,7 @@ def __exit__(self, type, value, traceback): self._pool.delete(self) def __cmp__(self, other): - if isinstance(other, LDAPUrl): + if isinstance(other, Connection): return self.uri.initializeUrl() == other.uri.initializeUrl() return False @@ -293,27 +303,33 @@ def set_uri(self, uri: LDAPUrl): if not isinstance(uri, LDAPUrl): uri = LDAPUrl(uri) if len(self._pool) > 0: - map( - lambda c: (c.set_uri(uri), c.giveback(force=True)), - filter(lambda cp: cp.uri != uri, self._pool), + list( + map( + lambda c: (c.set_uri(uri), c.giveback(force=True)), + filter(lambda cp: cp.uri != uri, self._pool), + ) ) self.uri = uri return True def set_binddn(self, binddn: str): if len(self._pool) > 0: - map( - lambda c: (c.set_binddn(binddn), c.giveback(force=True)), - filter(lambda cp: cp.binddn != binddn, self._pool), + list( + map( + lambda c: (c.set_binddn(binddn), c.giveback(force=True)), + filter(lambda cp: cp.binddn != binddn, self._pool), + ) ) self.binddn = binddn return True def set_bindpw(self, bindpw: str): if len(self._pool) > 0: - map( - lambda c: (c.set_bindpw(bindpw), c.giveback(force=True)), - filter(lambda cp: cp.bindpw != bindpw, self._pool), + list( + map( + lambda c: (c.set_bindpw(bindpw), c.giveback(force=True)), + filter(lambda cp: cp.bindpw != bindpw, self._pool), + ) ) self.bindpw = bindpw return True @@ -365,7 +381,7 @@ def ping(self): def get(self, binddn: str = "", bindpw: str = ""): if len(self._pool) == 0: self.scale - self._lock.acquire() + self._lock.acquire(timeout=1) if len(self._pool) == 0: self._lock.release() logging.warning( @@ -395,7 +411,7 @@ def get(self, binddn: str = "", bindpw: str = ""): return con def put(self, connection): - self._lock.acquire() + self._lock.acquire(timeout=1) if connection.inUse: connection.giveback() if not connection in self._pool: @@ -405,7 +421,7 @@ def put(self, connection): return True def status(self): - self._lock.acquire() + self._lock.acquire(timeout=1) for p in self._pool: if p.inUse: if sys.getrefcount(p) < 4: @@ -414,9 +430,12 @@ def status(self): self._lock.release() def delete(self, connection, force=True): - return - self._lock.acquire() + self._lock.acquire(timeout=1) if connection in self._pool: if any([not self.params.get("keep", False), force]): self._pool.remove(connection) + del connection self._lock.release() + + def __len__(self): + return len(self._pool) diff --git a/Tests/__init__.py b/Tests/__init__.py index ea28d0ce..b1391a87 100644 --- a/Tests/__init__.py +++ b/Tests/__init__.py @@ -4,20 +4,7 @@ See https://www.python-ldap.org/ for details. """ - -from . import t_bind -from . import t_cext -from . import t_cidict -from . import t_ldap_dn -from . import t_ldap_filter -from . import t_ldap_functions -from . import t_ldap_modlist -from . import t_ldap_schema_tokenizer -from . import t_ldapurl -from . import t_ldif -from . import t_ldapobject -from . import t_edit -from . import t_ldap_schema_subentry -from . import t_untested_mods -from . import t_ldap_controls_libldap -from . import t_ldap_options +from . import (t_bind, t_cext, t_cidict, t_edit, t_ldap_controls_libldap, t_ldap_dn, + t_ldap_filter, t_ldap_functions, t_ldap_modlist, t_ldap_options, + t_ldap_schema_subentry, t_ldap_schema_tokenizer, t_ldapobject, + t_ldappool, t_ldapurl, t_ldif, t_untested_mods) diff --git a/Tests/t_ldappool.py b/Tests/t_ldappool.py new file mode 100644 index 00000000..dbc57333 --- /dev/null +++ b/Tests/t_ldappool.py @@ -0,0 +1,513 @@ +""" +Automatic tests for python-ldap's module ldappool + +See https://www.python-ldap.org/ for details. +""" + +import os +import sys +import unittest +import time + +# Switch off processing .ldaprc or ldap.conf before importing _ldap +os.environ["LDAPNOINIT"] = "1" + +sys.path.append("../Lib") +import ldappool +from ldappool import Connection, ConnectionPool + +import ldap as _ldap +from ldapurl import LDAPUrl +import ldapurl + + +class ldapmock: + """Mocking some LDAP methods to avoid having a + full LDAP Setup for unittestst""" + + def __init__(self, fail=0, down=0): + self.fail = int(fail) + self.down = int(down) + + @property + def __whoami_s(self): + """if down was set when initializing + we fail for the count until we return success + """ + for _ in range(self.down): + self.down -= 1 + if self.down < 0: + self.down = 0 + raise _ldap.SERVER_DOWN() + return "cn=tester,dc=example,dc=com" + + def whoami_s(self): + """if down was set when initializing + we fail for the count until we return success + """ + for _ in range(self.down): + self.down -= 1 + if self.down < 0: + self.down = 0 + raise _ldap.SERVER_DOWN() + return "cn=tester,dc=example,dc=com" + + def initialize(self, uri): + return self + + def simple_bind_s(self, binddn, bindpw): + """if fail was set when initializing + we fail for the count until we return success + """ + for _ in range(self.fail): + self.fail -= 1 + if self.fail < 0: + self.fail = 0 + raise _ldap.INVALID_CREDENTIALS() + return (97, []) + + def set_option(self, *args, **kwargs): + return True + + def search_s(self, *args, **kwargs): + return True + + def authenticate(self, binddn, bindpw): + """if fail was set when initializing + we fail for the count until we return success + """ + for _ in range(self.fail): + self.fail -= 1 + if self.fail < 0: + self.fail = 0 + raise _ldap.INVALID_CREDENTIALS() + return (97, []) + + def unbind_s(self, *args, **kwargs): + return True + + def __enter__(self): + return + +class TestConnection(unittest.TestCase): + + def test_Connectionparams(self): + """test if a Connection handles parameters correctly""" + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + assert connection.params.get("retries") == 3 + assert connection.inUse == False + assert connection.established == False + assert connection.binddn == "cn=tester,dc=example,dc=com" + assert connection.bindpw == "changeme" + + def test_Connectionhandling(self): + """test if a Connection changes state when in use""" + ldap = ldapmock(fail=0) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + with connection as ctx: + assert connection.inUse == True + assert connection.established == True + assert connection.whoami == "cn=tester,dc=example,dc=com" + + def test_Connectionhandlingautherror(self): + """test if a connection raises exception if credentials are wrong""" + ldap = ldapmock(fail=2) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "allow_tls_fallback": True}, + ) + with self.assertRaises(_ldap.INVALID_CREDENTIALS) as ctx: + connection.conn() + + def test_Connectionhandlingauthentication(self): + """test if a connection can authenticate for someone""" + ldap = ldapmock(fail=0, down=0) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 1, "allow_tls_fallback": True}, + ) + connection._conn = ldap + with connection as ctx: + assert ctx.authenticate("test", "test") == (97, []) + + def test_Connectionhandlingserverdown(self): + """test if aconnection retries until params.retries reached""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + with connection as ctx: + assert connection.whoami == "cn=tester,dc=example,dc=com" + + def test_ConnectionhandlingserverdownExceed(self): + """test to ensure we raise ldap.SERVER_DOWN after max retries + has been reached and we have not succeeded""" + ldap = ldapmock(down=10) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + with self.assertRaises(_ldap.SERVER_DOWN) as ctx: + with connection as ctx: + connection.whoami + + """test to ensure without context ldap.SERVER_DOWN after max retries + has been reached and we have not succeeded""" + ldap = ldapmock(down=10) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + with self.assertRaises(_ldap.SERVER_DOWN) as ctx: + connection.conn.search_s() + + def test_Connectionconfigchange(self): + """test if a connection updates configuration changes accordingly""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + connection = Connection( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3}, + ) + connection.set_credentials("cn=another,dc=example,dc=com", "changetoo") + assert connection.binddn == "cn=another,dc=example,dc=com" + assert connection.bindpw == "changetoo" + + def test_Connectionlocktime(self): + """test locktime which ensures we do not stress the connections too often""" + conn = Connection(LDAPUrl("ldap:///"), "", "") + assert conn._Connection__locktime() == True + assert conn._Connection__locktime() == False + time.sleep(15) + assert conn._Connection__locktime() == True + + + def test_Connectionmethods(self): + """test Connection methods which are there for + simplifying handling with the class""" + + """check set_uri""" + conn = Connection(LDAPUrl("ldap://127.0.0.1/"), "", "") + conn.set_uri(LDAPUrl("ldap://localhost/dc=example,dc=com")) + assert conn.uri == Connection(LDAPUrl("ldap://localhost/dc=example,dc=com"), "", "").uri + + """check set_binddn""" + conn = Connection(LDAPUrl("ldap://127.0.0.1/"), "", "") + conn.set_binddn("cn=Directory Manager") + assert conn.binddn == "cn=Directory Manager" + + """check set_bindpw""" + conn = Connection(LDAPUrl("ldap://127.0.0.1/"), "", "") + conn.set_bindpw("changeme") + assert conn.bindpw == "changeme" + + """check set_credentials""" + conn = Connection(LDAPUrl("ldap://127.0.0.1/"), "", "") + conn.set_credentials("cn=Directory Manager", "changeme") + assert conn.binddn == "cn=Directory Manager" + assert conn.bindpw == "changeme" + + +class TestConnectionPool(unittest.TestCase): + + def test_ConnectionPoolParams(self): + """test if ConnectionPool handles parameters correctly""" + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "autoBind": True}, + max=3, + ) + assert pool.params.get("retries") == 3 + assert pool.params.get("autoBind") == True + assert pool.binddn == "cn=tester,dc=example,dc=com" + assert pool.bindpw == "changeme" + assert pool.basedn == "dc=example,dc=com" + assert pool.scope == _ldap.SCOPE_SUBTREE + assert pool.filter == "(uid=tester)" + assert pool.attributes == ["uid", "mail"] + + def test_ConnectionPoolhandling(self): + """test if ConnectionPool context for Connection + is handled correctly""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + assert len(pool._pool) == 3 + assert pool.ping + with pool.get() as conn: + assert conn.search_s("something") == True + assert conn.authenticate("test", "test") == (97, []) + + def test_ConnectionPoolCleanup(self): + """test if ConnectionPool removing Connection + without deleting it""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True, "autoBind": True}, + max=3, + ) + assert len(pool._pool) == 3 + assert pool.ping + conn = pool.get() + pool.delete(conn) + + def test_ConnectionPoolconfigchange(self): + """test if ConnectionPool delegates configuration changes + to Connections in pool during runtime change""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + pool.scale + assert len(pool) == 3 + pool.set_credentials("cn=another,dc=example,dc=com", "changetoo") + assert pool.binddn == "cn=another,dc=example,dc=com" + assert pool.bindpw == "changetoo" + for conn in pool._pool: + assert conn.binddn == "cn=another,dc=example,dc=com" + assert conn.bindpw == "changetoo" + + def test_ConnectionPoolLDAPPoolExhausted(self): + """test if ConnectionPool raises Exception when + all connections are in use""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + pool.scale + assert len(pool) == 3 + with self.assertRaises(ldappool.LDAPPoolExhausted) as ctx: + for _ in range(pool.max + 1): + c = pool.get() + + def test_ConnectionPoolLDAPPoolExhausted(self): + """test if all connections are free'ed when returned + to the Pool but connection kept established""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + pool.scale + assert len(pool) == 3 + + conn = [] + for _ in range(pool.max - 1): + conn.append(pool.get()) + for c in conn: + pool.put(c) + assert c.inUse == False + assert c.established == True + + def test_ConnectionPoolLDAPPoolGiveback(self): + """test if ConnectionPool giveback returns + connections to the pool""" + ldap = ldapmock() + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + pool.scale + assert len(pool) == 3 + conn = pool.get() + conn.giveback() + assert conn.inUse == False + + """test if all connections are free'ed when returned + to the Pool but connection kept established""" + conn = [] + for _ in range(pool.max - 1): + conn.append(pool.get()) + for c in conn: + pool.put(c) + assert c.inUse == False + assert c.established == True + + def test_ConnectionPoolLDAPLockTimeout(self): + """test if ConnectionPool raises Locktimeout accordingly + when connection has been locked""" + ldap = ldapmock(down=2) + ldappool.ldap.initialize = ldap.initialize + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + pool.scale + assert len(pool) == 1 + with self.assertRaises(ldappool.LDAPLockTimeout) as ctx: + c = pool.get() + if not c._lock.acquire(blocking=True, timeout=1): + raise ldappool.LDAPLockTimeout() + if not c._lock.acquire(blocking=True, timeout=1): + raise ldappool.LDAPLockTimeout() + c.authenticate("test", "test") == (97, []) + + """test if ConnectionPool releases lock when Connection + is used and returned""" + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + pool.scale + assert len(pool) == 1 + with pool.get() as ctx: + pass + with pool.get() as ctx: + pass + + def test_ConnectionPoolmethods(self): + """test ConnectionPool methods which are there for + simplifying handling with the class""" + + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?sub?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + assert pool.scope == _ldap.SCOPE_SUBTREE + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?base?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + assert pool.scope == _ldap.SCOPE_BASE + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?one?(uid=tester)" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + assert pool.scope == _ldap.SCOPE_ONELEVEL + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?one?(uid=tester)?extensiontest" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=1, + ) + assert isinstance(pool.extensions, ldapurl.LDAPUrlExtensions) + pool = ConnectionPool( + uri=LDAPUrl( + "ldaps://localhost:636/dc=example,dc=com?uid,mail?one?(uid=tester)?extensiontest" + ), + binddn="cn=tester,dc=example,dc=com", + bindpw="changeme", + params={"retries": 3, "prewarm": True}, + max=3, + ) + pool.status + +if __name__ == "__main__": + unittest.main() From 306c2e41adab0630bc0fa6933f9dae6bfdf31f83 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Tue, 28 Jan 2025 09:32:08 +0100 Subject: [PATCH 3/9] added ldappool to setup and deploy as otherwise the unittest will fail --- setup.py | 1 + tox.ini | 1 + 2 files changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 8e7963a1..92a2d4dc 100644 --- a/setup.py +++ b/setup.py @@ -142,6 +142,7 @@ class OpenLDAP2: py_modules = [ 'ldapurl', 'ldif', + 'ldappool', ], packages = [ diff --git a/tox.ini b/tox.ini index 22752067..c6def5c4 100644 --- a/tox.ini +++ b/tox.ini @@ -85,6 +85,7 @@ commands = Tests/t_ldap_modlist.py \ Tests/t_ldap_schema_tokenizer.py \ Tests/t_ldapurl.py \ + Tests/t_ldappool.py \ Tests/t_ldif.py \ Tests/t_untested_mods.py From 7576c47eef4510680cd73ade8793b9dd5dda4a4b Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Tue, 28 Jan 2025 19:26:31 +0100 Subject: [PATCH 4/9] fixed includes in ldappool unittest, fixed packages section --- Tests/t_ldappool.py | 1 - pyproject.toml | 2 +- setup.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Tests/t_ldappool.py b/Tests/t_ldappool.py index dbc57333..6c3ddc60 100644 --- a/Tests/t_ldappool.py +++ b/Tests/t_ldappool.py @@ -12,7 +12,6 @@ # Switch off processing .ldaprc or ldap.conf before importing _ldap os.environ["LDAPNOINIT"] = "1" -sys.path.append("../Lib") import ldappool from ldappool import Connection, ConnectionPool diff --git a/pyproject.toml b/pyproject.toml index dda8dbc1..89596235 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,5 +4,5 @@ target-version = ['py36', 'py37', 'py38'] [tool.isort] line_length=88 -known_first_party=['ldap', '_ldap', 'ldapurl', 'ldif', 'slapdtest'] +known_first_party=['ldap', '_ldap', 'ldapurl', 'ldif', 'slapdtest', 'ldappool'] sections=['FUTURE', 'STDLIB', 'THIRDPARTY', 'FIRSTPARTY', 'LOCALFOLDER'] diff --git a/setup.py b/setup.py index 92a2d4dc..46974b73 100644 --- a/setup.py +++ b/setup.py @@ -142,7 +142,6 @@ class OpenLDAP2: py_modules = [ 'ldapurl', 'ldif', - 'ldappool', ], packages = [ @@ -152,6 +151,7 @@ class OpenLDAP2: 'ldap.schema', 'slapdtest', 'slapdtest.certs', + 'ldappool', ], package_dir = {'': 'Lib',}, data_files = LDAP_CLASS.extra_files, From 1140996bf35c982607d122d573da738f93544188 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Tue, 28 Jan 2025 19:58:46 +0100 Subject: [PATCH 5/9] added import error for python3.6 on dataclasses, since its not mandatory we just ignore it --- Lib/ldappool/__init__.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index 0dffa137..b0afb15c 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -1,4 +1,8 @@ -import dataclasses +try: + import dataclasses +except ImportError: + # we are on python < 3.7 so ignore + pass import logging import sys import threading @@ -28,8 +32,12 @@ class LDAPLockTimeout(Exception): def e2c(entry): - cls = dataclasses.make_dataclass("", ["dn"] + list(entry[1].keys()), frozen=True) - return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) + try: + cls = dataclasses.make_dataclass("", ["dn"] + list(entry[1].keys()), frozen=True) + return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) + except NameError as dcerror: + print(f"dataclasses not supported") + return entry class Connection(object): From e5e335b9d7d5a419e05e6a8bcf9eda630e559554 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Tue, 28 Jan 2025 21:27:32 +0100 Subject: [PATCH 6/9] some ldap options are not available < 3.9 --- Lib/ldappool/__init__.py | 46 +++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index b0afb15c..f243bfaf 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -33,7 +33,9 @@ class LDAPLockTimeout(Exception): def e2c(entry): try: - cls = dataclasses.make_dataclass("", ["dn"] + list(entry[1].keys()), frozen=True) + cls = dataclasses.make_dataclass( + "", ["dn"] + list(entry[1].keys()), frozen=True + ) return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) except NameError as dcerror: print(f"dataclasses not supported") @@ -166,25 +168,29 @@ def __authenticate__(self): raise ldap.INVALID_CREDENTIALS def __set_connection_parameters__(self): - self._conn.set_option(ldap.OPT_REFERRALS, self.params.get("referrals", False)) - self._conn.set_option( - ldap.OPT_NETWORK_TIMEOUT, self.params.get("network_timeout", 10.0) - ) - self._conn.set_option(ldap.OPT_TIMEOUT, self.params.get("timeout", 10.0)) - self._conn.set_option( - ldap.OPT_X_KEEPALIVE_IDLE, self.params.get("keepalive_idle", 10) - ) - self._conn.set_option( - ldap.OPT_X_KEEPALIVE_INTERVAL, self.params.get("keepalive_interval", 5) - ) - self._conn.set_option( - ldap.OPT_X_KEEPALIVE_PROBES, self.params.get("keepalive_probes", 3) - ) - self._conn.set_option(ldap.OPT_RESTART, ldap.OPT_ON) - if self.params.get("allow_tls_fallback", False): - logging.debug("TLS Fallback enabled in LDAP") - self._conn.set_option(ldap.OPT_X_TLS_TRY, 1) - self._conn.set_option(ldap.OPT_X_TLS_NEWCTX, ldap.OPT_OFF) + try: + self._conn.set_option( + ldap.OPT_REFERRALS, self.params.get("referrals", False) + ) + self._conn.set_option( + ldap.OPT_NETWORK_TIMEOUT, self.params.get("network_timeout", 10.0) + ) + self._conn.set_option(ldap.OPT_TIMEOUT, self.params.get("timeout", 10.0)) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_IDLE, self.params.get("keepalive_idle", 10) + ) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_INTERVAL, self.params.get("keepalive_interval", 5) + ) + self._conn.set_option( + ldap.OPT_X_KEEPALIVE_PROBES, self.params.get("keepalive_probes", 3) + ) + self._conn.set_option(ldap.OPT_RESTART, ldap.OPT_ON) + if self.params.get("allow_tls_fallback", False): + self._conn.set_option(ldap.OPT_X_TLS_TRY, 1) + self._conn.set_option(ldap.OPT_X_TLS_NEWCTX, ldap.OPT_OFF) + except Exception as connerr: + logging.error(f"cannot set LDAP option {connerr}") def __enter__(self): self.inUse = True From 85eae2464efa5c5a1fdbe6b6e8126f7ccf892ae8 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Wed, 29 Jan 2025 07:40:24 +0100 Subject: [PATCH 7/9] added python 3.6-8 checks and exceptions --- Lib/ldappool/__init__.py | 44 +++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index f243bfaf..b4902cb0 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -1,14 +1,19 @@ -try: +import sys + +if sys.version_info.minor >= 3.8: import dataclasses -except ImportError: - # we are on python < 3.7 so ignore - pass import logging import sys import threading -import time from urllib.parse import urlparse +import sys + +if sys.version_info.minor > 6: + from time import perf_counter_ns as perf_counter +elif sys.version_info.minor == 6: + from time import perf_counter + import ldap from ldapurl import LDAPUrl @@ -31,15 +36,22 @@ class LDAPLockTimeout(Exception): pass -def e2c(entry): - try: - cls = dataclasses.make_dataclass( - "", ["dn"] + list(entry[1].keys()), frozen=True - ) - return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) - except NameError as dcerror: - print(f"dataclasses not supported") - return entry +if sys.version_info >= 3.8: + + def e2c(entry): + try: + cls = dataclasses.make_dataclass( + "", ["dn"] + list(entry[1].keys()), frozen=True + ) + return cls(**dict(list([("dn", entry[0])] + list(entry[1].items())))) + except NameError as dcerror: + print(f"dataclasses not supported") + return entry + +else: + + def e2c(entry): + return f"dataclasses not support on python < {sys.version_info.minor}" class Connection(object): @@ -67,9 +79,9 @@ def __init__( def __locktime(self): if self._health == 0.0: - self._health = time.perf_counter_ns() + self._health = perf_counter() return True - if (time.perf_counter_ns() - self._health) / ns < ns_locktimeout: + if (perf_counter() - self._health) / ns < ns_locktimeout: return False return True From e80b2f4cd5555d0a6ddbf8cdffe7fc71c812e447 Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Wed, 29 Jan 2025 07:59:45 +0100 Subject: [PATCH 8/9] fixed one misses version_info compare --- Lib/ldappool/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index b4902cb0..6c8e1c74 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -36,7 +36,7 @@ class LDAPLockTimeout(Exception): pass -if sys.version_info >= 3.8: +if sys.version_info.minor >= 3.8: def e2c(entry): try: From 5e9bd910f5bd33d328097fc2cc7b742dba50a68c Mon Sep 17 00:00:00 2001 From: Michaela Lang Date: Thu, 30 Jan 2025 06:56:11 +0100 Subject: [PATCH 9/9] minor compare is single int not float --- Lib/ldappool/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/ldappool/__init__.py b/Lib/ldappool/__init__.py index 6c8e1c74..8530e005 100644 --- a/Lib/ldappool/__init__.py +++ b/Lib/ldappool/__init__.py @@ -1,6 +1,6 @@ import sys -if sys.version_info.minor >= 3.8: +if sys.version_info.minor >= 8: import dataclasses import logging import sys @@ -36,7 +36,7 @@ class LDAPLockTimeout(Exception): pass -if sys.version_info.minor >= 3.8: +if sys.version_info.minor >= 8: def e2c(entry): try: