diff --git a/.gitignore b/.gitignore index b6e4761..e4a1406 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/README.md b/README.md index 33519d2..c9293c3 100644 --- a/README.md +++ b/README.md @@ -1 +1,91 @@ -# pydb \ No newline at end of file +# pydb + +a tool for facilitating connection to databases with python and performing basic operations + +## Purpose and intended audience + +For many APIs the responses are based upon data residing in a relational database; this module provides a standardized +way for python programmers to access a database, via straight SQL calls to `execute_query()` and `execute_update()` and +also permits users to access SQLAlchemy for ORM-based interactions with database objects. + +## Requirements + + python >= 3.7 + (see setup.py for additional packages) + +## Installation and setup + +In a suitable python3 (>=3.7) virtual env, using pip: +``` + pip install https://github.com/huit/pydb/archive/refs/tags/v0.0.2.tar.gz#egg=pydb + # import the module for the specific type of db you'd like to use + from pydb.oracle_db import OracleDB +``` +* creating an OracleDB instance requires host, port, service, user, pwd. + * other db types may have other requirements - see specific module for details +* logging_level is optional, and will default to `logging.CRITICAL` +* logging_format is optional, and will default to pylog default formatting +* see https://github.com/huit/pylog for details + +``` +db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="username", pwd="pwd") +# where 8003 is a valid port +``` +## Basic operations + + create_connection() + provides a connection to the db host for more 'direct' access + + get_session() + Specific to SQL Alchemy; allows interaction with SQL Alchemy entities + see https://docs.sqlalchemy.org/en/14/orm/session.html?highlight=session#module-sqlalchemy.orm.session + + execute_query(self, query_string: str, args: dict = None) -> list + executes a sql query, return a list of dictionaries representing rows + 'args' represents a dictionary of parameterized values for the query; see examples below + + execute_update(self, query_string, args: dict = None) + used to execute an insert, update, or delete sql statement + 'args' represents a dictionary of parameterized values for the query; see examples below + + health_check() + Performs a basic query against the db to ensure connectivity + + cleanup() + Attempts to release any 'live' objects/connections to the host - to be run before exiting program + +## Examples + +Given a valid connection, and a table called `EMP`... + +To query the `EMP` table for all records: +``` +result = db.execute_query("select * from emp") +``` +To query the `EMP` table for a specific record: +``` +result = db.execute_query("select * from emp where ename= :ename", {'ename':'JOHNSON'}) +``` +Results for the individual rows would be in the following form: +``` +{'EMPNO': 7935, 'ENAME': 'JOHNSON', 'JOB': 'CLERK', 'MGR': 7839, 'HIREDATE': datetime.datetime(1981, 5, 1, 0, 0), 'SAL': 2850.0, 'COMM': None, 'DEPTNO': 30} +``` +Row results may vary somewhat depending on the exact module... e.g., for SqlAlchemyOracleDB the following would be received: +``` +{'empno': 7935, 'ename': 'JOHNSON', 'job': 'CLERK', 'mgr': 7839, 'hiredate': datetime.datetime(1981, 5, 1, 0, 0), 'sal': Decimal('2850'), 'comm': None, 'deptno': 30} +``` + +### integrated example +``` +pip install https://github.com/huit/pydb/archive/refs/tags/v0.0.2.tar.gz#egg=pydb + +from pydb.oracle_db import OracleDB +db = OracleDb(host="valid_host", port=8003, service="SERVICE_NAME", user="username", pwd="pwd") + +result = db.execute_query("select * from emp where ename= :ename", {'ename':'JOHNSON'}) +print(result) +``` +produces +``` +{'EMPNO': 7935, 'ENAME': 'JOHNSON', 'JOB': 'CLERK', 'MGR': 7839, 'HIREDATE': datetime.datetime(1981, 5, 1, 0, 0), 'SAL': 2850.0, 'COMM': None, 'DEPTNO': 30} +``` diff --git a/pydb/__init__.py b/pydb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pydb/database.py b/pydb/database.py new file mode 100644 index 0000000..1cb84f8 --- /dev/null +++ b/pydb/database.py @@ -0,0 +1,68 @@ +import abc + +from enum import Enum + + +class DatabaseType(Enum): + """ + not implemented: other databases + """ + ORACLE = "oracle" + SQL_ALCHEMY_ORACLE = "sql_alchemy_oracle" + + +class DBInterface(metaclass=abc.ABCMeta): + + @abc.abstractmethod + def execute_query(self, query_string: str, args=None) -> list: + """ + executes a sql query, return a list of dictionaries representing rows + :param self: + :param query_string: + :param args: + :return: + """ + raise NotImplementedError + + @abc.abstractmethod + def execute_update(self, query_string, args=None): + """ + used to execute an insert, update, or delete sql statement + :param self: + :param query_string: + :param args: + :return: + """ + raise NotImplementedError + + @abc.abstractmethod + def health_check(): + """ + Performs a basic query against the db to ensure connectivity + :return: + """ + raise NotImplementedError + + @abc.abstractmethod + def cleanup(): + """ + Attempts to release any 'live' objects/connections to the host + :return: + """ + raise NotImplementedError + + @abc.abstractmethod + def create_connection(): + """ + provides a connection to the db host for more 'direct' access + :return: + """ + raise NotImplementedError + + @abc.abstractmethod + def get_session(): + """ + Specific to SQL Alchemy; allows interaction with SQL Alchemy entities + :return: + """ + raise NotImplementedError diff --git a/pydb/oracle_db.py b/pydb/oracle_db.py new file mode 100644 index 0000000..c282ab0 --- /dev/null +++ b/pydb/oracle_db.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +""" +Module for interacting with an oracle database +""" +# -*- encoding: utf-8 -*- + +#============================================================================================ +# Imports +#============================================================================================ +# Standard imports +import logging + +# Third-party imports +import cx_Oracle + +from pylog.pylog import get_common_logger_for_module + +from .database import DBInterface + +# Local imports + + +class OracleDB(DBInterface): + """ + Class for interacting with an oracle database + """ + + def __init__(self, host: str, port: int, service: str, user: str, pwd: str, + logging_level: int = 50, logging_format: logging.Formatter = None): + """ + Setup for oracle db connections. oracle_config must be a python dictionary with the following fields: + + :param host: + :param port: + :param service: + :param user: + :param pwd: + :param logging_level: + :param logging_format: + :param logging_level: defaults to logging.CRITICAL + :param logging_format: defaults to None here, which translates to the pylog.get_commong_logging_format + """ + + self.host = host + self.port = port + self.service = service + self.user = user + self.pwd = pwd + self.logger = get_common_logger_for_module(module_name=__name__, level=logging_level, log_format=logging_format) + + self._pool = self.set_up_session_pool() + + def set_up_session_pool(self): + try: + dsn_str = cx_Oracle.makedsn(self.host, self.port, service_name=self.service) + pool = cx_Oracle.SessionPool( + user=self.user, + password=self.pwd, + dsn=dsn_str, + min=2, + max=5, + increment=1, + threaded=True, + encoding="UTF-8" + ) + return pool + + except cx_Oracle.DatabaseError as err: + obj, = err.args + self.logger.error("Error creating pool") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) + raise Exception(f"Error creating pool: {obj.message}") + + def create_connection(self): + """ + Function for creating a connection with the database from a session pool + """ + try: + return self._pool.acquire() + + except cx_Oracle.DatabaseError as err: + obj, = err.args + self.logger.error("Error acquiring database connection from the session pool") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) + raise Exception("Error acquiring database connection from the session pool") + + @staticmethod + def make_dict(cursor): + """ + Function for converting a query result row into a dictionary + """ + column_names = [d[0] for d in cursor.description] + + def create_row(*args): + return dict(zip(column_names, args)) + return create_row + + def execute_query(self, query_string: str, args=None) -> dict: + """ + Function for executing a query against the database via the session pool + """ + try: + connection = self.create_connection() + cursor = connection.cursor() + if args is not None: + cursor.execute(query_string, args) + else: + cursor.execute(query_string) + cursor.rowfactory = self.make_dict(cursor) + query_result = cursor.fetchall() + return query_result + + except cx_Oracle.DatabaseError as err: + obj, = err.args + self.logger.error("Error in execute_query") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) + raise Exception(f"Error executing query: {query_string}") + + finally: + cursor.close() + self._pool.release(connection) + + def execute_update(self, query_string, args=None): + """ + Function for executing an insert/update query against the database via the session pool + """ + try: + connection = self.create_connection() + cursor = connection.cursor() + if args is not None: + cursor.execute(query_string, args) + else: + cursor.execute(query_string) + connection.commit() + + except cx_Oracle.DatabaseError as err: + obj, = err.args + self.logger.error("Error in execute_update") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) + raise Exception(f"Error executing update: {query_string}") + + finally: + cursor.close() + self._pool.release(connection) + + def health_check(self): + """ + provides a means to verify DB connectivity with a simple query + :return: + """ + return self.execute_query("SELECT 1 FROM DUAL") + + def cleanup(self): + if self._pool is not None: + self.logger.info("Active session pool found. Attempting to close session pool.") + try: + self._pool.close(force=True) + self.logger.info("Session pool successfully closed.") + + except cx_Oracle.Error as err: + self.logger.error("Unable to close the active session.", exc_info=True) + obj, = err.args + self.logger.error("Context:", obj.context) + self.logger.error("Message:", obj.message) + + def get_session(self): + return None diff --git a/pydb/sql_alchemy_oracle_db.py b/pydb/sql_alchemy_oracle_db.py new file mode 100644 index 0000000..4d2e113 --- /dev/null +++ b/pydb/sql_alchemy_oracle_db.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 +import cx_Oracle +import logging + +from sqlalchemy import create_engine, text +from sqlalchemy.pool import NullPool +from sqlalchemy.orm import sessionmaker + +from pylog.pylog import get_common_logger_for_module + +from .database import DBInterface + + +class SqlAlchemyOracleDB(DBInterface): + """ + Module for interacting with an Oracle DB via SQL Alchemy + """ + def __init__(self, host: str, port: int, service: str, user: str, pwd: str, + logging_level: int = 50, logging_format: logging.Formatter = None): + """ + initialize object with values required to create a connection to an Oracle DB + :param host: + :param port: + :param service: + :param user: + :param pwd: + :param logging_level: + :param logging_format: + """ + self.logger = get_common_logger_for_module(module_name=__name__, level=logging_level, log_format=logging_format) + self.host = host + self.port = port + self.service = service + self.user = user + self.pwd = pwd + self.engine = self.setup_engine() + + def setup_engine(self): + """ + Create a SQL Alchemy engine to connect to Oracle DB with a connection pool + :return: + """ + try: + dsn_str = cx_Oracle.makedsn(self.host, self.port, service_name=self.service) + pool = cx_Oracle.SessionPool( + user=self.user, password=self.pwd, dsn=dsn_str, + min=2, max=5, increment=1, threaded=True + ) + engine = create_engine("oracle://", + creator=pool.acquire, + poolclass=NullPool, + echo=True, + max_identifier_length=128) + + self.logger.info("Setup sqlalchemy engine successfully") + return engine + except Exception as err: + obj, = err.args + self.logger.critical("Cannot create sql alchemy engine") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) + raise Exception(f"Cannot create sql alchemy engine: {obj.message}") + + def get_engine(self): + if self.engine is None: + self.engine = self.setup_engine() + return self.engine + + def get_session(self): + """ + Specific to SQL Alchemy; allows interaction with SQL Alchemy entities + :return: + """ + Session = sessionmaker(bind=self.get_engine()) + return Session() + + def create_connection(self): + """ + provides a connection to the db host for more 'direct' access + :return: + """ + return self.get_engine().connect() + + def health_check(self): + """ + Performs a basic query against the db to ensure connectivity + """ + with self.create_connection() as conn: + return conn.scalar("select 1 from dual") + + def execute_query(self, query_string: str, args: dict = None) -> list: + """ + executes a sql query, return a list of dictionaries representing rows + + :param query_string: str + :param args: dict + :return: list of dict + """ + with self.create_connection() as conn: + statement = text(query_string) + + if args is not None and len(args.keys()) > 0: + query_result = conn.execute(statement, args).fetchall() + else: + query_result = conn.execute(statement).fetchall() + + return [dict(row) for row in query_result] + + def execute_update(self, query_string: str, args: dict): + """ + used to execute an insert, update, or delete sql statement + + :param query_string: str + :param args: dict + :return: + """ + with self.create_connection() as conn: + statement = text(query_string) + trans = conn.begin() + if args is not None and len(args.keys()) > 0: + conn.execute(statement, args) + else: + conn.execute(statement) + trans.commit() + + def cleanup(self): + """ + release any existing connections; to be called prior to exiting program + :return: + """ + if self.engine is not None: + self.logger.info("sql alchemy engine found") + try: + self.engine.dispose() + except Exception as err: + obj, = err.args + self.logger.critical("Cannot dispose sql alchemy engine") + self.logger.error("Context: %s", obj.context) + self.logger.error("Message: %s", obj.message) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..d6e0bab --- /dev/null +++ b/setup.py @@ -0,0 +1,30 @@ +import setuptools + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +setuptools.setup( + name="pydb", + version="0.0.2", + author="Michael Kerry", + author_email="michael_kerry@harvard.edu", + description="A package to facilitate connecting to an oracle DB from a python application", + long_description=long_description, + long_description_content_type="text/markdown", + url="https://github.com/huit/pydb", + project_urls={ + "Bug Tracker": "https://github.com/huit/pydb/issues", + }, + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + install_requires=[ + "cx-Oracle==8.1.0", + 'sqlalchemy==1.4.1', + 'pylog @ https://github.com/huit/pylog/archive/refs/tags/v0.0.2.tar.gz#egg=pylog', + ], + packages=setuptools.find_packages(), + python_requires=">=3.7", +)