From 0745ac2fd42a1e60edfd6ad0f99e990190d9da97 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Thu, 25 Apr 2024 16:16:15 -0400 Subject: [PATCH] Minor changes and updates for AAPOR with Confex --- aether_api_fastapi.code-workspace | 6 +++++- app/db_sql.py | 8 ++++++-- app/main.py | 3 --- app/routers/e_confex.py | 11 +++++++---- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/aether_api_fastapi.code-workspace b/aether_api_fastapi.code-workspace index 876a149..1621c78 100644 --- a/aether_api_fastapi.code-workspace +++ b/aether_api_fastapi.code-workspace @@ -4,5 +4,9 @@ "path": "." } ], - "settings": {} + "settings": { + "cSpell.words": [ + "poolclass" + ] + } } \ No newline at end of file diff --git a/app/db_sql.py b/app/db_sql.py index d8fa352..21b037f 100644 --- a/app/db_sql.py +++ b/app/db_sql.py @@ -6,6 +6,7 @@ from app.log import log, logging, logger_reset from sqlalchemy import create_engine, text, Time from sqlalchemy.exc import IntegrityError, OperationalError, ProgrammingError +from sqlalchemy.pool import NullPool db_uri = settings.SQLALCHEMY_DB_URI @@ -15,10 +16,12 @@ db_uri = settings.SQLALCHEMY_DB_URI # log.setLevel(logging.INFO) connection_string = db_uri -engine = create_engine(url=connection_string, pool_size=5, max_overflow=15, pool_recycle=settings.DB['pool_recycle'], pool_pre_ping=True, echo=False, echo_pool=True, isolation_level='READ UNCOMMITTED', connect_args={'connect_timeout': settings.DB['connect_timeout']}) +engine = create_engine(url=connection_string, poolclass=NullPool, echo=False, isolation_level='READ COMMITTED', connect_args={'connect_timeout': settings.DB['connect_timeout']}) +# engine = create_engine(url=connection_string, pool_size=5, max_overflow=15, pool_recycle=settings.DB['pool_recycle'], pool_pre_ping=True, echo=False, echo_pool=True, isolation_level='READ UNCOMMITTED', connect_args={'connect_timeout': settings.DB['connect_timeout']}) # NOTE: The default isolation_level is 'REPEATABLE READ'. This can sometimes not show updated data. # NOTE: The "echo" set to True option shows the SQL queries. # NOTE: Switching from READ COMMITTED to READ UNCOMMITTED (or REPEATABLE READ). Testing started 2024-04-23 +# levels: "REPEATABLE READ" "READ COMMITTED" "READ UNCOMMITTED" "SERIALIZABLE" log.info('DB SQL trying to connect...') try: @@ -60,7 +63,8 @@ def sql_connect( settings.SQLALCHEMY_DB_URI = 'mysql://'+DB['username']+':'+DB['password']+'@'+DB['server']+'/'+DB['name'] log.debug(settings.DB) - new_engine = create_engine(url=settings.SQLALCHEMY_DB_URI, pool_size=5, max_overflow=15, pool_recycle=settings.DB['pool_recycle'], pool_pre_ping=True, echo=False, echo_pool=True, isolation_level='READ UNCOMMITTED', connect_args={'connect_timeout': settings.DB['connect_timeout']}) + new_engine = create_engine(url=settings.SQLALCHEMY_DB_URI, poolclass=NullPool, echo=False, isolation_level='READ UNCOMMITTED', connect_args={'connect_timeout': settings.DB['connect_timeout']}) + # new_engine = create_engine(url=settings.SQLALCHEMY_DB_URI, pool_size=5, max_overflow=15, pool_recycle=settings.DB['pool_recycle'], pool_pre_ping=True, echo=False, echo_pool=True, isolation_level='READ UNCOMMITTED', connect_args={'connect_timeout': settings.DB['connect_timeout']}) current_db.engine = new_engine log.info(f'Created and connected to database: {settings.SQLALCHEMY_DB_URI}') diff --git a/app/main.py b/app/main.py index b8f8e0e..a9f47d7 100644 --- a/app/main.py +++ b/app/main.py @@ -10,9 +10,6 @@ from functools import lru_cache from pydantic import BaseModel, EmailStr, Field from typing import Dict, List, Optional, Set, Union -# from sqlalchemy import create_engine, text -# from sqlalchemy.exc import IntegrityError, OperationalError - from . import config # from app.lib_general import common_route_params, Common_Route_Params from app.log import log, logging diff --git a/app/routers/e_confex.py b/app/routers/e_confex.py index bb85914..c4784b5 100644 --- a/app/routers/e_confex.py +++ b/app/routers/e_confex.py @@ -27,13 +27,14 @@ router = APIRouter() # ### BEGIN ### API Confex ### import_event_session_list() ### -# Updated 2023-04-1 +# processing time: 156 records @ 1457 seconds (sessions, locations, presentations, presenters, files) +# Updated 2024-04-25 @router.get('/event/{e_confex_event_id}/import_reg', response_model=Resp_Body_Base) async def import_event_session_list( - e_confex_event_id: str = Query(..., min_length=5, max_length=22), # For AAPOR: aapor_2023 (2023-05) + e_confex_event_id: str = Query(..., min_length=5, max_length=22), # For AAPOR: aapor_2023 (2024-05); aapor_2023 (2023-05) - event_id: str = Query(..., min_length=11, max_length=22), # For AAPOR: 9jW-Db-SF-wt (1478 2023-05); x2H2P2MYlXU (1447 2022-05) - # Account ID For AAPOR: j5EBhRDqPuw + event_id: str = Query(..., min_length=11, max_length=22), # For AAPOR: MCz-Qm-48-j1 (1513 2024-05) 9jW-Db-SF-wt (1478 2023-05); x2H2P2MYlXU (1447 2022-05) + # Account ID For AAPOR: j5EBhRDqPuw (20) begin_loop: int = 1, end_loop: int = 250, @@ -507,6 +508,8 @@ async def import_event_session_list( # return False # Presentation File List for Presenter + # NOTE: Only the files under ChildList_VendorFiles where used for AAPOR 2023. + # NOTE: There are more files under ChildList_Files. This was ignored for AAPOR 2023. log.info('Get presentation files for a presenter...') log.debug(confex_presentation_detail.get('ChildList_VendorFiles')) event_presentation_data['confex_file_list'] = []