From 53c6c6b51c8585043bf67980efe934700ceace38 Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Tue, 19 Oct 2021 21:21:11 -0400 Subject: [PATCH] Work on event importing and related --- app/main.py | 7 +- app/methods/event_methods.py | 3 +- app/methods/event_session_methods.py | 3 +- app/models/event_presentation_models.py | 4 + app/models/event_presenter_models.py | 5 + app/models/event_session_models.py | 4 + app/routers/event.py | 8 +- app/routers/event_importing.py | 318 ++++++++++++++++++++++++ 8 files changed, 345 insertions(+), 7 deletions(-) create mode 100644 app/routers/event_importing.py diff --git a/app/main.py b/app/main.py index 6464a58..1251f2c 100644 --- a/app/main.py +++ b/app/main.py @@ -18,7 +18,7 @@ from app.lib_general import log, logging from app.log import log # Import the routers here first: -from app.routers import api_crud, api, importing, account, address, archive, archive_content, contact, cont_edu_cert, cont_edu_cert_person, event, event_badge, event_badge_template, event_exhibit, event_file, event_location, event_person, event_person_detail, event_person_tracking, event_presentation, event_presenter, event_registration, event_session, flask_cfg, hosted_file, journal, journal_entry, log_client_viewing, lookup, membership_cfg, membership_group, membership_group_person, membership_person, membership_person_profile, membership_type, membership_type_person, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets#, e_impexium +from app.routers import api_crud, api, importing, account, address, archive, archive_content, contact, cont_edu_cert, cont_edu_cert_person, event, event_badge, event_badge_template, event_exhibit, event_file, event_importing, event_location, event_person, event_person_detail, event_person_tracking, event_presentation, event_presenter, event_registration, event_session, flask_cfg, hosted_file, journal, journal_entry, log_client_viewing, lookup, membership_cfg, membership_group, membership_group_person, membership_person, membership_person_profile, membership_type, membership_type_person, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets#, e_impexium from app.db_sql import db @@ -132,6 +132,11 @@ app.include_router( prefix='/event/file', tags=['Event File'], ) +app.include_router( + event_importing.router, + # prefix='/event/importing', + tags=['Event Importing'], +) app.include_router( event_location.router, # prefix='/event/location', diff --git a/app/methods/event_methods.py b/app/methods/event_methods.py index a385f11..09391d8 100644 --- a/app/methods/event_methods.py +++ b/app/methods/event_methods.py @@ -62,7 +62,7 @@ def load_event_obj( exclude_unset: bool = True, model_as_dict: bool = False, ) -> Event_Base|bool: - log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if event_id := redis_lookup_id_random(record_id_random=event_id, table_name='event'): pass @@ -231,6 +231,7 @@ def load_event_obj( enabled = enabled, limit = limit, inc_event_file_list = inc_event_file_list, + inc_event_location = inc_event_location, inc_event_presentation_list = inc_event_presentation_list, inc_event_presenter_cat = inc_event_presenter_cat, inc_event_presenter_list = inc_event_presenter_list, diff --git a/app/methods/event_session_methods.py b/app/methods/event_session_methods.py index 0b137df..37cb424 100644 --- a/app/methods/event_session_methods.py +++ b/app/methods/event_session_methods.py @@ -53,7 +53,7 @@ def load_event_session_obj( exclude_unset: bool = True, model_as_dict: bool = False, ) -> Event_Session_Base|bool: - log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if event_session_id := redis_lookup_id_random(record_id_random=event_session_id, table_name='event_session'): pass @@ -136,6 +136,7 @@ def load_event_session_obj( # event_session_obj.event_file_list = event_file_dict_list # else: event_session_obj.event_file_list = [] + log.debug(f'Get event location? Include Event Location: {inc_event_location} Event Location ID: {event_location_id}') if inc_event_location and event_location_id: log.info('Need to include event location...') if event_location_obj := load_event_location_obj( diff --git a/app/models/event_presentation_models.py b/app/models/event_presentation_models.py index 66d1bac..cba1c31 100644 --- a/app/models/event_presentation_models.py +++ b/app/models/event_presentation_models.py @@ -29,6 +29,10 @@ class Event_Presentation_Base(BaseModel): alias = 'event_presentation_id' ) + external_id: Optional[str] = Field( + alias = 'event_presentation_external_id' + ) + code: Optional[str] = Field( alias = 'event_presentation_code' ) diff --git a/app/models/event_presenter_models.py b/app/models/event_presenter_models.py index ae609bc..73d6a7f 100644 --- a/app/models/event_presenter_models.py +++ b/app/models/event_presenter_models.py @@ -30,10 +30,15 @@ class Event_Presenter_Base(BaseModel): alias = 'event_presenter_id' ) + external_id: Optional[str] = Field( + alias = 'event_presenter_external_id' + ) + code: Optional[str] = Field( alias = 'event_presenter_code' ) + account_id_random: Optional[str] account_id: Optional[int] diff --git a/app/models/event_session_models.py b/app/models/event_session_models.py index 5f903c6..32a5aa4 100644 --- a/app/models/event_session_models.py +++ b/app/models/event_session_models.py @@ -29,6 +29,10 @@ class Event_Session_Base(BaseModel): alias = 'event_session_id' ) + external_id: Optional[str] = Field( + alias = 'event_session_external_id' + ) + code: Optional[str] = Field( alias = 'event_session_code' ) diff --git a/app/routers/event.py b/app/routers/event.py index 44ad161..fec91d1 100644 --- a/app/routers/event.py +++ b/app/routers/event.py @@ -208,7 +208,7 @@ async def get_event_obj_li( # Working well as of 2021-06-30. Using as a template for other routes. @router.get('/event/{event_id}', response_model=Resp_Body_Base) async def get_event_obj( - event_id: str = Query(..., min_length=1, max_length=22), + event_id: str = Query(..., min_length=11, max_length=22), enabled: str = 'enabled', # enabled, disabled, all approved: str = 'all', # approve(d), not_approved, all hidden: str = 'not_hidden', # hidden, not_hidden, all @@ -270,7 +270,7 @@ async def get_event_obj( # inc_event_device_list = inc_event_device_list, # inc_event_exhibit_list = inc_event_exhibit_list, inc_event_file_list = inc_event_file_list, - # inc_event_location = inc_event_location, + inc_event_location = inc_event_location, inc_event_location_list = inc_event_location_list, # inc_event_person = inc_event_person, inc_event_person_list = inc_event_person_list, @@ -322,7 +322,7 @@ async def get_account_obj_event_list( # inc_event_device_list: bool = False, # inc_event_exhibit_list: bool = False, inc_event_file_list: bool = False, - # inc_event_location: bool = False, + inc_event_location: bool = False, # Currently only specific to an event_session inc_event_location_list: bool = False, # inc_event_person: bool = False, inc_event_person_list: bool = False, @@ -380,7 +380,7 @@ async def get_account_obj_event_list( # inc_event_device_list = inc_event_device_list, # inc_event_exhibit_list = inc_event_exhibit_list, inc_event_file_list = inc_event_file_list, - # inc_event_location = inc_event_location, + inc_event_location = inc_event_location, inc_event_location_list = inc_event_location_list, # inc_event_person = inc_event_person, inc_event_person_list = inc_event_person_list, diff --git a/app/routers/event_importing.py b/app/routers/event_importing.py new file mode 100644 index 0000000..3881ddd --- /dev/null +++ b/app/routers/event_importing.py @@ -0,0 +1,318 @@ +import datetime, json, pytz, secrets, time +import pandas, xlrd # qrcode +from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, Response, status +from pydantic import BaseModel, EmailStr, Field +from typing import Dict, List, Optional, Set, Union + +from app.lib_general import log, logging, secure_hash_string +from app.config import settings +from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random + +from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template + +# from app.methods.event_session_methods import create_update_event_session_obj_v4, get_event_session_rec_list, load_event_session_obj, update_event_session_obj +# from app.methods.event_presentation_methods import create_update_event_presentation_obj_v4, get_event_presentation_rec_list, load_event_presentation_obj +# from app.methods.event_presenter_methods import create_update_event_presenter_obj_v4, get_event_presenter_rec_list, load_event_presenter_obj + +from app.models.event_models import Event_Base +from app.models.event_location_models import Event_Location_Base +from app.models.event_person_models import Event_Person_Base +from app.models.event_presentation_models import Event_Presentation_Base +from app.models.event_presenter_models import Event_Presenter_Base +from app.models.event_session_models import Event_Session_Base + +from app.models.response_models import Resp_Body_Base, mk_resp + + +router = APIRouter() + + +# Based on the program import template the clients are given. +# Ideally the import file should only contain records with new External IDs. Old records will be checked and only updated if needed. +# Updated 2021-10-19 +@router.get('/event/importing/create_update_w_external_id', response_model=Resp_Body_Base) +async def event_importing_create_update_w_external_id( + response: Response = Response, + x_account_id: str = Header(...), + ): + log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + account_id = 2 # CMSC 2 8Gfxbxr19Nw + event_id = 1438 # CMSC 2021 1438 8nMUlA93Ybw + full_file_path = 'admin/temp/import_event_program_external_id.csv' + + df = pandas.read_csv(full_file_path, na_filter=False, dtype={'external_id':str, 'External ID':str, 'event_presentation_external_id':str, 'event_presentation_code':str, 'event_presentation_sort':int, 'event_presenter_external_id':str, 'event_presenter_code':str, 'event_presenter_designations':str, 'event_presenter_sort':int, 'event_session_external_id':str, 'event_session_code':str, 'event_session_sort':int, 'source_id':str, 'Source ID':str, 'email':str, 'Email Address':str}) + log.debug(df) + + df_dict = df.to_dict(orient='records') + + event_session_data_li = [] + + for record in df_dict: + + if event_location_external_id := record.get('event_location_external_id', None): pass + elif event_location_external_id := record.get('event_location_code', None): + log.info('No event location external ID was found. Creating one with the location code...') + else: + log.info('No event location external ID was found. Creating one with the name...') + + event_location_name = record.get('event_location_name', '') + + event_location_external_id = (str(event_id) + ':' + event_location_name).strip() + log.debug(f'Event Location External ID: {event_location_external_id}') + + if event_session_external_id := record.get('event_session_external_id', None): pass + else: + log.error('No event session external ID was found.') + continue + + if event_presentation_external_id := record.get('event_presentation_external_id', None): pass + else: + log.error('No event presentation external ID was found.') + continue + + if event_presenter_external_id := record.get('event_presenter_external_id', None): pass + else: + log.info('No event presenter external ID was found. Creating one...') + + last_first_name = ', '.join( [record.get('event_presenter_family_name', ''), record.get('event_presenter_given_name', '')] ) + + event_presenter_external_id = (event_presentation_external_id + ':' + last_first_name).strip() + log.debug(f'Event Presenter External ID: {event_presenter_external_id}') + + data = {} + data['account_id'] = account_id + data['event_id'] = event_id + + # INSERT or UPDATE event_location + data['event_location_external_id'] = event_location_external_id + + sql = f""" + SELECT * + FROM `v_event_location` AS `event_location` + WHERE /*event_location.account_id = :account_id + AND*/ event_location.event_id = :event_id + AND event_location.external_id = :event_location_external_id + LIMIT 1; + """ + + event_location_data = {} + event_location_data['code'] = record.get('event_location_code', '') + + event_location_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case + + event_location_data['name'] = record.get('event_location_name', '') + + event_location_id = None + if event_location_rec_result := sql_select(data=data, sql=sql): + # Pull out IDs and UPDATE existing event_location record + event_location_rec = event_location_rec_result + event_location_id = event_location_rec.get('event_location_id', None) + event_location_id_random = event_location_rec.get('event_location_id_random', None) + event_location_name = event_location_rec.get('name', None) + log.info(f'Found one record. Event Location ID: {event_location_id_random} Event Location Name: {event_location_name}') + + event_location_data['id'] = event_location_id + + log.debug(event_location_data) + + if event_location_obj_up_result := sql_update(data=event_location_data, table_name='event_location'): + log.debug(event_location_obj_up_result) + else: + log.warning(event_location_obj_up_result) + pass # Probably nothing to update + else: + log.info(f'No record found that needs to be updated. Creating a new event location. Event Location External ID: {event_location_external_id}') + + event_location_data['external_id'] = event_location_external_id # NOTE: This should not change + + log.debug(event_location_data) + + if event_location_obj_in_result := sql_insert(data=event_location_data, table_name='event_location'): + log.debug(event_location_obj_in_result) + event_location_id = event_location_obj_in_result + else: + log.warning(event_location_obj_in_result) + continue # Something unexpected may have happened + + # INSERT or UPDATE event_session + data['event_session_external_id'] = event_session_external_id + + sql = f""" + SELECT * + FROM `v_event_session` AS `event_session` + WHERE event_session.account_id = :account_id + AND event_session.event_id = :event_id + AND event_session.external_id = :event_session_external_id + LIMIT 1; + """ + + event_session_data = {} + event_session_data['code'] = record.get('event_session_code', '') + + event_session_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case + event_session_data['event_location_id'] = event_location_id # INSERT and UPDATE the event_location_id in case the session was moved + + event_session_data['name'] = record.get('event_session_name', '') + + event_session_start_datetime = record.get('event_session_start_date', '') + ' ' + record.get('event_session_start_time', '') + event_session_data['start_datetime'] = event_session_start_datetime + + event_session_end_datetime = record.get('event_session_end_date', '') + ' ' + record.get('event_session_end_time', '') + event_session_data['end_datetime'] = event_session_end_datetime + + if event_session_rec_result := sql_select(data=data, sql=sql): + # Pull out IDs and UPDATE existing event_session record + event_session_rec = event_session_rec_result + event_session_id = event_session_rec.get('event_session_id', None) + event_session_id_random = event_session_rec.get('event_session_id_random', None) + event_session_name = event_session_rec.get('name', None) + log.info(f'Found one record. Event Session ID: {event_session_id_random} Event Session Name: {event_session_name}') + + event_session_data['id'] = event_session_id + + log.debug(event_session_data) + + if event_session_obj_up_result := sql_update(data=event_session_data, table_name='event_session'): + log.debug(event_session_obj_up_result) + else: + log.warning(event_session_obj_up_result) + pass # Probably nothing to update + else: + log.info(f'No record found that needs to be updated. Creating a new event session. Event Session External ID: {event_session_external_id}') + + event_session_data['external_id'] = event_session_external_id # NOTE: This should not change + + log.debug(event_session_data) + + if event_session_obj_in_result := sql_insert(data=event_session_data, table_name='event_session'): + log.debug(event_session_obj_in_result) + event_session_id = event_session_obj_in_result + else: + log.warning(event_session_obj_in_result) + continue # Something unexpected may have happened + + # INSERT or UPDATE event_presentation + data['event_presentation_external_id'] = event_presentation_external_id + + sql = f""" + SELECT * + FROM `v_event_presentation` AS `event_presentation` + WHERE event_presentation.event_id = :event_id + AND event_presentation.external_id = :event_presentation_external_id + LIMIT 1; + """ + + event_presentation_data = {} + event_presentation_data['code'] = record.get('event_presentation_code', '') + + event_presentation_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case + event_presentation_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved + + event_presentation_data['name'] = record.get('event_presentation_name', '') + + event_presentation_start_datetime = record.get('event_presentation_start_date', '') + ' ' + record.get('event_presentation_start_time', '') + event_presentation_data['start_datetime'] = event_presentation_start_datetime + + event_presentation_end_datetime = record.get('event_presentation_end_date', '') + ' ' + record.get('event_presentation_end_time', '') + event_presentation_data['end_datetime'] = event_presentation_end_datetime + + event_presentation_data['sort'] = record.get('event_presentation_sort', '') + + event_presentation_data['notes'] = record.get('event_presentation_notes', '') + + if event_presentation_rec_result := sql_select(data=data, sql=sql): + # Pull out IDs and UPDATE existing event_presentation record + event_presentation_rec = event_presentation_rec_result + event_presentation_id = event_presentation_rec.get('event_presentation_id', None) + event_presentation_id_random = event_presentation_rec.get('event_presentation_id_random', None) + event_presentation_name = event_presentation_rec.get('name', None) + log.info(f'Found one record. Event Presentation ID: {event_presentation_id_random} Event Presentation Name: {event_presentation_name}') + + event_presentation_data['id'] = event_presentation_id + + log.debug(event_presentation_data) + + if event_presentation_obj_up_result := sql_update(data=event_presentation_data, table_name='event_presentation'): + log.debug(event_presentation_obj_up_result) + else: + log.warning(event_presentation_obj_up_result) + pass # Probably nothing to update + else: + log.info(f'No record found that needs to be updated. Creating a new event presentation. Event Presentation External ID: {event_presentation_external_id}') + + event_presentation_data['external_id'] = event_presentation_external_id # NOTE: This should not change + + log.debug(event_presentation_data) + + if event_presentation_obj_in_result := sql_insert(data=event_presentation_data, table_name='event_presentation'): + log.debug(event_presentation_obj_in_result) + event_presentation_id = event_presentation_obj_in_result + else: + log.warning(event_presentation_obj_in_result) + continue # Something unexpected may have happened + + # INSERT or UPDATE event_presenter + data['event_presenter_external_id'] = event_presenter_external_id + + sql = f""" + SELECT * + FROM `v_event_presenter` AS `event_presenter` + WHERE event_presenter.event_id = :event_id + AND event_presenter.external_id = :event_presenter_external_id + LIMIT 1; + """ + + event_presenter_data = {} + event_presenter_data['event_id'] = event_id # INSERT and UPDATE the event_id just in case + event_presenter_data['event_session_id'] = event_session_id # UPDATE the event_session_id in case the presentation was moved + event_presenter_data['event_presentation_id'] = event_presentation_id # UPDATE the event_presentation_id in case the presenter was moved + + event_presenter_data['given_name'] = record.get('event_presenter_given_name', '') + event_presenter_data['family_name'] = record.get('event_presenter_family_name', '') + event_presenter_full_name = (event_presenter_data['given_name'] + ' ' + event_presenter_data['family_name']).strip() + event_presenter_data['designations'] = record.get('event_presenter_designations', '') + + event_presenter_data['email'] = record.get('event_presenter_email', '') + + event_presenter_data['notes'] = record.get('event_presenter_notes', '') + + if event_presenter_rec_result := sql_select(data=data, sql=sql): + # Pull out IDs and UPDATE existing event_presenter record + event_presenter_rec = event_presenter_rec_result + event_presenter_id = event_presenter_rec.get('event_presenter_id', None) + event_presenter_id_random = event_presenter_rec.get('event_presenter_id_random', None) + event_presenter_given_name = event_presenter_rec.get('given_name', None) + log.info(f'Found one record. Event Presenter ID: {event_presenter_id_random} Event Presenter Given Name: {event_presenter_given_name}') + + event_presenter_data['id'] = event_presenter_id + + log.debug(event_presenter_data) + + if event_presenter_obj_up_result := sql_update(data=event_presenter_data, table_name='event_presenter'): + log.debug(event_presenter_obj_up_result) + else: + log.warning(event_presenter_obj_up_result) + pass # Probably nothing to update + else: + log.info(f'No record found that needs to be updated. Creating a new event presenter. Event Presenter External ID: {event_presenter_external_id}') + + event_presenter_data['external_id'] = event_presenter_external_id # NOTE: This should not change + + log.debug(event_presenter_data) + + if event_presenter_obj_in_result := sql_insert(data=event_presenter_data, table_name='event_presenter'): + log.debug(event_presenter_obj_in_result) + event_presenter_id = event_presenter_obj_in_result + else: + log.warning(event_presenter_obj_in_result) + continue # Something unexpected may have happened + + + + + event_session_data_li.append(data) + log.debug(f'Record processed: {event_session_id} {event_presentation_id} {event_presenter_full_name}') + + return mk_resp(data=event_session_data_li, response=response)