Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improving the CRUD v2 file export

Browse Source
This commit is contained in:
Scott Idem
2024-05-03 14:49:55 -04:00
parent 5a107031e6
commit da86206a24
3 changed files with 95 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
app/ae_obj_types_def.py
View File

@@ -59,7 +59,40 @@ from app.models.e_stripe_models import *
obj_type_kv_li = { obj_type_kv_li = {
'sponsorship': { 'sponsorship': {
'tbl': 'sponsorship', 'tbl_default': 'v_sponsorship', 'tbl_update': 'sponsorship', 'tbl': 'sponsorship', 'tbl_default': 'v_sponsorship', 'tbl_update': 'sponsorship',
'mdl': Sponsorship_Base, 'mdl_default': Sponsorship_Base, 'mdl_in': Sponsorship_Base, 'mdl_out': Sponsorship_Base 'mdl': Sponsorship_Base, 'mdl_default': Sponsorship_Base, 'mdl_in': Sponsorship_Base, 'mdl_out': Sponsorship_Base,
'exp_default': [
'sponsorship_id_random',
# 'account_id_random', 'sponsorship_cfg_id_random',
'name', 'description', 'website_url',
# 'logo_li_json', 'media_li_json',
# 'social_li_json', 'address_li_json', 'contact_li_json',
# 'other',
'level_num', 'level_str',
# 'amount', 'paid',
'agree',
# 'selected_option_li_json', 'questions_li_json',
'guest_li_json',
# 'staff_notes',
'comments',
'enable', 'hide', 'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on',
# 'poc__full_name', 'poc__email', 'poc__phone_mobile',
# 'address__mailing',
# 'logo__primary', 'logo__light', 'logo__dark',
# 'questions__accommodations', 'questions__accommodations_text',
# 'questions__public_recognition', 'questions__social_email', 'questions__dedicated_promos', 'questions__table_exhibit', 'questions__table_power', 'questions__plenary_session', 'questions__plenary_opening', 'questions__virtual_session',
# 'social__facebook', 'social__twitter', 'social__instagram', 'social__linkedin', 'social__org',
# 'organization_json', 'person_json', 'group_li_json', 'poc_json',
# 'sponsorship_cfg_id_random', 'sponsorship_cfg_name', , 'account_name',
# 'contact__full_name', 'contact__email', 'contact__phone',
# 'contact__1', 'contact__2', 'contact__3',
# 'guest__title',
# 'guest__dietary', 'guest__full_name', 'guest__comments', 'guest__given_name', , 'guest__ada', 'guest__family_name', 'guest__affiliations',
# 'guest__1', 'guest__2', 'guest__3', 'guest__4', 'guest__5', 'guest__6', 'guest__7', 'guest__8',
],
}, },
'sponsorship_cfg': { 'sponsorship_cfg': {
'tbl': 'sponsorship_cfg', 'tbl_default': 'v_sponsorship_cfg', 'tbl_update': 'sponsorship_cfg', 'tbl': 'sponsorship_cfg', 'tbl_default': 'v_sponsorship_cfg', 'tbl_update': 'sponsorship_cfg',

18
app/lib_general.py
View File

@@ -388,13 +388,27 @@ def create_export_file(
full_dest_path = file_dest_w_subdir+'.csv' full_dest_path = file_dest_w_subdir+'.csv'
filename_w_ext = filename+'.csv' filename_w_ext = filename+'.csv'
tmp_file_path = os.path.join(subdir_path,filename_w_ext) tmp_file_path = os.path.join(subdir_path,filename_w_ext)
data_dataframe.to_csv(full_dest_path, columns=column_name_li, index=False) data_dataframe.to_csv(
full_dest_path,
# na_rep='NULL',
columns=column_name_li,
index=False,
# errors='ignore',
)
elif export_type == 'Excel': elif export_type == 'Excel':
log.info('Saving dataframe to Excel file') log.info('Saving dataframe to Excel file')
full_dest_path = file_dest_w_subdir+'.xlsx' full_dest_path = file_dest_w_subdir+'.xlsx'
filename_w_ext = filename+'.xlsx' filename_w_ext = filename+'.xlsx'
tmp_file_path = os.path.join(subdir_path,filename_w_ext) tmp_file_path = os.path.join(subdir_path,filename_w_ext)
data_dataframe.to_excel(full_dest_path, columns=column_name_li, index=False) # sheet_name='Sheet_name_1' # This should ignore unknown columns
data_dataframe.to_excel(
full_dest_path,
# na_rep='NULL',
columns=column_name_li,
index=False,
# engine='openpyxl',
# errors='ignore',
)
except: except:
log.exception('Something went wrong while trying to save the export file.') log.exception('Something went wrong while trying to save the export file.')
return False return False

58
app/routers/api_crud_v2.py
View File

@@ -70,6 +70,7 @@ async def get_obj_li_l1(
tbl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real SQL database table or view name to use. tbl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real SQL database table or view name to use.
mdl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real Python Pydantic model name to use. mdl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real Python Pydantic model name to use.
exp_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for a list of column names to use.
# use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17 # use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17
# use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17 # use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17
@@ -114,6 +115,7 @@ async def get_obj_li_l1(
tbl_alt=tbl_alt, tbl_alt=tbl_alt,
mdl_alt=mdl_alt, mdl_alt=mdl_alt,
exp_alt=exp_alt,
hidden=hidden, hidden=hidden,
order_by_li=order_by_li, order_by_li=order_by_li,
@@ -137,6 +139,7 @@ async def get_obj_li_l2(
tbl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real SQL database table or view name to use. tbl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real SQL database table or view name to use.
mdl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real Python Pydantic model name to use. mdl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real Python Pydantic model name to use.
exp_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for a list of column names to use.
# use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17 # use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17
# use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17 # use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17
@@ -182,6 +185,7 @@ async def get_obj_li_l2(
tbl_alt=tbl_alt, tbl_alt=tbl_alt,
mdl_alt=mdl_alt, mdl_alt=mdl_alt,
exp_alt=exp_alt,
hidden=hidden, hidden=hidden,
order_by_li=order_by_li, order_by_li=order_by_li,
@@ -206,6 +210,7 @@ async def get_obj_li_l3(
tbl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real SQL database table or view name to use. tbl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real SQL database table or view name to use.
mdl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real Python Pydantic model name to use. mdl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real Python Pydantic model name to use.
exp_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for a list of column names to use.
# use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17 # use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17
# use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17 # use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17
@@ -252,6 +257,7 @@ async def get_obj_li_l3(
tbl_alt=tbl_alt, tbl_alt=tbl_alt,
mdl_alt=mdl_alt, mdl_alt=mdl_alt,
exp_alt=exp_alt,
hidden=hidden, hidden=hidden,
order_by_li=order_by_li, order_by_li=order_by_li,
@@ -275,6 +281,7 @@ def handle_get_obj_li(
tbl_alt: Optional[str] = 'default', tbl_alt: Optional[str] = 'default',
mdl_alt: Optional[str] = 'default', mdl_alt: Optional[str] = 'default',
exp_alt: Optional[str] = 'default',
hidden: str = 'not_hidden', hidden: str = 'not_hidden',
order_by_li: Optional[str] = None, order_by_li: Optional[str] = None,
@@ -353,6 +360,7 @@ def handle_get_obj_li(
debug_data['for_obj_id'] = for_obj_id debug_data['for_obj_id'] = for_obj_id
debug_data['tbl_alt'] = tbl_alt debug_data['tbl_alt'] = tbl_alt
debug_data['mdl_alt'] = mdl_alt debug_data['mdl_alt'] = mdl_alt
debug_data['exp_alt'] = exp_alt
# debug_data['use_alt_table'] = use_alt_table # debug_data['use_alt_table'] = use_alt_table
# debug_data['use_alt_base'] = use_alt_base # debug_data['use_alt_base'] = use_alt_base
debug_data['jp_obj'] = jp_obj debug_data['jp_obj'] = jp_obj
@@ -401,6 +409,9 @@ def handle_get_obj_li(
base_name = obj_type_kv_li[obj_name][f'mdl_{mdl_alt}'] base_name = obj_type_kv_li[obj_name][f'mdl_{mdl_alt}']
# base_name = obj_type_kv_li[obj_name].get(f'mdl_{mdl_alt}') # base_name = obj_type_kv_li[obj_name].get(f'mdl_{mdl_alt}')
log.info(f'mdl_alt was found. Using {base_name} model.') log.info(f'mdl_alt was found. Using {base_name} model.')
if exp_alt:
column_name_li = obj_type_kv_li[obj_name].get(f'exp_{exp_alt}')
log.info(f'exp_alt was found. Using {column_name_li} column list.')
# if use_alt_table: # if use_alt_table:
# table_name = obj_type_kv_li[obj_name]['table_name_alt'] # table_name = obj_type_kv_li[obj_name]['table_name_alt']
@@ -475,10 +486,12 @@ def handle_get_obj_li(
log.warning('base_name model was not found. Returning raw data.') log.warning('base_name model was not found. Returning raw data.')
resp_data = record resp_data = record
resp_data_li.append(resp_data) resp_data_li.append(resp_data)
column_name_li = list(sql_result[0].keys()) # This should be the same for all records in the list. if not column_name_li:
column_name_li = list(sql_result[0].keys()) # This should be the same for all records in the list.
additional_column_name_li = []
if return_file: if return_file:
log.setLevel(logging.DEBUG) log.setLevel(logging.INFO)
# We want to handle any field that has a suffix of _json. It should be expanded into a list of fields that are prefixed with the original field name minus _json. # We want to handle any field that has a suffix of _json. It should be expanded into a list of fields that are prefixed with the original field name minus _json.
@@ -490,8 +503,7 @@ def handle_get_obj_li(
for field_name in record.keys(): for field_name in record.keys():
field_value = record[field_name] field_value = record[field_name]
if field_name.endswith('li_json') and record[field_name]: if field_name.endswith('li_json') and record[field_name]:
log.info(f'Found a field that ends with li_json: {field_name}') log.info(f'Found a field that ends with li_json: {field_name}: {field_value}')
log.info(f'Field value is a list??: {field_value}')
# field_value = json.loads(record[key]) # Convert the string to a list of dictionaries # field_value = json.loads(record[key]) # Convert the string to a list of dictionaries
@@ -501,25 +513,38 @@ def handle_get_obj_li(
if isinstance(field_value, list): if isinstance(field_value, list):
# Loop through the list of dictionaries # Loop through the list of dictionaries
log.info(f'Field value is a list: {field_value}') log.info(f'Field value is a list (int key): {field_value}')
for item in field_value: key = 1
# item = json.loads(item) # Convert the string to a dictionary for value in field_value:
for key, value in item.items(): new_field_name = field_name[:-8]+'__'+str(key)
new_field_name = field_name[:-8]+'__'+key new_record[new_field_name] = value
new_record[new_field_name] = value additional_column_name_li.append(new_field_name)
key += 1
# for item in field_value:
# # item = json.loads(item) # Convert the string to a dictionary
# for key, value in item.items():
# new_field_name = field_name[:-8]+'__'+key
# new_record[new_field_name] = value
# additional_column_name_li.append(new_field_name)
# We want to keep the original field in this case. This way the list of data will be displayed in one single field and expanded to one field per item in the list.
# new_record.pop(field_name) # Remove the original field
else: else:
# Loop through key value pairs in the dictionary # Loop through key value pairs in the dictionary
log.info(f'Field value is a dict: {field_value}') log.info(f'Field value is a dict (list): {field_value}')
for key, value in field_value.items(): for key, value in field_value.items():
log.debug(f'Key: {key}') log.debug(f'Key: {key}')
new_field_name = field_name[:-8]+'__'+key new_field_name = field_name[:-8]+'__'+key
new_record[new_field_name] = value new_record[new_field_name] = value
additional_column_name_li.append(new_field_name)
new_record.pop(field_name) # Remove the original field
# Create a new field for each and value to the record # Create a new field for each and value to the record
# new_field_name = key[:-8]+'__cust_li' # new_field_name = key[:-8]+'__cust_li'
# new_record[new_field_name] = record[key] # new_record[new_field_name] = record[key]
new_record.pop(field_name) # Remove the original field
elif field_name.endswith('_json') and record[field_name]: elif field_name.endswith('_json') and record[field_name]:
log.info(f'Found a field that ends with _json: {field_name}') log.info(f'Found a field that ends with _json: {field_name}')
log.info(f'Field value is a dict???: {field_value}') log.info(f'Field value is a dict???: {field_value}')
@@ -527,6 +552,7 @@ def handle_get_obj_li(
for key, value in field_value.items(): for key, value in field_value.items():
new_field_name = field_name[:-5]+'__'+key new_field_name = field_name[:-5]+'__'+key
new_record[new_field_name] = value new_record[new_field_name] = value
additional_column_name_li.append(new_field_name)
new_record.pop(field_name) # Remove the original field new_record.pop(field_name) # Remove the original field
elif field_name.endswith('li_json') or field_name.endswith('_json'): elif field_name.endswith('li_json') or field_name.endswith('_json'):
@@ -548,7 +574,13 @@ def handle_get_obj_li(
filename_w_ext = filename+'.xlsx' filename_w_ext = filename+'.xlsx'
log.setLevel(logging.INFO) log.setLevel(logging.INFO)
if result := create_export_file(data_dict_list=new_resp_data_li, column_name_li=[], subdir_path=obj_name, filename=filename, rm_id=True, export_type=file_type): if not exp_alt and additional_column_name_li:
column_name_li = column_name_li + sorted(list(set(additional_column_name_li)))
elif additional_column_name_li:
column_name_li = column_name_li + sorted(list(set(additional_column_name_li)))
log.info(f'Column names: {column_name_li}')
log.info(f'Additional column names: {sorted(set(additional_column_name_li))}')
if result := create_export_file(data_dict_list=new_resp_data_li, column_name_li=column_name_li, subdir_path=obj_name, filename=filename, rm_id=True, export_type=file_type):
log.info(f'Export file created and saved: {result}') log.info(f'Export file created and saved: {result}')
else: else:
log.error('Something went wrong while creating or saving the export file') log.error('Something went wrong while creating or saving the export file')