diff --git a/app/ae_obj_types_def.py b/app/ae_obj_types_def.py index 59317fc..c099b47 100644 --- a/app/ae_obj_types_def.py +++ b/app/ae_obj_types_def.py @@ -59,7 +59,40 @@ from app.models.e_stripe_models import * obj_type_kv_li = { 'sponsorship': { 'tbl': 'sponsorship', 'tbl_default': 'v_sponsorship', 'tbl_update': 'sponsorship', - 'mdl': Sponsorship_Base, 'mdl_default': Sponsorship_Base, 'mdl_in': Sponsorship_Base, 'mdl_out': Sponsorship_Base + 'mdl': Sponsorship_Base, 'mdl_default': Sponsorship_Base, 'mdl_in': Sponsorship_Base, 'mdl_out': Sponsorship_Base, + 'exp_default': [ + 'sponsorship_id_random', + # 'account_id_random', 'sponsorship_cfg_id_random', + 'name', 'description', 'website_url', + # 'logo_li_json', 'media_li_json', + # 'social_li_json', 'address_li_json', 'contact_li_json', + # 'other', + 'level_num', 'level_str', + # 'amount', 'paid', + 'agree', + # 'selected_option_li_json', 'questions_li_json', + 'guest_li_json', + # 'staff_notes', + 'comments', + 'enable', 'hide', 'priority', 'sort', 'group', 'notes', 'created_on', 'updated_on', + + # 'poc__full_name', 'poc__email', 'poc__phone_mobile', + # 'address__mailing', + # 'logo__primary', 'logo__light', 'logo__dark', + # 'questions__accommodations', 'questions__accommodations_text', + # 'questions__public_recognition', 'questions__social_email', 'questions__dedicated_promos', 'questions__table_exhibit', 'questions__table_power', 'questions__plenary_session', 'questions__plenary_opening', 'questions__virtual_session', + + # 'social__facebook', 'social__twitter', 'social__instagram', 'social__linkedin', 'social__org', + + # 'organization_json', 'person_json', 'group_li_json', 'poc_json', + # 'sponsorship_cfg_id_random', 'sponsorship_cfg_name', , 'account_name', + # 'contact__full_name', 'contact__email', 'contact__phone', + # 'contact__1', 'contact__2', 'contact__3', + + # 'guest__title', + # 'guest__dietary', 'guest__full_name', 'guest__comments', 'guest__given_name', , 'guest__ada', 'guest__family_name', 'guest__affiliations', + # 'guest__1', 'guest__2', 'guest__3', 'guest__4', 'guest__5', 'guest__6', 'guest__7', 'guest__8', + ], }, 'sponsorship_cfg': { 'tbl': 'sponsorship_cfg', 'tbl_default': 'v_sponsorship_cfg', 'tbl_update': 'sponsorship_cfg', diff --git a/app/lib_general.py b/app/lib_general.py index 2c5f14d..cf52cc3 100644 --- a/app/lib_general.py +++ b/app/lib_general.py @@ -388,13 +388,27 @@ def create_export_file( full_dest_path = file_dest_w_subdir+'.csv' filename_w_ext = filename+'.csv' tmp_file_path = os.path.join(subdir_path,filename_w_ext) - data_dataframe.to_csv(full_dest_path, columns=column_name_li, index=False) + data_dataframe.to_csv( + full_dest_path, + # na_rep='NULL', + columns=column_name_li, + index=False, + # errors='ignore', + ) elif export_type == 'Excel': log.info('Saving dataframe to Excel file') full_dest_path = file_dest_w_subdir+'.xlsx' filename_w_ext = filename+'.xlsx' tmp_file_path = os.path.join(subdir_path,filename_w_ext) - data_dataframe.to_excel(full_dest_path, columns=column_name_li, index=False) # sheet_name='Sheet_name_1' + # This should ignore unknown columns + data_dataframe.to_excel( + full_dest_path, + # na_rep='NULL', + columns=column_name_li, + index=False, + # engine='openpyxl', + # errors='ignore', + ) except: log.exception('Something went wrong while trying to save the export file.') return False diff --git a/app/routers/api_crud_v2.py b/app/routers/api_crud_v2.py index 3628880..be2abf6 100644 --- a/app/routers/api_crud_v2.py +++ b/app/routers/api_crud_v2.py @@ -70,6 +70,7 @@ async def get_obj_li_l1( tbl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real SQL database table or view name to use. mdl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real Python Pydantic model name to use. + exp_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for a list of column names to use. # use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17 # use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17 @@ -114,6 +115,7 @@ async def get_obj_li_l1( tbl_alt=tbl_alt, mdl_alt=mdl_alt, + exp_alt=exp_alt, hidden=hidden, order_by_li=order_by_li, @@ -137,6 +139,7 @@ async def get_obj_li_l2( tbl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real SQL database table or view name to use. mdl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real Python Pydantic model name to use. + exp_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for a list of column names to use. # use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17 # use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17 @@ -182,6 +185,7 @@ async def get_obj_li_l2( tbl_alt=tbl_alt, mdl_alt=mdl_alt, + exp_alt=exp_alt, hidden=hidden, order_by_li=order_by_li, @@ -206,6 +210,7 @@ async def get_obj_li_l3( tbl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real SQL database table or view name to use. mdl_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for the real Python Pydantic model name to use. + exp_alt: Optional[str] = Query('default', max_length=50), # This is used as a lookup for a list of column names to use. # use_alt_table: bool = False, # NOTE: This will use table_name_alt if they exist. -2023-11-17 # use_alt_base: bool = False, # NOTE: This will use base_name_alt if they exist. -2023-11-17 @@ -252,6 +257,7 @@ async def get_obj_li_l3( tbl_alt=tbl_alt, mdl_alt=mdl_alt, + exp_alt=exp_alt, hidden=hidden, order_by_li=order_by_li, @@ -275,6 +281,7 @@ def handle_get_obj_li( tbl_alt: Optional[str] = 'default', mdl_alt: Optional[str] = 'default', + exp_alt: Optional[str] = 'default', hidden: str = 'not_hidden', order_by_li: Optional[str] = None, @@ -353,6 +360,7 @@ def handle_get_obj_li( debug_data['for_obj_id'] = for_obj_id debug_data['tbl_alt'] = tbl_alt debug_data['mdl_alt'] = mdl_alt + debug_data['exp_alt'] = exp_alt # debug_data['use_alt_table'] = use_alt_table # debug_data['use_alt_base'] = use_alt_base debug_data['jp_obj'] = jp_obj @@ -401,6 +409,9 @@ def handle_get_obj_li( base_name = obj_type_kv_li[obj_name][f'mdl_{mdl_alt}'] # base_name = obj_type_kv_li[obj_name].get(f'mdl_{mdl_alt}') log.info(f'mdl_alt was found. Using {base_name} model.') + if exp_alt: + column_name_li = obj_type_kv_li[obj_name].get(f'exp_{exp_alt}') + log.info(f'exp_alt was found. Using {column_name_li} column list.') # if use_alt_table: # table_name = obj_type_kv_li[obj_name]['table_name_alt'] @@ -475,10 +486,12 @@ def handle_get_obj_li( log.warning('base_name model was not found. Returning raw data.') resp_data = record resp_data_li.append(resp_data) - column_name_li = list(sql_result[0].keys()) # This should be the same for all records in the list. + if not column_name_li: + column_name_li = list(sql_result[0].keys()) # This should be the same for all records in the list. + additional_column_name_li = [] if return_file: - log.setLevel(logging.DEBUG) + log.setLevel(logging.INFO) # We want to handle any field that has a suffix of _json. It should be expanded into a list of fields that are prefixed with the original field name minus _json. @@ -490,8 +503,7 @@ def handle_get_obj_li( for field_name in record.keys(): field_value = record[field_name] if field_name.endswith('li_json') and record[field_name]: - log.info(f'Found a field that ends with li_json: {field_name}') - log.info(f'Field value is a list??: {field_value}') + log.info(f'Found a field that ends with li_json: {field_name}: {field_value}') # field_value = json.loads(record[key]) # Convert the string to a list of dictionaries @@ -501,25 +513,38 @@ def handle_get_obj_li( if isinstance(field_value, list): # Loop through the list of dictionaries - log.info(f'Field value is a list: {field_value}') - for item in field_value: - # item = json.loads(item) # Convert the string to a dictionary - for key, value in item.items(): - new_field_name = field_name[:-8]+'__'+key - new_record[new_field_name] = value + log.info(f'Field value is a list (int key): {field_value}') + key = 1 + for value in field_value: + new_field_name = field_name[:-8]+'__'+str(key) + new_record[new_field_name] = value + additional_column_name_li.append(new_field_name) + key += 1 + # for item in field_value: + # # item = json.loads(item) # Convert the string to a dictionary + # for key, value in item.items(): + # new_field_name = field_name[:-8]+'__'+key + # new_record[new_field_name] = value + # additional_column_name_li.append(new_field_name) + + # We want to keep the original field in this case. This way the list of data will be displayed in one single field and expanded to one field per item in the list. + # new_record.pop(field_name) # Remove the original field else: # Loop through key value pairs in the dictionary - log.info(f'Field value is a dict: {field_value}') + log.info(f'Field value is a dict (list): {field_value}') for key, value in field_value.items(): log.debug(f'Key: {key}') new_field_name = field_name[:-8]+'__'+key new_record[new_field_name] = value + additional_column_name_li.append(new_field_name) + + new_record.pop(field_name) # Remove the original field # Create a new field for each and value to the record # new_field_name = key[:-8]+'__cust_li' # new_record[new_field_name] = record[key] - new_record.pop(field_name) # Remove the original field + elif field_name.endswith('_json') and record[field_name]: log.info(f'Found a field that ends with _json: {field_name}') log.info(f'Field value is a dict???: {field_value}') @@ -527,6 +552,7 @@ def handle_get_obj_li( for key, value in field_value.items(): new_field_name = field_name[:-5]+'__'+key new_record[new_field_name] = value + additional_column_name_li.append(new_field_name) new_record.pop(field_name) # Remove the original field elif field_name.endswith('li_json') or field_name.endswith('_json'): @@ -548,7 +574,13 @@ def handle_get_obj_li( filename_w_ext = filename+'.xlsx' log.setLevel(logging.INFO) - if result := create_export_file(data_dict_list=new_resp_data_li, column_name_li=[], subdir_path=obj_name, filename=filename, rm_id=True, export_type=file_type): + if not exp_alt and additional_column_name_li: + column_name_li = column_name_li + sorted(list(set(additional_column_name_li))) + elif additional_column_name_li: + column_name_li = column_name_li + sorted(list(set(additional_column_name_li))) + log.info(f'Column names: {column_name_li}') + log.info(f'Additional column names: {sorted(set(additional_column_name_li))}') + if result := create_export_file(data_dict_list=new_resp_data_li, column_name_li=column_name_li, subdir_path=obj_name, filename=filename, rm_id=True, export_type=file_type): log.info(f'Export file created and saved: {result}') else: log.error('Something went wrong while creating or saving the export file')