diff --git a/admin/requirements.txt b/admin/requirements.txt index 2e4f967..417fcd7 100644 --- a/admin/requirements.txt +++ b/admin/requirements.txt @@ -8,8 +8,11 @@ redis aioredis html2text pytz -#mypy stripe passlib argon2_cffi PyJWT +pandas +openpyxl +# xlrd +# mypy diff --git a/app/main.py b/app/main.py index 5cd888f..473dd21 100644 --- a/app/main.py +++ b/app/main.py @@ -18,7 +18,7 @@ from app.lib_general import log, logging from app.log import log # Import the routers here first: -from app.routers import api_crud, api, account, address, archive, archive_content, contact, cont_edu_cert, cont_edu_cert_person, event, event_exhibit, event_file, event_person, event_person_detail, event_presentation, event_presenter, event_registration, event_session, flask_cfg, hosted_file, journal, journal_entry, log_client_viewing, lookup, membership_cfg, membership_group, membership_group_person, membership_person, membership_person_profile, membership_type, membership_type_person, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets # , items, journals +from app.routers import api_crud, api, importing, account, address, archive, archive_content, contact, cont_edu_cert, cont_edu_cert_person, event, event_exhibit, event_file, event_person, event_person_detail, event_presentation, event_presenter, event_registration, event_session, flask_cfg, hosted_file, journal, journal_entry, log_client_viewing, lookup, membership_cfg, membership_group, membership_group_person, membership_person, membership_person_profile, membership_type, membership_type_person, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets # , items, journals from app.db_sql import db @@ -63,6 +63,11 @@ app.include_router( prefix='/flask_cfg', tags=['Flask CFG'], ) +app.include_router( + importing.router, + prefix='/importing', + tags=['Importing'], +) # app.include_router( # flask_cfg.router, # prefix='/redis', diff --git a/app/models/contact_models.py b/app/models/contact_models.py index 4cd6ebe..936863e 100644 --- a/app/models/contact_models.py +++ b/app/models/contact_models.py @@ -44,6 +44,8 @@ class Contact_Base(BaseModel): timezone_name: Optional[str] email: Optional[str] + email_active: Optional[bool] + email_status: Optional[str] phone_mobile: Optional[str] phone_home: Optional[str] diff --git a/app/models/person_models.py b/app/models/person_models.py index fb62ae1..855040d 100644 --- a/app/models/person_models.py +++ b/app/models/person_models.py @@ -42,13 +42,16 @@ class Person_Base(BaseModel): membership_person_id_random: Optional[str] # Linked from membership_person using the v_person view membership_person_id: Optional[int] # Linked from membership_person using the v_person view + informal_name: Optional[str] given_name: Optional[str] family_name: Optional[str] middle_name: Optional[str] prefix: Optional[str] suffix: Optional[str] + full_name: Optional[str] - informal_name: Optional[str] + informal_full_name: Optional[str] + last_first_name: Optional[str] title: Optional[str] @@ -57,9 +60,14 @@ class Person_Base(BaseModel): tagline: Optional[str] notes: Optional[str] + created_on: Optional[datetime.datetime] = None updated_on: Optional[datetime.datetime] = None + # Including JSON data + other_json: Optional[Json] + meta_json: Optional[Json] + # Including other related objects # archive_list: Optional[list] # Archive_Base() event_list: Optional[list] # Event_Base() # Priority l1 diff --git a/app/models/response_models.py b/app/models/response_models.py index 9bfd139..d57bed9 100644 --- a/app/models/response_models.py +++ b/app/models/response_models.py @@ -41,7 +41,7 @@ def mk_resp( exclude_unset: bool = True, response = None ): - # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) if data is None: data_out = { 'result': data } diff --git a/app/models/user_models.py b/app/models/user_models.py index ce425d8..c96dcee 100644 --- a/app/models/user_models.py +++ b/app/models/user_models.py @@ -62,6 +62,9 @@ class User_New_Base(BaseModel): notes: Optional[str] + # Including JSON data + other_json: Optional[Json] + _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now) #@validator('user_id_random', always=True) diff --git a/app/routers/importing.py b/app/routers/importing.py new file mode 100644 index 0000000..bffd551 --- /dev/null +++ b/app/routers/importing.py @@ -0,0 +1,285 @@ +import datetime, json, pytz, secrets, time +import pandas, xlrd # qrcode +from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, status +from pydantic import BaseModel, EmailStr, Field +from typing import Dict, List, Optional, Set, Union + +from app.lib_general import log, logging, secure_hash_string +from app.config import settings +from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random + +from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template + +from app.methods.contact_methods import load_contact_obj, update_contact_obj +from app.methods.person_methods import create_update_person_obj, get_person_rec_list, load_person_obj, update_person_obj +from app.methods.user_methods import load_user_obj + +from app.models.contact_models import Contact_Base +from app.models.person_models import Person_Base +from app.models.user_models import User_Base, User_New_Base, User_Out_Base + +from app.models.response_models import Resp_Body_Base, mk_resp + + +router = APIRouter() + + +@router.post('/person_data', response_model=Resp_Body_Base) +async def importing_person_data( + ): + log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL + log.debug(locals()) + + account_id = 19 + full_file_path = 'admin/temp/import_person_data.xlsx' + + df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str}) + #df = df.fillna('') # replace NaN with '' + # df = df.fillna(None) + # df = df.fillna('', inplace=True) + #return str(df.info()) + log.debug(df) + + df_dict = df.to_dict(orient='records') + # log.debug(df_dict) + + # return mk_resp(data=False, status_code=500) + + person_data_li = [] + # for i in df.index: + for record in df_dict: + person_new = None + person_id = None + contact_id = None + address_id = None + user_id = None + + person_data = {} + person_data['external_id'] = record['external_id'] + # person_data['informal_name'] = record['informal_name'] + person_data['given_name'] = record['given_name'] + if record['middle_name']: + person_data['middle_name'] = record['middle_name'] + else: + person_data['middle_name'] = None + if record['family_name']: + person_data['family_name'] = record['family_name'] + else: + person_data['family_name'] = None + # person_data['prefix'] = record['prefix'] + person_data['suffix'] = record['suffix'] + if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']: + person_data['full_name'] = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name'] + elif person_data['given_name'] and person_data['family_name']: + person_data['full_name'] = person_data['given_name']+' '+person_data['family_name'] + elif person_data['given_name']: + person_data['full_name'] = person_data['family_name'] + elif record['informal_full_name']: + person_data['full_name'] = record['informal_full_name'] + elif record['informal_name']: + person_data['full_name'] = record['informal_name'] + else: + person_data['full_name'] = None + if record['informal_full_name']: + person_data['informal_full_name'] = record['informal_full_name'] + else: + person_data['informal_full_name'] = None + person_data['last_first_name'] = record['last_first_name'] + person_data['created_on'] = record['created_on'] + person_data['updated_on'] = record['updated_on'] + + other_data = {} + other_data['contact_type'] = record['contact_type'] # ???? + + meta_data = {} + meta_data['created_by_method'] = record['created_by_method'] + meta_data['created_by_name'] = record['created_by_name'] + meta_data['modified_by'] = record['modified_by'] + meta_data['created_by_method'] = record['created_by_method'] + + person_data['other_json'] = json.dumps(other_data, indent=4) + person_data['meta_json'] = json.dumps(meta_data, indent=4) + + + # Look up by email address or external ID and INSERT or UPDATE new person record + # INSERT or UPDATE a contact record and address record if needed + # INSERT or UPDATE a user record if needed + # Process the person data + log.debug(person_data) + # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') + # continue + if person_rec_li_result := sql_select(table_name='v_person', field_name='external_id', field_value=person_data['external_id']): + if not isinstance(person_rec_li_result, list): + # Pull out IDs and UPDATE existing person record + log.debug('Found one record') + person_rec = person_rec_li_result + person_id = person_rec.get('person_id', None) + contact_id = person_rec.get('contact_id_new', None) # Using _new from view until old contact_id is removed from person table + address_id = person_rec.get('address_id', None) + user_id = person_rec.get('user_id', None) + person_data['id'] = person_id + if person_obj_up_result := sql_update(data=person_data, table_name='person'): + log.debug(person_obj_up_result) + else: + log.warning(person_obj_up_result) + continue # Something unexpected may have happened + else: + log.warning('Found more than one record') + # Do nothing + continue # Something unexpected may have happened + person_rec_li = person_rec_li_result + else: + # INSERT new record + log.debug('Found no records or something went wrong') + person_data['account_id'] = account_id + if person_obj_in_result := sql_insert(data=person_data, table_name='person'): + log.debug(person_obj_in_result) + person_id = person_obj_in_result # Should be an int + person_new = True # Need to UPDATE this record after the contact, address, and user data is processed + else: + log.warning(person_obj_in_result) + continue # Something unexpected may have happened + + # Process the contact data + log.debug('Process the contact data') + contact_data = {} + contact_data['email'] = record['email'] + if record['email_status'] != 'Undeliverable': + contact_data['email_active'] = True + else: + contact_data['email_active'] = False + contact_data['email_status'] = record['email_status'] + contact_data['phone_mobile'] = record['phone_mobile'] + contact_data['phone_home'] = record['phone_home'] + if record['phone_fax']: + contact_data['phone_fax'] = record['phone_fax'] + elif record['phone_fax'] and record['phone_work_fax']: + contact_data['phone_fax'] = record['phone_fax'] + contact_data['phone_other'] = record['phone_work_fax'] + contact_data['phone_office'] = record['phone_work'] + + log.debug(contact_data) + if contact_id: + # UPDATE existing contact record + contact_data['id'] = contact_id + if contact_obj_up_result := sql_update(data=contact_data, table_name='contact'): + log.debug(contact_obj_up_result) + else: + log.warning(contact_obj_up_result) + continue # Something unexpected may have happened + elif person_id: + # INSERT new contact record and link to person record + contact_data['account_id'] = account_id + contact_data['for_type'] = 'person' + contact_data['for_id'] = person_id + if contact_obj_in_result := sql_insert(data=contact_data, table_name='contact'): + log.debug(contact_obj_in_result) + contact_id = contact_obj_in_result # Should be an int + else: + log.debug(contact_obj_in_result) + continue # Something unexpected may have happened + + # Process the contact address data + log.debug('Process the contact address data') + address_data = {} + address_data['line_1'] = record['address_line_1'] + address_data['line_2'] = record['address_line_2'] + address_data['line_3'] = record['address_line_3'] + address_data['city'] = record['address_city'] + address_data['country_subdivision_code'] = record['address_country_code']+'-'+record['address_state_province_code'] + address_data['postal_code'] = record['address_postal_code'] + address_data['country_alpha_2_code'] = record['address_country_code'] + + log.debug(address_data) + if address_id: + # UPDATE existing address record + log.debug('UPDATE existing address record') + address_data['id'] = address_id + if address_obj_up_result := sql_update(data=address_data, table_name='address'): + log.debug(address_obj_up_result) + else: + log.warning(address_obj_up_result) + # continue # Something unexpected may have happened + elif contact_id: + # INSERT new address record and link to contact record + log.debug('INSERT new address record and link to contact record') + address_data['account_id'] = account_id + address_data['for_type'] = 'contact' + address_data['for_id'] = contact_id + if address_obj_in_result := sql_insert(data=address_data, table_name='address'): + log.debug(address_obj_in_result) + address_id = address_obj_in_result # Should be an int + else: + log.debug(address_obj_in_result) + # break + continue # Something unexpected may have happened + + # Process the user data + log.debug('Process the user data') + user_data = {} + user_data['name'] = person_data['full_name'] + user_data['username'] = record['email'] + user_data['email'] = record['email'] + user_data['email_verified'] = contact_data['email_active'] # Not perfect, but a good start + random_password_string = secrets.token_urlsafe(8) + user_data['password'] = secure_hash_string(string=random_password_string) + + user_data['enable'] = False + user_data['enable_from'] = datetime.datetime.now(datetime.timezone.utc) + user_data['enable_to'] = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365) + + user_data['super'] = False + user_data['manager'] = False + user_data['administrator'] = False + + user_data['public'] = False + user_data['verified'] = True + user_data['notes'] = 'Created by importing list' + + other_data = {} + other_data['temp_password'] = random_password_string + + user_data['other_json'] = json.dumps(other_data, indent=4) + + log.debug(user_data) + if user_id: + # UPDATE existing user record + user_data['id'] = user_id + if user_obj_up_result := sql_update(data=user_data, table_name='user'): + log.debug(user_obj_up_result) + else: + log.warning(user_obj_up_result) + continue # Something unexpected may have happened + elif person_id: + # INSERT new user record and link to person record + user_data['account_id'] = account_id + user_data['person_id'] = person_id + if user_obj_in_result := sql_insert(data=user_data, table_name='user'): + log.debug(user_obj_in_result) + user_id = user_obj_in_result # Should be an int + else: + log.debug(user_obj_in_result) + # break + continue # Something unexpected may have happened + + if person_new: + log.debug('Updating person record one more time since this is a new person') + person_data_up = {} + person_data_up['id'] = person_id + person_data_up['user_id'] = user_id + random_password_string + # Don't need to update with the new contact or address IDs that were just created. + + if person_obj_up_result := sql_update(data=person_data_up, table_name='person'): + log.debug(person_obj_up_result) + else: + log.warning(person_obj_up_result) + # break + continue # Something unexpected may have happened + + person_data_li.append(person_data) + log.debug(f"Record processed: {person_id} {person_data['full_name']}") + # log.debug('*** *** *** *** END TEST RUN *** *** *** ***') + # break + + return mk_resp(data=person_data_li)