Working on importing IDAA data
This commit is contained in:
Scott Idem
@@ -8,8 +8,11 @@ redis
|
||||
aioredis
|
||||
html2text
|
||||
pytz
|
||||
#mypy
|
||||
stripe
|
||||
passlib
|
||||
argon2_cffi
|
||||
PyJWT
|
||||
pandas
|
||||
openpyxl
|
||||
# xlrd
|
||||
# mypy
|
||||
|
||||
@@ -18,7 +18,7 @@ from app.lib_general import log, logging
|
||||
from app.log import log
|
||||
|
||||
# Import the routers here first:
|
||||
from app.routers import api_crud, api, account, address, archive, archive_content, contact, cont_edu_cert, cont_edu_cert_person, event, event_exhibit, event_file, event_person, event_person_detail, event_presentation, event_presenter, event_registration, event_session, flask_cfg, hosted_file, journal, journal_entry, log_client_viewing, lookup, membership_cfg, membership_group, membership_group_person, membership_person, membership_person_profile, membership_type, membership_type_person, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets # , items, journals
|
||||
from app.routers import api_crud, api, importing, account, address, archive, archive_content, contact, cont_edu_cert, cont_edu_cert_person, event, event_exhibit, event_file, event_person, event_person_detail, event_presentation, event_presenter, event_registration, event_session, flask_cfg, hosted_file, journal, journal_entry, log_client_viewing, lookup, membership_cfg, membership_group, membership_group_person, membership_person, membership_person_profile, membership_type, membership_type_person, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets # , items, journals
|
||||
|
||||
from app.db_sql import db
|
||||
|
||||
@@ -63,6 +63,11 @@ app.include_router(
|
||||
prefix='/flask_cfg',
|
||||
tags=['Flask CFG'],
|
||||
)
|
||||
app.include_router(
|
||||
importing.router,
|
||||
prefix='/importing',
|
||||
tags=['Importing'],
|
||||
)
|
||||
# app.include_router(
|
||||
# flask_cfg.router,
|
||||
# prefix='/redis',
|
||||
|
||||
@@ -44,6 +44,8 @@ class Contact_Base(BaseModel):
|
||||
timezone_name: Optional[str]
|
||||
|
||||
email: Optional[str]
|
||||
email_active: Optional[bool]
|
||||
email_status: Optional[str]
|
||||
|
||||
phone_mobile: Optional[str]
|
||||
phone_home: Optional[str]
|
||||
|
||||
@@ -42,13 +42,16 @@ class Person_Base(BaseModel):
|
||||
membership_person_id_random: Optional[str] # Linked from membership_person using the v_person view
|
||||
membership_person_id: Optional[int] # Linked from membership_person using the v_person view
|
||||
|
||||
informal_name: Optional[str]
|
||||
given_name: Optional[str]
|
||||
family_name: Optional[str]
|
||||
middle_name: Optional[str]
|
||||
prefix: Optional[str]
|
||||
suffix: Optional[str]
|
||||
|
||||
full_name: Optional[str]
|
||||
informal_name: Optional[str]
|
||||
informal_full_name: Optional[str]
|
||||
last_first_name: Optional[str]
|
||||
|
||||
title: Optional[str]
|
||||
|
||||
@@ -57,9 +60,14 @@ class Person_Base(BaseModel):
|
||||
tagline: Optional[str]
|
||||
|
||||
notes: Optional[str]
|
||||
|
||||
created_on: Optional[datetime.datetime] = None
|
||||
updated_on: Optional[datetime.datetime] = None
|
||||
|
||||
# Including JSON data
|
||||
other_json: Optional[Json]
|
||||
meta_json: Optional[Json]
|
||||
|
||||
# Including other related objects
|
||||
# archive_list: Optional[list] # Archive_Base()
|
||||
event_list: Optional[list] # Event_Base() # Priority l1
|
||||
|
||||
@@ -41,7 +41,7 @@ def mk_resp(
|
||||
exclude_unset: bool = True,
|
||||
response = None
|
||||
):
|
||||
# log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
if data is None: data_out = { 'result': data }
|
||||
|
||||
@@ -62,6 +62,9 @@ class User_New_Base(BaseModel):
|
||||
|
||||
notes: Optional[str]
|
||||
|
||||
# Including JSON data
|
||||
other_json: Optional[Json]
|
||||
|
||||
_processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
|
||||
|
||||
#@validator('user_id_random', always=True)
|
||||
|
||||
285
app/routers/importing.py
Normal file
285
app/routers/importing.py
Normal file
@@ -0,0 +1,285 @@
|
||||
import datetime, json, pytz, secrets, time
|
||||
import pandas, xlrd # qrcode
|
||||
from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, status
|
||||
from pydantic import BaseModel, EmailStr, Field
|
||||
from typing import Dict, List, Optional, Set, Union
|
||||
|
||||
from app.lib_general import log, logging, secure_hash_string
|
||||
from app.config import settings
|
||||
from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
|
||||
|
||||
from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
|
||||
|
||||
from app.methods.contact_methods import load_contact_obj, update_contact_obj
|
||||
from app.methods.person_methods import create_update_person_obj, get_person_rec_list, load_person_obj, update_person_obj
|
||||
from app.methods.user_methods import load_user_obj
|
||||
|
||||
from app.models.contact_models import Contact_Base
|
||||
from app.models.person_models import Person_Base
|
||||
from app.models.user_models import User_Base, User_New_Base, User_Out_Base
|
||||
|
||||
from app.models.response_models import Resp_Body_Base, mk_resp
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post('/person_data', response_model=Resp_Body_Base)
|
||||
async def importing_person_data(
|
||||
):
|
||||
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
|
||||
log.debug(locals())
|
||||
|
||||
account_id = 19
|
||||
full_file_path = 'admin/temp/import_person_data.xlsx'
|
||||
|
||||
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
|
||||
#df = df.fillna('') # replace NaN with ''
|
||||
# df = df.fillna(None)
|
||||
# df = df.fillna('', inplace=True)
|
||||
#return str(df.info())
|
||||
log.debug(df)
|
||||
|
||||
df_dict = df.to_dict(orient='records')
|
||||
# log.debug(df_dict)
|
||||
|
||||
# return mk_resp(data=False, status_code=500)
|
||||
|
||||
person_data_li = []
|
||||
# for i in df.index:
|
||||
for record in df_dict:
|
||||
person_new = None
|
||||
person_id = None
|
||||
contact_id = None
|
||||
address_id = None
|
||||
user_id = None
|
||||
|
||||
person_data = {}
|
||||
person_data['external_id'] = record['external_id']
|
||||
# person_data['informal_name'] = record['informal_name']
|
||||
person_data['given_name'] = record['given_name']
|
||||
if record['middle_name']:
|
||||
person_data['middle_name'] = record['middle_name']
|
||||
else:
|
||||
person_data['middle_name'] = None
|
||||
if record['family_name']:
|
||||
person_data['family_name'] = record['family_name']
|
||||
else:
|
||||
person_data['family_name'] = None
|
||||
# person_data['prefix'] = record['prefix']
|
||||
person_data['suffix'] = record['suffix']
|
||||
if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']:
|
||||
person_data['full_name'] = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name']
|
||||
elif person_data['given_name'] and person_data['family_name']:
|
||||
person_data['full_name'] = person_data['given_name']+' '+person_data['family_name']
|
||||
elif person_data['given_name']:
|
||||
person_data['full_name'] = person_data['family_name']
|
||||
elif record['informal_full_name']:
|
||||
person_data['full_name'] = record['informal_full_name']
|
||||
elif record['informal_name']:
|
||||
person_data['full_name'] = record['informal_name']
|
||||
else:
|
||||
person_data['full_name'] = None
|
||||
if record['informal_full_name']:
|
||||
person_data['informal_full_name'] = record['informal_full_name']
|
||||
else:
|
||||
person_data['informal_full_name'] = None
|
||||
person_data['last_first_name'] = record['last_first_name']
|
||||
person_data['created_on'] = record['created_on']
|
||||
person_data['updated_on'] = record['updated_on']
|
||||
|
||||
other_data = {}
|
||||
other_data['contact_type'] = record['contact_type'] # ????
|
||||
|
||||
meta_data = {}
|
||||
meta_data['created_by_method'] = record['created_by_method']
|
||||
meta_data['created_by_name'] = record['created_by_name']
|
||||
meta_data['modified_by'] = record['modified_by']
|
||||
meta_data['created_by_method'] = record['created_by_method']
|
||||
|
||||
person_data['other_json'] = json.dumps(other_data, indent=4)
|
||||
person_data['meta_json'] = json.dumps(meta_data, indent=4)
|
||||
|
||||
|
||||
# Look up by email address or external ID and INSERT or UPDATE new person record
|
||||
# INSERT or UPDATE a contact record and address record if needed
|
||||
# INSERT or UPDATE a user record if needed
|
||||
# Process the person data
|
||||
log.debug(person_data)
|
||||
# log.debug('*** *** *** *** END TEST RUN *** *** *** ***')
|
||||
# continue
|
||||
if person_rec_li_result := sql_select(table_name='v_person', field_name='external_id', field_value=person_data['external_id']):
|
||||
if not isinstance(person_rec_li_result, list):
|
||||
# Pull out IDs and UPDATE existing person record
|
||||
log.debug('Found one record')
|
||||
person_rec = person_rec_li_result
|
||||
person_id = person_rec.get('person_id', None)
|
||||
contact_id = person_rec.get('contact_id_new', None) # Using _new from view until old contact_id is removed from person table
|
||||
address_id = person_rec.get('address_id', None)
|
||||
user_id = person_rec.get('user_id', None)
|
||||
person_data['id'] = person_id
|
||||
if person_obj_up_result := sql_update(data=person_data, table_name='person'):
|
||||
log.debug(person_obj_up_result)
|
||||
else:
|
||||
log.warning(person_obj_up_result)
|
||||
continue # Something unexpected may have happened
|
||||
else:
|
||||
log.warning('Found more than one record')
|
||||
# Do nothing
|
||||
continue # Something unexpected may have happened
|
||||
person_rec_li = person_rec_li_result
|
||||
else:
|
||||
# INSERT new record
|
||||
log.debug('Found no records or something went wrong')
|
||||
person_data['account_id'] = account_id
|
||||
if person_obj_in_result := sql_insert(data=person_data, table_name='person'):
|
||||
log.debug(person_obj_in_result)
|
||||
person_id = person_obj_in_result # Should be an int
|
||||
person_new = True # Need to UPDATE this record after the contact, address, and user data is processed
|
||||
else:
|
||||
log.warning(person_obj_in_result)
|
||||
continue # Something unexpected may have happened
|
||||
|
||||
# Process the contact data
|
||||
log.debug('Process the contact data')
|
||||
contact_data = {}
|
||||
contact_data['email'] = record['email']
|
||||
if record['email_status'] != 'Undeliverable':
|
||||
contact_data['email_active'] = True
|
||||
else:
|
||||
contact_data['email_active'] = False
|
||||
contact_data['email_status'] = record['email_status']
|
||||
contact_data['phone_mobile'] = record['phone_mobile']
|
||||
contact_data['phone_home'] = record['phone_home']
|
||||
if record['phone_fax']:
|
||||
contact_data['phone_fax'] = record['phone_fax']
|
||||
elif record['phone_fax'] and record['phone_work_fax']:
|
||||
contact_data['phone_fax'] = record['phone_fax']
|
||||
contact_data['phone_other'] = record['phone_work_fax']
|
||||
contact_data['phone_office'] = record['phone_work']
|
||||
|
||||
log.debug(contact_data)
|
||||
if contact_id:
|
||||
# UPDATE existing contact record
|
||||
contact_data['id'] = contact_id
|
||||
if contact_obj_up_result := sql_update(data=contact_data, table_name='contact'):
|
||||
log.debug(contact_obj_up_result)
|
||||
else:
|
||||
log.warning(contact_obj_up_result)
|
||||
continue # Something unexpected may have happened
|
||||
elif person_id:
|
||||
# INSERT new contact record and link to person record
|
||||
contact_data['account_id'] = account_id
|
||||
contact_data['for_type'] = 'person'
|
||||
contact_data['for_id'] = person_id
|
||||
if contact_obj_in_result := sql_insert(data=contact_data, table_name='contact'):
|
||||
log.debug(contact_obj_in_result)
|
||||
contact_id = contact_obj_in_result # Should be an int
|
||||
else:
|
||||
log.debug(contact_obj_in_result)
|
||||
continue # Something unexpected may have happened
|
||||
|
||||
# Process the contact address data
|
||||
log.debug('Process the contact address data')
|
||||
address_data = {}
|
||||
address_data['line_1'] = record['address_line_1']
|
||||
address_data['line_2'] = record['address_line_2']
|
||||
address_data['line_3'] = record['address_line_3']
|
||||
address_data['city'] = record['address_city']
|
||||
address_data['country_subdivision_code'] = record['address_country_code']+'-'+record['address_state_province_code']
|
||||
address_data['postal_code'] = record['address_postal_code']
|
||||
address_data['country_alpha_2_code'] = record['address_country_code']
|
||||
|
||||
log.debug(address_data)
|
||||
if address_id:
|
||||
# UPDATE existing address record
|
||||
log.debug('UPDATE existing address record')
|
||||
address_data['id'] = address_id
|
||||
if address_obj_up_result := sql_update(data=address_data, table_name='address'):
|
||||
log.debug(address_obj_up_result)
|
||||
else:
|
||||
log.warning(address_obj_up_result)
|
||||
# continue # Something unexpected may have happened
|
||||
elif contact_id:
|
||||
# INSERT new address record and link to contact record
|
||||
log.debug('INSERT new address record and link to contact record')
|
||||
address_data['account_id'] = account_id
|
||||
address_data['for_type'] = 'contact'
|
||||
address_data['for_id'] = contact_id
|
||||
if address_obj_in_result := sql_insert(data=address_data, table_name='address'):
|
||||
log.debug(address_obj_in_result)
|
||||
address_id = address_obj_in_result # Should be an int
|
||||
else:
|
||||
log.debug(address_obj_in_result)
|
||||
# break
|
||||
continue # Something unexpected may have happened
|
||||
|
||||
# Process the user data
|
||||
log.debug('Process the user data')
|
||||
user_data = {}
|
||||
user_data['name'] = person_data['full_name']
|
||||
user_data['username'] = record['email']
|
||||
user_data['email'] = record['email']
|
||||
user_data['email_verified'] = contact_data['email_active'] # Not perfect, but a good start
|
||||
random_password_string = secrets.token_urlsafe(8)
|
||||
user_data['password'] = secure_hash_string(string=random_password_string)
|
||||
|
||||
user_data['enable'] = False
|
||||
user_data['enable_from'] = datetime.datetime.now(datetime.timezone.utc)
|
||||
user_data['enable_to'] = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365)
|
||||
|
||||
user_data['super'] = False
|
||||
user_data['manager'] = False
|
||||
user_data['administrator'] = False
|
||||
|
||||
user_data['public'] = False
|
||||
user_data['verified'] = True
|
||||
user_data['notes'] = 'Created by importing list'
|
||||
|
||||
other_data = {}
|
||||
other_data['temp_password'] = random_password_string
|
||||
|
||||
user_data['other_json'] = json.dumps(other_data, indent=4)
|
||||
|
||||
log.debug(user_data)
|
||||
if user_id:
|
||||
# UPDATE existing user record
|
||||
user_data['id'] = user_id
|
||||
if user_obj_up_result := sql_update(data=user_data, table_name='user'):
|
||||
log.debug(user_obj_up_result)
|
||||
else:
|
||||
log.warning(user_obj_up_result)
|
||||
continue # Something unexpected may have happened
|
||||
elif person_id:
|
||||
# INSERT new user record and link to person record
|
||||
user_data['account_id'] = account_id
|
||||
user_data['person_id'] = person_id
|
||||
if user_obj_in_result := sql_insert(data=user_data, table_name='user'):
|
||||
log.debug(user_obj_in_result)
|
||||
user_id = user_obj_in_result # Should be an int
|
||||
else:
|
||||
log.debug(user_obj_in_result)
|
||||
# break
|
||||
continue # Something unexpected may have happened
|
||||
|
||||
if person_new:
|
||||
log.debug('Updating person record one more time since this is a new person')
|
||||
person_data_up = {}
|
||||
person_data_up['id'] = person_id
|
||||
person_data_up['user_id'] = user_id
|
||||
random_password_string
|
||||
# Don't need to update with the new contact or address IDs that were just created.
|
||||
|
||||
if person_obj_up_result := sql_update(data=person_data_up, table_name='person'):
|
||||
log.debug(person_obj_up_result)
|
||||
else:
|
||||
log.warning(person_obj_up_result)
|
||||
# break
|
||||
continue # Something unexpected may have happened
|
||||
|
||||
person_data_li.append(person_data)
|
||||
log.debug(f"Record processed: {person_id} {person_data['full_name']}")
|
||||
# log.debug('*** *** *** *** END TEST RUN *** *** *** ***')
|
||||
# break
|
||||
|
||||
return mk_resp(data=person_data_li)
|
||||
Reference in New Issue
Block a user