Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Working on data importing. Mainly for IDAA membership.

Browse Source
This commit is contained in:
Scott Idem
2021-08-19 15:10:59 -04:00
parent 02f719d206
commit 4eaeb152fe
6 changed files with 222 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/methods/membership_person_methods.py
View File

@@ -46,7 +46,7 @@ def load_membership_person_obj(
inc_product_list: bool = False, # The list of products that give access to a member_type or member_group
inc_user: bool = False,
) -> Membership_Person_Base:
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.WARING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
if membership_person_id := redis_lookup_id_random(record_id_random=membership_person_id, table_name='membership_person'): pass

77
app/methods/person_methods.py
View File

@@ -52,8 +52,9 @@ def load_person_obj(
inc_post_comment_list: bool = False,
inc_product: bool = False,
inc_user: bool = False,
inc_user_role_list: bool = False,
) -> Person_Base|dict|bool:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
if person_id := redis_lookup_id_random(record_id_random=person_id, table_name='person'): pass
@@ -62,15 +63,20 @@ def load_person_obj(
if person_rec := sql_select(table_name='v_person', record_id=person_id): pass
else: return False
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(person_rec)
try:
person_obj = Person_Base(**person_rec)
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(person_obj)
except ValidationError as e:
log.error(e.json())
return False
# Updated 2021-07-09
if inc_contact:
log.info('Need to include contact data...')
contact_id = person_rec.get('contact_id', None)
log.debug(contact_id)
if contact_result := load_contact_obj(
@@ -116,6 +122,7 @@ def load_person_obj(
# Updated 2021-07-09
if inc_membership_person:
log.info('Need to include membership person data...')
from app.methods.membership_person_methods import load_membership_person_obj
membership_person_id = person_rec.get('membership_person_id', None)
log.debug(membership_person_id)
@@ -174,6 +181,7 @@ def load_person_obj(
# Updated 2021-06-18
if inc_order_list:
log.info('Need to include order list data...')
if order_rec_list_result := get_order_rec_list(
for_obj_type = 'person',
for_obj_id = person_id,
@@ -199,6 +207,7 @@ def load_person_obj(
# Updated 2021-06-18
if inc_organization:
log.info('Need to include organization data...')
organization_id = person_rec.get('organization_id', None)
log.debug(organization_id)
if organization_dict := load_organization_obj(
@@ -214,22 +223,25 @@ def load_person_obj(
person_obj.organization = organization_dict
else: person_obj.organization = None
# Updated 2021-06-18
# Updated 2021-08-19
if inc_user:
# user_id = person_rec.get('user_id', None)
# log.debug(user_id)
# if user_result := load_user_obj(
# user_id = user_id,
# limit = limit,
# by_alias = by_alias,
# exclude_unset = exclude_unset,
# model_as_dict = model_as_dict,
# enabled = enabled,
# ):
# person_obj.user = user_result
# else: person_obj.user = None
pass
log.info('Need to include user data...')
user_id = person_rec.get('user_id', None)
log.debug(user_id)
from app.methods.user_methods import load_user_obj
if user_result := load_user_obj(
user_id = user_id,
limit = limit,
by_alias = by_alias,
exclude_unset = exclude_unset,
model_as_dict = model_as_dict,
enabled = enabled,
inc_user_role_list = inc_user_role_list,
):
person_obj.user = user_result
else: person_obj.user = None
print(person_obj)
if model_as_dict:
return person_obj.dict(by_alias=by_alias, exclude_unset=exclude_unset) # pylint: disable=no-member
else:
@@ -292,6 +304,41 @@ def get_person_rec_list(
# ### END ### API Person Methods ### get_person_rec_list() ###
# ### BEGIN ### API Person Methods ### get_person_rec_w_external_id() ###
def get_person_rec_w_external_id(
account_id: str,
external_id: str,
enabled: str = 'enabled', # enabled, disabled, all
) -> dict|bool:
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
# if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
# else: return False
account_id = 99
data = {}
data['account_id'] = account_id
data['external_id'] = external_id
sql = f"""
SELECT `person`.id AS 'person_id', `person`.id_random AS 'person_id_random'
FROM `person` AS `person`
WHERE person.account_id = :account_id
AND person.external_id = :external_id
LIMIT 1;
"""
if person_rec_result := sql_select(data=data, sql=sql):
person_rec = person_rec_result
else:
person_rec = None
log.debug(person_rec_result)
return person_rec
# ### END ### API Person Methods ### get_person_rec_w_external_id() ###
# ### BEGIN ### API Person Methods ### create_person_obj() ###
# NOTE: This will create a person and then also create a linked contact if person_obj.contact data is passed. The create_contact_obj will create a contact and then also create a linked address if person_obj.contact.address data is passed.
# Reviewed and updated 2021-08-10

5
app/models/person_models.py
View File

@@ -61,13 +61,16 @@ class Person_Base(BaseModel):
tagline: Optional[str]
birth_date: Optional[datetime.datetime]
birth_date: Optional[datetime.date]
lu_gender_id: Optional[int]
lu_gender_name: Optional[str]
email_allowed: Optional[bool]
paper_mail_allowed: Optional[bool]
external_id: Optional[str]
external_import_id: Optional[str]
notes: Optional[str]
created_on: Optional[datetime.datetime] = None

122
app/routers/importing.py
View File

@@ -24,6 +24,84 @@ from app.models.response_models import Resp_Body_Base, mk_resp
router = APIRouter()
# First round: Working importing the Cvent Address Book export data
@router.post('/person_data_v2_external_id', response_model=Resp_Body_Base)
async def importing_person_data_v2_external_id(
response: Response = Response,
):
log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
account_id = 99
full_file_path = 'admin/temp/import_person_external_id.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'email':str, 'External ID':str, 'Email Address':str})
log.debug(df)
df_dict = df.to_dict(orient='records')
person_data_li = []
for record in df_dict:
person_id = None
if external_id := record.get('external_id', None): pass
elif external_id := record.get('External ID', None): pass
else:
log.info('No external ID was found.')
continue
if email := record.get('email', None): pass
elif email := record.get('Email Address', None): pass
else:
log.info('No email address was found.')
continue
data = {}
data['account_id'] = account_id
data['email'] = email
sql = f"""
SELECT *
FROM `v_person` AS `person`
WHERE person.account_id = :account_id
AND person.email = :email
ORDER BY `person`.created_on DESC, `person`.updated_on DESC
LIMIT 1;
"""
if person_rec_result := sql_select(data=data, sql=sql):
# Pull out IDs and UPDATE existing person record
log.debug('Found one record')
person_rec = person_rec_result
person_id = person_rec.get('person_id', None)
# contact_id = person_rec.get('contact_id', None)
# address_id = person_rec.get('address_id', None)
# user_id = person_rec.get('user_id', None)
# external_id = person_rec.get('external_id', None)
# external_import_id = person_rec.get('external_import_id', None)
person_data = {}
person_data['id'] = person_id
person_data['external_id'] = external_id
if person_obj_up_result := sql_update(data=person_data, table_name='person'):
log.debug(person_obj_up_result)
else:
log.warning(person_obj_up_result)
continue # Something unexpected may have happened
person_rec = person_rec_result
else:
continue # Something unexpected may have happened
person_data_min = {}
person_data_min['person_id'] = person_id
person_data_min['full_name'] = person_rec.get('full_name', None)
person_data_min['email'] = email
person_data_li.append(person_data_min)
log.debug(f"Record processed: {person_id} {person_rec.get('full_name', None)}")
return mk_resp(data=person_data_li)
# First round: Working importing the Cvent Address Book export data
@router.post('/person_data_v2', response_model=Resp_Body_Base)
async def importing_person_data_v2(
@@ -48,7 +126,7 @@ async def importing_person_data_v2(
account_id = 99
full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
#df = df.fillna('') # replace NaN with ''
# df = df.fillna(None)
# df = df.fillna('', inplace=True)
@@ -72,11 +150,11 @@ async def importing_person_data_v2(
# membership_person_id = None
person_data = {}
if record['external_id']:
person_data['external_id'] = record['external_id']
if record['external_import_id']:
person_data['external_import_id'] = record['external_import_id']
else:
person_data['external_id'] = record['email']
external_id = person_data['external_id']
person_data['external_import_id'] = record['email']
external_import_id = person_data['external_import_id']
if nickname := record.get('nickname', None): person_data['informal_name'] = nickname
if given_name := record.get('given_name', None): person_data['given_name'] = given_name
@@ -160,12 +238,12 @@ async def importing_person_data_v2(
# continue
data = {}
data['account_id'] = account_id
data['external_id'] = external_id
data['external_import_id'] = external_import_id
sql = f"""
SELECT *
FROM `v_person` AS `person`
WHERE person.account_id = :account_id
AND person.external_id = :external_id
AND person.external_import_id = :external_import_id
ORDER BY `person`.created_on DESC, `person`.updated_on DESC
LIMIT 1;
"""
@@ -447,7 +525,7 @@ async def importing_person_data_v2_membership(
account_id = 99
full_file_path = 'admin/temp/import_person_contact_address_user_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
log.debug(df)
df_dict = df.to_dict(orient='records')
@@ -466,23 +544,23 @@ async def importing_person_data_v2_membership(
person_data = {}
person_data['account_id'] = account_id
if record['external_id']:
person_data['external_id'] = record['external_id']
if record['external_import_id']:
person_data['external_import_id'] = record['external_import_id']
else:
person_data['external_id'] = record['email']
external_id = person_data['external_id']
person_data['external_import_id'] = record['email']
external_import_id = person_data['external_import_id']
if membership_type_name := record.get('membership_type', None): pass
else: continue
data = {}
data['account_id'] = account_id
data['external_id'] = external_id
data['external_import_id'] = external_import_id
sql = f"""
SELECT *
FROM `v_person` AS `person`
WHERE person.account_id = :account_id
AND person.external_id = :external_id
AND person.external_import_id = :external_import_id
ORDER BY `person`.created_on DESC, `person`.updated_on DESC
LIMIT 1;
"""
@@ -661,7 +739,7 @@ async def importing_person_data(
account_id = 99
full_file_path = 'admin/temp/import_person_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
#df = df.fillna('') # replace NaN with ''
# df = df.fillna(None)
# df = df.fillna('', inplace=True)
@@ -683,10 +761,10 @@ async def importing_person_data(
user_id = None
person_data = {}
if record['external_id']:
person_data['external_id'] = record['external_id']
if record['external_import_id']:
person_data['external_import_id'] = record['external_import_id']
else:
person_data['external_id'] = record['email']
person_data['external_import_id'] = record['email']
# person_data['informal_name'] = record['informal_name']
person_data['given_name'] = record['given_name']
if record['middle_name']:
@@ -732,14 +810,14 @@ async def importing_person_data(
person_data['meta_json'] = json.dumps(meta_data, indent=4)
# Look up by email address or external ID and INSERT or UPDATE new person record
# Look up by email address or external import ID and INSERT or UPDATE new person record
# INSERT or UPDATE a contact record and address record if needed
# INSERT or UPDATE a user record if needed
# Process the person data
log.debug(person_data)
# log.debug('*** *** *** *** END TEST RUN *** *** *** ***')
# continue
if person_rec_li_result := sql_select(table_name='v_person', field_name='external_id', field_value=person_data['external_id']):
if person_rec_li_result := sql_select(table_name='v_person', field_name='external_import_id', field_value=person_data['external_import_id']):
if not isinstance(person_rec_li_result, list):
# Pull out IDs and UPDATE existing person record
log.debug('Found one record')
@@ -982,7 +1060,7 @@ async def importing_cont_edu_cert_person_data(
cont_edu_cert_id = 3
full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
log.debug(df)
df_dict = df.to_dict(orient='records')
@@ -1114,7 +1192,7 @@ async def importing_cont_edu_cert_person_data_touch(
cont_edu_cert_id = 3
full_file_path = 'admin/temp/import_cont_edu_cert_person_data.xlsx'
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_import_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
log.debug(df)
df_dict = df.to_dict(orient='records')

4
app/routers/membership_person.py
View File

@@ -214,7 +214,7 @@ async def get_person_obj_membership_person(
else: return mk_resp(data=None, status_code=404)
if membership_person_rec := sql_select(table_name='v_membership_person', field_name='person_id', field_value=person_id): pass
else: return False
else: return mk_resp(data=None, status_code=404)
membership_person_id = membership_person_rec.get('membership_person_id', None)
log.debug(membership_person_id)
@@ -243,7 +243,7 @@ async def get_person_obj_membership_person(
# inc_user = inc_user,
):
response_data = membership_person_result
else: return mk_resp(data=False, status_code=400, response=response) # Bad Request
else: return mk_resp(data=False, status_code=404, response=response) # Bad Request
return mk_resp(data=response_data)
# ### END ### API Membership Person ### get_person_obj_membership_person() ###

55
app/routers/person.py
View File

@@ -11,7 +11,7 @@ from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_
# from app.methods.membership_person_methods import load_membership_person_obj
from app.methods.order_methods import get_order_rec_list, load_order_obj
from app.methods.person_methods import create_update_person_obj, get_person_rec_list, load_person_obj, update_person_obj
from app.methods.person_methods import create_update_person_obj, get_person_rec_list, get_person_rec_w_external_id, load_person_obj, update_person_obj
from app.models.person_models import Person_Base
from app.models.response_models import Resp_Body_Base, mk_resp
@@ -179,6 +179,57 @@ async def get_person_obj_li(
return result
# ### BEGIN ### API Person ### person_obj_external_id() ###
# NOTE: This endpoint is a little different. It will likely be used to help log in the person/user coming from an external service (Cvent).
# Updated 2021-08-19
@router.get('/person/external_id', response_model=Resp_Body_Base)
async def person_obj_external_id(
account_id: str = Query(..., min_length=1, max_length=22),
external_id: str = Query(..., min_length=1, max_length=75),
limit: int = 500,
enabled: str = 'enabled',
inc_address: bool = False,
inc_contact: bool = False,
# inc_person: bool = False,
inc_user: bool = False,
inc_user_role_list: bool = False,
x_account_id: str = Header(...),
by_alias: Optional[bool] = True,
exclude_unset: Optional[bool] = True,
response: Response = Response,
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
# if account_id := redis_lookup_id_random(record_id_random=account_id, table_name='account'): pass
# else: return mk_resp(data=None, status_code=404, response=response)
account_id = 99 # WARNING!!!! Get rid of 99!
if person_data := get_person_rec_w_external_id(account_id=account_id, external_id=external_id): pass
else: return mk_resp(data=None, status_code=404, response=response)
person_id = person_data.get('person_id')
if person_dict := load_person_obj(
person_id = person_id,
limit = limit,
model_as_dict = True, # NOTE: returning model as a dict
enabled = enabled,
inc_address = inc_address,
inc_contact = inc_contact,
inc_user = inc_user,
inc_user_role_list = inc_user_role_list,
):
if isinstance(person_dict, dict):
response_data = person_dict
else:
response_data = person_dict
else:
return mk_resp(data=False, status_code=400, response=response) # Bad Request
return mk_resp(data=response_data)
# ### END ### API Person ### person_obj_external_id() ###
# ### BEGIN ### API Person ### get_person_obj() ###
# Working well as of 2021-06-25. Using as a template for other routes.
@router.get('/person/{person_id}', response_model=Resp_Body_Base)
@@ -216,7 +267,7 @@ async def get_person_obj(
exclude_unset: Optional[bool] = True,
response: Response = Response,
):
log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
log.debug(locals())
if person_id := redis_lookup_id_random(record_id_random=person_id, table_name='person'): pass