Working on importing IDAA data

admin/requirements.txt

@@ -8,8 +8,11 @@ redis
 aioredis
 html2text
 pytz
-#mypy
 stripe
 passlib
 argon2_cffi
 PyJWT
+pandas
+openpyxl
+# xlrd
+# mypy

app/main.py

@@ -18,7 +18,7 @@ from app.lib_general import log, logging
 from app.log import log
 
 # Import the routers here first:
-from app.routers import api_crud, api, account, address, archive, archive_content, contact, cont_edu_cert, cont_edu_cert_person, event, event_exhibit, event_file, event_person, event_person_detail, event_presentation, event_presenter, event_registration, event_session, flask_cfg, hosted_file, journal, journal_entry, log_client_viewing, lookup, membership_cfg, membership_group, membership_group_person, membership_person, membership_person_profile, membership_type, membership_type_person, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets # , items, journals
+from app.routers import api_crud, api, importing, account, address, archive, archive_content, contact, cont_edu_cert, cont_edu_cert_person, event, event_exhibit, event_file, event_person, event_person_detail, event_presentation, event_presenter, event_registration, event_session, flask_cfg, hosted_file, journal, journal_entry, log_client_viewing, lookup, membership_cfg, membership_group, membership_group_person, membership_person, membership_person_profile, membership_type, membership_type_person, order, order_cart, organization, page, person, post, post_comment, product, site, site_domain, user, user_person, websockets # , items, journals
 
 from app.db_sql import db
 
@@ -63,6 +63,11 @@ app.include_router(
     prefix='/flask_cfg',
     tags=['Flask CFG'],
 )
+app.include_router(
+    importing.router,
+    prefix='/importing',
+    tags=['Importing'],
+)
 # app.include_router(
 #     flask_cfg.router,
 #     prefix='/redis',

app/models/contact_models.py

@@ -44,6 +44,8 @@ class Contact_Base(BaseModel):
     timezone_name: Optional[str]
 
     email: Optional[str]
+    email_active: Optional[bool]
+    email_status: Optional[str]
 
     phone_mobile: Optional[str]
     phone_home: Optional[str]

app/models/person_models.py

@@ -42,13 +42,16 @@ class Person_Base(BaseModel):
     membership_person_id_random: Optional[str] # Linked from membership_person using the v_person view
     membership_person_id: Optional[int] # Linked from membership_person using the v_person view
 
+    informal_name: Optional[str]
     given_name: Optional[str]
     family_name: Optional[str]
     middle_name: Optional[str]
     prefix: Optional[str]
     suffix: Optional[str]
+
     full_name: Optional[str]
-    informal_name: Optional[str]
+    informal_full_name: Optional[str]
+    last_first_name: Optional[str]
 
     title: Optional[str]
 
@@ -57,9 +60,14 @@ class Person_Base(BaseModel):
     tagline: Optional[str]
 
     notes: Optional[str]
+
     created_on: Optional[datetime.datetime] = None
     updated_on: Optional[datetime.datetime] = None
 
+    # Including JSON data
+    other_json: Optional[Json]
+    meta_json: Optional[Json]
+
     # Including other related objects
     # archive_list: Optional[list] # Archive_Base()
     event_list: Optional[list] # Event_Base() # Priority l1

app/models/response_models.py

@@ -41,7 +41,7 @@ def mk_resp(
         exclude_unset: bool = True,
         response = None
         ):
-    # log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
     log.debug(locals())
 
     if data is None: data_out = { 'result': data }

app/models/user_models.py

@@ -62,6 +62,9 @@ class User_New_Base(BaseModel):
 
     notes: Optional[str]
 
+    # Including JSON data
+    other_json: Optional[Json]
+
     _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
 
     #@validator('user_id_random', always=True)

app/routers/importing.py

@@ -0,0 +1,285 @@
+import datetime, json, pytz, secrets, time
+import pandas, xlrd # qrcode
+from fastapi import APIRouter, Body, Depends, Header, HTTPException, Query, status
+from pydantic import BaseModel, EmailStr, Field
+from typing import Dict, List, Optional, Set, Union
+
+from app.lib_general import log, logging, secure_hash_string
+from app.config import settings
+from app.db_sql import sql_insert, sql_update, sql_insert_or_update, sql_select, sql_delete, redis_lookup_id_random
+
+from app.routers.api_crud import delete_obj_template, get_obj_template, get_obj_li_template, patch_obj_template, post_obj_template
+
+from app.methods.contact_methods import load_contact_obj, update_contact_obj
+from app.methods.person_methods import create_update_person_obj, get_person_rec_list, load_person_obj, update_person_obj
+from app.methods.user_methods import load_user_obj
+
+from app.models.contact_models import Contact_Base
+from app.models.person_models import Person_Base
+from app.models.user_models import User_Base, User_New_Base, User_Out_Base
+
+from app.models.response_models import Resp_Body_Base, mk_resp
+
+
+router = APIRouter()
+
+
+@router.post('/person_data', response_model=Resp_Body_Base)
+async def importing_person_data(
+        ):
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    account_id = 19
+    full_file_path = 'admin/temp/import_person_data.xlsx'
+
+    df = pandas.read_excel(full_file_path, na_filter=False, dtype={'external_id':str, 'phone_home':str, 'phone_mobile':str, 'city':str, 'state_province':str, 'address_postal_code':str, 'country':str})
+    #df = df.fillna('') # replace NaN with ''
+    # df = df.fillna(None)
+    # df = df.fillna('', inplace=True)
+    #return str(df.info())
+    log.debug(df)
+
+    df_dict = df.to_dict(orient='records')
+    # log.debug(df_dict)
+
+    # return mk_resp(data=False, status_code=500)
+
+    person_data_li = []
+    # for i in df.index:
+    for record in df_dict:
+        person_new = None
+        person_id = None
+        contact_id = None
+        address_id = None
+        user_id = None
+
+        person_data = {}
+        person_data['external_id'] = record['external_id']
+        # person_data['informal_name'] = record['informal_name']
+        person_data['given_name'] = record['given_name']
+        if record['middle_name']:
+            person_data['middle_name'] = record['middle_name']
+        else:
+            person_data['middle_name'] = None
+        if record['family_name']:
+            person_data['family_name'] = record['family_name']
+        else:
+            person_data['family_name'] = None
+        # person_data['prefix'] = record['prefix']
+        person_data['suffix'] = record['suffix']
+        if person_data['given_name'] and person_data['middle_name'] and person_data['family_name']:
+            person_data['full_name'] = person_data['given_name']+' '+person_data['middle_name']+' '+person_data['family_name']
+        elif person_data['given_name'] and person_data['family_name']:
+            person_data['full_name'] = person_data['given_name']+' '+person_data['family_name']
+        elif person_data['given_name']:
+            person_data['full_name'] = person_data['family_name']
+        elif record['informal_full_name']:
+            person_data['full_name'] = record['informal_full_name']
+        elif record['informal_name']:
+            person_data['full_name'] = record['informal_name']
+        else:
+            person_data['full_name'] = None
+        if record['informal_full_name']:
+            person_data['informal_full_name'] = record['informal_full_name']
+        else:
+            person_data['informal_full_name'] = None
+        person_data['last_first_name'] = record['last_first_name']
+        person_data['created_on'] = record['created_on']
+        person_data['updated_on'] = record['updated_on']
+
+        other_data = {}
+        other_data['contact_type'] = record['contact_type'] # ????
+
+        meta_data = {}
+        meta_data['created_by_method'] = record['created_by_method']
+        meta_data['created_by_name'] = record['created_by_name']
+        meta_data['modified_by'] = record['modified_by']
+        meta_data['created_by_method'] = record['created_by_method']
+
+        person_data['other_json'] = json.dumps(other_data, indent=4)
+        person_data['meta_json'] = json.dumps(meta_data, indent=4)
+
+
+        # Look up by email address or external ID and INSERT or UPDATE new person record
+        # INSERT or UPDATE a contact record and address record if needed
+        # INSERT or UPDATE a user record if needed
+        # Process the person data
+        log.debug(person_data)
+        # log.debug('*** *** *** *** END TEST RUN *** *** *** ***')
+        # continue
+        if person_rec_li_result := sql_select(table_name='v_person', field_name='external_id', field_value=person_data['external_id']):
+            if not isinstance(person_rec_li_result, list):
+                # Pull out IDs and UPDATE existing person record
+                log.debug('Found one record')
+                person_rec = person_rec_li_result
+                person_id = person_rec.get('person_id', None)
+                contact_id = person_rec.get('contact_id_new', None) # Using _new from view until old contact_id is removed from person table
+                address_id = person_rec.get('address_id', None)
+                user_id = person_rec.get('user_id', None)
+                person_data['id'] = person_id
+                if person_obj_up_result := sql_update(data=person_data, table_name='person'):
+                    log.debug(person_obj_up_result)
+                else:
+                    log.warning(person_obj_up_result)
+                    continue # Something unexpected may have happened
+            else:
+                log.warning('Found more than one record')
+                # Do nothing
+                continue # Something unexpected may have happened
+            person_rec_li = person_rec_li_result
+        else:
+            # INSERT new record
+            log.debug('Found no records or something went wrong')
+            person_data['account_id'] = account_id
+            if person_obj_in_result := sql_insert(data=person_data, table_name='person'):
+                log.debug(person_obj_in_result)
+                person_id = person_obj_in_result # Should be an int
+                person_new = True # Need to UPDATE this record after the contact, address, and user data is processed
+            else:
+                log.warning(person_obj_in_result)
+                continue # Something unexpected may have happened
+
+        # Process the contact data
+        log.debug('Process the contact data')
+        contact_data = {}
+        contact_data['email'] = record['email']
+        if record['email_status'] != 'Undeliverable':
+            contact_data['email_active'] = True
+        else:
+            contact_data['email_active'] = False
+        contact_data['email_status'] = record['email_status']
+        contact_data['phone_mobile'] = record['phone_mobile']
+        contact_data['phone_home'] = record['phone_home']
+        if record['phone_fax']:
+            contact_data['phone_fax'] = record['phone_fax']
+        elif record['phone_fax'] and record['phone_work_fax']:
+            contact_data['phone_fax'] = record['phone_fax']
+            contact_data['phone_other'] = record['phone_work_fax']
+        contact_data['phone_office'] = record['phone_work']
+
+        log.debug(contact_data)
+        if contact_id:
+            # UPDATE existing contact record
+            contact_data['id'] = contact_id
+            if contact_obj_up_result := sql_update(data=contact_data, table_name='contact'):
+                log.debug(contact_obj_up_result)
+            else:
+                log.warning(contact_obj_up_result)
+                continue # Something unexpected may have happened
+        elif person_id:
+            # INSERT new contact record and link to person record
+            contact_data['account_id'] = account_id
+            contact_data['for_type'] = 'person'
+            contact_data['for_id'] = person_id
+            if contact_obj_in_result := sql_insert(data=contact_data, table_name='contact'):
+                log.debug(contact_obj_in_result)
+                contact_id = contact_obj_in_result # Should be an int
+            else:
+                log.debug(contact_obj_in_result)
+                continue # Something unexpected may have happened
+
+        # Process the contact address data
+        log.debug('Process the contact address data')
+        address_data = {}
+        address_data['line_1'] = record['address_line_1']
+        address_data['line_2'] = record['address_line_2']
+        address_data['line_3'] = record['address_line_3']
+        address_data['city'] = record['address_city']
+        address_data['country_subdivision_code'] = record['address_country_code']+'-'+record['address_state_province_code']
+        address_data['postal_code'] = record['address_postal_code']
+        address_data['country_alpha_2_code'] = record['address_country_code']
+
+        log.debug(address_data)
+        if address_id:
+            # UPDATE existing address record
+            log.debug('UPDATE existing address record')
+            address_data['id'] = address_id
+            if address_obj_up_result := sql_update(data=address_data, table_name='address'):
+                log.debug(address_obj_up_result)
+            else:
+                log.warning(address_obj_up_result)
+                # continue # Something unexpected may have happened
+        elif contact_id:
+            # INSERT new address record and link to contact record
+            log.debug('INSERT new address record and link to contact record')
+            address_data['account_id'] = account_id
+            address_data['for_type'] = 'contact'
+            address_data['for_id'] = contact_id
+            if address_obj_in_result := sql_insert(data=address_data, table_name='address'):
+                log.debug(address_obj_in_result)
+                address_id = address_obj_in_result # Should be an int
+            else:
+                log.debug(address_obj_in_result)
+                # break
+                continue # Something unexpected may have happened
+
+        # Process the user data
+        log.debug('Process the user data')
+        user_data = {}
+        user_data['name'] = person_data['full_name']
+        user_data['username'] = record['email']
+        user_data['email'] = record['email']
+        user_data['email_verified'] = contact_data['email_active'] # Not perfect, but a good start
+        random_password_string = secrets.token_urlsafe(8)
+        user_data['password'] = secure_hash_string(string=random_password_string)
+
+        user_data['enable'] = False
+        user_data['enable_from'] = datetime.datetime.now(datetime.timezone.utc)
+        user_data['enable_to'] = datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=365)
+
+        user_data['super'] = False
+        user_data['manager'] = False
+        user_data['administrator'] = False
+
+        user_data['public'] = False
+        user_data['verified'] = True
+        user_data['notes'] = 'Created by importing list'
+
+        other_data = {}
+        other_data['temp_password'] = random_password_string
+
+        user_data['other_json'] = json.dumps(other_data, indent=4)
+
+        log.debug(user_data)
+        if user_id:
+            # UPDATE existing user record
+            user_data['id'] = user_id
+            if user_obj_up_result := sql_update(data=user_data, table_name='user'):
+                log.debug(user_obj_up_result)
+            else:
+                log.warning(user_obj_up_result)
+                continue # Something unexpected may have happened
+        elif person_id:
+            # INSERT new user record and link to person record
+            user_data['account_id'] = account_id
+            user_data['person_id'] = person_id
+            if user_obj_in_result := sql_insert(data=user_data, table_name='user'):
+                log.debug(user_obj_in_result)
+                user_id = user_obj_in_result # Should be an int
+            else:
+                log.debug(user_obj_in_result)
+                # break
+                continue # Something unexpected may have happened
+
+        if person_new:
+            log.debug('Updating person record one more time since this is a new person')
+            person_data_up = {}
+            person_data_up['id'] = person_id
+            person_data_up['user_id'] = user_id
+            random_password_string
+            # Don't need to update with the new contact or address IDs that were just created.
+
+            if person_obj_up_result := sql_update(data=person_data_up, table_name='person'):
+                log.debug(person_obj_up_result)
+            else:
+                log.warning(person_obj_up_result)
+                # break
+                continue # Something unexpected may have happened
+
+        person_data_li.append(person_data)
+        log.debug(f"Record processed: {person_id} {person_data['full_name']}")
+        # log.debug('*** *** *** *** END TEST RUN *** *** *** ***')
+        # break
+
+    return mk_resp(data=person_data_li)