Finally updating this...

README.md

@@ -1,2 +1,2 @@
 # Aether API Python FastAPI
-The Aether API was created and is being developed by Scott Idem.
+The Aether API was created and is being developed by Scott Idem using the Python FastAPI framework.

admin/requirements.txt

@@ -5,3 +5,6 @@ SQLAlchemy
 mysqlclient
 redis
 aioredis
+html2text
+pytz
+#mypy

app/config.py.default

@@ -14,5 +14,6 @@ class Settings(BaseSettings):
     AETHER_DB_PASSWORD = 'xxx'
     SQLALCHEMY_DATABASE_URI = 'mysql://'+AETHER_DB_USERNAME+':'+AETHER_DB_PASSWORD+'@'+AETHER_DB_SERVER+'/'+AETHER_DB_NAME
 
+    DB_CFG_FASTAPI_ID = 0
 
 settings = Settings()

app/db_sql.py

@@ -0,0 +1,499 @@
+from app.config import settings
+from .log import *
+
+from sqlalchemy import create_engine, text
+from sqlalchemy.exc import IntegrityError, OperationalError
+
+db_uri = settings.SQLALCHEMY_DATABASE_URI
+
+connection_string = db_uri
+engine = create_engine(name_or_url=connection_string, pool_size=25, pool_recycle=60, pool_pre_ping=True, echo=True, echo_pool=True, isolation_level='READ COMMITTED')
+# NOTE: The default isolation_level is 'REPEATABLE READ'. This can sometimes not show updated data.
+
+db = engine.connect()
+
+
+# #### ### ## # BEGIN SQL # ## ### ####
+# Create, Read/Get, Update, Delete
+# CRUD or CGUD
+
+
+# ### BEGIN ### Core Help CRUD ### sql_insert() ###
+def sql_insert(sql=None, data=None, table_name=None, rm_id_random=None, id_random_length=None):
+    log.setLevel(logging.ERROR) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    return False
+# ### END ### Core Help CRUD ### sql_insert() ###
+
+
+# ### BEGIN ### Core Help CRUD ### sql_update() ###
+def sql_update(sql=None, data=None, table_name=None, rm_id_random=None, id_random_length=None):
+    log.setLevel(logging.ERROR) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    return False
+# ### END ### Core Help CRUD ### sql_update() ###
+
+
+# ### BEGIN ### Core Help CRUD ### sql_insert_or_update() ###
+# The catch all SQL INSERT or UPDATE function - STI 2021-02-17
+# This one does it all for SQL INSERT and UPDATE queries
+def sql_insert_or_update(sql:str=None, data:dict=None, table_name:str=None, rm_id_random:bool=None, id_random_length:int=None):
+    log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    #if sql: pass
+    #else:
+        #log.error('SQL text is missing')
+        #return False
+
+    if sql:
+        sql_insert_or_update = text(sql)
+    elif table_name and data:
+        if rm_id_random:
+            data = lookup_id_random_pop(obj_data=data)
+        if not data.get('id_random', None) and id_random_length:
+            data['id_random'] = secrets.token_urlsafe(id_random_length)
+
+        fields = []
+        values = []
+        for key, value in data.items():
+            if key != 'id': # A special exception for the id auto increment field.
+                fields.append('`'+str(key)+'`')
+                values.append(':'+str(key))
+        fields_string = ', '.join(fields)
+        values_string = ', '.join(values)
+
+        field_list = []
+        for key, value in data.items():
+            if key != 'id': # Creating a special exception for the id field.
+                field_list.append('`'+str(key) + '` = :' + str(key))
+        set_values_string = ', '.join(field_list)
+
+        sql_insert_or_update = text(
+            f"""
+            INSERT INTO `{table_name}` ({fields_string}) VALUES ({values_string})
+            ON DUPLICATE KEY UPDATE
+                {set_values_string}
+            ;
+            """
+        )
+
+        log.debug(f"""
+            INSERT INTO `{table_name}` ({fields_string}) VALUES ({values_string})
+            ON DUPLICATE KEY UPDATE
+                {set_values_string}
+            ;
+            """)
+
+    trans = db.begin()
+    try:
+        result_insert = db.execute(sql_insert_or_update, data)
+        trans.commit()
+    except Exception as e:
+        trans.rollback()
+        log.exception('*** An exception happened. ***')
+        log.exception(repr(e))
+        log.exception('***')
+        log.exception(str(e))
+        log.exception('^^^ exception ^^^')
+        return False
+    else:
+        if result_insert.rowcount == 1 and result_insert.lastrowid > 0: # insert
+            log.info('Insert record')
+            record_id = result_insert.lastrowid
+            return record_id
+        elif result_insert.rowcount == 1 and result_insert.lastrowid == 0: # update with no change
+            log.info('Update record with no change')
+            return True
+        elif result_insert.rowcount == 2 and result_insert.lastrowid > 0: # update with change
+            log.info('Update record with changes')
+            record_id = result_insert.lastrowid
+            return record_id
+        else:
+            log.debug(result_insert)
+            log.debug(vars(result_insert))
+            log.debug(dir(result_insert))
+            log.debug(result_insert.rowcount) # returns 1 on insert and 2 on update with change
+            log.debug(result_insert.lastrowid) # returns last row ID on insert and update with a change and returns 0 if nothing changed
+            return False
+    return False
+# ### END ### Core Help CRUD ### sql_insert_or_update() ###
+
+
+# ### BEGIN ### Core Help CRUD ### sql_select() ###
+# The catch all SQL SELECT function - STI 2021-02-17
+# This one does it all for SQL SELECT queries
+def sql_select(table_name=None, record_id=None, record_id_random=None, field_name=None, field_value=None, sql=None, data=None, rm_id_random=None, as_dict=True, as_list=None):
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    if table_name and not (record_id or record_id_random or field_name or field_value or sql or data):
+        # Select all records from a table
+        log.info('Select all records from a table')
+        sql = text(
+            f"""
+            SELECT *
+            FROM `{table_name}`
+            ;
+            """
+        )
+    elif table_name and (record_id or record_id_random) and not (field_name or field_value or sql or data):
+        # Select all records from a table with an ID (auto or random)
+        log.info('Select all records from a table with an ID (auto or random)')
+        data = {}
+        if record_id:
+            data['record_id'] = record_id
+
+            sql = text(
+                f"""
+                SELECT *
+                FROM `{table_name}`
+                WHERE `{table_name}`.id = :record_id
+                ;
+                """
+            )
+        elif record_id_random:
+            data['record_id_random'] = record_id_random
+
+            sql = text(
+                f"""
+                SELECT *
+                FROM `{table_name}`
+                WHERE `{table_name}`.id_random = :record_id_random
+                ;
+                """
+            )
+    elif table_name and field_name and field_value and not (record_id or record_id_random or sql or data):
+        # Select all records from a table with a specific field and field value
+        log.info('Select all records from a table with a specific field and field value')
+        data = {}
+        data[field_name] = field_value
+
+        sql = text(
+            f"""
+            SELECT *
+            FROM `{table_name}`
+            WHERE `{table_name}`.{field_name} = :{field_name}
+            ;
+            """
+        )
+    elif table_name and data and not (record_id or record_id_random or field_name or field_value or sql):
+        # Select all records from a table with a specific list of fields and field values (list of dicts)
+        log.info('Select all records from a table with a specific list of fields and field values (list of dicts)')
+
+        if rm_id_random:
+            data = lookup_id_random_pop(obj_data=data)
+
+        sql_where = []
+        for field_name in data:
+            sql_where_line = f"""`{table_name}`.{field_name} = :{field_name}"""
+            sql_where.append(sql_where_line)
+
+        sql_where_string = ' AND '.join(sql_where)
+        log.debug(sql_where_string)
+
+        sql = text(
+            f"""
+            SELECT *
+            FROM `{table_name}`
+            WHERE {sql_where_string}
+            ;
+            """
+        )
+    elif sql and not (table_name or record_id or record_id_random or field_name or field_value or data):
+        # Select records based on the SQL statement given
+        log.info('Select records based on the SQL statement given')
+        sql = text(sql)
+    elif sql and data and not (table_name or record_id or record_id_random or field_name or field_value):
+        # Select records based on the SQL statement given and with the matching data dict fields and values
+
+        if rm_id_random:
+            data = lookup_id_random_pop(obj_data=data)
+
+        log.info('Select records based on the SQL statement given and with the matching data dict fields and values')
+        sql = text(sql)
+    else:
+        # Nothing matched the expected combination of parameters passed to this function
+        log.warning('Nothing matched the expected combination of parameters passed to this function')
+        return False # Not successful
+
+    #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(sql)
+    log.debug(data)
+
+    try:
+        if data:
+            log.info('Executing with SQL statement and data...')
+            result = db.execute(sql, data)
+        else:
+            log.info('Executing with SQL statement only...')
+            result = db.execute(sql)
+    except OperationalError as e:
+        log.warning('*** An exception happened: OperationalError ***')
+        log.warning('* This is likely a "MySQL server has gone away" error. Going to try again... *')
+        log.warning(repr(e))
+        log.warning('***')
+        log.warning(str(e))
+        log.warning('^^^ exception ^^^')
+
+        log.warning('Trying to recreate the pool...')
+        log.debug('############## ############')
+        log.debug(dir(db))
+        log.debug(vars(db))
+        log.debug('############## ############')
+        log.debug(dir(db.engine))
+        log.debug(vars(db.engine))
+        log.debug('############## ############')
+        log.debug(dir(db.engine.pool))
+        log.debug(vars(db.engine.pool))
+        log.debug('############## ############')
+        db.engine.dispose()
+        log.warning('Now trying the query again...')
+        try:
+            if data:
+                log.warning('2x Executing with SQL statement and data...')
+                result = db.execute(sql, data)
+            else:
+                log.warning('2x Executing with SQL statement only...')
+                result = db.execute(sql)
+        except Exception as e:
+            log.warning('2x A *second* exception happened. Returning False.')
+            log.exception(repr(e))
+            log.exception('***')
+            log.exception(str(e))
+            log.exception('^^^ exception ^^^')
+            return False # Not successful
+        else:
+            log.info('Successfully executed the SQL on the second try.')
+            pass
+    except Exception as e:
+        log.info('An exception happened. Returning False.')
+        log.exception(repr(e))
+        log.exception('***')
+        log.exception(str(e))
+        log.exception('^^^ exception ^^^')
+        return False # Not successful
+    else:
+        log.info('Successfully executed the SQL on the first try.')
+        pass
+
+    #log.debug(result.fetchall()) # Uncommenting this breaks things?
+    # BEGIN NOTE: Check this out later! ###
+    #header = result.keys()
+    #for row in result:
+    #    yield dict(zip(header, row))
+    # END NOTE: Check this out later! ###
+
+    #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(result.rowcount)
+    log.debug(vars(result))
+    log.debug(dir(result))
+    if result.rowcount == 1:
+        log.info(f'Found one record. as_dict={as_dict}, as_list={as_list}')
+        if as_dict:
+            record = sql_result_proxy_to_dict_simple(result_proxy=result.first())
+        else:
+            record = result.first()
+        if as_list:
+            record_li = []
+            record_li.append(record)
+            #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+            log.debug(record_li)
+            return record_li # Successful
+        else:
+            #log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+            log.debug(record)
+            return record # Successful
+    elif result.rowcount > 1:
+        log.info(f'Found {result.rowcount} records. as_dict={as_dict}, as_list={as_list}')
+        #log.info('Found more than one record. Returning as a list of dicts.')
+        if as_dict:
+            record_li = sql_result_proxy_to_dict_simple(result_proxy=result.fetchall())
+        else:
+            record_li = result.fetchall()
+        log.debug(record_li)
+        return record_li # Successful
+    else:
+        log.info('No records found. Returning None.')
+        log.debug(result)
+        return None # Successful
+# ### END ### Core Help CRUD ### sql_select() ###
+
+
+# ### BEGIN ### Core Help CRUD ### sql_delete() ###
+# The catch all SQL DELETE function - STI 2021-02-17
+# This one does it all for SQL DELETE queries
+def sql_delete(table_name:str=None, record_id:int=None, record_id_random:str=None, field_name:str=None, field_value=None, sql:str=None, data:dict=None):
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    if table_name and (record_id or record_id_random) and not (field_name or field_value or sql or data):
+        # Delete all records from a table with an ID (auto or random)
+        log.info('Delete all records from a table with an ID (auto or random)')
+        data = {}
+        if record_id:
+            data['record_id'] = record_id
+
+            sql = text(
+                f"""
+                DELETE FROM `{table_name}`
+                WHERE `{table_name}`.id = :record_id
+                """
+            )
+        elif record_id_random:
+            data['record_id_random'] = record_id_random
+
+            sql = text(
+                f"""
+                DELETE FROM `{table_name}`
+                WHERE `{table_name}`.id_random = :record_id_random
+                """
+            )
+    elif table_name and field_name and field_value and not (record_id or record_id_random or sql or data):
+        # Delete all records from a table with a specific field and field value
+        log.info('Delete all records from a table with a specific field and field value')
+        data = {}
+        data[field_name] = field_value
+
+        sql = text(
+            f"""
+            DELETE FROM `{table_name}`
+            WHERE `{table_name}`.{field_name} = :{field_name}
+            """
+        )
+    elif table_name and data and not (record_id or record_id_random or field_name or field_value or sql):
+        # Delete all records from a table with a specific list of fields and field values (list of dicts)
+        log.info('Delete all records from a table with a specific list of fields and field values (list of dicts)')
+        sql_where = []
+        for field_name in data:
+            sql_where_line = f"""`{table_name}`.{field_name} = :{field_name}"""
+            sql_where.append(sql_where_line)
+
+        sql_where_string = ' AND '.join(sql_where)
+        log.debug(sql_where_string)
+
+        sql = text(
+            f"""
+            DELETE FROM `{table_name}`
+            WHERE {sql_where_string}
+            """
+        )
+        log.debug(sql)
+    elif sql and not (table_name or record_id or record_id_random or field_name or field_value or data):
+        # Delete records based on the SQL statement given
+        log.info('Delete records based on the SQL statement given')
+        sql = text(sql)
+    elif sql and data and not (table_name or record_id or record_id_random or field_name or field_value):
+        # Delete records based on the SQL statement given and with the matching data dict fields and values
+        log.info('Delete records based on the SQL statement given and with the matching data dict fields and values')
+        sql = text(sql)
+    else:
+        # Nothing matched the expected combination of parameters passed to this function
+        log.warning('Nothing matched the expected combination of parameters passed to this function')
+        return False # Not successful
+
+    log.debug(sql)
+
+    try:
+        log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+        if data:
+            log.info('Executing with SQL (DELETE) statement and data...')
+            result = db.execute(sql, data)
+        else:
+            log.info('Executing with SQL (DELETE?) statement only...')
+            result = db.execute(sql)
+        log.debug(result)
+        log.debug(dir(result))
+        log.debug(vars(result))
+    except OperationalError as e:
+        log.warning('*** An exception happened: OperationalError ***')
+        log.warning('* This is likely a "MySQL server has gone away" error. Going to try again... *')
+        log.warning(repr(e))
+        log.warning('***')
+        log.warning(str(e))
+        log.warning('^^^ exception ^^^')
+
+        log.warning('Trying to recreate the pool...')
+        log.debug('############## ############')
+        log.debug(dir(db))
+        log.debug(vars(db))
+        log.debug('############## ############')
+        log.debug(dir(db.engine))
+        log.debug(vars(db.engine))
+        log.debug('############## ############')
+        log.debug(dir(db.engine.pool))
+        log.debug(vars(db.engine.pool))
+        log.debug('############## ############')
+        db.engine.dispose()
+        log.warning('Now trying the query again...')
+        try:
+            log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+            if data:
+                log.warning('2x Executing with SQL statement and data...')
+                result = db.execute(sql, data)
+            else:
+                log.warning('2x Executing with SQL statement only...')
+                result = db.execute(sql)
+            log.debug(result)
+        except Exception as e:
+            log.warning('2x A *second* exception happened. Returning False.')
+            log.exception(repr(e))
+            log.exception('***')
+            log.exception(str(e))
+            log.exception('^^^ exception ^^^')
+            return False # Not successful
+        else:
+            log.info('Successfully executed the SQL on the second try.')
+            pass
+    except Exception as e:
+        log.info('An exception happened. Returning False.')
+        log.exception(repr(e))
+        log.exception('***')
+        log.exception(str(e))
+        log.exception('^^^ exception ^^^')
+        return False # Not successful
+    else:
+        log.info('Successfully executed the SQL on the first try.')
+        pass
+
+    # NOTE: Need to deal with 0 rows affected when the WHERE clause was not satisfied and there was no error.
+    return True # Successful
+
+
+# NOTE WARNING: This is a near duplicate of what is under lib_rest (was lib_general). WARNING
+# Change SQL SELECT result RowProxy record to a dict (named key/value)
+# Change SQL SELECT list result RowProxy records to a list of dicts (named key/value)
+def sql_result_proxy_to_dict_simple(result_proxy=None):
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    log.debug(type(result_proxy))
+
+    if isinstance(result_proxy, list):
+        log.info('Processing a SQL list...')
+
+        record_li = []
+        for row_proxy in result_proxy:
+            log.debug(row_proxy)
+
+            record = {}
+            for key, value in row_proxy.items():
+                record[key] = value
+            record_li.append(record)
+        return record_li
+
+    # Must import sqlalchemy to check the type correctly.
+    # Or convert it to a string and compare.
+    if str(type(result_proxy)) == '<class \'sqlalchemy.engine.result.RowProxy\'>':
+    #if isinstance(result_proxy, sqlalchemy.engine.result.RowProxy):
+        log.info('Processing a SQL record (sqlalchemy.engine.result.RowProxy)')
+
+        row_proxy = result_proxy
+
+        record = {}
+        for key, value in row_proxy.items():
+            record[key] = value
+        return record
+    return False

app/lib_general.py

@@ -1,12 +1,12 @@
-import redis
+import datetime, redis
 
-from datetime import datetime, time, timedelta
+#from datetime import datetime, time, timedelta
 from fastapi import APIRouter, Depends, Header, HTTPException, status
 from pydantic import BaseModel, EmailStr, Field
 from typing import Dict, List, Optional, Set, Union
 
 from .log import *
-from .db import *
+from .db_sql import *
 
 
 async def get_token_header(x_token: str = Header(...)):
@@ -38,78 +38,82 @@ async def get_account_header(x_account_id: str = Header(...)):
     return account
 
 
-#Add the processing time to the response header.
-#@app.middleware('http')
-#async def add_process_time_header(request: Request, call_next):
-    #import time
-    #start_time = time.time()
-    #response = await call_next(request)
-    #process_time = time.time() - start_time
-    #response.headers['X-Process-Time'] = str(process_time)
-    #return response
-
-
-#async def get_token_header(x_token: str = Header(...)):
-    #if x_token != 'fake-super-secret-token':
-        #raise HTTPException(status_code=400, detail='X-Token header invalid')
-
-
-#async def get_account_header(x_account_id: str = Header(...)):
-#@app.middleware("http")
-#async def get_account_header(x_account_id: str = Header(...)):
-    #return x_account_id
-    #x_account_id: str = Header(...)
-    #x_account_id = 'static random ID...'
-    #response = await call_next(request)
-
-    #print(x_account_id)
-
-    #return x_account_id
-
-
-#async def get_account_header(x_account_id: str = Header(...)):
-    #print('get_account_header(): '+x_account_id+'z9999z')
-    #return x_account_id+'z9999z'
-
-
-
-
-
-# Attempt to look up id_random key
-# If success then return the id number
+# Just return the value if it is an integer
+# Check if the id_random value is a string and the correct length
+# Attempt to look up id_random key in Redis
+# If success then return the ID number
 # If not success and there is a table_name then check the database table passed
-# If found in database table then store in Redis
+# If found in database table then store in Redis and return the ID number
 def redis_lookup_id_random(record_id_random=None, table_name=None):
-    log.setLevel(logging.DEBUG) # DEBUG, INFO, WARN, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
     log.debug(locals())
 
+    if record_id_random is None: return False
+    if isinstance(record_id_random, bool): return False
+    if isinstance(record_id_random, int):
+        return record_id_random
+    elif isinstance(record_id_random, str):
+        pass
+    else:
+        log.warning(f'Unexpected data type: {str(type(record_id_random))} Expected type is a string 11 or 22 characters long.')
+        return False
+
+    if record_id_random and table_name:
+        # WARNING: The record_id_random string length should be checked just in case?
+        if len(record_id_random) < 11:
+            log.warning(f'The length of id_random is too short: {str(record_id_random)} ({len(record_id_random)} chars)')
+            return False
+        elif len(record_id_random) > 22:
+            log.warning(f'The length of id_random is too long {str(record_id_random)} ({len(record_id_random)} chars)')
+            return False
+        else:
+            pass
+    else:
+        log.warning('Missing table_name to select from for id_random')
+        return False
+
     r = redis.Redis(host='localhost', port=6379, db=7, password=None, decode_responses=True)
 
     key_name = 'record_id:'+record_id_random
 
     record_id = r.get(key_name)
-    #print('Record ID? '+str(record_id))
+    log.debug(f'Record ID? {str(record_id)}')
 
     if record_id:
-        print('TTL for: '+key_name+' : '+str(record_id)+' is '+str(r.ttl(key_name))+' seconds')
-        return record_id
+        log.info('The record ID was found using the record_id_random value.')
+        log.info(f'TTL for: {key_name} : {str(record_id)} is {str(r.ttl(key_name))} seconds')
+        return int(record_id)
     elif table_name:
         data = { 'id_random': record_id_random }
-        sql = """
+        sql = f"""
             SELECT id
-            FROM `"""+table_name+"""` AS `table`
-            WHERE table.id_random = :id_random
+            FROM `{table_name}` AS `table`
+            WHERE `table`.id_random = :id_random;
             """
 
-        if select_results := sql_select(table_name=table_name, record_id_random=record_id_random): # sql_select(sql=sql, data=data)
-            #print('Record ID random found: '+str(select_results['id']))
-            record_id = select_results['id']
-            r.setex(key_name, timedelta(minutes=2), value=record_id)
-            return record_id
+        if select_results := sql_select(sql=sql, data=data):
+            log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+            log.debug(select_results)
+            log.debug(type(select_results))
+            if isinstance(select_results, dict):
+                log.info(f"""Record ID random found: {str(select_results['id'])}""")
+                if record_id := select_results.get('id'):
+                    r.setex(key_name, datetime.timedelta(minutes=90), value=record_id)
+                    return int(record_id)
+                else:
+                    log.setLevel(logging.ERROR) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+                    log.error('The SQL result was not what was expected.')
+                    return False
+            else:
+                log.setLevel(logging.ERROR) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+                log.error('More than one record may have been found. There may be a duplicate id_random.')
+                log.error(select_results)
+                return False
         else:
-            #print('Record ID random was not found')
+            #log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+            log.info('Record ID random was not found')
             return None
-    else:
-        print('Missing table_name to select from for id_random')
-        return False
-    #return False
+
+    log.setLevel(logging.ERROR) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.error('We should not be here. Something unexpected happened.')
+    return False # Just in case

app/main.py

@@ -1,7 +1,7 @@
 import logging, random # , uvicorn
 
-from datetime import datetime, time, timedelta
 from enum import Enum
+#from datetime import datetime, time, timedelta
 from fastapi import Body, Cookie, Depends, FastAPI, File, Form, Header, HTTPException, Path, Query, Request, status, UploadFile
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, PlainTextResponse
@@ -18,14 +18,14 @@ from .lib_general import *
 from .log import *
 
 # Import the routers here first:
-from .routers import items, journals, users, websockets
+from .routers import api_crud, items, journals, users, websockets
 
 
 # TEST TEST TEST
-print('**** Calling db.py ... ****')
-#from .db import engine, SessionLocal, Base
-from .db import db
-print('**** Called db.py ****')
+print('**** Calling db_sql.py ... ****')
+#from .db_sql import engine, SessionLocal, Base
+from .db_sql import db
+print('**** Called db_sql.py ****')
 # TEST TEST TEST
 
 
@@ -48,6 +48,13 @@ app.mount('/static', StaticFiles(directory='static'), name='static')
 
 
 # Set up each route once the router has been imported
+app.include_router(
+    api_crud.router,
+    prefix='/crud',
+    tags=['CRUD'],
+    #dependencies=[Depends(get_token_header)],
+    #responses={404: {'description': 'Not found'}},
+)
 app.include_router(
     items.router,
     prefix='/item',

app/redis.py

@@ -1,41 +0,0 @@
-from app.db import *
-import redis
-from datetime import timedelta
-
-# Attempt to look up id_random key
-# If success then return the id number
-# If not success and there is a table_name then check the database table passed
-# If found in database table then store in Redis
-def redis_lookup_id_random(record_id_random=None, table_name=None):
-    print('*** redis_lookup_id_random() ***')
-
-    r = redis.Redis(host='localhost', port=6379, db=7, password=None, decode_responses=True)
-
-    key_name = 'record_id:'+record_id_random
-
-    record_id = r.get(key_name)
-    #print('Record ID? '+str(record_id))
-
-    if record_id:
-        print('TTL for: '+key_name+' : '+str(record_id)+' is '+str(r.ttl(key_name))+' seconds')
-        return record_id
-    elif table_name:
-        data = { 'id_random': record_id_random }
-        sql = """
-            SELECT id
-            FROM `"""+table_name+"""` AS `table`
-            WHERE table.id_random = :id_random
-            """
-
-        if select_results := sql_select(table_name=table_name, record_id_random=record_id_random): # sql_select(sql=sql, data=data)
-            #print('Record ID random found: '+str(select_results['id']))
-            record_id = select_results['id']
-            r.setex(key_name, timedelta(minutes=2), value=record_id)
-            return record_id
-        else:
-            #print('Record ID random was not found')
-            return False
-    else:
-        print('Missing table_name to select from for id_random')
-        return False
-    #return False

app/routers/address_model.py

@@ -0,0 +1,119 @@
+from __future__ import annotations
+import datetime, hashlib, logging, os, pytz, redis, secrets
+
+from typing import Dict, List, Optional, Set, Union
+from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator
+
+from ..lib_general import *
+from ..log import *
+from .common_field_schema import base_fields, default_num_bytes
+#from .account_model import Account_Base
+
+
+class Address_Base(BaseModel):
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    #from .account_model import Account_Base
+
+    id_random: Optional[str] = Field(
+        **base_fields['address_id_random'],
+        alias='address_id_random',
+        default_factory=lambda:secrets.token_urlsafe(default_num_bytes),
+    )
+    id: Optional[int] = Field(
+        #alias='address_id'
+    )
+    account_id_random: Optional[str]
+    account_id: Optional[int]
+
+    for_type: Optional[str]
+    for_id_random: Optional[str]
+    for_id: Optional[int]    #organization: Optional[Organization_Base] = Organization_Base()
+
+    name: Optional[str]
+    attention_to: Optional[str]
+    organization_name: Optional[str]
+
+    line_1: Optional[str]
+    line_2: Optional[str]
+    line_3: Optional[str]
+    city: Optional[str]
+    country_subdivision_code: Optional[str]
+    state_province: Optional[str]
+    postal_code: Optional[str]
+    country_alpha_2_code: Optional[str]
+    country: Optional[str]
+
+    lu_time_zone_id: Optional[str]
+    timezone: Optional[str]
+
+    latitude: Optional[str]
+    longitude: Optional[str]
+
+    map_url: Optional[str]
+
+    congressional_district: Optional[str]
+
+    #priority: Optional[int]
+    #sort: Optional[int]
+    #group: Optional[str]
+
+    created_on: Optional[datetime.datetime] = None
+    updated_on: Optional[datetime.datetime] = None
+
+    #account: Optional[Account_Base] = Account_Base()
+
+    _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
+
+    #@validator('address_id_random', always=True)
+    def address_id_random_copy(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['id_random']:
+            return values['id_random']
+        return None
+
+    @validator('id', always=True)
+    def address_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['id_random']:
+            log.debug(values['id_random'])
+            return redis_lookup_id_random(record_id_random=values['id_random'], table_name='address')
+        return None
+
+    @validator('account_id', always=True)
+    def account_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['account_id_random']:
+            return redis_lookup_id_random(record_id_random=values['account_id_random'], table_name='account')
+        return None
+
+    #@validator('organization_id', always=True)
+    #def organization_id_lookup(cls, v, values, **kwargs):
+        #log.setLevel(logging.WARNING)
+        #log.debug(locals())
+
+        #if values['organization_id']:
+            #return redis_lookup_id_random(record_id_random=values['organization_id'], table_name='organization')
+        #return None
+
+    @validator('for_id', always=True)
+    def for_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['for_id_random'] and values['for_type']:
+            return redis_lookup_id_random(record_id_random=values['for_id_random'], table_name=values['for_type'])
+        return None
+
+    class Config:
+        underscore_attrs_are_private = True
+        fields = base_fields
+
+Address_Base.update_forward_refs()

app/routers/api_crud.py

@@ -0,0 +1,80 @@
+import datetime
+#from datetime import datetime, time, timedelta
+from fastapi import APIRouter, Depends, Header, HTTPException, Query, status
+from pydantic import BaseModel, EmailStr, Field
+from typing import Dict, List, Optional, Set, Union
+
+from ..lib_general import *
+from ..log import *
+from app.config import settings
+from app.db import *
+#from .journal_models import *
+#from .user_models import *
+from .user_model import *
+from .response_model import *
+
+router = APIRouter()
+
+
+# Working on the basic API CRUD - STI 2021-03-05
+
+#@router.get('/{object_l1}/list')
+@router.get('/{object_l1}/{object_id}/list')
+@router.get('/{object_l1}/{object_l2}/{object_id}/list')
+@router.get('/{object_l1}/{object_l2}/{object_id}/{object_l3}/list')
+async def get_obj_li(object_l1: str=None, object_l2: str=None, object_l3: str=None, object_id: str=None, x_account_id: str = Header(...)):
+    response_data = {}
+    response_data['object_l1'] = object_l1
+    response_data['object_l2'] = object_l2
+    response_data['object_l3'] = object_l3
+    response_data['object_id'] = object_id
+    response_data['list'] = 'li'
+
+    sql_result = sql_select(table_name='user', record_id=1)
+
+    response_data['sql_result'] = sql_result
+
+    return response_data
+
+
+@router.get('/{object_l1}/{object_id}')
+@router.get('/{object_l1}/{object_l2}/{object_id}')
+@router.get('/{object_l1}/{object_l2}/{object_l3}/{object_id}')
+async def get_obj(object_l1: str=None, object_l2: str=None, object_l3: str=None, object_id: str=None, x_account_id: str = Header(...),
+                  qry_str: Optional[str] = Query(None, max_length=50),
+                  qry_int: Optional[int] = None,
+                  by_alias: Optional[bool] = True,
+                  exclude_unset: Optional[bool] = True,
+                  ):
+    log.setLevel(logging.DEBUG) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    log.debug(by_alias)
+    log.debug(exclude_unset)
+    log.debug(qry_str)
+    log.debug(qry_int)
+
+    response_data = {}
+    response_data['object_l1'] = object_l1
+    response_data['object_l2'] = object_l2
+    response_data['object_l3'] = object_l3
+    response_data['object_id'] = object_id
+
+    data = {}
+    data['id_random'] = 1
+
+    sql_select_str = f"""
+        SELECT `user`.id AS 'user_id', `user`.id_random AS 'user_id_random', username, name, email, super
+        FROM `user` AS `user`
+        WHERE `user`.id = :id_random;
+        """
+
+    #sql_result = sql_select(table_name='user', record_id=1)
+    sql_result = sql_select(sql=sql_select_str, data=data)
+    resp_data = User_Base(**sql_result).dict(by_alias=by_alias, exclude_unset=exclude_unset)
+
+    #response_data['sql_result'] = sql_result
+
+    resp = mk_resp(data=resp_data)
+
+    return resp

app/routers/common_field_schema.py

@@ -0,0 +1,69 @@
+import copy, datetime, hashlib, logging, os, pytz, redis, secrets
+
+default_num_bytes = 8 # URL safe 8 bytes is 11 characters long and 16 bytes is 22 characters long
+
+xxx_id_random_field_schema: dict = {
+    'title': 'XXX ID Random',
+    'description': 'This is an id_random field for this object.',
+    'min_length': 11,
+    'max_length': 22,
+    'example': secrets.token_urlsafe(8) # random each reloading of app with: secrets.token_urlsafe(8)
+}
+
+xxx_id_random_field_schema_default: dict = copy.copy(xxx_id_random_field_schema)
+xxx_id_random_field_schema_default['default_factory'] = lambda:secrets.token_urlsafe(8)
+
+created_updated_on_field_schema: dict = {
+    'title': 'Created or Updated On',
+    'description': 'This is the created or updated on timestamp field for this object. It is filled in by the SQL DB.',
+    'example': '2021-12-31T21:10:10'
+}
+
+base_fields = {}
+#base_fields['id_random'] = xxx_id_random_field_schema_default
+base_fields['obj_id_random'] = xxx_id_random_field_schema # General or generic object_id_random
+base_fields['account_id_random'] = xxx_id_random_field_schema
+base_fields['address_id_random'] = xxx_id_random_field_schema
+base_fields['archive_id_random'] = xxx_id_random_field_schema
+base_fields['contact_id_random'] = xxx_id_random_field_schema
+base_fields['event_exhibit_id_random'] = xxx_id_random_field_schema
+base_fields['event_file_id_random'] = xxx_id_random_field_schema
+base_fields['event_id_random'] = xxx_id_random_field_schema
+base_fields['event_presentation_id_random'] = xxx_id_random_field_schema
+base_fields['event_registration_id_random'] = xxx_id_random_field_schema
+base_fields['fundraising_id_random'] = xxx_id_random_field_schema
+base_fields['hosted_file_id_random'] = xxx_id_random_field_schema
+base_fields['membership_id_random'] = xxx_id_random_field_schema
+base_fields['membership_profile_id_random'] = xxx_id_random_field_schema
+base_fields['order_cart_id_random'] = xxx_id_random_field_schema
+base_fields['order_cart_line_id_random'] = xxx_id_random_field_schema
+base_fields['order_id_random'] = xxx_id_random_field_schema
+base_fields['order_line_id_random'] = xxx_id_random_field_schema
+base_fields['order_transaction_id_random'] = xxx_id_random_field_schema
+base_fields['organization_id_random'] = xxx_id_random_field_schema
+base_fields['page_id_random'] = xxx_id_random_field_schema
+base_fields['person_id_random'] = xxx_id_random_field_schema
+base_fields['post_id_random'] = xxx_id_random_field_schema
+base_fields['post_comment_id_random'] = xxx_id_random_field_schema
+base_fields['product_id_random'] = xxx_id_random_field_schema
+base_fields['site_id_random'] = xxx_id_random_field_schema
+base_fields['site_domain_id_random'] = xxx_id_random_field_schema
+base_fields['user_id_random'] = xxx_id_random_field_schema
+
+base_fields['created_on'] = created_updated_on_field_schema
+base_fields['updated_on'] = created_updated_on_field_schema
+
+base_fields['obj_type'] = {}
+base_fields['obj_id_random'] = xxx_id_random_field_schema_default
+base_fields['obj_id_rand'] = xxx_id_random_field_schema_default
+base_fields['obj_id'] = {}
+base_fields['obj_name'] = {}
+base_fields['obj_notes'] = {}
+
+base_fields['for_id_random'] = xxx_id_random_field_schema_default
+
+#xxx_id_random_field_schema['alias'] = 'order_id_random'
+#base_fields['id_random'] = xxx_id_random_field_schema_default
+#xxx_id_random_field_schema['alias'] = 'user_id_random_x'
+#c = {'alias': 'user_id_random_x'}
+#base_fields['user_id_random'] = combine_dict(xxx_id_random_field_schema, c)

app/routers/contact_model.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+import datetime, hashlib, logging, os, pytz, redis, secrets
+
+from typing import Dict, List, Optional, Set, Union
+from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator
+
+from ..lib_general import *
+from ..log import *
+from .common_field_schema import base_fields, default_num_bytes
+#from .account_model import Account_Base
+from .address_model import Address_Base
+
+
+class Contact_Base(BaseModel):
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    #from .account_model import Account_Base
+    #from .address_model import Address_Base
+
+    id_random: Optional[str] = Field(
+        **base_fields['contact_id_random'],
+        alias='contact_id_random',
+        default_factory=lambda:secrets.token_urlsafe(default_num_bytes),
+    )
+    id: Optional[int] = Field(
+        #alias='contact_id'
+    )
+    account_id_random: Optional[str]
+    account_id: Optional[int]
+    address_id_random: Optional[str]
+    address_id: Optional[int]
+
+    for_type: Optional[str]
+    for_id_random: Optional[str]
+    for_id: Optional[int]
+
+    name: Optional[str]
+    title: Optional[str]
+    tagline: Optional[str]
+
+    description: Optional[str]
+    lu_time_zone_id: Optional[str]
+    timezone: Optional[str]
+
+    email: Optional[str]
+    website: Optional[str]
+    website_name: Optional[str]
+
+    phone_mobile: Optional[str]
+    phone_home: Optional[str]
+    phone_office: Optional[str]
+    phone_land: Optional[str]
+    phone_fax: Optional[str]
+
+    facebook: Optional[str]
+    instagram: Optional[str]
+    twitter: Optional[str]
+    linkedin: Optional[str]
+
+    other_site_url: Optional[str]
+    other_site_name: Optional[str]
+
+    other_text: Optional[str]
+    other_json: Optional[Json]
+
+    priority: Optional[int]
+    sort: Optional[int]
+    group: Optional[str]
+
+    #account: Optional[Account_Base] = Account_Base()
+    address: Optional[Address_Base] = Address_Base()
+
+    created_on: Optional[datetime.datetime] = None
+    updated_on: Optional[datetime.datetime] = None
+
+    _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
+
+    #@validator('contact_id_random', always=True)
+    def contact_id_random_copy(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['id_random']:
+            return values['id_random']
+        return None
+
+    @validator('id', always=True)
+    def contact_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['id_random']:
+            log.debug(values['id_random'])
+            return redis_lookup_id_random(record_id_random=values['id_random'], table_name='contact')
+        return None
+
+    @validator('account_id', always=True)
+    def account_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['account_id_random']:
+            return redis_lookup_id_random(record_id_random=values['account_id_random'], table_name='account')
+        return None
+
+    @validator('address_id', always=True)
+    def address_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values.get('address_id_random', None):
+            return redis_lookup_id_random(record_id_random=values['address_id_random'], table_name='address')
+        return None
+
+    @validator('for_id', always=True)
+    def for_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['for_id_random'] and values['for_type']:
+            return redis_lookup_id_random(record_id_random=values['for_id_random'], table_name=values['for_type'])
+        return None
+
+    class Config:
+        underscore_attrs_are_private = True
+        fields = base_fields
+
+Contact_Base.update_forward_refs()

app/routers/core_object_model.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+import datetime, hashlib, logging, os, pytz, redis, secrets
+
+from typing import Dict, List, Optional, Set, Union
+from pydantic import BaseModel, EmailStr, Field, PrivateAttr, ValidationError, validator
+
+from .common_field_schema import base_fields
+
+
+class Core_Object_Base(BaseModel):
+    app.logger.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    app.logger.debug(locals())
+
+    obj_type: str
+    obj_id_random: str # alias this one based on obj_type?
+    obj_id_rand: str # alias this one based on obj_type?
+    obj_id: int # alias this one?
+    obj_name: Optional[str]
+    id_random: str # alias this one?
+    id: int # alias this one?
+
+    account_id_random: Optional[str]
+    account_id: Optional[str]
+
+    notes: Optional[str]
+    created_on: Optional[datetime.datetime] = None
+    updated_on: Optional[datetime.datetime] = None
+
+    _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
+
+
+class Example_Object_Base(Core_Object_Base): # Based on Core_Object_Base
+    title: Optional[str] = None
+    description: Optional[str] = None
+    password_set_on: Optional[datetime.datetime] = None
+    archive_on: Optional[datetime.datetime] = None
+    logged_in_on: Optional[datetime.datetime] = None
+    last_activity_on: Optional[datetime.datetime] = None
+    other_random_fields: dict
+    list_of_: Optional[dict] = {}
+
+# Create, Read/Get, Update, Delete
+# CRUD or CGUD
+
+# def create_object(object_data):
+#     return False # True, False, or None or object_data
+
+# def get_object(object_id):
+#     return object_data # False or None
+
+# def update_object(object_id, object_data):
+#     return False # True, False, or None or object_data
+
+# def delete_object(object_id):
+#     return False # True, False, or None or object_data

app/routers/response_model.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+import datetime, hashlib, logging, os, pytz, redis, secrets
+
+from typing import Dict, List, Optional, Set, Union
+from pydantic import BaseModel, EmailStr, Field, PrivateAttr, ValidationError, validator
+
+from ..lib_general import *
+from ..log import *
+from .common_field_schema import base_fields
+from app.config import settings
+
+
+# The pydantic BaseModel to help make consistent REST responses - STI 2021-03-05
+class Resp_Body_Base(BaseModel):
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    data: Union[dict, list]
+    meta: Optional[dict]
+
+
+# The make response function for REST - STI 2021-03-05
+def mk_resp(data={}, dict_to_json=None, status_code=200, status_message=None, status_name=None, success=True, details=None, by_alias=True, exclude_unset=True):
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    if data is None: data = { 'result': None }
+    elif data == False: data = { 'result': False }
+    elif data == True: data = { 'result': True }
+
+    resp_body = {}
+    resp_body['data'] = data
+    resp_body['meta'] = {}
+    resp_body['meta']['details'] = details
+    resp_body['meta']['status_code'] = status_code
+    resp_body['meta']['status_message'] = settings.HTTP_STATUS_LI[status_code]['message']
+    resp_body['meta']['status_name'] = settings.HTTP_STATUS_LI[status_code]['name']
+    resp_body['meta']['success'] = success
+
+    if isinstance(data, bool):
+        resp_body['meta']['data_type'] = 'bool'
+    elif isinstance(data, int):
+        resp_body['meta']['data_type'] = 'int'
+    elif isinstance(data, str):
+        resp_body['meta']['data_type'] = 'str'
+    elif isinstance(data, dict):
+        resp_body['meta']['data_type'] = 'dict'
+    elif isinstance(data, list):
+        resp_body['meta']['data_type'] = 'list'
+        resp_body['meta']['data_list_count'] = len(data)
+
+    log.debug(type(resp_body['data']))
+
+    resp_body = Resp_Body_Base(**resp_body).dict(by_alias=by_alias, exclude_unset=exclude_unset)
+    #resp_body_json = resp_body.json(by_alias=True, exclude_unset=False)
+
+    #response = app.response_class(
+        #response=resp_body_json,
+        #status=status_code,
+        #mimetype='application/json'
+    #)
+    return resp_body

app/routers/user_model.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+import datetime, hashlib, logging, os, pytz, redis, secrets
+
+from typing import Dict, List, Optional, Set, Union
+from pydantic import BaseModel, EmailStr, Field, Json, PrivateAttr, ValidationError, validator
+
+from ..lib_general import *
+from ..log import *
+from .common_field_schema import base_fields, default_num_bytes
+#from .account_model import Account_Base
+from .contact_model import Contact_Base
+#from .organization_model import Organization_Base
+#from .person_model import Person_Base
+
+
+class User_Base(BaseModel):
+    log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL
+    log.debug(locals())
+
+    #from .account_model import Account_Base
+    #from .contact_model import Contact_Base
+    #from .organization_model import Organization_Base
+    #from .person_model import Person_Base
+
+    #if TYPE_CHECKING:
+        #from .person_model import Person_Base
+
+    id_random: Optional[str] = Field(
+        **base_fields['user_id_random'],
+        alias='user_id_random',
+        default_factory=lambda:secrets.token_urlsafe(default_num_bytes),
+    )
+    id: Optional[int] = Field(
+        #alias='user_id'
+    )
+    account_id_random: Optional[str]
+    account_id: Optional[int]
+    contact_id_random: Optional[str]
+    contact_id: Optional[int]
+    organization_id_random: Optional[str]
+    organization_id: Optional[int]
+    person_id_random: Optional[str]
+    person_id: Optional[int]
+
+    username: Optional[str]
+    name: Optional[str]
+    email: Optional[str]
+    email_verified: Optional[bool]
+    password: Optional[str]
+    auth_key: Optional[str]
+
+    enable: Optional[bool]
+    enable_from: Optional[datetime.datetime] = None
+    enable_to: Optional[datetime.datetime] = None
+
+    super: Optional[bool]
+    manager: Optional[bool]
+    administrator: Optional[bool]
+    public: Optional[bool]
+    verified: Optional[bool]
+    status_id: Optional[int]
+    status_name: Optional[str]
+
+    password_set_on: Optional[datetime.datetime] = None
+    password_reset_token: Optional[str] = None
+    password_reset_expire_on: Optional[datetime.datetime] = None
+    logged_in_on: Optional[datetime.datetime] = None
+    last_activity_on: Optional[datetime.datetime] = None
+
+    #account: Optional[Account_Base]# = Account_Base()
+    contact: Optional[Contact_Base]# = Contact_Base()
+    #organization: Optional[Organization_Base]# = Organization_Base()
+    #person: Optional[Person_Base]# = Person_Base()
+
+    notes: Optional[str]
+    created_on: Optional[datetime.datetime] = None
+    updated_on: Optional[datetime.datetime] = None
+
+    _processed_at: datetime.datetime = PrivateAttr(default_factory=datetime.datetime.now)
+
+    #@validator('user_id_random', always=True)
+    def user_id_random_copy(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['id_random']:
+            return values['id_random']
+        return None
+
+    @validator('id', always=True)
+    def user_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['id_random']:
+            log.debug(values['id_random'])
+            return redis_lookup_id_random(record_id_random=values['id_random'], table_name='user')
+        return None
+
+    @validator('account_id', always=True)
+    def account_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['account_id_random']:
+            return redis_lookup_id_random(record_id_random=values['account_id_random'], table_name='account')
+        return None
+
+    @validator('contact_id', always=True)
+    def contact_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['contact_id_random']:
+            return redis_lookup_id_random(record_id_random=values['contact_id_random'], table_name='contact')
+        return None
+
+    @validator('organization_id', always=True)
+    def organization_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['organization_id_random']:
+            return redis_lookup_id_random(record_id_random=values['organization_id_random'], table_name='organization')
+        return None
+
+    @validator('person_id', always=True)
+    def person_id_lookup(cls, v, values, **kwargs):
+        log.setLevel(logging.WARNING)
+        log.debug(locals())
+
+        if values['person_id_random']:
+            return redis_lookup_id_random(record_id_random=values['person_id_random'], table_name='person')
+        return None
+
+    class Config:
+        underscore_attrs_are_private = True
+        fields = base_fields
+
+User_Base.update_forward_refs()