diff --git a/app/config.py.default b/app/config.py.default index 23dc956..1eacb42 100644 --- a/app/config.py.default +++ b/app/config.py.default @@ -12,6 +12,7 @@ class Settings(BaseSettings): AETHER_CFG['id'] = 0 # AETHER_CFG['api_id'] = 0 # NOT CURRENTLY NEED OR USED + JWT_KEY = '' # 22 characters; super secret Aether JWT signing key # APP_NAME: str = "Aether API (FastAPI)" # SUPER_EMAIL: EmailStr = 'Aether.Super@oneskyit.com' diff --git a/app/lib_general.py b/app/lib_general.py index 3936807..cbad113 100644 --- a/app/lib_general.py +++ b/app/lib_general.py @@ -123,6 +123,9 @@ class Common_Route_Params: @logger_reset # This breaks things for some reason when the function is async. Do not use async def common_route_params()! def common_route_params( x_account_id: str = Header(..., min_length=11, max_length=22), + # x_aether_api_key: Optional[str] = Header(..., min_length=11, max_length=22), + # x_aether_api_token: Optional[str] = Header(..., min_length=11, max_length=22), + # x_aether_jwt_token: Optional[str] = Header(..., min_length=11, max_length=50), enabled: str = 'enabled', # all, enabled, disabled limit: int = 100, offset: int = 0, @@ -246,12 +249,14 @@ def verify_secure_hash_string(string: str, string_hash: str): @logger_reset def sign_jwt( secret_key: str, # Secret/Private/Password - public_key: str, # Will be part of the token. Use to look up secret when verifying. ttl: int = 60, # Default to 60 seconds max_renew: int = 0, # Default to 0 + public_key: str = None, # Will be part of the token. Use to look up secret when verifying.??? account_id: str = None, person_id: str = None, user_id: str = None, + json_str: str = None, + b64_str: str = None, ) -> Dict[str, str]: log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) @@ -264,6 +269,8 @@ def sign_jwt( 'account_id': account_id, 'person_id': person_id, 'user_id': user_id, + 'json_str': json_str, + 'b64_str': b64_str, } secret = secret_key algorithm = 'HS256' diff --git a/app/routers/api.py b/app/routers/api.py index 5edea85..bfbf3e1 100644 --- a/app/routers/api.py +++ b/app/routers/api.py @@ -17,7 +17,15 @@ router = APIRouter() # ### BEGIN ### API API ### request_jwt() ### -# Generate JWT using associated API private key +# This can be used to generate JWTs for various purposes: +# * for end client browser API access +# * for proof of sign in +# * newer/better version of sign in by URL +# Generate (sign) JWT using Aether platform super secret key or x_aether_signing_key sort of secret key if passed. The Aether platform super secret JWT signing key must be used API access token +# If x_aether_api_key is passed then set higher TTL +# If old and valid x_aether_api_jwt_token is passed then decode and decrease TTL by 1 +# Updated 2023-03-24 + # Verify JWT using the API public key's associated API private key # API server or trusted app can generate JWTs # JWT contains: @@ -33,14 +41,27 @@ router = APIRouter() # Updated 2021-07-14 @router.get('/request_jwt', response_model=Resp_Body_Base) async def request_jwt( - x_aether_api_secret_key: Optional[str] = Header(None, min_length=22, max_length=22), # If passed then can also set TTL - x_aether_api_public_key: Optional[str] = Header(None, min_length=22, max_length=22), # Used to look up the API secret if not given - x_aether_api_token: Optional[str] = Header(None), # Token given to client by an API key holder (short max TTL) - account_id: str = None, - session_id: str = None, # End client (web browser) - client_id: str = None, # End client (web browser) - person_id: str = None, - user_id: str = None, + x_aether_signing_key: Optional[str] = Header(None, min_length=22, max_length=22), # The (secret) signing key. Keep safe!!! If passed then use to sign JWT. Otherwise need to get from system/environment. + + # x_aether_secret_key: Optional[str] = Header(None, min_length=22, max_length=22), # The Aether secret key. Keep safe!!! If passed then can also set TTL + + x_aether_api_key: Optional[str] = Header(None, min_length=22, max_length=22), # The client side API key. This should be kept secret by the client. If passed then store with JWT and can set TTL. + + # x_aether_api_public_key: Optional[str] = Header(None, min_length=22, max_length=22), # Used to look up the API secret if not given + + x_aether_jwt: Optional[str] = Header(None), # A JWT that was created and given to client browser or server in the past. It may or may not be valid. If the x_aether_signing_key was not passed, then assume it was signed with the Aether super secret key. + + account_id: str = None, # Handle this different because it is special + json_str: str = None, # This is what should be stored + b64_str: str = None, # This is what should be stored + # I would like payload to be a dict, but then we have to use POST instead of GET... + # Maybe base64 encode and decode? + + # session_id: str = None, # End client (web browser) + # client_id: str = None, # End client (web browser) + # person_id: str = None, + # user_id: str = None, + max_ttl: int = 300, # Number of seconds to live. Only use if given the API secret key. # Seconds: 3600 = 1 hr; 300 = 5 min max_renew: int = 5, # Decrease count by 1 until 0 if only sent a current API token. @@ -49,16 +70,43 @@ async def request_jwt( log.setLevel(logging.WARNING) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - if x_aether_api_secret_key or x_aether_api_token: pass + # One of these is required + if account_id or json_str or b64_str: pass else: return mk_resp(data=False, status_code=400, response=response) # Bad Request - if not x_aether_api_secret_key: max_ttl = 300 # Override any max_ttl if no API secret - if not x_aether_api_secret_key: max_renew = 5 # Override any max_rewnew if no API secret + # Possible overrides and checks go here + if x_aether_signing_key: pass + elif x_aether_api_key: + # Override any if for API JWT??? + max_ttl = 3600 + max_renew = 5 + # if not x_aether_secret_key: max_renew = 5 # Override any max_rewnew if no API secret + # api_secret_key = x_aether_secret_key - api_secret_key = x_aether_api_secret_key + signing_key = None + if x_aether_signing_key: + signing_key = x_aether_signing_key + elif settings.JWT_KEY: + signing_key = settings.JWT_KEY + else: + log.error('No key found to sign the JWT with!') + return mk_resp(data=False, status_code=400, response=response) # Bad Request - if x_aether_api_secret_key: - log.debug(f'Contains a value in x_aether_api_secret_key: {x_aether_api_secret_key}') + + payload = {} + payload['account_id'] = account_id + payload['json_str'] = json_str + payload['b64_str'] = b64_str + + token = sign_jwt(secret_key=signing_key, public_key=x_aether_api_key, ttl=max_ttl, max_renew=max_renew, **payload) + + response_data = { 'jwt': token } + + return mk_resp(data=response_data) + + + if x_aether_secret_key: + log.debug(f'Contains a value in x_aether_secret_key: {x_aether_secret_key}') table_name_select = 'api_key' field_name = 'secret_key' @@ -125,7 +173,7 @@ async def request_jwt( payload['user_id'] = user_id token = sign_jwt(secret_key=api_secret_key, public_key=api_public_key, ttl=max_ttl, max_renew=max_renew, **payload) - response_data = { 'api_access_jwt': token } + response_data = { 'jwt': token } return mk_resp(data=response_data) # ### END ### API API ### request_jwt() ### diff --git a/app/routers/hosted_file.py b/app/routers/hosted_file.py index 356f6db..7f8d2ff 100644 --- a/app/routers/hosted_file.py +++ b/app/routers/hosted_file.py @@ -20,7 +20,10 @@ router = APIRouter() # ### BEGIN ### API Hosted File ### directory_check() ### -# Updated 2022-08-09 +# This can be used to clean up the hosted_files directory. Currently it only looks for hashed files in the root, but that is kind of useless now. 2023-03-28 +# This needs to be updated to delete orphan files (no records in the DB (dev, test, prod)). Careful... +# I also need to clean up the DB side if there is no file in the hosted_files directory. Less concerning? +# Updated 2023-03-28 @router.get('/directory_check', response_model=Resp_Body_Base) async def directory_check( rm_orphan: bool = False, @@ -30,11 +33,6 @@ async def directory_check( log.setLevel(logging.INFO) # DEBUG, INFO, WARNING, ERROR, EXCEPTION, CRITICAL log.debug(locals()) - # print('HERE HERE HERE') - - # return mk_resp(data=True, response=commons.response, status_message='HERE HERE HERE The hosted file directory check.') - - # ### Orphan file: ### Delete file from server hosted_files_path = settings.FILES_PATH['hosted_files_root'] # hosted_files_path = '/home/scott/tmp/hosted_files_dev/' @@ -49,12 +47,17 @@ async def directory_check( log.info('Path exists! Going to get a list of files...') directory_list = os.listdir(full_directory_path) + count = 0 + result_list = [] for directory_item in directory_list: + if count >= 100: break + file_path_w_item = os.path.join(full_directory_path, directory_item) # log.info(f'Full file path with directory item: {file_path_w_item}') - log.info(f'Checking directory item: {directory_item}') + # log.info(f'Checking directory item: {directory_item}') if os.path.isfile(file_path_w_item): + # ### Found file ### # log.debug(f'File: {directory_item}') # result_list.append(file_path_w_item) @@ -63,66 +66,82 @@ async def directory_check( log.warning(f'Not a hashed file! File: {directory_item}') continue - if lookup_file_hash_result := lookup_file_hash(file_hash=directory_item.replace('.file', '')): - # log.info('DB record found') - # result_list.append(file_path_w_item) - pass - else: - log.warning(f'Hosted File record not found!!! File: {directory_item}') - result_list.append(file_path_w_item) - if rm_orphan: - log.info('Going remove the hosted file from server...') + log.info(f'Hosted hashed file found: {directory_item}') + result_list.append(file_path_w_item) - try: - # log.warning('DELETE') - pathlib.Path(file_path_w_item).unlink() - # continue - except OSError as e: - log.error("Error: %s : %s" % (file_path, e.strerror)) - # return False - continue + # Create a subdirectory with the first 2 characters of the hash + full_subdirectory_path = os.path.join(full_directory_path, directory_item[:2]) + log.info(f'Making directory: {full_subdirectory_path}') + os.makedirs(full_subdirectory_path, exist_ok=True) + + # Move the file to the subdirectory + log.info(f'Moving to: {full_subdirectory_path}') + shutil.move(os.path.join(full_directory_path, directory_item), os.path.join(full_subdirectory_path, directory_item)) + + # if lookup_file_hash_result := lookup_file_hash(file_hash=directory_item.replace('.file', '')): + # log.info('DB record found') + # # result_list.append(file_path_w_item) + # pass + # else: + # log.warning(f'Hosted File record not found!!! File: {directory_item}') + # result_list.append(file_path_w_item) + # if rm_orphan: + # log.info('Going remove the hosted file from server...') + + # try: + # # log.warning('DELETE') + # pathlib.Path(file_path_w_item).unlink() + # # continue + # except OSError as e: + # log.error("Error: %s : %s" % (file_path, e.strerror)) + # # return False + # continue else: + # ### Found directory ### + # continue # log.debug(f'Directory: {directory_item}') # pass - log.info('Subdirectory Path exists! Going to get a list of files...') - full_subdirectory_path = os.path.join(full_directory_path, directory_item) - subdirectory_list = os.listdir(full_subdirectory_path) + log.info('Subdirectory Path exists! Going to get a list of files... [LATER]') + # full_subdirectory_path = os.path.join(full_directory_path, directory_item) + # subdirectory_list = os.listdir(full_subdirectory_path) - subdirectory_result_list = [] - for subdirectory_item in subdirectory_list: - file_path_w_item = os.path.join(full_subdirectory_path, subdirectory_item) - # log.info(f'Full file path with directory item: {file_path_w_item}') - log.info(f'Checking subdirectory item: {subdirectory_item}') - if os.path.isfile(file_path_w_item): - # log.debug(f'File: {subdirectory_item}') - # subdirectory_result_list.append(file_path_w_item) + # subdirectory_result_list = [] + # for subdirectory_item in subdirectory_list: + # file_path_w_item = os.path.join(full_subdirectory_path, subdirectory_item) + # # log.info(f'Full file path with directory item: {file_path_w_item}') + # log.info(f'Checking subdirectory item: {subdirectory_item}') + # if os.path.isfile(file_path_w_item): + # # log.debug(f'File: {subdirectory_item}') + # # subdirectory_result_list.append(file_path_w_item) - if '.file' in subdirectory_item: pass - else: - log.warning(f'Not a hashed file! File: {subdirectory_item}') - continue + # if '.file' in subdirectory_item: pass + # else: + # log.warning(f'Not a hashed file! File: {subdirectory_item}') + # continue - if lookup_file_hash_result := lookup_file_hash(file_hash=subdirectory_item.replace('.file', '')): - # log.info('DB record found') - # subdirectory_result_list.append(file_path_w_item) - pass - else: - log.warning(f'Hosted File record not found!!! File: {subdirectory_item}') - result_list.append(file_path_w_item) - if rm_orphan: - log.info('Going remove the hosted file from server...') + # if lookup_file_hash_result := lookup_file_hash(file_hash=subdirectory_item.replace('.file', '')): + # # log.info('DB record found') + # # subdirectory_result_list.append(file_path_w_item) + # pass + # else: + # log.warning(f'Hosted File record not found!!! File: {subdirectory_item}') + # result_list.append(file_path_w_item) + # if rm_orphan: + # log.info('Going remove the hosted file from server...') - try: - # log.warning('DELETE') - pathlib.Path(file_path_w_item).unlink() - # continue - except OSError as e: - log.error("Error: %s : %s" % (file_path, e.strerror)) - # return False - continue - else: - log.warning(f'Subdirectory: {subdirectory_item}') - pass + # try: + # # log.warning('DELETE') + # pathlib.Path(file_path_w_item).unlink() + # # continue + # except OSError as e: + # log.error("Error: %s : %s" % (file_path, e.strerror)) + # # return False + # continue + # else: + # log.warning(f'Subdirectory: {subdirectory_item}') + # pass + + count = count + 1 return mk_resp(data=result_list, response=commons.response, status_message='The hosted file directory check.') else: diff --git a/app/routers/websockets.py b/app/routers/websockets.py index 48dc58a..264f0ef 100644 --- a/app/routers/websockets.py +++ b/app/routers/websockets.py @@ -66,6 +66,7 @@ async def get(response: Response = Response): class ConnectionManager: def __init__(self): + # NOTE: The active_connections list should be in Redis self.active_connections: List[WebSocket] = [] async def connect(self, websocket: WebSocket): @@ -76,6 +77,10 @@ class ConnectionManager: def disconnect(self, websocket: WebSocket): self.active_connections.remove(websocket) + + # Targets: echo, direct, group, broadcast + # send_ text, bytes, json + # receive_ text, bytes, json async def echo(self, message: str, websocket: WebSocket): log.setLevel(logging.DEBUG) # log.debug(dir(websocket)) @@ -94,7 +99,7 @@ class ConnectionManager: log.debug(connection) await connection.send_text(message) - async def group(self, group_id: str, data: str): + async def group(self, group_id: str, data: dict): log.setLevel(logging.DEBUG) log.debug(locals()) @@ -205,6 +210,9 @@ async def ws_client_id( log.debug(f'Command: {cmd}') log.debug(f'Message: {msg}') + data['client_id'] = client_id + data['group_id'] = group_id + await manager.group(group_id=group_id, data=data) # if msg_type: