Scott.Idem/OSIT-AE-API-FastAPI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Docs: Consolidate admin documentation and migrate reference data

Browse Source 
- Created LOCAL_DEVELOPMENT_GUIDE.md and DEPLOYMENT_GUIDE_MANUAL.md from legacy txt files.
- Migrated country/time_zone data and requirements.txt to documentation/reference_data/.
- Removed redundant admin/documentation/ and admin/data_files/ directories.
- Enhanced app/lib_schema_v3.py to explicitly capture 'required' fields from DB 'NOT NULL' constraint.
- Added verification tests for schema logic and standalone DB connectivity.
This commit is contained in:
Scott Idem
2026-01-16 10:06:51 -05:00
parent db5cf2502a
commit 31fd384704
17 changed files with 465 additions and 222 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
admin/documentation/gunicorn.service.default
View File

@@ -1,24 +0,0 @@
[Unit]
Description=gunicorn daemon
Requires=gunicorn.socket
After=network.target
[Service]
Type=notify
# the specific user that our service will run as
User=root
Group=root
# another option for an even more restricted service is
# DynamicUser=yes
# see http://0pointer.net/blog/dynamic-users-with-systemd.html
RuntimeDirectory=gunicorn
WorkingDirectory=/srv/http/dev_fastapi.oneskyit.com
Environment="PATH=/srv/http/dev_fastapi.oneskyit.com/environment/bin"
ExecStart=/srv/http/dev_fastapi.oneskyit.com/environment/bin/gunicorn --bind unix:/srv/http/dev_fastapi.oneskyit.com/gunicorn.sock -m 007 app.main:app --workers 4 -k uvicorn.workers.UvicornWorker --access-logfile admin/log/access.log --error-logfile admin/log/error.log, --log-file admin/log/log.log --capture-output --keep-alive 5
ExecReload=/bin/kill -s HUP $MAINPID
KillMode=mixed
TimeoutStopSec=5
PrivateTmp=true
[Install]
WantedBy=multi-user.target

14
admin/documentation/gunicorn.socket.default
View File

@@ -1,14 +0,0 @@
[Unit]
Description=gunicorn socket
[Socket]
ListenStream=/run/gunicorn.sock
# Our service won't need permissions for the socket, since it
# inherits the file descriptor by socket activation
# only the nginx daemon will need access to the socket
User=http
# Optionally restrict the socket permissions even more.
# Mode=600
[Install]
WantedBy=sockets.target

85
admin/documentation/nginx_fastapi_server.example.default
View File

@@ -1,85 +0,0 @@
server {
access_log /var/log/nginx/access_dev_fastapi.oneskyit.com.log;
listen 443 ssl; # managed by Certbot
listen [::]:443 ssl http2; # managed by Certbot
#listen 443 http3 reuseport; # UDP listener for QUIC+HTTP/3
server_name dev-fastapi.oneskyit.com;
ssl_certificate /etc/letsencrypt/live/oneskyit.com-0001/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/oneskyit.com-0001/privkey.pem; # managed by Certbot
include /etc/letsencrypt/options-ssl-nginx.conf; # managed by Certbot
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; # managed by Certbot
#add_header Alt-Svc 'quic=":443"'; # Advertise that QUIC is available
#add_header QUIC-Status $quic; # Sent when QUIC was used
include brotli.conf;
include gzip.conf;
client_max_body_size 4096M; # or 4G
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_redirect off;
proxy_buffering off;
proxy_pass http://unix:/run/gunicorn.sock;
}
location /ws {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
#proxy_read_timeout 600;
#proxy_headers_hash_max_size 1024;
proxy_pass http://unix:/run/gunicorn.sock;
}
location /ws_redis {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
#proxy_read_timeout 600;
#proxy_headers_hash_max_size 1024;
proxy_pass http://unix:/run/gunicorn.sock;
}
}
server {
if ($host = dev-fastapi.oneskyit.com) {
return 301 https://$host$request_uri;
} # managed by Certbot
listen 80;
listen [::]:80;
server_name dev-fastapi.oneskyit.com;
return 404; # managed by Certbot
}

55
admin/documentation/run locally.txt
View File

@@ -1,55 +0,0 @@
# Go to root of application
cd ~/path/to/directory/my_application/
# Create new application environment
virtualenv environment
# Activate application environment
source environment/bin/activate
# Install application requirements
pip install -r admin/requirements.txt
pip install --upgrade --force-reinstall -r admin/requirements.txt
pip install --ignore-installed -r admin/requirements.txt
pip list
# Start application
uvicorn app.main:app --host 0.0.0.0 --port 5005 --reload
# View app
http://localhost:5005
# Deactivate environment when done
deactivate
# Use git
# Go to root of application
cd ~/path/to/directory/my_application/
git init
git remote add origin https://scott_idem@bitbucket.org/oneskyit/one-sky-it-api.git
git add .
git commit -m 'Initial commit'
git push -u origin master
git push -u origin development
git push -u origin new-branch-name
# List branches
git branch -a
# Create new branch
git branch new-branch-name
# Switch branch
git switch new-branch-name
# Clone from Bitbucket:
git clone https://scott_idem@bitbucket.org/oneskyit/one-sky-it-api-fastapi.git /srv/http/the_path_to_create
gunicorn --bind unix:/home/scott/OSIT_dev/aether_api_fastapi/gunicorn.sock --umask 007 app.main:app --workers 2 --worker-class uvicorn.workers.UvicornWorker --access-logfile admin/log/access.log --error-logfile admin/log/error.log, --log-file admin/log/log.log --capture-output --keep-alive 5 --reload

42
admin/documentation/setup_server.txt
View File

@@ -1,42 +0,0 @@
sudo git clone https://scott_idem@bitbucket.org/oneskyit/one-sky-it-api-fastapi.git /srv/http/dev_fastapi.oneskyit.com
sudo mkdir admin/log
sudo ls -lha /srv/http/
sudo chown http:http -R /srv/http/dev_fastapi.oneskyit.com/
sudo chmod 775 -R /srv/http/dev_fastapi.oneskyit.com/
sudo ls -lha /srv/http/
cd /srv/http/dev_fastapi.oneskyit.com/
rm .gitignore
git branch -a
git switch development
virtualenv environment
source environment/bin/activate
pip install -U -r admin/requirements.txt
sudo vim /etc/systemd/system/gunicorn.socket
sudo vim /etc/systemd/system/gunicorn.service
sudo systemctl daemon-reload
sudo systemctl enable gunicorn.socket
sudo systemctl start gunicorn.socket
sudo systemctl status gunicorn.socket
sudo systemctl status gunicorn.service
# Do not: sudo systemctl enable gunicorn.service
# Do not? sudo systemctl start gunicorn.service
sudo vim /etc/nginx/sites-available/dev_fastapi.oneskyit.com
sudo ln -s /etc/nginx/sites-available/dev_fastapi.oneskyit.com /etc/nginx/sites-enabled/dev_fastapi.oneskyit.com
sudo systemctl restart nginx.service
sudo systemctl status nginx.service
# Troubleshooting:
systemctl list-units --type=service --state=active
systemctl list-units --type=service --state=running
sudo systemctl | grep running
sudo systemctl list-unit-files | grep enabled

10
app/lib_schema_v3.py
View File

@@ -38,9 +38,15 @@ def get_object_schema_info(obj_type: str, view: str = 'default', variant: str =
try:
db_result = db.execute(text(f"DESCRIBE `{table_name}`"))
for row in db_result.fetchall():
# row format: (Field, Type, Null, Key, Default, Extra)
schema_info["database"]["columns"].append({
"field": row[0], "type": row[1], "nullable": row[2] == 'YES',
"key": row[3], "default": row[4], "extra": row[5]
"field": row[0],
"db_type": row[1],
"nullable": row[2] == 'YES',
"required": row[2] == 'NO', # Explicitly capture NOT NULL
"key": row[3],
"db_default": row[4],
"extra": row[5]
})
except Exception as e:
schema_info["database"]["error"] = str(e)

153
documentation/DEPLOYMENT_GUIDE_MANUAL.md Normal file
View File

@@ -0,0 +1,153 @@
# Manual Server Deployment Guide (Non-Docker)
This guide describes the manual process for deploying the Aether API on a Linux server using Nginx, Gunicorn, and Systemd.
## 1. Initial Server Setup
### Clone the Repository
```bash
sudo git clone https://scott_idem@bitbucket.org/oneskyit/one-sky-it-api-fastapi.git /srv/http/dev_fastapi.oneskyit.com
```
### Configure Permissions
```bash
sudo mkdir admin/log
sudo chown http:http -R /srv/http/dev_fastapi.oneskyit.com/
sudo chmod 775 -R /srv/http/dev_fastapi.oneskyit.com/
```
### Environment Preparation
```bash
cd /srv/http/dev_fastapi.oneskyit.com/
git switch development
virtualenv environment
source environment/bin/activate
pip install -U -r admin/requirements.txt
```
## 2. Gunicorn Configuration (Systemd)
### Socket Configuration (`/etc/systemd/system/gunicorn.socket`)
```ini
[Unit]
Description=gunicorn socket
[Socket]
ListenStream=/run/gunicorn.sock
User=http
[Install]
WantedBy=sockets.target
```
### Service Configuration (`/etc/systemd/system/gunicorn.service`)
```ini
[Unit]
Description=gunicorn daemon
Requires=gunicorn.socket
After=network.target
[Service]
Type=notify
User=root
Group=root
RuntimeDirectory=gunicorn
WorkingDirectory=/srv/http/dev_fastapi.oneskyit.com
Environment="PATH=/srv/http/dev_fastapi.oneskyit.com/environment/bin"
ExecStart=/srv/http/dev_fastapi.oneskyit.com/environment/bin/gunicorn \
--bind unix:/srv/http/dev_fastapi.oneskyit.com/gunicorn.sock \
-m 007 app.main:app \
--workers 4 \
-k uvicorn.workers.UvicornWorker \
--access-logfile admin/log/access.log \
--error-logfile admin/log/error.log \
--capture-output --keep-alive 5
ExecReload=/bin/kill -s HUP $MAINPID
KillMode=mixed
TimeoutStopSec=5
PrivateTmp=true
[Install]
WantedBy=multi-user.target
```
### Activation
```bash
sudo systemctl daemon-reload
sudo systemctl enable gunicorn.socket
sudo systemctl start gunicorn.socket
```
## 3. Nginx Configuration
Create a site configuration file at `/etc/nginx/sites-available/dev_fastapi.oneskyit.com`:
```nginx
server {
access_log /var/log/nginx/access_dev_fastapi.oneskyit.com.log;
listen 443 ssl;
listen [::]:443 ssl http2;
server_name dev-fastapi.oneskyit.com;
ssl_certificate /etc/letsencrypt/live/oneskyit.com-0001/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/oneskyit.com-0001/privkey.pem;
client_max_body_size 4096M;
location / {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_redirect off;
proxy_buffering off;
proxy_pass http://unix:/run/gunicorn.sock;
}
# WebSocket Support
location ~ ^/(ws|ws_redis) {
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_http_version 1.1;
proxy_redirect off;
proxy_buffering off;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_pass http://unix:/run/gunicorn.sock;
}
}
server {
listen 80;
listen [::]:80;
server_name dev-fastapi.oneskyit.com;
return 301 https://$host$request_uri;
}
```
Enable and restart Nginx:
```bash
sudo ln -s /etc/nginx/sites-available/dev_fastapi.oneskyit.com /etc/nginx/sites-enabled/
sudo systemctl restart nginx.service
```
## 4. Troubleshooting
```bash
# Check status
sudo systemctl status gunicorn.socket
sudo systemctl status gunicorn.service
sudo systemctl status nginx.service
# List active units
systemctl list-units --type=service --state=running
```

78
documentation/LOCAL_DEVELOPMENT_GUIDE.md Normal file
View File

@@ -0,0 +1,78 @@
# Local Development Guide
This guide provides instructions for setting up and running the Aether API locally for development.
## 1. Prerequisites
- Python 3.9+ (or as specified in `requirements.txt`)
- `virtualenv` or `venv`
## 2. Setup Procedure
### Create and Activate Environment
```bash
# Go to root of application
cd ~/path/to/directory/aether_api_fastapi/
# Create new application environment
virtualenv environment
# Activate application environment
source environment/bin/activate
```
### Install Requirements
```bash
# Install application requirements
pip install -r admin/requirements.txt
# Troubleshooting/Force Reinstall if needed:
# pip install --upgrade --force-reinstall -r admin/requirements.txt
# pip install --ignore-installed -r admin/requirements.txt
```
## 3. Running the Application
### Start with Uvicorn (Reload enabled)
```bash
uvicorn app.main:app --host 0.0.0.0 --port 5005 --reload
```
### Accessing the API
The application will be available at:
- API Root: [http://localhost:5005](http://localhost:5005)
- Swagger Docs: [http://localhost:5005/docs](http://localhost:5005/docs)
## 4. Git Workflow Basics
### Initialization
```bash
git init
git remote add origin https://scott_idem@bitbucket.org/oneskyit/one-sky-it-api.git
```
### Basic Commands
```bash
# Adding and committing
git add .
git commit -m 'Your commit message'
# Pushing to branches
git push -u origin master
git push -u origin development
```
### Branch Management
```bash
# List branches
git branch -a
# Create and switch to new branch
git branch new-branch-name
git switch new-branch-name
```
## 5. Deployment Basics (Manual)
If you are cloning for the first time on a server:
```bash
git clone https://scott_idem@bitbucket.org/oneskyit/one-sky-it-api-fastapi.git /srv/http/destination_path
```

0
admin/data_files/country.txt → documentation/reference_data/country.txt
View File

0
admin/requirements.txt → documentation/reference_data/requirements.txt
View File

0
admin/requirements_current.txt → documentation/reference_data/requirements_current.txt
View File

0
admin/data_files/time_zone.txt → documentation/reference_data/time_zone.txt
View File

10
tests/conftest_mock.py Normal file
View File

@@ -0,0 +1,10 @@
import sys
from unittest.mock import MagicMock
# Create a mock for app.config before any other app imports
mock_config = MagicMock()
mock_config.settings = MagicMock()
# Manually inject into sys.modules
sys.modules["app.config"] = mock_config
print("MOCKED: app.config")

70
tests/generate_registry_live.py Normal file
View File

@@ -0,0 +1,70 @@
#!/usr/bin/env python3
import sys
import os
import json
import datetime
import sqlalchemy
from unittest.mock import MagicMock
# 1. Setup Path to ensure we can import the app
sys.path.append("/srv/aether_api")
sys.path.append(os.getcwd()) # For local execution
# 2. Mock app.config BEFORE importing app modules
mock_config = MagicMock()
mock_config.settings = MagicMock()
sys.modules["app.config"] = mock_config
# 3. Setup REAL DB connection for introspection
DB_USER = "aether_dev"
DB_PASS = "$1sky.AE_dev.2023"
DB_SERVER = "vpn-db.oneskyit.com"
DB_PORT = "3306"
DB_NAME = "aether_dev"
DB_URI = f"mysql://{DB_USER}:{DB_PASS}@{DB_SERVER}:{DB_PORT}/{DB_NAME}"
mock_db = MagicMock()
mock_db.execute = lambda stmt: engine.execute(stmt)
engine = sqlalchemy.create_engine(DB_URI)
sys.modules["app.db_sql"] = MagicMock(db=mock_db)
def type_handler(x):
if isinstance(x, (datetime.datetime, datetime.date)):
return x.isoformat()
if isinstance(x, type):
return x.__name__
return str(x)
def export_registry():
try:
from app.ae_obj_types_def import obj_type_kv_li
from app.lib_schema_v3 import get_object_schema_info
except ImportError as e:
print(f"Error: Could not import Aether API definitions. {e}", file=sys.stderr)
sys.exit(1)
final_output = {}
allowed_keys = ["tbl", "tbl_default", "tbl_update", "table_name", "tbl_name_update", "searchable_fields", "exp_default"]
for obj_key, obj_def in obj_type_kv_li.items():
final_output[obj_key] = {}
for k in allowed_keys:
if k in obj_def: final_output[obj_key][k] = obj_def[k]
for k in ["mdl", "mdl_out", "mdl_in", "mdl_default"]:
if k in obj_def:
model = obj_def[k]
try:
if hasattr(model, "schema"): final_output[obj_key][f"{k}_schema"] = model.schema()
final_output[obj_key][k] = model.__name__
except Exception: final_output[obj_key][k] = str(model)
try:
schema_info = get_object_schema_info(obj_key)
if "database" in schema_info: final_output[obj_key]["db_schema"] = schema_info["database"]
except Exception as e: final_output[obj_key]["db_schema_error"] = str(e)
print(json.dumps(final_output, indent=2, default=type_handler))
if __name__ == "__main__":
export_registry()

46
tests/standalone_db_test.py Normal file
View File

@@ -0,0 +1,46 @@
import sqlalchemy
from sqlalchemy import text
import json
# Connection details from .env
DB_USER = "aether_dev"
DB_PASS = "$1sky.AE_dev.2023"
DB_SERVER = "vpn-db.oneskyit.com"
DB_PORT = "3306"
DB_NAME = "aether_dev"
# Construct URI
DB_URI = f"mysql://{DB_USER}:{DB_PASS}@{DB_SERVER}:{DB_PORT}/{DB_NAME}"
def test_raw_describe():
engine = sqlalchemy.create_engine(DB_URI)
print(f"Connecting to {DB_SERVER}...")
try:
with engine.connect() as conn:
result = conn.execute(text("DESCRIBE journal"))
columns = []
for row in result.fetchall():
columns.append({
"Field": row[0],
"Type": row[1],
"Null": row[2],
"Default": row[4]
})
# Print as a nice JSON snippet
print("\n--- Raw DB Metadata for 'journal' (Sample) ---")
print(json.dumps(columns[:5], indent=2))
# Highlight the specific fields mentioned in the task
print("\n--- Target Fields ---")
targets = ["name", "enable", "passcode_timeout", "created_on"]
for col in columns:
if col["Field"] in targets:
print(f"Field: {col['Field']:16} | Type: {col['Type']:15} | Null: {col['Null']:3} | Default: {col['Default']}")
except Exception as e:
print(f"Error: {e}")
if __name__ == "__main__":
test_raw_describe()

44
tests/test_lib_schema_isolated.py Normal file
View File

@@ -0,0 +1,44 @@
import sys
import os
import json
from unittest.mock import MagicMock
# Add current directory to path
sys.path.append(os.getcwd())
# 1. Mock EVERYTHING before importing the target
mock_config = MagicMock()
mock_config.settings = MagicMock()
sys.modules["app.config"] = mock_config
mock_db = MagicMock()
sys.modules["app.db_sql"] = MagicMock(db=mock_db)
# 2. Mock DESCRIBE result
mock_row = [
("created_on", "timestamp", "NO", "", "current_timestamp()", "")
]
mock_db.execute.return_value.fetchall.return_value = mock_row
# 3. Import and test
from app.lib_schema_v3 import get_object_schema_info
def test_isolated_logic():
# We need to mock obj_type_kv_li as well
import app.lib_schema_v3
app.lib_schema_v3.obj_type_kv_li = {"journal": {"tbl": "journal", "mdl": MagicMock()}}
print("Testing isolated logic for get_object_schema_info...")
info = get_object_schema_info("journal")
col = info["database"]["columns"][0]
print(f"Resulting column info: {json.dumps(col, indent=2)}")
assert col["field"] == "created_on"
assert col["db_type"] == "timestamp"
assert col["required"] is True
assert col["db_default"] == "current_timestamp()"
print("\nIsolated test passed! The logic in lib_schema_v3 is correct.")
if __name__ == "__main__":
test_isolated_logic()

56
tests/verify_schema_logic.py Normal file
View File

@@ -0,0 +1,56 @@
import sys
import os
import json
import sqlalchemy
from sqlalchemy import text
# Add current directory to path
sys.path.append(os.getcwd())
# MUST BE FIRST: Mock app.config
import tests.conftest_mock
# Connection details from .env (Bypassing app.config)
DB_USER = "aether_dev"
DB_PASS = "$1sky.AE_dev.2023"
DB_SERVER = "vpn-db.oneskyit.com"
DB_PORT = "3306"
DB_NAME = "aether_dev"
DB_URI = f"mysql://{DB_USER}:{DB_PASS}@{DB_SERVER}:{DB_PORT}/{DB_NAME}"
# Mock the db object before importing lib_schema_v3
from app.db_sql import db
db.engine = sqlalchemy.create_engine(DB_URI)
from app.lib_schema_v3 import get_object_schema_info
def verify_enhanced_schema():
print("Testing enhanced get_object_schema_info for 'journal'...")
try:
info = get_object_schema_info("journal")
if "error" in info:
print(f"Error: {info['error']}")
return
cols = info["database"]["columns"]
targets = ["name", "enable", "passcode_timeout", "created_on"]
print("\n--- Verified Enhanced Metadata ---")
for col in cols:
if col["field"] in targets:
print(f"Field: {col['field']:16} | DB Type: {col['db_type']:15} | Required: {str(col['required']):5} | Default: {col['db_default']}")
# Basic check to ensure new keys exist
assert "db_type" in cols[0]
assert "required" in cols[0]
assert "db_default" in cols[0]
print("\nSuccess: New schema keys are present and populated.")
except Exception as e:
print(f"Test failed: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
verify_enhanced_schema()