From 235e076e63cb3ad7b01e0d2b4009e78c1bc57512 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Fri, 7 Apr 2017 00:52:48 -0700 Subject: [PATCH] Change all the configuration variables to ENVIRONMENT VARS. Following 12factor principle for app development, all the configuration variables for the docker images can now be setup using environment vars. The default values are set using the run.sh script in both mailman-web and mailman-core containers. Mailman-core is now run by "run.sh" which traps SIGTERM and performs mailman stop for graceful exit of mailman core. --- README.md | 70 +++++++++++++++++ core/assets/run.sh | 147 ++++++++++++++++++++++++++++++------ web/assets/run.sh | 68 +++++++++++------ web/mailman-web/settings.py | 28 ++++--- 4 files changed, 254 insertions(+), 59 deletions(-) diff --git a/README.md b/README.md index 2ddb1e9..ec19159 100644 --- a/README.md +++ b/README.md @@ -188,6 +188,76 @@ exit 0 Please do not forget to make the script executable (chmod +x certbot-renew.sh). +Configuration +============= + +Most of the configuraiton is supposed to be handled through environment +variables in the `docker-compose.yaml`. + +### Mailman-web +These are the settings that you MUST change before deploying: + +- `SERVE_FROM_DOMAIN`: The domain name from which Django will be served. To be + added to `ALLOWED_HOSTS` in django settings. Default value is not set. + +- `HYPERKITT_API_KEY`: Hyperkitty's API Key, should be set to the same value as + set for the mailman-core. + +These are the settings that are set to sane default and you do not need to +change them unless you know what you want. + +- `DATABASE_URL`: URL of the type + `driver://user:password@hostname:port/databasename` for the django to use. If + not set, the default is set to + `sqlite:///opt/mailman-web-data/mailmanweb.db`. The standard + docker-compose.yaml comes with it set to a postgres database. It is not must + to change this if you are happy with postgresql. + +- `MAILMAN_REST_URL`: The URL to the Mailman core's REST API server. Defaut + value is `http://mailman-core:8001`. + +- `MAILMAN_REST_USER`: Mailman's REST API username. Default value is `restadmin` + +- `MAILMAN_REST_PASSWORD`: Mailman's REST API user's password. Default value is + `restpass` + +- `DJANGO_HOST_IP`: IP of the container from which the django will be + served. Default value is `172.19.199.3`. + +- `SMTP_HOST`: IP Address/hostname from which you will be sending + emails. Default value is `172.19.199.1`, which is the address of the Host OS. + +- `SMTP_PORT`: Port used for SMTP. Default is `25`. + + +### Mailman-Core + +These are the variables that you MUST change before deploying: + +- `HYPERKITT_API_KEY`: Hyperkitty's API Key, should be set to the same value as + set for the mailman-core. + +- `DATABASE_CLASS`: Default value is `mailman.database.sqlite.SQLiteDatabase`. + +These are the variables that you don't need to change if you are using a +standard version of docker-compose.yaml from this repository. + +- `MM_HOSTNAME`: Default value is `mailman-core` + +- `SMTP_HOST`: IP Address/hostname from which you will be sending + emails. Default value is `172.19.199.1`, which is the address of the Host OS. + +- `SMTP_PORT`: Port used for SMTP. Default is `25`. + +- `HYPERKITTY_API_URL`: Default value is `http://mailman-web:8000/hyperkitty` + +- `DATABASE_URL`: URL of the type + `driver://user:password@hostname:port/databasename` for the django to use. If + not set, the default is set to + `sqlite:///opt/mailman-web-data/mailmanweb.db`. The standard + docker-compose.yaml comes with it set to a postgres database. It is not must + to change this if you are happy with postgresql. + LICENSE ======= diff --git a/core/assets/run.sh b/core/assets/run.sh index 487b83a..4e1fa3a 100755 --- a/core/assets/run.sh +++ b/core/assets/run.sh @@ -1,17 +1,6 @@ #! /bin/bash set -e -# Check if the configuration file is present. -if [[ ! -e /opt/mailman/mailman.cfg ]]; then - echo "/opt/mailman/mailman.cfg configuration file not found..." - exit 1 -fi - -if [[ ! -e /opt/mailman/mailman-hyperkitty.cfg ]]; then - echo "/opt/mailman/mailman-hyperkitty.cfg configuration file not found..." - echo "Hyperkitty will not be enabled or will not work properly..." -fi - # Check if the master lock exists for the mailman. # It means that that either some other mailman process is running or # the last time mailman did not exit clean. @@ -21,16 +10,128 @@ if [[ -e /opt/mailman/core/var/locks/master.lck ]]; then exit 1 fi -# Check if the database is available yet. Do not start the container before the -# postgresql boots up. -until psql $DATABASE_URL -c '\l'; do - >&2 echo "Postgres is unavailable - sleeping" - sleep 1 +function wait_for_postgres () { + # Check if the postgres database is up and accepting connections before + # moving forward. + # TODO: Use python's psycopg2 module to do this in python instead of + # installing postgres-client in the image. + until psql $DATABASE_URL -c '\l'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 + done + >&2 echo "Postgres is up - continuing" +} + + +# Check if $DATABASE_URL is defined, if not, use a standard sqlite database. +# +# If the $DATABASE_URL is defined and is postgres, check if it is available +# yet. Do not start the container before the postgresql boots up. +# +# If the $DATABASE_URL is defined and is mysql, check if the database is +# available before the container boots up. +# +# TODO: Check the database type and detect if it is up based on that. For now, +# assume that postgres is being used if DATABASE_URL is defined. +if [[ -z "$DATABASES_URL" ]]; then + echo "DATABASE_URL is not defined. Using sqlite database..." + export DATABASE_URL=sqlite:///mailman.db + export DATABASE_TYPE='sqlite' + export DATABASE_CLASS='mailman.database.sqlite.SQLiteDatabase' +fi + + +if [[ "DATABASE_TYPE" = 'postgres' ]] +then + wait_for_postgres +fi + +# Check if $MM_HOSTNAME is set, if not, set it to a default value. +# TODO: Factor this out to a function. +if [[ -z "$MM_HOSTNAME" ]]; then + export MM_HOSTNAME=mailman-core +fi + +if [[ -z "$SMTP_HOST" ]]; then + export SMTP_HOST='172.19.199.1' +fi + +if [[ -z "$SMTP_PORT" ]]; then + export SMTP_PORT=25 +fi + +if [[ ! -d /config/ ]]; then + mkdir /config/ +fi + +# Generate a basic mailman.cfg. +cat > /config/mailman.cfg <> /config/mailman.cfg +fi + + +if [[ -z "$HYPERKITTY_API_KEY" ]]; then + echo "HYPERKITTY_API_KEY not defined, please set this environment variable..." + echo "exiting..." + exit 1 +fi + +if [[ -z "$HYPERKITTY_URL" ]]; then + echo "HYPERKITTY_URL not set, using the default value of http://mailman-web:8000/hyperkitty" + HYPERKITTY_URL="http://mailman-web:8000/hyperkitty/" +fi + +# Generate a basic mailman-hyperkitty.cfg. +cat > /config/mailman-hyperkitty.cfg <&2 echo "Postgres is up - continuing" - - -# Run mailman using the pidproxy command which spawns off mailman -# and forwards any signal you send it to the master runner in mailman. -/opt/pidproxy.py /opt/mailman/var/master.pid mailman -C /opt/mailman/mailman.cfg start --force diff --git a/web/assets/run.sh b/web/assets/run.sh index 361ced3..12e2f39 100755 --- a/web/assets/run.sh +++ b/web/assets/run.sh @@ -1,29 +1,8 @@ #! /bin/bash set -e -# Check if $DATABASE_URL is defined, if not, use a standard sqlite database. -# -# If the $DATABASE_URL is defined and is postgres, check if it is available -# yet. Do not start the container before the postgresql boots up. -# -# If the $DATABASE_URL is defined and is mysql, check if the database is -# available before the container boots up. -# -# TODO: Check the database type and detect if it is up based on that. For now, -# assume that postgres is being used if DATABASE_URL is defined. -if [[ -z "$DATABASES_URL" ]]; then - echo "$DATABASE_URL is not defined. Using sqlite database..." - DATABASE_URL="sqlite:///opt/mailman-web-data/database/mailmanweb.db" - DATABASE_TYPE='sqlite' - if [[ ! -e "/opt/mailman-web-data/database" ]]; then - mkdir -p /opt/mailman-web-data/database/ - fi -else - DATABASE_TYPE='postgres' - wait_for_postgres() -fi -function wait_for_postgres { +function wait_for_postgres () { # Check if the postgres database is up and accepting connections before # moving forward. # TODO: Use python's psycopg2 module to do this in python instead of @@ -35,7 +14,7 @@ function wait_for_postgres { >&2 echo "Postgres is up - continuing" } -function check_or_create { +function check_or_create () { # Check if the path exists, if not, create the directory. if [[ ! -e dir ]]; then echo "$1 does not exist, creating ..." @@ -43,6 +22,38 @@ function check_or_create { fi } +# function postgres_ready(){ +# python << END +# import sys +# import psycopg2 +# try: +# conn = psycopg2.connect(dbname="$POSTGRES_DB", user="$POSTGRES_USER", password="$POSTGRES_PASSWORD", host="postgres") +# except psycopg2.OperationalError: +# sys.exit(-1) +# sys.exit(0) +# END +# } + +# Check if $DATABASE_URL is defined, if not, use a standard sqlite database. +# +# If the $DATABASE_URL is defined and is postgres, check if it is available +# yet. Do not start the container before the postgresql boots up. +# +# If the $DATABASE_URL is defined and is mysql, check if the database is +# available before the container boots up. +# +# TODO: Check the database type and detect if it is up based on that. For now, +# assume that postgres is being used if DATABASE_URL is defined. + +if [[ ! -v DATABASE_URL ]]; then + echo "DATABASE_URL is not defined. Using sqlite database..." + export DATABASE_URL=sqlite://mailmanweb.db + export DATABASE_TYPE='sqlite' +else + export DATABASE_TYPE='postgres' + wait_for_postgres +fi + # Check if we are in the correct directory before running commands. if [[ ! $(pwd) == '/opt/mailman-web' ]]; then echo "Running in the wrong directory...switching to /opt/mailman-web" @@ -57,7 +68,7 @@ if [[ ! -e /opt/mailman-web-data/logs/mailmanweb.log ]]; then touch /opt/mailman-web-data/logs/mailmanweb.log fi -if [[ ! -e /opt/mailman-web-data/logs/mailmanweb.log ]]; then +if [[ ! -e /opt/mailman-web-data/logs/uwsgi.log ]]; then echo "Creating log file for uwsgi.." touch /opt/mailman-web-data/logs/uwsgi.log fi @@ -84,11 +95,18 @@ python manage.py migrate # It can also point to a logging daemon accessible at a URL. if [[ -z "$UWSGI_LOG_URL" ]]; then echo "No $UWSGI_LOG_URL defined, logging uwsgi to /opt/mailman-web-data/logs/uwsgi.log ..." - UWSGI_LOG_URL='/opt/mailman-web-data/logs/uwsgi.log' + export UWSGI_LOG_URL='/opt/mailman-web-data/logs/uwsgi.log' if [[ ! -e "$UWSGI_LOG_URL" ]]; then touch "$UWSGI_LOG_URL" fi fi +if [[ -z "$UWSGI_WSGI_FILE" ]]; then + export UWSGI_WSGI_FILE="wsgi.py" + export UWSGI_HTTP=":8000" + export UWSGI_WORKERS=2 + export UWSGI_THREADS=4 +fi + # Run the web server. uwsgi --http-auto-chunked --http-keepalive --logto "$UWSGI_LOG_URL" diff --git a/web/mailman-web/settings.py b/web/mailman-web/settings.py index 0f18287..3c697d4 100644 --- a/web/mailman-web/settings.py +++ b/web/mailman-web/settings.py @@ -27,6 +27,7 @@ https://docs.djangoproject.com/en/1.8/ref/settings/ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) import os +import dj_database_url BASE_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -47,16 +48,17 @@ SITE_ID = 1 ALLOWED_HOSTS = [ "localhost", # Archiving API from Mailman, keep it. # "lists.your-domain.org", + os.environ.get('SERVE_FROM_DOMAIN'), # Add here all production URLs you may have. "mailman-web", ] # Mailman API credentials -MAILMAN_REST_API_URL = 'http://mailman-core:8001' -MAILMAN_REST_API_USER = 'restadmin' -MAILMAN_REST_API_PASS = 'restpass' -MAILMAN_ARCHIVER_KEY = 'ASmallAPIKey' -MAILMAN_ARCHIVER_FROM = ('mailman-web', '172.19.199.3') +MAILMAN_REST_API_URL = os.environ.get('MAILMAN_REST_URL', 'http://mailman-core:8001') +MAILMAN_REST_API_USER = os.environ.get('MAILMAN_REST_USER', 'restadmin') +MAILMAN_REST_API_PASS = os.environ.get('MAILMAN_REST_PASSWORD', 'restpass') +MAILMAN_ARCHIVER_KEY = os.environ.get('HYPERKITTY_API_KEY') +MAILMAN_ARCHIVER_FROM = ('mailman-web', os.environ.get('DJANGO_HOST_IP', '172.19.199.3')) # Application definition @@ -142,8 +144,12 @@ WSGI_APPLICATION = 'wsgi.application' # https://docs.djangoproject.com/en/1.8/ref/settings/#databases -DATABASES['default'] = dj-database-url.config(conn_max_age=600) - +# This uses $DATABASE_URL from the environment variable to create a +# django-style-config-dict. +# https://github.com/kennethreitz/dj-database-url +DATABASES = { + 'default': dj_database_url.config(conn_max_age=600) +} # If you're behind a proxy, use the X-Forwarded-Host header # See https://docs.djangoproject.com/en/1.8/ref/settings/#use-x-forwarded-host @@ -262,13 +268,11 @@ SERVER_EMAIL = 'root@localhost.local' # Change this when you have a real email backend EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' -EMAIL_HOST = '172.19.199.1' -EMAIL_PORT = 25 +EMAIL_HOST = os.environ.get('SMTP_HOST', '172.19.199.1') +EMAIL_PORT = os.environ.get('SMTP_PORT', 25) EMAIL_HOST_USER = '' EMAIL_HOST_PASSWORD = '' EMAIL_USE_TLS = False -DEFAULT_FROM_EMAIL = 'Admin ' - # Compatibility with Bootstrap 3 from django.contrib.messages import constants as messages # flake8: noqa @@ -404,6 +408,8 @@ LOGGING = { 'class': 'logging.StreamHandler', 'formatter': 'simple', }, + # TODO: use an environment variable $DJ_LOG_URL to configure the logging + # using an environment variable. }, 'loggers': { 'django.request': {