From ad1aaff6244502317b9938bc4a21275b5a7f6c23 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Thu, 6 Apr 2017 17:06:12 -0700 Subject: [PATCH] Several change to mailman-web image. - Download source code as zip from gitlab instead of git+https because speed. - Check if the $DATABASE_URL is not defined, use a default sqlite database. - Use dj-database-url package to set database in django settings using DATABASE_URL environment variable. - Use UWSGI_LOG_URL as the URL for UWSGI logging and set it to a default value if it has not been defined. --- web/Dockerfile | 28 +++++++++++----- web/assets/run.sh | 67 ++++++++++++++++++++++++++++++++----- web/mailman-web/settings.py | 26 ++------------ 3 files changed, 79 insertions(+), 42 deletions(-) diff --git a/web/Dockerfile b/web/Dockerfile index 29844f5..be28308 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -2,17 +2,27 @@ FROM python:2.7 MAINTAINER Abhilash Raj +ENV DEBIAN_FRONTEND=noninteractive + RUN apt-get update \ && apt-get install -y ruby-sass \ - git \ - postgresql-client \ - && python -m pip install git+https://gitlab.com/mailman/mailmanclient.git \ - git+https://gitlab.com/mailman/postorius.git \ - git+https://gitlab.com/mailman/django-mailman3.git \ - git+https://gitlab.com/mailman/hyperkitty.git \ - whoosh \ - uwsgi \ - psycopg2 + wget \ + postgresql-client + +RUN wget -O mailmanclient.zip https://gitlab.com/mailman/mailmanclient/repository/archive.zip?ref=master \ + && wget -O postorius.zip https://gitlab.com/mailman/postorius/repository/archive.zip?ref=master \ + && wget -O django-mailman3.zip https://gitlab.com/mailman/django-mailman3/repository/archive.zip?ref=master \ + && wget -O hyperkitty.zip https://gitlab.com/mailman/hyperkitty/repository/archive.zip?ref=master \ + && python -m pip install mailmanclient.zip \ + postorius.zip \ + django-mailman3.zip \ + hyperkitty.zip \ + whoosh \ + uwsgi \ + psycopg2 \ + dj-database-url \ + pymysql \ + && rm mailmanclient.zip postorius.zip hyperkitty.zip django-mailman3.zip ADD mailman-web /opt/mailman-web diff --git a/web/assets/run.sh b/web/assets/run.sh index 02942bc..361ced3 100755 --- a/web/assets/run.sh +++ b/web/assets/run.sh @@ -1,14 +1,47 @@ #! /bin/bash set -e -# Check if the database is available yet. Do not start the container before the -# postgresql boots up. -until psql $DATABASE_URL -c '\l'; do - >&2 echo "Postgres is unavailable - sleeping" - sleep 1 -done +# Check if $DATABASE_URL is defined, if not, use a standard sqlite database. +# +# If the $DATABASE_URL is defined and is postgres, check if it is available +# yet. Do not start the container before the postgresql boots up. +# +# If the $DATABASE_URL is defined and is mysql, check if the database is +# available before the container boots up. +# +# TODO: Check the database type and detect if it is up based on that. For now, +# assume that postgres is being used if DATABASE_URL is defined. +if [[ -z "$DATABASES_URL" ]]; then + echo "$DATABASE_URL is not defined. Using sqlite database..." + DATABASE_URL="sqlite:///opt/mailman-web-data/database/mailmanweb.db" + DATABASE_TYPE='sqlite' + if [[ ! -e "/opt/mailman-web-data/database" ]]; then + mkdir -p /opt/mailman-web-data/database/ + fi +else + DATABASE_TYPE='postgres' + wait_for_postgres() +fi ->&2 echo "Postgres is up - continuing" +function wait_for_postgres { + # Check if the postgres database is up and accepting connections before + # moving forward. + # TODO: Use python's psycopg2 module to do this in python instead of + # installing postgres-client in the image. + until psql $DATABASE_URL -c '\l'; do + >&2 echo "Postgres is unavailable - sleeping" + sleep 1 + done + >&2 echo "Postgres is up - continuing" +} + +function check_or_create { + # Check if the path exists, if not, create the directory. + if [[ ! -e dir ]]; then + echo "$1 does not exist, creating ..." + mkdir "$1" + fi +} # Check if we are in the correct directory before running commands. if [[ ! $(pwd) == '/opt/mailman-web' ]]; then @@ -19,11 +52,15 @@ fi # Check if the logs directory is setup. if [[ ! -e /opt/mailman-web-data/logs/mailmanweb.log ]]; then - echo "Create log file..." + echo "Creating log file for mailman web" mkdir -p /opt/mailman-web-data/logs/ touch /opt/mailman-web-data/logs/mailmanweb.log fi +if [[ ! -e /opt/mailman-web-data/logs/mailmanweb.log ]]; then + echo "Creating log file for uwsgi.." + touch /opt/mailman-web-data/logs/uwsgi.log +fi # Check if the settings_local.py file exists, if yes, copy it too. if [[ -e /opt/mailman-web-data/settings_local.py ]]; then @@ -41,5 +78,17 @@ python manage.py collectstatic --noinput # this command will upgrade the database. python manage.py migrate +# Check if there is a non-standard location for logging defined. +# It can be changed by $UWSGI_LOG_URL environment variable, which if not set points +# to /opt/mailman-web/logs/uwsgi.log +# It can also point to a logging daemon accessible at a URL. +if [[ -z "$UWSGI_LOG_URL" ]]; then + echo "No $UWSGI_LOG_URL defined, logging uwsgi to /opt/mailman-web-data/logs/uwsgi.log ..." + UWSGI_LOG_URL='/opt/mailman-web-data/logs/uwsgi.log' + if [[ ! -e "$UWSGI_LOG_URL" ]]; then + touch "$UWSGI_LOG_URL" + fi +fi + # Run the web server. -uwsgi --http-auto-chunked --http-keepalive --static-map /static/=/opt/mailman-web-data/static/ +uwsgi --http-auto-chunked --http-keepalive --logto "$UWSGI_LOG_URL" diff --git a/web/mailman-web/settings.py b/web/mailman-web/settings.py index e7e0922..0f18287 100644 --- a/web/mailman-web/settings.py +++ b/web/mailman-web/settings.py @@ -141,30 +141,8 @@ WSGI_APPLICATION = 'wsgi.application' # Database # https://docs.djangoproject.com/en/1.8/ref/settings/#databases -DATABASES = { - #'default': { - # Use 'sqlite3', 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. - #'ENGINE': 'django.db.backends.sqlite3', - # DB name or path to database file if using sqlite3. - #'NAME': '/opt/mailman-web-data/mailmansuite.db', - # The following settings are not used with sqlite3: - #'USER': 'mailmansuite', - #'PASSWORD': 'mmpass', - # HOST: empty for localhost through domain sockets or '127.0.0.1' for - # localhost through TCP. - #'HOST': '', - # PORT: set to empty string for default. - #'PORT': '', - #} - # Example for PostgreSQL (recommanded for production): - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'mailmandb', - 'USER': 'mailman', - 'PASSWORD': 'mailmanpass', - 'HOST': 'database', - } -} + +DATABASES['default'] = dj-database-url.config(conn_max_age=600) # If you're behind a proxy, use the X-Forwarded-Host header