From 7bc5bf5dc46ded22b0de3e98edcc8cdebb2706f0 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Sun, 28 May 2017 14:20:39 -0700 Subject: [PATCH 1/5] Use a config file for uwsgi. --- docker-compose.yaml | 4 ---- web/Dockerfile | 26 +++++++++++--------------- web/assets/run.sh | 21 +++------------------ web/mailman-web/uwsgi.ini | 10 ++++++++++ 4 files changed, 24 insertions(+), 37 deletions(-) create mode 100644 web/mailman-web/uwsgi.ini diff --git a/docker-compose.yaml b/docker-compose.yaml index 61a87d7..034078f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -32,10 +32,6 @@ services: volumes: - /opt/mailman/web:/opt/mailman-web-data environment: - - UWSGI_WSGI_FILE=wsgi.py - - UWSGI_HTTP=:8000 - - UWSGI_WORKERS=2 - - UWSGI_THREADS=4 - DATABASE_URL=postgres://mailman:mailmanpass@database/mailmandb - HYPERKITTY_API_KEY=someapikey networks: diff --git a/web/Dockerfile b/web/Dockerfile index 1951c0a..be7a532 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -10,20 +10,16 @@ RUN apt-get update \ postgresql-client \ && rm -rf /var/lib/apt/lists/* -RUN wget -O mailmanclient.zip https://gitlab.com/mailman/mailmanclient/repository/archive.zip?ref=master \ - && wget -O postorius.zip https://gitlab.com/mailman/postorius/repository/archive.zip?ref=master \ - && wget -O django-mailman3.zip https://gitlab.com/mailman/django-mailman3/repository/archive.zip?ref=master \ - && wget -O hyperkitty.zip https://gitlab.com/mailman/hyperkitty/repository/archive.zip?ref=master \ - && python -m pip install -U mailmanclient.zip \ - postorius.zip \ - hyperkitty.zip \ - whoosh \ - uwsgi \ - psycopg2 \ - dj-database-url \ - pymysql \ - && python -m pip install -U django-mailman3.zip \ - && rm mailmanclient.zip postorius.zip hyperkitty.zip django-mailman3.zip +RUN python -m pip install -U mailmanclient==3.1.0 \ + postorius==1.1.0 \ + hyperkitty==1.1.0 \ + django-mailman3==1.1.0 \ + whoosh \ + uwsgi \ + psycopg2 \ + dj-database-url \ + pymysql \ + && python -m pip install -U django==1.10 ADD mailman-web /opt/mailman-web @@ -35,4 +31,4 @@ EXPOSE 8000 ENTRYPOINT ["/opt/run.sh"] -CMD ["uwsgi", "--http-auto-chunked","--http-keepalive"] +CMD ["uwsgi", "--ini", "/opt/mailman-web/uwsgi.ini"] diff --git a/web/assets/run.sh b/web/assets/run.sh index b34ddc9..dafb162 100755 --- a/web/assets/run.sh +++ b/web/assets/run.sh @@ -49,8 +49,9 @@ if [[ ! -v DATABASE_URL ]]; then echo "DATABASE_URL is not defined. Using sqlite database..." export DATABASE_URL=sqlite://mailmanweb.db export DATABASE_TYPE='sqlite' -else - export DATABASE_TYPE='postgres' +fi + +if [[ "$DATABASE_TYPE" = 'postgres' ]]; then wait_for_postgres fi @@ -88,20 +89,4 @@ python manage.py collectstatic --noinput # this command will upgrade the database. python manage.py migrate - -# Log to the default location /opt/mailman-web-data/logs/uwsgi.log if the -# logging variable is not set. -if [[ ! -v UWSGI_LOGTO ]]; then - echo "No UWSGI_LOGTO defined, logging uwsgi to /opt/mailman-web-data/logs/uwsgi.log ..." - export UWSGI_LOGTO='/opt/mailman-web-data/logs/uwsgi.log' - touch "$UWSGI_LOGTO" -fi - -if [[ ! -v UWSGI_WSGI_FILE ]]; then - export UWSGI_WSGI_FILE="wsgi.py" - export UWSGI_HTTP=:8000 - export UWSGI_WORKERS=2 - export UWSGI_THREADS=4 -fi - exec $@ diff --git a/web/mailman-web/uwsgi.ini b/web/mailman-web/uwsgi.ini new file mode 100644 index 0000000..659d978 --- /dev/null +++ b/web/mailman-web/uwsgi.ini @@ -0,0 +1,10 @@ +[uwsgi] +http = :8000 +chdir = /opt/mailman-web +wsgi-file = wsgi.py +process = 2 +threads = 2 +uid = 1000 +gid = 1000 +req-log = file:/opt/mailman-web-data/uwsgi.log +error-log = file:/opt/mailman-web-data/uwsgi-error.log From a21097d68e3891ff9e03ce45e47c850a5459161e Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Sun, 28 May 2017 14:28:02 -0700 Subject: [PATCH 2/5] Spin of qcluster command from uwsgi. --- web/mailman-web/uwsgi.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/web/mailman-web/uwsgi.ini b/web/mailman-web/uwsgi.ini index 659d978..db329a8 100644 --- a/web/mailman-web/uwsgi.ini +++ b/web/mailman-web/uwsgi.ini @@ -8,3 +8,4 @@ uid = 1000 gid = 1000 req-log = file:/opt/mailman-web-data/uwsgi.log error-log = file:/opt/mailman-web-data/uwsgi-error.log +attach-daemon = ./manage.py qcluster From 4bac2fb2361ea7514b639de91a39ca65d85f87b3 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Sun, 28 May 2017 14:52:09 -0700 Subject: [PATCH 3/5] Add crons and update the uwsgi configuration. --- web/mailman-web/uwsgi.ini | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/web/mailman-web/uwsgi.ini b/web/mailman-web/uwsgi.ini index db329a8..c757d1c 100644 --- a/web/mailman-web/uwsgi.ini +++ b/web/mailman-web/uwsgi.ini @@ -1,11 +1,32 @@ [uwsgi] +# Port on which uwsgi will be listening. http = :8000 + +# Move to the directory wher the django files are. chdir = /opt/mailman-web + +# Use the wsgi file provided with the django project. wsgi-file = wsgi.py + +# Setup default number of processes and threads per process. +master = true process = 2 threads = 2 + +# Drop privielges and don't run as root. uid = 1000 gid = 1000 + +# Setup the log files. req-log = file:/opt/mailman-web-data/uwsgi.log error-log = file:/opt/mailman-web-data/uwsgi-error.log + +# Setup the django_q related worker processes. attach-daemon = ./manage.py qcluster + +# Setup hyperkitty's cron jobs. +cron2 = unique=1 ./manage.py runjobs minutely +cron2 = hour=1,unique=1 ./manage.py runjobs hourly +cron2 = day=1,unique=1 ./manage.py runjobs monthly +cron2 = week=1,unique=1 ./manage.py runjobs weekly +cron2 = month=1,unique=1 ./manage.py runjobs yearly From 0ae0a55a4211536a2d258e70c549dfd6bbe64019 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Sun, 28 May 2017 14:59:52 -0700 Subject: [PATCH 4/5] Specify the database type along with the URL. --- docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 034078f..81bdcea 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -32,6 +32,7 @@ services: volumes: - /opt/mailman/web:/opt/mailman-web-data environment: + - DATABASE_TYPE=postgres - DATABASE_URL=postgres://mailman:mailmanpass@database/mailmandb - HYPERKITTY_API_KEY=someapikey networks: From 2025f733a9685f6ffe7dcab137ea2a13a30c4c6d Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Sun, 28 May 2017 17:11:41 -0700 Subject: [PATCH 5/5] Add a mailman user and setup loggers. --- web/assets/run.sh | 6 ++++++ web/mailman-web/uwsgi.ini | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/web/assets/run.sh b/web/assets/run.sh index dafb162..d69a8da 100755 --- a/web/assets/run.sh +++ b/web/assets/run.sh @@ -89,4 +89,10 @@ python manage.py collectstatic --noinput # this command will upgrade the database. python manage.py migrate +# Create a mailman user with the specific UID and GID and do not create home +# directory for it. Also chown the logs directory to write the files. +useradd -M -U -u 1000 mailman +chown mailman:mailman /opt/mailman-web-data -R + + exec $@ diff --git a/web/mailman-web/uwsgi.ini b/web/mailman-web/uwsgi.ini index c757d1c..0b488bc 100644 --- a/web/mailman-web/uwsgi.ini +++ b/web/mailman-web/uwsgi.ini @@ -17,10 +17,6 @@ threads = 2 uid = 1000 gid = 1000 -# Setup the log files. -req-log = file:/opt/mailman-web-data/uwsgi.log -error-log = file:/opt/mailman-web-data/uwsgi-error.log - # Setup the django_q related worker processes. attach-daemon = ./manage.py qcluster @@ -30,3 +26,17 @@ cron2 = hour=1,unique=1 ./manage.py runjobs hourly cron2 = day=1,unique=1 ./manage.py runjobs monthly cron2 = week=1,unique=1 ./manage.py runjobs weekly cron2 = month=1,unique=1 ./manage.py runjobs yearly + +# Setup the request log. +req-logger = file:/opt/mailman-web-data/logs/uwsgi.log + +# Log cron seperately. +logger = cron file:/opt/mailman-web-data/logs/uwsgi-cron.log +log-route = cron uwsgi-cron + +# Log qcluster commands seperately. +logger = qcluster file:/opt/mailman-web-data/logs/uwsgi-qcluster.log +log-route = qcluster uwsgi-daemons + +# Last log and it logs the rest of the stuff. +logger = file:/opt/mailman-web-data/logs/uwsgi-error.log