Merge branch 'main' into fix_sqlite-url

.circleci/config.yml

@@ -45,7 +45,7 @@ jobs:
 
 workflows:
   version: 2
-  build-and-deploy-workflow:
+  test-stable:
     jobs:
       - build:
           rolling: "no"
@@ -53,7 +53,7 @@ workflows:
             tags:
               only: /^v\d+\.\d+\.\d+$/
 
-  cron-workflow:
+  cron-builds:
     triggers:
       - schedule:
           cron: "0 0 * * *"
@@ -65,7 +65,7 @@ workflows:
           rolling: "yes"
           context: org-global
 
-  rolling-workflow:
+  test-rolling:
     jobs:
       - build:
           rolling: "yes"

README.md

@@ -89,36 +89,24 @@ $ docker pull docker.io/maxking/mailman-web:rolling
 $ docker pull docker.io/maxking/mailman-core:rolling
 ```
 
-Rolling releases are built with every commit and also re-generated weekly. You
+Rolling releases are built with every commit and also re-generated nightly. You
 can inspect the images to get which commit it was built using:
 
 ```bash
 $ docker inspect --format '{{json .Config.Labels }}' mailman-core | python -m json.tool
 {
-    "version.core": "31f434d0",
     "version.git_commit": "45a4d7805b2b3d0e7c51679f59682d64ba02f05f",
-    "version.mm3-hk": "c625bfd2"
 }
 
 $ docker inspect --format '{{json .Config.Labels }}' mailman-web | python -m json.tool
 {
-    "version.client": "d9e9cb73",
-    "version.dj-mm3": "72a7d6c4",
     "version.git_commit": "45a4d7805b2b3d0e7c51679f59682d64ba02f05f",
-    "version.hyperkitty": "b67ca8a8",
-    "version.postorius": "73328ad4"
 }
 
 ```
 
 - `version.git_commit` : This is the commit hash of the Dockerfile in the
   [Github repo](https://github.com/maxking/docker-mailman)
-- `version.core`: The commit hash of Mailman Core
-- `version.mm3-hk`: The commit hash of Mailman3-hyperkitty plugin.
-- `version.client`: The commit hash of Mailman Client.
-- `version.hyperkitty`: The commit hash of Hyperkitty.
-- `version.postorius`: The commit hash of Postorius.
-- `version.dj-mm3`: The commit hash of Django-Mailman3 project.
 
 ## Dependencies
 

build.sh

@@ -12,49 +12,19 @@ DOCKER=docker
 
 if [ "$BUILD_ROLLING" = "yes" ]; then
     echo "Building rolling releases..."
-    python3 -m venv venv
-    source venv/bin/activate
-    pip install python-gitlab
-    # Get the latest commit for repositories and set their reference values to be
-    # used in the development builds.
-    CORE_REF=$(python get_latest_ref.py mailman/mailman)
-    CLIENT_REF=$(python get_latest_ref.py mailman/mailmanclient)
-    POSTORIUS_REF=$(python get_latest_ref.py mailman/postorius)
-    HYPERKITTY_REF=$(python get_latest_ref.py mailman/hyperkitty)
-    DJ_MM3_REF=$(python get_latest_ref.py mailman/django-mailman3)
-    MM3_HK_REF=$(python get_latest_ref.py mailman/mailman-hyperkitty)
-
     # Build the mailman-core image.
     $DOCKER build -f core/Dockerfile.dev \
-            --build-arg CORE_REF=$CORE_REF \
-            --build-arg MM3_HK_REF=$MM3_HK_REF \
-            --label version.core="$CORE_REF" \
-            --label version.mm3-hk="$MM3_HK_REF" \
             --label version.git_commit="$COMMIT_ID" \
             -t maxking/mailman-core:rolling core/
 
     # Build the mailman-web image.
     $DOCKER build -f web/Dockerfile.dev \
             --label version.git_commit="$COMMIT_ID" \
-            --label version.postorius="$POSTORIUS_REF" \
-            --label version.hyperkitty="$HYPERKITTY_REF" \
-            --label version.client="$CLIENT_REF" \
-            --label version.dj-mm3="$DJ_MM3_REF" \
-            --build-arg POSTORIUS_REF=$POSTORIUS_REF \
-            --build-arg CLIENT_REF=$CLIENT_REF \
-            --build-arg HYPERKITTY_REF=$HYPERKITTY_REF \
-            --build-arg DJ_MM3_REF=$DJ_MM3_REF \
             -t maxking/mailman-web:rolling web/
 
     # build the postorius image.
     $DOCKER build -f postorius/Dockerfile.dev\
 			--label version.git_commit="$COMMIT_ID"\
-            --label version.postorius="$POSTORIUS_REF" \
-            --label version.client="$CLIENT_REF" \
-            --label version.dj-mm3="$DJ_MM3_REF" \
-            --build-arg POSTORIUS_REF=$POSTORIUS_REF \
-            --build-arg CLIENT_REF=$CLIENT_REF \
-            --build-arg DJ_MM3_REF=$DJ_MM3_REF \
 			-t maxking/postorius:rolling postorius/
 else
     echo "Building stable releases..."

core/Dockerfile

@@ -10,12 +10,12 @@ RUN --mount=type=cache,target=/root/.cache \
     && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \
        libffi-dev \
   # psutil needs linux-headers to compile on musl c library.
-    && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers \
+    && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c \
     && python3 -m pip install -U pip setuptools wheel \
         && python3 -m pip install psycopg2 \
                    gunicorn==19.9.0 \
                    mailman==3.3.5 \
-                   mailman-hyperkitty==1.1.0 \
+                   mailman-hyperkitty==1.2.0 \
                    pymysql \
                    'sqlalchemy<1.4.0' \
     && apk del build-deps \

core/Dockerfile.dev

@@ -15,11 +15,11 @@ RUN --mount=type=cache,target=/root/.cache \
     apk update \
 	&& apk add --no-cache --virtual build-deps gcc python3-dev musl-dev \
    	    postgresql-dev git libffi-dev  \
-    && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers \
+    && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c \
     && python3 -m pip install -U psycopg2 pymysql setuptools wheel \
     && python3 -m pip install \
-        git+https://gitlab.com/mailman/mailman@${CORE_REF} \
+        git+https://gitlab.com/mailman/mailman \
-        git+https://gitlab.com/mailman/mailman-hyperkitty@${MM3_HK_REF} \
+        git+https://gitlab.com/mailman/mailman-hyperkitty \
         gunicorn==19.9.0 \
     && apk del build-deps \
     && adduser -S mailman

core/docker-entrypoint.sh

@@ -15,8 +15,8 @@ function wait_for_postgres () {
 
 function wait_for_mysql () {
 	# Check if MySQL is up and accepting connections.
-	HOSTNAME=$(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print(o.hostname);")
+	readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));")
-	until mysqladmin ping --host "$HOSTNAME" --silent; do
+	until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do
 		>&2 echo "MySQL is unavailable - sleeping"
 		sleep 1
 	done
@@ -36,7 +36,7 @@ fi
 # SMTP_HOST defaults to the gateway
 if [[ ! -v SMTP_HOST ]]; then
 	export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }')
-	echo "SMPT_HOST not specified, using the gateway ($SMTP_HOST) as default"
+	echo "SMTP_HOST not specified, using the gateway ($SMTP_HOST) as default"
 fi
 
 if [[ ! -v SMTP_PORT ]]; then

docker-compose-mysql.yaml

@@ -7,6 +7,7 @@ services:
     hostname: mailman-core
     volumes:
     - /opt/mailman/core:/opt/mailman/
+    stop_grace_period: 30s
     links:
     - database:database
     depends_on:
@@ -34,8 +35,8 @@ services:
     volumes:
     - /opt/mailman/web:/opt/mailman-web-data
     environment:
-    - DATABASE_URL=mysql://mailman:mailmanpass@database/mailmandb?charset=utf8mb4
     - DATABASE_TYPE=mysql
+    - DATABASE_URL=mysql://mailman:mailmanpass@database/mailmandb?charset=utf8mb4
     - HYPERKITTY_API_KEY=someapikey
     - SECRET_KEY=thisisaverysecretkey
     - DYLD_LIBRARY_PATH=/usr/local/mysql/lib/
@@ -51,7 +52,6 @@ services:
       MYSQL_USER: mailman
       MYSQL_PASSWORD: mailmanpass
       MYSQL_RANDOM_ROOT_PASSWORD: "yes"
-    restart: always
     image: mariadb:10.5
     command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci
     volumes:
@@ -60,7 +60,7 @@ services:
       mailman:
 
 networks:
-   mailman:
+  mailman:
     driver: bridge
     ipam:
       driver: default

get_latest_ref.py

@@ -1,41 +0,0 @@
-#! /usr/bin/env python
-import os
-import sys
-import gitlab
-
-
-def usage():
-    print("{} <project_name> <project_branch>")
-
-
-def main():
-    if 2 > len(sys.argv) > 3:
-        usage()
-
-    project_name = sys.argv[1]
-    if len(sys.argv) > 2:
-        branch_name = sys.argv[2]
-    else:
-        branch_name = 'master'
-
-    gl_token = os.getenv('GITLAB_TOKEN')
-    if gl_token is None:
-        print('GITLAB_TOKEN not set!')
-        exit(1)
-    gl = gitlab.Gitlab('https://gitlab.com/', gl_token)
-
-    project = gl.projects.get(project_name)
-
-    # Find the last commit in the branch that passed the CI
-    # successfully and return the reference to it.
-    for commit in project.commits.list(ref=branch_name):
-        stasues = list(status.status == 'success' for status in
-                        commit.statuses.list() if status.allow_failure == False)
-        if len(stasues) == 0:
-            continue
-        if all(stasues):
-            print(commit.short_id)
-            break
-
-if __name__ == '__main__':
-    main()

postorius/Dockerfile

@@ -12,10 +12,10 @@ COPY docker-entrypoint.sh /usr/local/bin/
 RUN --mount=type=cache,target=/root/.cache \
     set -ex \
 	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \
-		postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \
+		postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \
 	&& apk add --no-cache --virtual .mailman-rundeps bash sassc \
 		postgresql-client mysql-client py3-mysqlclient curl mailcap gettext \
-		python3 py3-pip libffi libuuid pcre-dev \
+		python3 py3-pip libffi libuuid pcre-dev py-cryptography \
 	&& python3 -m pip install -U 'Django<3.2' pip setuptools wheel \
 	&& python3 -m pip install postorius==1.3.6 \
         uwsgi \

postorius/Dockerfile.dev

@@ -16,14 +16,14 @@ ARG CLIENT_REF
 RUN --mount=type=cache,target=/root/.cache \
     set -ex \
 	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \
-		postgresql-dev mariadb-dev python3-dev libffi-dev git cargo rust \
+		postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev git cargo rust \
 	&& apk add --no-cache --virtual .mailman-rundeps bash sassc \
 		postgresql-client mysql-client py3-mysqlclient curl mailcap \
-		python3 py3-pip libffi gettext \
+		python3 py3-pip libffi gettext py-cryptography \
     && python3 -m pip install -U pip setuptools wheel \
     && python3 -m pip install -U \
-        git+https://gitlab.com/mailman/mailmanclient@${CLIENT_REF} \
+        git+https://gitlab.com/mailman/mailmanclient \
-        git+https://gitlab.com/mailman/postorius@${POSTORIUS_REF} \
+        git+https://gitlab.com/mailman/postorius \
 		uwsgi \
 		'psycopg2<2.9' \
 		dj-database-url \
@@ -32,7 +32,7 @@ RUN --mount=type=cache,target=/root/.cache \
 		django-utils-six \
     && python3 -m pip install -U 'Django<3.2' \
     && python3 -m pip install -U \
-       git+https://gitlab.com/mailman/django-mailman3@${DJ_MM3_REF} \
+       git+https://gitlab.com/mailman/django-mailman3 \
 	&& apk del .build-deps \
 	&& addgroup -S mailman \
 	&& adduser -S -G mailman mailman \

postorius/docker-entrypoint.sh

@@ -16,23 +16,14 @@ function wait_for_postgres () {
 
 function wait_for_mysql () {
 	# Check if MySQL is up and accepting connections.
-	HOSTNAME=$(python3 <<EOF
+	readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));")
-try:
+	until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do
-    from urllib.parse import urlparse
-except ImportError:
-    from urlparse import urlparse
-o = urlparse('$DATABASE_URL')
-print(o.hostname)
-EOF
-)
-	until mysqladmin ping --host "$HOSTNAME" --silent; do
 		>&2 echo "MySQL is unavailable - sleeping"
 		sleep 1
 	done
 	>&2 echo "MySQL is up - continuing"
 }
 
-
 function check_or_create () {
 	# Check if the path exists, if not, create the directory.
 	if [[ ! -e dir ]]; then

web/Dockerfile

@@ -12,15 +12,16 @@ COPY docker-entrypoint.sh /usr/local/bin/
 RUN --mount=type=cache,target=/root/.cache \
     set -ex \
 	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \
-           postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \
+           postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \
 	&& apk add --no-cache --virtual .mailman-rundeps bash sassc \
 	   postgresql-client mysql-client py3-mysqlclient curl mailcap gettext \
-	   python3 py3-pip xapian-core xapian-bindings-python3 libffi pcre-dev \
+	   python3 py3-pip xapian-core xapian-bindings-python3 libffi pcre-dev py-cryptography \
 	&& python3 -m pip install -U 'Django<3.2' pip setuptools wheel \
 	&& pip install mailmanclient==3.3.3 \
 		postorius==1.3.6 \
 		hyperkitty==1.3.5 \
 		django-mailman3==1.3.7 \
+		mistune==2.0.0rc1 \
 		whoosh \
 		uwsgi \
 		'psycopg2<2.9' \

web/Dockerfile.dev

@@ -17,15 +17,15 @@ ARG CLIENT_REF
 RUN --mount=type=cache,target=/root/.cache \
     set -ex \
     && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers git \
-        postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \
+        postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \
     && apk add --no-cache --virtual .mailman-rundeps bash sassc pcre-dev \
         python3 py3-pip postgresql-client mysql-client py3-mysqlclient \
-        curl mailcap xapian-core xapian-bindings-python3 libffi gettext \
+        curl mailcap xapian-core xapian-bindings-python3 libffi gettext py-cryptography \
     && python3 -m pip install -U pip setuptools wheel \
     && python3 -m pip install -U \
-        git+https://gitlab.com/mailman/mailmanclient@${CLIENT_REF} \
+        git+https://gitlab.com/mailman/mailmanclient \
-        git+https://gitlab.com/mailman/postorius@${POSTORIUS_REF} \
+        git+https://gitlab.com/mailman/postorius \
-        git+https://gitlab.com/mailman/hyperkitty@${HYPERKITTY_REF} \
+        git+https://gitlab.com/mailman/hyperkitty \
         whoosh \
         uwsgi \
         'psycopg2<2.9' \
@@ -38,7 +38,7 @@ RUN --mount=type=cache,target=/root/.cache \
         django-utils-six \
     && python3 -m pip install -U 'Django<3.2' \
     && python3 -m pip install -U \
-       git+https://gitlab.com/mailman/django-mailman3@${DJ_MM3_REF} \
+       git+https://gitlab.com/mailman/django-mailman3 \
     && apk del .build-deps \
     && addgroup -S mailman \
     && adduser -S -G mailman mailman \

web/docker-entrypoint.sh

@@ -16,23 +16,14 @@ function wait_for_postgres () {
 
 function wait_for_mysql () {
 	# Check if MySQL is up and accepting connections.
-	HOSTNAME=$(python3 <<EOF
+	readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));")
-try:
+	until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do
-    from urllib.parse import urlparse
-except ImportError:
-    from urlparse import urlparse
-o = urlparse('$DATABASE_URL')
-print(o.hostname)
-EOF
-)
-	until mysqladmin ping --host "$HOSTNAME" --silent; do
 		>&2 echo "MySQL is unavailable - sleeping"
 		sleep 1
 	done
 	>&2 echo "MySQL is up - continuing"
 }
 
-
 function check_or_create () {
 	# Check if the path exists, if not, create the directory.
 	if [[ ! -e dir ]]; then