diff --git a/.circleci/config.yml b/.circleci/config.yml index 4078009..c2aaaa5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -45,7 +45,7 @@ jobs: workflows: version: 2 - build-and-deploy-workflow: + test-stable: jobs: - build: rolling: "no" @@ -53,7 +53,7 @@ workflows: tags: only: /^v\d+\.\d+\.\d+$/ - cron-workflow: + cron-builds: triggers: - schedule: cron: "0 0 * * *" @@ -65,7 +65,7 @@ workflows: rolling: "yes" context: org-global - rolling-workflow: + test-rolling: jobs: - build: rolling: "yes" diff --git a/README.md b/README.md index 133ebf3..d67df98 100644 --- a/README.md +++ b/README.md @@ -89,36 +89,24 @@ $ docker pull docker.io/maxking/mailman-web:rolling $ docker pull docker.io/maxking/mailman-core:rolling ``` -Rolling releases are built with every commit and also re-generated weekly. You +Rolling releases are built with every commit and also re-generated nightly. You can inspect the images to get which commit it was built using: ```bash $ docker inspect --format '{{json .Config.Labels }}' mailman-core | python -m json.tool { - "version.core": "31f434d0", "version.git_commit": "45a4d7805b2b3d0e7c51679f59682d64ba02f05f", - "version.mm3-hk": "c625bfd2" } $ docker inspect --format '{{json .Config.Labels }}' mailman-web | python -m json.tool { - "version.client": "d9e9cb73", - "version.dj-mm3": "72a7d6c4", "version.git_commit": "45a4d7805b2b3d0e7c51679f59682d64ba02f05f", - "version.hyperkitty": "b67ca8a8", - "version.postorius": "73328ad4" } ``` - `version.git_commit` : This is the commit hash of the Dockerfile in the [Github repo](https://github.com/maxking/docker-mailman) -- `version.core`: The commit hash of Mailman Core -- `version.mm3-hk`: The commit hash of Mailman3-hyperkitty plugin. -- `version.client`: The commit hash of Mailman Client. -- `version.hyperkitty`: The commit hash of Hyperkitty. -- `version.postorius`: The commit hash of Postorius. -- `version.dj-mm3`: The commit hash of Django-Mailman3 project. ## Dependencies @@ -414,7 +402,7 @@ smtp_port: $SMTP_PORT configuration: /etc/postfix-mailman.cfg ``` -So, if you need to update the values, you can set `SMTP_HOST`, `SMTP_PORT`, +So, if you need to update the values, you can set `SMTP_HOST`, `SMTP_PORT`, `MM_HOSTNAME` environment variables in `mailman-core` container. Please verify the output for `[mta]` section to ensure that it points to diff --git a/build.sh b/build.sh index 370d621..9d8fe7a 100755 --- a/build.sh +++ b/build.sh @@ -12,49 +12,19 @@ DOCKER=docker if [ "$BUILD_ROLLING" = "yes" ]; then echo "Building rolling releases..." - python3 -m venv venv - source venv/bin/activate - pip install python-gitlab - # Get the latest commit for repositories and set their reference values to be - # used in the development builds. - CORE_REF=$(python get_latest_ref.py mailman/mailman) - CLIENT_REF=$(python get_latest_ref.py mailman/mailmanclient) - POSTORIUS_REF=$(python get_latest_ref.py mailman/postorius) - HYPERKITTY_REF=$(python get_latest_ref.py mailman/hyperkitty) - DJ_MM3_REF=$(python get_latest_ref.py mailman/django-mailman3) - MM3_HK_REF=$(python get_latest_ref.py mailman/mailman-hyperkitty) - # Build the mailman-core image. $DOCKER build -f core/Dockerfile.dev \ - --build-arg CORE_REF=$CORE_REF \ - --build-arg MM3_HK_REF=$MM3_HK_REF \ - --label version.core="$CORE_REF" \ - --label version.mm3-hk="$MM3_HK_REF" \ --label version.git_commit="$COMMIT_ID" \ -t maxking/mailman-core:rolling core/ # Build the mailman-web image. $DOCKER build -f web/Dockerfile.dev \ --label version.git_commit="$COMMIT_ID" \ - --label version.postorius="$POSTORIUS_REF" \ - --label version.hyperkitty="$HYPERKITTY_REF" \ - --label version.client="$CLIENT_REF" \ - --label version.dj-mm3="$DJ_MM3_REF" \ - --build-arg POSTORIUS_REF=$POSTORIUS_REF \ - --build-arg CLIENT_REF=$CLIENT_REF \ - --build-arg HYPERKITTY_REF=$HYPERKITTY_REF \ - --build-arg DJ_MM3_REF=$DJ_MM3_REF \ -t maxking/mailman-web:rolling web/ # build the postorius image. $DOCKER build -f postorius/Dockerfile.dev\ --label version.git_commit="$COMMIT_ID"\ - --label version.postorius="$POSTORIUS_REF" \ - --label version.client="$CLIENT_REF" \ - --label version.dj-mm3="$DJ_MM3_REF" \ - --build-arg POSTORIUS_REF=$POSTORIUS_REF \ - --build-arg CLIENT_REF=$CLIENT_REF \ - --build-arg DJ_MM3_REF=$DJ_MM3_REF \ -t maxking/postorius:rolling postorius/ else echo "Building stable releases..." diff --git a/core/Dockerfile b/core/Dockerfile index 9b2cd53..ca904e6 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -10,12 +10,12 @@ RUN --mount=type=cache,target=/root/.cache \ && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \ libffi-dev \ # psutil needs linux-headers to compile on musl c library. - && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers \ + && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c \ && python3 -m pip install -U pip setuptools wheel \ && python3 -m pip install psycopg2 \ gunicorn==19.9.0 \ mailman==3.3.5 \ - mailman-hyperkitty==1.1.0 \ + mailman-hyperkitty==1.2.0 \ pymysql \ 'sqlalchemy<1.4.0' \ && apk del build-deps \ diff --git a/core/Dockerfile.dev b/core/Dockerfile.dev index 20a391f..bb0c0b8 100644 --- a/core/Dockerfile.dev +++ b/core/Dockerfile.dev @@ -15,11 +15,11 @@ RUN --mount=type=cache,target=/root/.cache \ apk update \ && apk add --no-cache --virtual build-deps gcc python3-dev musl-dev \ postgresql-dev git libffi-dev \ - && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers \ + && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers py-cryptography mariadb-connector-c \ && python3 -m pip install -U psycopg2 pymysql setuptools wheel \ && python3 -m pip install \ - git+https://gitlab.com/mailman/mailman@${CORE_REF} \ - git+https://gitlab.com/mailman/mailman-hyperkitty@${MM3_HK_REF} \ + git+https://gitlab.com/mailman/mailman \ + git+https://gitlab.com/mailman/mailman-hyperkitty \ gunicorn==19.9.0 \ && apk del build-deps \ && adduser -S mailman diff --git a/core/docker-entrypoint.sh b/core/docker-entrypoint.sh index 38338ad..80aa6c7 100755 --- a/core/docker-entrypoint.sh +++ b/core/docker-entrypoint.sh @@ -15,8 +15,8 @@ function wait_for_postgres () { function wait_for_mysql () { # Check if MySQL is up and accepting connections. - HOSTNAME=$(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print(o.hostname);") - until mysqladmin ping --host "$HOSTNAME" --silent; do + readarray -d' ' -t ENDPOINT <<< $(python3 -c "from urllib.parse import urlparse; o = urlparse('$DATABASE_URL'); print('%s %s' % (o.hostname, o.port if o.port else '3306'));") + until mysqladmin ping --host ${ENDPOINT[0]} --port ${ENDPOINT[1]} --silent; do >&2 echo "MySQL is unavailable - sleeping" sleep 1 done @@ -36,7 +36,7 @@ fi # SMTP_HOST defaults to the gateway if [[ ! -v SMTP_HOST ]]; then export SMTP_HOST=$(/sbin/ip route | awk '/default/ { print $3 }') - echo "SMPT_HOST not specified, using the gateway ($SMTP_HOST) as default" + echo "SMTP_HOST not specified, using the gateway ($SMTP_HOST) as default" fi if [[ ! -v SMTP_PORT ]]; then diff --git a/docker-compose-mysql.yaml b/docker-compose-mysql.yaml index e97c602..5d8358e 100644 --- a/docker-compose-mysql.yaml +++ b/docker-compose-mysql.yaml @@ -7,6 +7,7 @@ services: hostname: mailman-core volumes: - /opt/mailman/core:/opt/mailman/ + stop_grace_period: 30s links: - database:database depends_on: @@ -34,8 +35,8 @@ services: volumes: - /opt/mailman/web:/opt/mailman-web-data environment: + - DATABASE_TYPE=mysql - DATABASE_URL=mysql://mailman:mailmanpass@database/mailmandb?charset=utf8mb4 - - DATABASE_TYPE=mysql - HYPERKITTY_API_KEY=someapikey - SECRET_KEY=thisisaverysecretkey - DYLD_LIBRARY_PATH=/usr/local/mysql/lib/ @@ -51,7 +52,6 @@ services: MYSQL_USER: mailman MYSQL_PASSWORD: mailmanpass MYSQL_RANDOM_ROOT_PASSWORD: "yes" - restart: always image: mariadb:10.5 command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci volumes: @@ -60,7 +60,7 @@ services: mailman: networks: - mailman: + mailman: driver: bridge ipam: driver: default diff --git a/get_latest_ref.py b/get_latest_ref.py deleted file mode 100755 index a761d7e..0000000 --- a/get_latest_ref.py +++ /dev/null @@ -1,41 +0,0 @@ -#! /usr/bin/env python -import os -import sys -import gitlab - - -def usage(): - print("{} ") - - -def main(): - if 2 > len(sys.argv) > 3: - usage() - - project_name = sys.argv[1] - if len(sys.argv) > 2: - branch_name = sys.argv[2] - else: - branch_name = 'master' - - gl_token = os.getenv('GITLAB_TOKEN') - if gl_token is None: - print('GITLAB_TOKEN not set!') - exit(1) - gl = gitlab.Gitlab('https://gitlab.com/', gl_token) - - project = gl.projects.get(project_name) - - # Find the last commit in the branch that passed the CI - # successfully and return the reference to it. - for commit in project.commits.list(ref=branch_name): - stasues = list(status.status == 'success' for status in - commit.statuses.list() if status.allow_failure == False) - if len(stasues) == 0: - continue - if all(stasues): - print(commit.short_id) - break - -if __name__ == '__main__': - main() diff --git a/postorius/Dockerfile b/postorius/Dockerfile index f8d719b..408c832 100644 --- a/postorius/Dockerfile +++ b/postorius/Dockerfile @@ -12,10 +12,10 @@ COPY docker-entrypoint.sh /usr/local/bin/ RUN --mount=type=cache,target=/root/.cache \ set -ex \ && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \ - postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \ + postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \ && apk add --no-cache --virtual .mailman-rundeps bash sassc \ postgresql-client mysql-client py3-mysqlclient curl mailcap gettext \ - python3 py3-pip libffi libuuid pcre-dev \ + python3 py3-pip libffi libuuid pcre-dev py-cryptography \ && python3 -m pip install -U 'Django<3.2' pip setuptools wheel \ && python3 -m pip install postorius==1.3.6 \ uwsgi \ diff --git a/postorius/Dockerfile.dev b/postorius/Dockerfile.dev index f1560cb..e427a1c 100644 --- a/postorius/Dockerfile.dev +++ b/postorius/Dockerfile.dev @@ -16,14 +16,14 @@ ARG CLIENT_REF RUN --mount=type=cache,target=/root/.cache \ set -ex \ && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \ - postgresql-dev mariadb-dev python3-dev libffi-dev git cargo rust \ + postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev git cargo rust \ && apk add --no-cache --virtual .mailman-rundeps bash sassc \ postgresql-client mysql-client py3-mysqlclient curl mailcap \ - python3 py3-pip libffi gettext \ + python3 py3-pip libffi gettext py-cryptography \ && python3 -m pip install -U pip setuptools wheel \ && python3 -m pip install -U \ - git+https://gitlab.com/mailman/mailmanclient@${CLIENT_REF} \ - git+https://gitlab.com/mailman/postorius@${POSTORIUS_REF} \ + git+https://gitlab.com/mailman/mailmanclient \ + git+https://gitlab.com/mailman/postorius \ uwsgi \ 'psycopg2<2.9' \ dj-database-url \ @@ -32,7 +32,7 @@ RUN --mount=type=cache,target=/root/.cache \ django-utils-six \ && python3 -m pip install -U 'Django<3.2' \ && python3 -m pip install -U \ - git+https://gitlab.com/mailman/django-mailman3@${DJ_MM3_REF} \ + git+https://gitlab.com/mailman/django-mailman3 \ && apk del .build-deps \ && addgroup -S mailman \ && adduser -S -G mailman mailman \ diff --git a/postorius/docker-entrypoint.sh b/postorius/docker-entrypoint.sh index 3327d16..eef042c 100755 --- a/postorius/docker-entrypoint.sh +++ b/postorius/docker-entrypoint.sh @@ -16,23 +16,14 @@ function wait_for_postgres () { function wait_for_mysql () { # Check if MySQL is up and accepting connections. - HOSTNAME=$(python3 <&2 echo "MySQL is unavailable - sleeping" sleep 1 done >&2 echo "MySQL is up - continuing" } - function check_or_create () { # Check if the path exists, if not, create the directory. if [[ ! -e dir ]]; then diff --git a/web/Dockerfile b/web/Dockerfile index 011b4c5..297f074 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -12,15 +12,16 @@ COPY docker-entrypoint.sh /usr/local/bin/ RUN --mount=type=cache,target=/root/.cache \ set -ex \ && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \ - postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \ + postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \ && apk add --no-cache --virtual .mailman-rundeps bash sassc \ postgresql-client mysql-client py3-mysqlclient curl mailcap gettext \ - python3 py3-pip xapian-core xapian-bindings-python3 libffi pcre-dev \ + python3 py3-pip xapian-core xapian-bindings-python3 libffi pcre-dev py-cryptography \ && python3 -m pip install -U 'Django<3.2' pip setuptools wheel \ && pip install mailmanclient==3.3.3 \ postorius==1.3.6 \ hyperkitty==1.3.5 \ django-mailman3==1.3.7 \ + mistune==2.0.0rc1 \ whoosh \ uwsgi \ 'psycopg2<2.9' \ diff --git a/web/Dockerfile.dev b/web/Dockerfile.dev index 2c5951c..9b2809c 100644 --- a/web/Dockerfile.dev +++ b/web/Dockerfile.dev @@ -17,15 +17,15 @@ ARG CLIENT_REF RUN --mount=type=cache,target=/root/.cache \ set -ex \ && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers git \ - postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \ + postgresql-dev mariadb-dev mariadb-connector-c python3-dev libffi-dev openldap-dev cargo rust \ && apk add --no-cache --virtual .mailman-rundeps bash sassc pcre-dev \ python3 py3-pip postgresql-client mysql-client py3-mysqlclient \ - curl mailcap xapian-core xapian-bindings-python3 libffi gettext \ + curl mailcap xapian-core xapian-bindings-python3 libffi gettext py-cryptography \ && python3 -m pip install -U pip setuptools wheel \ && python3 -m pip install -U \ - git+https://gitlab.com/mailman/mailmanclient@${CLIENT_REF} \ - git+https://gitlab.com/mailman/postorius@${POSTORIUS_REF} \ - git+https://gitlab.com/mailman/hyperkitty@${HYPERKITTY_REF} \ + git+https://gitlab.com/mailman/mailmanclient \ + git+https://gitlab.com/mailman/postorius \ + git+https://gitlab.com/mailman/hyperkitty \ whoosh \ uwsgi \ 'psycopg2<2.9' \ @@ -38,7 +38,7 @@ RUN --mount=type=cache,target=/root/.cache \ django-utils-six \ && python3 -m pip install -U 'Django<3.2' \ && python3 -m pip install -U \ - git+https://gitlab.com/mailman/django-mailman3@${DJ_MM3_REF} \ + git+https://gitlab.com/mailman/django-mailman3 \ && apk del .build-deps \ && addgroup -S mailman \ && adduser -S -G mailman mailman \ diff --git a/web/docker-entrypoint.sh b/web/docker-entrypoint.sh index 0f74f9f..4c86313 100755 --- a/web/docker-entrypoint.sh +++ b/web/docker-entrypoint.sh @@ -16,23 +16,14 @@ function wait_for_postgres () { function wait_for_mysql () { # Check if MySQL is up and accepting connections. - HOSTNAME=$(python3 <&2 echo "MySQL is unavailable - sleeping" sleep 1 done >&2 echo "MySQL is up - continuing" } - function check_or_create () { # Check if the path exists, if not, create the directory. if [[ ! -e dir ]]; then