Make several changes in the release and CI setup. (#509)

* Make several changes in the release and CI setup.

- Remove .travis/ directory completely.
- Update deploy.py script to tag with major.minor version
  and remove the v in the tag which it didn't before.
- Fix the CircleCI cron setup.

* Update config.yml

* Use buildkit if possible.

* Use machine executor which supports buildkit.

* Remove setup docker step

* Use newer image.

* Use buildkit syntax for caching pip dependencies.

This should significantly speed up the build process by not having
to build python wheels multiple times.

* Use plain output from buildkit

* Make deploy script work with Python 2.7

Also cache python deps across builds.

* Fix more python3 things

* Install dependency in python3

.circleci/config.yml

@@ -2,23 +2,30 @@ version: '2'
 
 jobs:
   build:
-    docker:
+    machine:
-      - image: circleci/python:3.6-stretch-browsers
+      image: ubuntu-2004:202010-01
+      docker_layer_caching: true
     environment:
-      REGISTRY: QUAY
+      DOCKER_BUILDKIT: 1
-      QUAY_URL: quay.io
+      BUILDKIT_PROGRESS: plain
     steps:
       - checkout
-      - setup_remote_docker
       - run:
           name: Setup Environment
           command: source setup_env.sh
       - run:
-          name: Install Docker Compose
+          name: Install Python dependencies
-          command: sudo pip install docker-compose
+          command: python3 -m pip install packaging
+      - restore_cache:
+          keys:
+            - python-deps-cache-v1
       - run:
           name: Building Container Images
           command: ./build.sh
+      - save_cache:
+          key: python-deps-cache-v1
+          paths:
+            - /root/.cache
       - run:
           name: Generate Tests
           command: bash tests/generate_tests.sh
@@ -34,7 +41,7 @@ jobs:
           command: bash tests/test.sh
       - deploy:
           command: |
-              python .travis/deploy.py
+              python3 deploy.py
 
 workflows:
   version: 2
@@ -51,7 +58,7 @@ workflows:
           cron: "0 0 * * *"
           filters:
             branches:
-              only: master
+              only: main
     jobs:
       - build:
           context: org-global

.travis/deploy.sh

@@ -1,28 +0,0 @@
-#! /bin/bash
-
-set -e
-
-USER=maxking
-
-deploy() {
-	URL="$1"
-	TAG="$2"
-	PASSWORD="$3"
-	# Tag the containers correctly.
-	docker tag maxking/mailman-core:rolling "$URL/maxking/mailman-core:$TAG"
-	docker tag maxking/mailman-web:rolling  "$URL/maxking/mailman-web:$TAG"
-	docker tag maxking/postorius:rolling "$URL/maxking/postorius:$TAG"
-	# Login to the registry and push.
-	docker login -u $USER -p $PASSWORD $URL
-	docker push "$URL/maxking/mailman-web:$TAG"
-	docker push "$URL/maxking/mailman-core:$TAG"
-	docker push "$URL/maxking/postorius:$TAG"
-}
-
-if [ ! "$BRANCH" = "master" ] && [ "$PULL_REQUEST" ]; then
-	echo "Deploy only from master branch. This is $TRAVIS_BRANCH"
-	exit 0
-fi
-
-deploy "quay.io" "rolling" "$DOCKER_PASSWORD"
-deploy "docker.io" "rolling" "$QUAY_PASSWORD"

.travis/deploy_dockerhub.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-deploy () {
-		docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS
-		docker push maxking/mailman-web:$1
-		docker push maxking/mailman-core:$1
-}
-
-
-if [ "$EVENT_TYPE" = "cron" ]; then
-		deploy "rolling"
-elif [ "$BRANCH" = "master" ]; then
-		deploy "latest"
-else
-		# If the branch isn't master and this was not a cron job, no need to deploy.
-		echo "EVENT_TYPE = $EVENT_TYPE, BRANCH = $BRANCH"
-		exit 0
-fi

build.py

@@ -1,79 +0,0 @@
-#/usr/bin/env python3
-
-# This is the build script to build container images for Mailman.
-import sys
-import subprocess
-from pathlib import Path
-
-
-STABLE_DOCKERFILES = {
-    'core-stable': Path('core/Dockerfile'),
-    'web-stable': Path('web/Dockerfile'),
-    'postorius-stable': Path('postorius/Dockerfile'),
-}
-
-ROLLING_DOCKERFILES = {
-    'core-rolling': Path('core/Dockerfile.dev'),
-    'web-rolling': Path('web/Dockerfile.dev'),
-    'postorius-rolling': Path('postorius/Dockerfile.dev'),
-}
-
-VARIANTS = {
-    'stable': STABLE_DOCKERFILES,
-    'rolling': ROLLING_DOCKERFILES,
-}
-
-def run_command(args):
-    print(' '.join(args))
-    subprocess.run(
-        args,
-        stdout=sys.stdout,
-        stderr=sys.stderr,
-        check=True)
-
-
-def docker_build(dockerfile, tag, args=None, labels=None):
-    cmd = [
-        'docker', 'build',
-        '-t', tag,
-        '-f', str(dockerfile),
-        str(dockerfile.parent)
-    ]
-
-    if args:
-        for arg in args:
-            cmd.append('--build-arg')
-            cmd.append(arg)
-
-    if labels:
-        for label in labels:
-            cmd.append('--label')
-            cmd.append(label)
-
-    return run_command(cmd)
-
-
-def docker_tag(from_tag, to_tag):
-    cmd = [
-        'docker', 'tag',
-        from_tag, to_tag,
-    ]
-
-    return run_command(cmd)
-
-def usage():
-    print('usage: python build.py (stable|rolling)')
-
-
-if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        usage()
-        sys.exit(1)
-
-    variant = sys.argv[1]
-    if variant not in VARIANTS:
-        usage()
-        sys.exit(1)
-
-    for name, path in VARIANTS[variant].items():
-        docker_build(dockerfile=path, tag=name)

core/Dockerfile

@@ -1,12 +1,12 @@
+# syntax = docker/dockerfile:1.3
 FROM alpine:3.12
 
-MAINTAINER Abhilash Raj
-
 #Add startup script to container
 COPY docker-entrypoint.sh /usr/local/bin/
 
 #Install all required packages, add user for executing mailman and set execution rights for startup script
-RUN apk update \
+RUN --mount=type=cache,target=/root/.cache \
+    apk update \
     && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \
        libffi-dev \
   # psutil needs linux-headers to compile on musl c library.

core/Dockerfile.dev

@@ -1,7 +1,6 @@
+# syntax = docker/dockerfile:1.3
 FROM alpine:3.12
 
-MAINTAINER Abhilash Raj
-
 #Add startup script to container
 COPY docker-entrypoint.sh /usr/local/bin/
 
@@ -12,7 +11,8 @@ ARG MM3_HK_REF
 
 #Install all required packages, add user for executing mailman and set execution
 #rights for startup script
-RUN apk update \
+RUN --mount=type=cache,target=/root/.cache \
+    apk update \
 	&& apk add --no-cache --virtual build-deps gcc python3-dev musl-dev \
    	    postgresql-dev git libffi-dev  \
     && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers \

deploy.py

@@ -26,6 +26,7 @@
 
 import os
 import subprocess
+from packaging import version
 
 #: Default user, which owns the repositories.
 USER = 'maxking'
@@ -81,35 +82,63 @@ def tag_and_push(image_names, url, img_tag):
     push(final)
 
 
+def get_tag_without_patch(tag):
+    """Given A.B.C return A.B"""
+    v = version.parse(tag)
+    return '{}.{}'.format(v.major, v.minor)
+
+
+def get_urls(url, img_tag):
+    core = ('maxking/mailman-core:rolling',
+            '{0}/maxking/mailman-core:{1}'.format(url, img_tag))
+    web = ('maxking/mailman-web:rolling',
+        '{0}/maxking/mailman-web:{1}'.format(url, img_tag))
+    postorius = ('maxking/postorius:rolling',
+        '{0}/maxking/postorius:{1}'.format(url, img_tag))
+
+    return (core, web, postorius)
+                     
+
+
 def main():
     """Primary entrypoint to this script."""
-    
+    # Boolean signifying if this is a stable release tag or just a branch.
+    is_release = False
+
     if os.environ.get(TAG_VAR) not in (None, ''):
         img_tag = os.environ.get(TAG_VAR)
+        # Released versions are tagged vA.B.C, so remove
+        # v from the tag when creating the release.
+        if img_tag.startswith('v'):
+            img_tag = img_tag[1:]
+            is_release = True
+
     elif os.environ.get(BRANCH_VAR) == PRIMARY_BRANCH:
         img_tag = 'rolling'
     else:
-        print(f'Not running on {PRIMARY_BRANCH} branch or Git tag so not publishing...')
+        print('Not running on {PRIMARY_BRANCH} branch or Git tag so not publishing...'.format(
+            PRIMARY_BRANCH=PRIMARY_BRANCH))
         exit(0)
 
+    # All the registries we are pushing to.
     for url in ('quay.io', 'docker.io', 'ghcr.io'):
 
-        core = ('maxking/mailman-core:rolling',
-                '{0}/maxking/mailman-core:{1}'.format(url, img_tag))
-        web = ('maxking/mailman-web:rolling',
-               '{0}/maxking/mailman-web:{1}'.format(url, img_tag))
-        postorius = ('maxking/postorius:rolling',
-                     '{0}/maxking/postorius:{1}'.format(url, img_tag))
-
         try:
             login(url)
         except subprocess.CalledProcessError:
             print('Failed to login to {}'.format(url))
             continue
+        
+        # Push all the container images.
+        for each in get_urls(url, img_tag):
+            tag_and_push(each, url, img_tag)
+
+        # If this is a release tag, tag them also with a.b version.
+        if is_release:
+            rel_tag = get_tag_without_patch(img_tag)
+            for each in get_urls(url, rel_tag):
+                tag_and_push(each, url, rel_tag)
 
-        tag_and_push(core, url, img_tag)
-        tag_and_push(web, url, img_tag)
-        tag_and_push(postorius, url, img_tag)
 
 
 if __name__ == '__main__':

postorius/Dockerfile

@@ -1,7 +1,6 @@
+# syntax = docker/dockerfile:1.3
 FROM alpine:3.12
 
-MAINTAINER Abhilash Raj
-
 # Add needed files for uwsgi server + settings for django
 COPY mailman-web /opt/mailman-web
 # Add startup script to container
@@ -10,7 +9,8 @@ COPY docker-entrypoint.sh /usr/local/bin/
 # Install packages and dependencies for postorius and hyperkitty Add user for
 # executing apps, change ownership for uwsgi+django files and set execution
 # rights for management script
-RUN set -ex \
+RUN --mount=type=cache,target=/root/.cache \
+    set -ex \
 	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \
 		postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \
 	&& apk add --no-cache --virtual .mailman-rundeps bash sassc \

postorius/Dockerfile.dev

@@ -1,7 +1,6 @@
+# syntax = docker/dockerfile:1.3
 FROM alpine:3.12
 
-MAINTAINER Abhilash Raj
-
 # Add needed files for uwsgi server + settings for django
 COPY mailman-web /opt/mailman-web
 # Add startup script to container
@@ -14,7 +13,8 @@ ARG CLIENT_REF
 # Install packages and dependencies for postorius and hyperkitty Add user for
 # executing apps, change ownership for uwsgi+django files and set execution
 # rights for management script
-RUN set -ex \
+RUN --mount=type=cache,target=/root/.cache \
+    set -ex \
 	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \
 		postgresql-dev mariadb-dev python3-dev libffi-dev git cargo rust \
 	&& apk add --no-cache --virtual .mailman-rundeps bash sassc \

web/Dockerfile

@@ -1,7 +1,6 @@
+# syntax = docker/dockerfile:1.3
 FROM alpine:3.12
 
-MAINTAINER Abhilash Raj
-
 # Add needed files for uwsgi server + settings for django
 COPY mailman-web /opt/mailman-web
 # Add startup script to container
@@ -10,7 +9,8 @@ COPY docker-entrypoint.sh /usr/local/bin/
 # Install packages and dependencies for postorius and hyperkitty Add user for
 # executing apps, change ownership for uwsgi+django files and set execution
 # rights for management script
-RUN set -ex \
+RUN --mount=type=cache,target=/root/.cache \
+    set -ex \
 	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \
            postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \
 	&& apk add --no-cache --virtual .mailman-rundeps bash sassc \

web/Dockerfile.dev

@@ -1,7 +1,6 @@
+# syntax = docker/dockerfile:1.3
 FROM alpine:3.12
 
-MAINTAINER Abhilash Raj
-
 # Add needed files for uwsgi server + settings for django
 COPY mailman-web /opt/mailman-web
 # Add startup script to container
@@ -15,7 +14,8 @@ ARG CLIENT_REF
 # Install packages and dependencies for postorius and hyperkitty Add user for
 # executing apps, change ownership for uwsgi+django files and set execution
 # rights for management script
-RUN set -ex \
+RUN --mount=type=cache,target=/root/.cache \
+    set -ex \
     && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers git \
         postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \
     && apk add --no-cache --virtual .mailman-rundeps bash sassc pcre-dev \