From fd837d04f194f8e4988cc2c075430c4ed0a000cc Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Fri, 17 Sep 2021 11:10:27 -0700 Subject: [PATCH] Make several changes in the release and CI setup. (#509) * Make several changes in the release and CI setup. - Remove .travis/ directory completely. - Update deploy.py script to tag with major.minor version and remove the v in the tag which it didn't before. - Fix the CircleCI cron setup. * Update config.yml * Use buildkit if possible. * Use machine executor which supports buildkit. * Remove setup docker step * Use newer image. * Use buildkit syntax for caching pip dependencies. This should significantly speed up the build process by not having to build python wheels multiple times. * Use plain output from buildkit * Make deploy script work with Python 2.7 Also cache python deps across builds. * Fix more python3 things * Install dependency in python3 --- .circleci/config.yml | 25 +++++++---- .travis/deploy.sh | 28 ------------ .travis/deploy_dockerhub.sh | 18 -------- build.py | 79 ---------------------------------- core/Dockerfile | 6 +-- core/Dockerfile.dev | 6 +-- .travis/deploy.py => deploy.py | 53 +++++++++++++++++------ postorius/Dockerfile | 6 +-- postorius/Dockerfile.dev | 6 +-- web/Dockerfile | 6 +-- web/Dockerfile.dev | 6 +-- 11 files changed, 75 insertions(+), 164 deletions(-) delete mode 100644 .travis/deploy.sh delete mode 100644 .travis/deploy_dockerhub.sh delete mode 100755 build.py rename .travis/deploy.py => deploy.py (68%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13f32ba..13e8269 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,23 +2,30 @@ version: '2' jobs: build: - docker: - - image: circleci/python:3.6-stretch-browsers + machine: + image: ubuntu-2004:202010-01 + docker_layer_caching: true environment: - REGISTRY: QUAY - QUAY_URL: quay.io + DOCKER_BUILDKIT: 1 + BUILDKIT_PROGRESS: plain steps: - checkout - - setup_remote_docker - run: name: Setup Environment command: source setup_env.sh - run: - name: Install Docker Compose - command: sudo pip install docker-compose + name: Install Python dependencies + command: python3 -m pip install packaging + - restore_cache: + keys: + - python-deps-cache-v1 - run: name: Building Container Images command: ./build.sh + - save_cache: + key: python-deps-cache-v1 + paths: + - /root/.cache - run: name: Generate Tests command: bash tests/generate_tests.sh @@ -34,7 +41,7 @@ jobs: command: bash tests/test.sh - deploy: command: | - python .travis/deploy.py + python3 deploy.py workflows: version: 2 @@ -51,7 +58,7 @@ workflows: cron: "0 0 * * *" filters: branches: - only: master + only: main jobs: - build: context: org-global diff --git a/.travis/deploy.sh b/.travis/deploy.sh deleted file mode 100644 index 54e0600..0000000 --- a/.travis/deploy.sh +++ /dev/null @@ -1,28 +0,0 @@ -#! /bin/bash - -set -e - -USER=maxking - -deploy() { - URL="$1" - TAG="$2" - PASSWORD="$3" - # Tag the containers correctly. - docker tag maxking/mailman-core:rolling "$URL/maxking/mailman-core:$TAG" - docker tag maxking/mailman-web:rolling "$URL/maxking/mailman-web:$TAG" - docker tag maxking/postorius:rolling "$URL/maxking/postorius:$TAG" - # Login to the registry and push. - docker login -u $USER -p $PASSWORD $URL - docker push "$URL/maxking/mailman-web:$TAG" - docker push "$URL/maxking/mailman-core:$TAG" - docker push "$URL/maxking/postorius:$TAG" -} - -if [ ! "$BRANCH" = "master" ] && [ "$PULL_REQUEST" ]; then - echo "Deploy only from master branch. This is $TRAVIS_BRANCH" - exit 0 -fi - -deploy "quay.io" "rolling" "$DOCKER_PASSWORD" -deploy "docker.io" "rolling" "$QUAY_PASSWORD" diff --git a/.travis/deploy_dockerhub.sh b/.travis/deploy_dockerhub.sh deleted file mode 100644 index 9c73f09..0000000 --- a/.travis/deploy_dockerhub.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -deploy () { - docker login -e $DOCKER_EMAIL -u $DOCKER_USER -p $DOCKER_PASS - docker push maxking/mailman-web:$1 - docker push maxking/mailman-core:$1 -} - - -if [ "$EVENT_TYPE" = "cron" ]; then - deploy "rolling" -elif [ "$BRANCH" = "master" ]; then - deploy "latest" -else - # If the branch isn't master and this was not a cron job, no need to deploy. - echo "EVENT_TYPE = $EVENT_TYPE, BRANCH = $BRANCH" - exit 0 -fi diff --git a/build.py b/build.py deleted file mode 100755 index 2b95c61..0000000 --- a/build.py +++ /dev/null @@ -1,79 +0,0 @@ -#/usr/bin/env python3 - -# This is the build script to build container images for Mailman. -import sys -import subprocess -from pathlib import Path - - -STABLE_DOCKERFILES = { - 'core-stable': Path('core/Dockerfile'), - 'web-stable': Path('web/Dockerfile'), - 'postorius-stable': Path('postorius/Dockerfile'), -} - -ROLLING_DOCKERFILES = { - 'core-rolling': Path('core/Dockerfile.dev'), - 'web-rolling': Path('web/Dockerfile.dev'), - 'postorius-rolling': Path('postorius/Dockerfile.dev'), -} - -VARIANTS = { - 'stable': STABLE_DOCKERFILES, - 'rolling': ROLLING_DOCKERFILES, -} - -def run_command(args): - print(' '.join(args)) - subprocess.run( - args, - stdout=sys.stdout, - stderr=sys.stderr, - check=True) - - -def docker_build(dockerfile, tag, args=None, labels=None): - cmd = [ - 'docker', 'build', - '-t', tag, - '-f', str(dockerfile), - str(dockerfile.parent) - ] - - if args: - for arg in args: - cmd.append('--build-arg') - cmd.append(arg) - - if labels: - for label in labels: - cmd.append('--label') - cmd.append(label) - - return run_command(cmd) - - -def docker_tag(from_tag, to_tag): - cmd = [ - 'docker', 'tag', - from_tag, to_tag, - ] - - return run_command(cmd) - -def usage(): - print('usage: python build.py (stable|rolling)') - - -if __name__ == '__main__': - if len(sys.argv) < 2: - usage() - sys.exit(1) - - variant = sys.argv[1] - if variant not in VARIANTS: - usage() - sys.exit(1) - - for name, path in VARIANTS[variant].items(): - docker_build(dockerfile=path, tag=name) diff --git a/core/Dockerfile b/core/Dockerfile index 9579f28..a583a7f 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -1,12 +1,12 @@ +# syntax = docker/dockerfile:1.3 FROM alpine:3.12 -MAINTAINER Abhilash Raj - #Add startup script to container COPY docker-entrypoint.sh /usr/local/bin/ #Install all required packages, add user for executing mailman and set execution rights for startup script -RUN apk update \ +RUN --mount=type=cache,target=/root/.cache \ + apk update \ && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \ libffi-dev \ # psutil needs linux-headers to compile on musl c library. diff --git a/core/Dockerfile.dev b/core/Dockerfile.dev index 8ea5cc3..20a391f 100644 --- a/core/Dockerfile.dev +++ b/core/Dockerfile.dev @@ -1,7 +1,6 @@ +# syntax = docker/dockerfile:1.3 FROM alpine:3.12 -MAINTAINER Abhilash Raj - #Add startup script to container COPY docker-entrypoint.sh /usr/local/bin/ @@ -12,7 +11,8 @@ ARG MM3_HK_REF #Install all required packages, add user for executing mailman and set execution #rights for startup script -RUN apk update \ +RUN --mount=type=cache,target=/root/.cache \ + apk update \ && apk add --no-cache --virtual build-deps gcc python3-dev musl-dev \ postgresql-dev git libffi-dev \ && apk add --no-cache bash su-exec postgresql-client mysql-client curl python3 py3-pip linux-headers \ diff --git a/.travis/deploy.py b/deploy.py similarity index 68% rename from .travis/deploy.py rename to deploy.py index 7670131..069f282 100644 --- a/.travis/deploy.py +++ b/deploy.py @@ -26,6 +26,7 @@ import os import subprocess +from packaging import version #: Default user, which owns the repositories. USER = 'maxking' @@ -81,35 +82,63 @@ def tag_and_push(image_names, url, img_tag): push(final) +def get_tag_without_patch(tag): + """Given A.B.C return A.B""" + v = version.parse(tag) + return '{}.{}'.format(v.major, v.minor) + + +def get_urls(url, img_tag): + core = ('maxking/mailman-core:rolling', + '{0}/maxking/mailman-core:{1}'.format(url, img_tag)) + web = ('maxking/mailman-web:rolling', + '{0}/maxking/mailman-web:{1}'.format(url, img_tag)) + postorius = ('maxking/postorius:rolling', + '{0}/maxking/postorius:{1}'.format(url, img_tag)) + + return (core, web, postorius) + + + def main(): """Primary entrypoint to this script.""" - + # Boolean signifying if this is a stable release tag or just a branch. + is_release = False + if os.environ.get(TAG_VAR) not in (None, ''): img_tag = os.environ.get(TAG_VAR) + # Released versions are tagged vA.B.C, so remove + # v from the tag when creating the release. + if img_tag.startswith('v'): + img_tag = img_tag[1:] + is_release = True + elif os.environ.get(BRANCH_VAR) == PRIMARY_BRANCH: img_tag = 'rolling' else: - print(f'Not running on {PRIMARY_BRANCH} branch or Git tag so not publishing...') + print('Not running on {PRIMARY_BRANCH} branch or Git tag so not publishing...'.format( + PRIMARY_BRANCH=PRIMARY_BRANCH)) exit(0) + # All the registries we are pushing to. for url in ('quay.io', 'docker.io', 'ghcr.io'): - core = ('maxking/mailman-core:rolling', - '{0}/maxking/mailman-core:{1}'.format(url, img_tag)) - web = ('maxking/mailman-web:rolling', - '{0}/maxking/mailman-web:{1}'.format(url, img_tag)) - postorius = ('maxking/postorius:rolling', - '{0}/maxking/postorius:{1}'.format(url, img_tag)) - try: login(url) except subprocess.CalledProcessError: print('Failed to login to {}'.format(url)) continue + + # Push all the container images. + for each in get_urls(url, img_tag): + tag_and_push(each, url, img_tag) + + # If this is a release tag, tag them also with a.b version. + if is_release: + rel_tag = get_tag_without_patch(img_tag) + for each in get_urls(url, rel_tag): + tag_and_push(each, url, rel_tag) - tag_and_push(core, url, img_tag) - tag_and_push(web, url, img_tag) - tag_and_push(postorius, url, img_tag) if __name__ == '__main__': diff --git a/postorius/Dockerfile b/postorius/Dockerfile index e674295..8a77cd5 100644 --- a/postorius/Dockerfile +++ b/postorius/Dockerfile @@ -1,7 +1,6 @@ +# syntax = docker/dockerfile:1.3 FROM alpine:3.12 -MAINTAINER Abhilash Raj - # Add needed files for uwsgi server + settings for django COPY mailman-web /opt/mailman-web # Add startup script to container @@ -10,7 +9,8 @@ COPY docker-entrypoint.sh /usr/local/bin/ # Install packages and dependencies for postorius and hyperkitty Add user for # executing apps, change ownership for uwsgi+django files and set execution # rights for management script -RUN set -ex \ +RUN --mount=type=cache,target=/root/.cache \ + set -ex \ && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \ postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \ && apk add --no-cache --virtual .mailman-rundeps bash sassc \ diff --git a/postorius/Dockerfile.dev b/postorius/Dockerfile.dev index c1f9913..f1560cb 100644 --- a/postorius/Dockerfile.dev +++ b/postorius/Dockerfile.dev @@ -1,7 +1,6 @@ +# syntax = docker/dockerfile:1.3 FROM alpine:3.12 -MAINTAINER Abhilash Raj - # Add needed files for uwsgi server + settings for django COPY mailman-web /opt/mailman-web # Add startup script to container @@ -14,7 +13,8 @@ ARG CLIENT_REF # Install packages and dependencies for postorius and hyperkitty Add user for # executing apps, change ownership for uwsgi+django files and set execution # rights for management script -RUN set -ex \ +RUN --mount=type=cache,target=/root/.cache \ + set -ex \ && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \ postgresql-dev mariadb-dev python3-dev libffi-dev git cargo rust \ && apk add --no-cache --virtual .mailman-rundeps bash sassc \ diff --git a/web/Dockerfile b/web/Dockerfile index 882103d..7f236bc 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -1,7 +1,6 @@ +# syntax = docker/dockerfile:1.3 FROM alpine:3.12 -MAINTAINER Abhilash Raj - # Add needed files for uwsgi server + settings for django COPY mailman-web /opt/mailman-web # Add startup script to container @@ -10,7 +9,8 @@ COPY docker-entrypoint.sh /usr/local/bin/ # Install packages and dependencies for postorius and hyperkitty Add user for # executing apps, change ownership for uwsgi+django files and set execution # rights for management script -RUN set -ex \ +RUN --mount=type=cache,target=/root/.cache \ + set -ex \ && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \ postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \ && apk add --no-cache --virtual .mailman-rundeps bash sassc \ diff --git a/web/Dockerfile.dev b/web/Dockerfile.dev index 1771c7f..2c5951c 100644 --- a/web/Dockerfile.dev +++ b/web/Dockerfile.dev @@ -1,7 +1,6 @@ +# syntax = docker/dockerfile:1.3 FROM alpine:3.12 -MAINTAINER Abhilash Raj - # Add needed files for uwsgi server + settings for django COPY mailman-web /opt/mailman-web # Add startup script to container @@ -15,7 +14,8 @@ ARG CLIENT_REF # Install packages and dependencies for postorius and hyperkitty Add user for # executing apps, change ownership for uwsgi+django files and set execution # rights for management script -RUN set -ex \ +RUN --mount=type=cache,target=/root/.cache \ + set -ex \ && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers git \ postgresql-dev mariadb-dev python3-dev libffi-dev openldap-dev cargo rust \ && apk add --no-cache --virtual .mailman-rundeps bash sassc pcre-dev \