From 3434446987bc96686a3a70e05a6686fe67166114 Mon Sep 17 00:00:00 2001 From: Abhilash Raj Date: Fri, 3 Nov 2017 18:43:59 -0700 Subject: [PATCH] Create rolling releases using the Gitlab API. (#171) * Create rolling releases using the Gitlab API. This commit builds rolling releases of Container images using the latest commit on master branch if the pipeline passed for it. The script which gets the references is still un-tested and should be tested. The latest commit hashes are passed as arguments to the Dockerfile, which is then used by PIP to install the specific version of the dependency. --- .travis.yml | 5 ++-- build.sh | 61 ++++++++++++++++++++++++++++------------- core/Dockerfile | 2 +- core/Dockerfile.dev | 16 +++++++---- get_latest_ref.py | 38 +++++++++++++++++++++++++ tests/generate_tests.sh | 13 +++++++-- tests/test.sh | 9 +++--- web/Dockerfile | 2 +- web/Dockerfile.dev | 26 +++++++++++------- 9 files changed, 127 insertions(+), 45 deletions(-) create mode 100755 get_latest_ref.py diff --git a/.travis.yml b/.travis.yml index 1058527..8f345b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,11 +8,12 @@ language: python env: - DB=postgres - DB=mysql + - DEV=true install: - - docker --version - - ./build.sh + - docker --version && python --version - sudo pip install docker-compose + - ./build.sh - docker-compose --version before_script: diff --git a/build.sh b/build.sh index f5e917a..f31a4da 100755 --- a/build.sh +++ b/build.sh @@ -1,24 +1,47 @@ #!/bin/bash -set -e - -# Use this script to build docker images. - -if [[ "$TRAVIS" ]] -then - if [ "$TRAVIS_BRANCH" = "master" ]; then - CORE_TAG="latest" - WEB_TAG="latest" - else - CORE_TAG="$TRAVIS_BRANCH" - WEB_TAG="$TRAVIS_BRANCH" - fi -else - CORE_TAG=`cat core/VERSION` - WEB_TAG=`cat web/VERSION` -fi +set -ex DOCKER=docker -$DOCKER build -t maxking/mailman-core:$CORE_TAG core/ -$DOCKER build -t maxking/mailman-web:$WEB_TAG web/ +# Set the env variable to later test this release before it is deployed. +if [ "$1" = "dev" ]; then + export DEV=true +fi + + +if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ "$DEV" = "true" ] ; then + python -m pip install python-gitlab + # Get the latest commit for repositories and set their reference values to be + # used in the development builds. + CORE_REF=$(python get_latest_ref.py mailman/mailman) + CLIENT_REF=$(python get_latest_ref.py mailman/mailmanclient) + POSTORIUS_REF=$(python get_latest_ref.py mailman/postorius) + HYPERKITTY_REF=$(python get_latest_ref.py mailman/hyperkitty) + DJ_MM3_REF=$(python get_latest_ref.py mailman/django-mailman3) + MM3_HK_REF=$(python get_latest_ref.py mailman/mailman-hyperkitty) + + # Build the mailman-core image. + $DOCKER build -f core/Dockerfile.dev \ + --build-arg CORE_REF=$CORE_REF \ + --build-arg MM3_HK_REF=$MM3_HK_REF \ + -t maxking/mailman-core:rolling core/ + + # Build the mailman-web image. + $DOCKER build -f web/Dockerfile.dev \ + --build-arg POSTORIUS_REF=$POSTORIUS_REF \ + --build-arg CLIENT_REF=$CLIENT_REF \ + --build-arg HYPERKITTY_REF=$HYPERKITTY_REF \ + --build-arg DJ_MM3_REF=$DJ_MM3_REF \ + -t maxking/mailman-web:rolling web/ +else + # Do the normal building process. + if [ "$TRAVIS_BRANCH" = "master" ]; then + TAG="latest" + else + TAG="$TRAVIS_BRANCH" + fi + + $DOCKER build -t maxking/mailman-core:$TAG core/ + $DOCKER build -t maxking/mailman-web:$TAG web/ +fi diff --git a/core/Dockerfile b/core/Dockerfile index 6e21e51..fcae376 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -8,7 +8,7 @@ COPY docker-entrypoint.sh /usr/local/bin/ #Install all required packages, add user for executing mailman and set execution rights for startup script RUN apk update \ && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \ - && apk add bash su-exec postgresql-client mysql-client \ + && apk add bash su-exec postgresql-client mysql-client curl \ && pip install psycopg2 \ mailman==3.1.0 \ mailman-hyperkitty==1.1.0 \ diff --git a/core/Dockerfile.dev b/core/Dockerfile.dev index 6bcbc02..66589c3 100644 --- a/core/Dockerfile.dev +++ b/core/Dockerfile.dev @@ -5,15 +5,19 @@ MAINTAINER Abhilash Raj #Add startup script to container COPY docker-entrypoint.sh /usr/local/bin/ +# Set the commits that we are building. +ARG CORE_REF +ARG MM3_HK_REF + + #Install all required packages, add user for executing mailman and set execution #rights for startup script RUN apk update \ && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev git \ - && apk add bash su-exec postgresql-client mysql-client \ - && pip install -U psycopg2 \ - git+https://gitlab.com/mailman/mailman.git \ - git+https://gitlab.com/mailman/mailman-hyperkitty.git \ - pymysql \ + && apk add bash su-exec postgresql-client mysql-client curl \ + && pip install -U psycopg2 pymysql \ + git+https://gitlab.com/mailman/mailman@${CORE_REF} \ + git+https://gitlab.com/mailman/mailman-hyperkitty@${MM3_HK_REF} \ && apk del build-deps \ && adduser -S mailman @@ -23,7 +27,9 @@ WORKDIR /opt/mailman #Expose the ports for the api (8001) and lmtp (8024) EXPOSE 8001 8024 +# Set the default configuration file. ENV MAILMAN_CONFIG_FILE /etc/mailman.cfg ENTRYPOINT ["docker-entrypoint.sh"] + CMD ["master"] diff --git a/get_latest_ref.py b/get_latest_ref.py new file mode 100755 index 0000000..4276e44 --- /dev/null +++ b/get_latest_ref.py @@ -0,0 +1,38 @@ +#! /usr/bin/env python +import os +import sys +import gitlab + + +def usage(): + print("{} ") + + +def main(): + if 2 > len(sys.argv) > 3: + usage() + + project_name = sys.argv[1] + if len(sys.argv) > 2: + branch_name = sys.argv[2] + else: + branch_name = 'master' + + gl_token = os.getenv('GITLAB_TOKEN') + if gl_token is None: + print('GITLAB_TOKEN not set!') + exit(1) + gl = gitlab.Gitlab('https://gitlab.com/', gl_token) + + project = gl.projects.get(project_name) + branch = project.branches.get(branch_name) + top_commit = project.commits.get(branch.commit['short_id']) + + if top_commit.last_pipeline['status'] == 'success': + print(top_commit.short_id) + else: + exit(1) + + +if __name__ == '__main__': + main() diff --git a/tests/generate_tests.sh b/tests/generate_tests.sh index 7412dd4..25284e6 100644 --- a/tests/generate_tests.sh +++ b/tests/generate_tests.sh @@ -1,9 +1,16 @@ #!/bin/sh -if [ "$TRAVIS_BRANCH" = "master" ]; then - TAG="latest" + +if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ ! -z $DEV ] ; then + echo "Travis event type is: $TRAVIS_EVENT_TYPE" + echo "This is a development version build: $DEV" + TAG="rolling" else - TAG="$TRAVIS_BRANCH" + if [ "$TRAVIS_BRANCH" = "master" ]; then + TAG="latest" + else + TAG="$TRAVIS_BRANCH" + fi fi diff --git a/tests/test.sh b/tests/test.sh index a752d3a..5ee8b58 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -1,7 +1,8 @@ #!/bin/bash set -e -if [ "$DB" = "postgres" ] +# If the DB environment variable is not set, use postgres.x +if [ "$DB" = "postgres" ] || [ -z $DB ] then docker-compose -f docker-compose.yaml -f docker-test.yaml up -d elif [ "$DB" = "mysql" ] @@ -26,10 +27,10 @@ docker logs mailman-core # Check to see if the core is working as expected. -curl -u restadmin:restpass http://172.19.199.2:8001/3.1/system +docker exec -it mailman-core curl -u restadmin:restpass http://172.19.199.2:8001/3.1/system # Check to see if postorius is working. -curl -L http://172.19.199.3:8000/postorius/lists | grep "Mailing List" +docker exec -it mailman-web curl -L http://172.19.199.3:8000/postorius/lists | grep "Mailing List" # Check to see if hyperkitty is working. -curl -L http://172.19.199.3:8000/hyperkitty/ | grep "Available lists" +docker exec -it mailman-web curl -L http://172.19.199.3:8000/hyperkitty/ | grep "Available lists" diff --git a/web/Dockerfile b/web/Dockerfile index 15aae65..3f6f382 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -14,7 +14,7 @@ RUN set -ex \ && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \ postgresql-dev mariadb-dev \ && apk add --no-cache --virtual .mailman-rundeps bash sassc \ - postgresql-client mysql-client py-mysqldb \ + postgresql-client mysql-client py-mysqldb curl \ && pip install -U mailmanclient==3.1.1 \ postorius==1.1.0 \ hyperkitty==1.1.4 \ diff --git a/web/Dockerfile.dev b/web/Dockerfile.dev index 26e1f22..7607bd6 100644 --- a/web/Dockerfile.dev +++ b/web/Dockerfile.dev @@ -7,6 +7,11 @@ COPY mailman-web /opt/mailman-web # Add startup script to container COPY docker-entrypoint.sh /usr/local/bin/ +ARG POSTORIUS_REF +ARG HYPERKITTY_REF +ARG DJ_MM3_REF +ARG CLIENT_REF + # Install packages and dependencies for postorius and hyperkitty Add user for # executing apps, change ownership for uwsgi+django files and set execution # rights for management script @@ -14,16 +19,16 @@ RUN set -ex \ && apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers git \ postgresql-dev mariadb-dev \ && apk add --no-cache --virtual .mailman-rundeps bash sassc \ - postgresql-client mysql-client py-mysqldb \ - && pip install -U git+https://gitlab.com/mailman/mailmanclient \ - git+https://gitlab.com/mailman/postorius \ - git+https://gitlab.com/mailman/hyperkitty \ - git+https://gitlab.com/mailman/django-mailman3 \ - whoosh \ - uwsgi \ - psycopg2 \ - dj-database-url \ - mysqlclient \ + postgresql-client mysql-client py-mysqldb curl \ + && pip install -U git+https://gitlab.com/mailman/mailmanclient@${CLIENT_REF} \ + git+https://gitlab.com/mailman/postorius@${POSTORIUS_REF} \ + git+https://gitlab.com/mailman/hyperkitty@${HYPERKITTY_REF} \ + git+https://gitlab.com/mailman/django-mailman3@${DJ_MM3_REF} \ + whoosh \ + uwsgi \ + psycopg2 \ + dj-database-url \ + mysqlclient \ && pip install -U django==1.10 \ && apk del .build-deps \ && addgroup -S mailman \ @@ -40,4 +45,5 @@ EXPOSE 8000 8080 STOPSIGNAL SIGINT ENTRYPOINT ["docker-entrypoint.sh"] + CMD ["uwsgi", "--ini", "/opt/mailman-web/uwsgi.ini"]