Create rolling releases using the Gitlab API. (#171)

* Create rolling releases using the Gitlab API. This commit builds rolling releases of Container images using the latest commit on master branch if the pipeline passed for it. The script which gets the references is still un-tested and should be tested. The latest commit hashes are passed as arguments to the Dockerfile, which is then used by PIP to install the specific version of the dependency.
2017-11-03 18:43:59 -07:00
parent 743bc8522c
commit 3434446987
9 changed files with 127 additions and 45 deletions
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,11 +8,12 @@ language: python
 env:
  - DB=postgres
  - DB=mysql
+  - DEV=true

 install:
-  - docker --version
-  - ./build.sh
+  - docker --version && python --version
  - sudo pip install docker-compose
+  - ./build.sh
  - docker-compose --version

 before_script:
--- a/build.sh
+++ b/build.sh
@@ -1,24 +1,47 @@
 #!/bin/bash

-set -e
-
-# Use this script to build docker images.
-
-if [[ "$TRAVIS" ]]
-then
-	if [ "$TRAVIS_BRANCH" = "master" ]; then
-		CORE_TAG="latest"
-		WEB_TAG="latest"
-	else
-		CORE_TAG="$TRAVIS_BRANCH"
-		WEB_TAG="$TRAVIS_BRANCH"
-	fi
-else
-    CORE_TAG=`cat core/VERSION`
-    WEB_TAG=`cat web/VERSION`
-fi
+set -ex

 DOCKER=docker

-$DOCKER build -t maxking/mailman-core:$CORE_TAG core/
-$DOCKER build -t maxking/mailman-web:$WEB_TAG web/
+# Set the env variable to later test this release before it is deployed.
+if [ "$1" = "dev" ]; then
+		export DEV=true
+fi
+
+
+if [ "$TRAVIS_EVENT_TYPE" = "cron" ]  || [ "$DEV" = "true" ] ; then
+		python -m pip install python-gitlab
+		# Get the latest commit for repositories and set their reference values to be
+		# used in the development builds.
+		CORE_REF=$(python get_latest_ref.py mailman/mailman)
+		CLIENT_REF=$(python get_latest_ref.py mailman/mailmanclient)
+		POSTORIUS_REF=$(python get_latest_ref.py mailman/postorius)
+		HYPERKITTY_REF=$(python get_latest_ref.py mailman/hyperkitty)
+		DJ_MM3_REF=$(python get_latest_ref.py mailman/django-mailman3)
+		MM3_HK_REF=$(python get_latest_ref.py mailman/mailman-hyperkitty)
+
+		# Build the mailman-core image.
+		$DOCKER build -f core/Dockerfile.dev \
+						--build-arg CORE_REF=$CORE_REF \
+						--build-arg MM3_HK_REF=$MM3_HK_REF \
+						-t maxking/mailman-core:rolling core/
+
+		# Build the mailman-web image.
+		$DOCKER build -f web/Dockerfile.dev \
+						--build-arg POSTORIUS_REF=$POSTORIUS_REF \
+						--build-arg CLIENT_REF=$CLIENT_REF \
+						--build-arg HYPERKITTY_REF=$HYPERKITTY_REF \
+						--build-arg DJ_MM3_REF=$DJ_MM3_REF \
+						-t maxking/mailman-web:rolling web/
+else
+		# Do the normal building process.
+		if [ "$TRAVIS_BRANCH" = "master" ]; then
+				TAG="latest"
+		else
+				TAG="$TRAVIS_BRANCH"
+		fi
+
+		$DOCKER build -t maxking/mailman-core:$TAG core/
+		$DOCKER build -t maxking/mailman-web:$TAG web/
+fi
--- a/core/Dockerfile
+++ b/core/Dockerfile
@@ -8,7 +8,7 @@ COPY docker-entrypoint.sh /usr/local/bin/
 #Install all required packages, add user for executing mailman and set execution rights for startup script
 RUN apk update \
    && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \
-    && apk add bash su-exec postgresql-client mysql-client \
+    && apk add bash su-exec postgresql-client mysql-client curl \
    && pip install psycopg2 \
                   mailman==3.1.0 \
                   mailman-hyperkitty==1.1.0 \
--- a/core/Dockerfile.dev
+++ b/core/Dockerfile.dev
@@ -5,15 +5,19 @@ MAINTAINER Abhilash Raj
 #Add startup script to container
 COPY docker-entrypoint.sh /usr/local/bin/

+# Set the commits that we are building.
+ARG CORE_REF
+ARG MM3_HK_REF
+
+
 #Install all required packages, add user for executing mailman and set execution
 #rights for startup script
 RUN apk update \
    && apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev git \
-    && apk add bash su-exec postgresql-client mysql-client \
-    && pip install -U psycopg2 \
-                      git+https://gitlab.com/mailman/mailman.git \
-                      git+https://gitlab.com/mailman/mailman-hyperkitty.git \
-                      pymysql \
+    && apk add bash su-exec postgresql-client mysql-client curl \
+    && pip install -U psycopg2 pymysql \
+                      git+https://gitlab.com/mailman/mailman@${CORE_REF} \
+											git+https://gitlab.com/mailman/mailman-hyperkitty@${MM3_HK_REF} \
    && apk del build-deps \
    && adduser -S mailman

@@ -23,7 +27,9 @@ WORKDIR /opt/mailman
 #Expose the ports for the api (8001) and lmtp (8024)
 EXPOSE 8001 8024

+# Set the default configuration file.
 ENV MAILMAN_CONFIG_FILE /etc/mailman.cfg

 ENTRYPOINT ["docker-entrypoint.sh"]
+
 CMD ["master"]
--- a/get_latest_ref.py
+++ b/get_latest_ref.py
@@ -0,0 +1,38 @@
+#! /usr/bin/env python
+import os
+import sys
+import gitlab
+
+
+def usage():
+    print("{} <project_name> <project_branch>")
+
+
+def main():
+    if 2 > len(sys.argv) > 3:
+        usage()
+
+    project_name = sys.argv[1]
+    if len(sys.argv) > 2:
+        branch_name = sys.argv[2]
+    else:
+        branch_name = 'master'
+
+    gl_token = os.getenv('GITLAB_TOKEN')
+    if gl_token is None:
+        print('GITLAB_TOKEN not set!')
+        exit(1)
+    gl = gitlab.Gitlab('https://gitlab.com/', gl_token)
+
+    project = gl.projects.get(project_name)
+    branch = project.branches.get(branch_name)
+    top_commit = project.commits.get(branch.commit['short_id'])
+
+    if top_commit.last_pipeline['status'] == 'success':
+        print(top_commit.short_id)
+    else:
+        exit(1)
+
+
+if __name__ == '__main__':
+    main()
--- a/tests/generate_tests.sh
+++ b/tests/generate_tests.sh
@@ -1,9 +1,16 @@
 #!/bin/sh

-if [ "$TRAVIS_BRANCH" = "master" ]; then
-    TAG="latest"
+
+if [ "$TRAVIS_EVENT_TYPE" = "cron" ] || [ ! -z $DEV ] ; then
+		echo "Travis event type is: $TRAVIS_EVENT_TYPE"
+		echo "This is a development version build: $DEV"
+		TAG="rolling"
 else
-    TAG="$TRAVIS_BRANCH"
+		if [ "$TRAVIS_BRANCH" = "master" ]; then
+				TAG="latest"
+		else
+				TAG="$TRAVIS_BRANCH"
+		fi
 fi


--- a/tests/test.sh
+++ b/tests/test.sh
@@ -1,7 +1,8 @@
 #!/bin/bash
 set -e

-if [ "$DB" = "postgres" ]
+# If the DB environment variable is not set, use postgres.x
+if [ "$DB" = "postgres" ] || [ -z $DB ]
 then
 	docker-compose -f docker-compose.yaml -f docker-test.yaml up -d
 elif [ "$DB" = "mysql" ]
@@ -26,10 +27,10 @@ docker logs mailman-core


 # Check to see if the core is working as expected.
-curl -u restadmin:restpass http://172.19.199.2:8001/3.1/system
+docker exec -it mailman-core curl -u restadmin:restpass http://172.19.199.2:8001/3.1/system

 # Check to see if postorius is working.
-curl -L http://172.19.199.3:8000/postorius/lists | grep "Mailing List"
+docker exec -it mailman-web curl -L http://172.19.199.3:8000/postorius/lists | grep "Mailing List"

 # Check to see if hyperkitty is working.
-curl -L http://172.19.199.3:8000/hyperkitty/ | grep "Available lists"
+docker exec -it mailman-web  curl -L http://172.19.199.3:8000/hyperkitty/ | grep "Available lists"
--- a/web/Dockerfile
+++ b/web/Dockerfile
@@ -14,7 +14,7 @@ RUN set -ex \
 	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers \
 	   	postgresql-dev mariadb-dev \
 	&& apk add --no-cache --virtual .mailman-rundeps bash sassc \
-	   postgresql-client mysql-client py-mysqldb \
+	   postgresql-client mysql-client py-mysqldb curl \
 	&& pip install -U mailmanclient==3.1.1 \
 		postorius==1.1.0 \
 		hyperkitty==1.1.4 \
--- a/web/Dockerfile.dev
+++ b/web/Dockerfile.dev
@@ -7,6 +7,11 @@ COPY mailman-web /opt/mailman-web
 # Add startup script to container
 COPY docker-entrypoint.sh /usr/local/bin/

+ARG POSTORIUS_REF
+ARG HYPERKITTY_REF
+ARG DJ_MM3_REF
+ARG CLIENT_REF
+
 # Install packages and dependencies for postorius and hyperkitty Add user for
 # executing apps, change ownership for uwsgi+django files and set execution
 # rights for management script
@@ -14,16 +19,16 @@ RUN set -ex \
 	&& apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers git \
 	   	postgresql-dev mariadb-dev \
 	&& apk add --no-cache --virtual .mailman-rundeps bash sassc \
-	   postgresql-client mysql-client py-mysqldb \
-	&& pip install -U git+https://gitlab.com/mailman/mailmanclient \
-		git+https://gitlab.com/mailman/postorius \
-		git+https://gitlab.com/mailman/hyperkitty \
-		git+https://gitlab.com/mailman/django-mailman3 \
-		whoosh \
-		uwsgi \
-		psycopg2 \
-		dj-database-url \
-		mysqlclient \
+	   postgresql-client mysql-client py-mysqldb curl \
+	&& pip install -U git+https://gitlab.com/mailman/mailmanclient@${CLIENT_REF} \
+		                git+https://gitlab.com/mailman/postorius@${POSTORIUS_REF} \
+		                git+https://gitlab.com/mailman/hyperkitty@${HYPERKITTY_REF} \
+		                git+https://gitlab.com/mailman/django-mailman3@${DJ_MM3_REF} \
+		                whoosh \
+		                uwsgi \
+		                psycopg2 \
+		                dj-database-url \
+		                mysqlclient \
 	&& pip install -U django==1.10 \
 	&& apk del .build-deps \
 	&& addgroup -S mailman \
@@ -40,4 +45,5 @@ EXPOSE 8000 8080
 STOPSIGNAL SIGINT

 ENTRYPOINT ["docker-entrypoint.sh"]
+
 CMD ["uwsgi", "--ini", "/opt/mailman-web/uwsgi.ini"]