Speedup docker builds

.dockerignore

@@ -6,6 +6,13 @@ __pycache__
 **/__pycache__
 
 **/.env
-**/db.sqlite3
-**/db.sqlite3*
-docker-compose.yaml
+db.sqlite3*
+docker-compose.yaml
+sent_emails
+media
+VRE/env.example
+clouds.yaml*
+VRE/db.sqlite3*
+.idea
+docker/project.env
+media/

.drone.yml

@@ -30,7 +30,7 @@ name: build-docker-image
 type: kubernetes
 
 steps:
-  - name: build-docker-image-branch
+  - name: scheduler-and-api
     image: plugins/docker
     settings:
       cache_from:
@@ -41,7 +41,7 @@ steps:
         from_secret: HARBOR_PASSWORD
       repo: registry.webhosting.rug.nl/${DRONE_REPO,,}
       registry: registry.webhosting.rug.nl
-      dockerfile: docker/Dockerfile.api
+      dockerfile: docker/Dockerfile
       # During building the images, we use an internal RUG caching server for getting the raw docker images
       # Without this, we will hit a rate limit of Docker. This only works on the RUG Drones building setup
       build_args:
@@ -79,29 +79,29 @@ steps:
         exclude:
           - tag
 
-  - name: scheduler
-    image: plugins/docker
-    settings:
-      cache_from:
-        - ${DRONE_REPO,,}:${DRONE_SOURCE_BRANCH/\//-}
-      username:
-        from_secret: HARBOR_USER
-      password:
-        from_secret: HARBOR_PASSWORD
-      repo: registry.webhosting.rug.nl/${DRONE_REPO,,}-scheduler
-      registry: registry.webhosting.rug.nl
-      dockerfile: docker/Dockerfile.scheduler
-      # During building the images, we use an internal RUG caching server for getting the raw docker images
-      # Without this, we will hit a rate limit of Docker. This only works on the RUG Drones building setup
-      build_args:
-        - DOCKER_CACHE=registry.webhosting.rug.nl/cache/library/
-      tags:
-        - ${DRONE_SOURCE_BRANCH/\//-}
-        - ${DRONE_SOURCE_BRANCH/\//-}-${DRONE_COMMIT_SHA:0:8}
-    when:
-      event:
-        exclude:
-          - tag
+  # - name: scheduler
+  #   image: plugins/docker
+  #   settings:
+  #     cache_from:
+  #       - ${DRONE_REPO,,}:${DRONE_SOURCE_BRANCH/\//-}
+  #     username:
+  #       from_secret: HARBOR_USER
+  #     password:
+  #       from_secret: HARBOR_PASSWORD
+  #     repo: registry.webhosting.rug.nl/${DRONE_REPO,,}-scheduler
+  #     registry: registry.webhosting.rug.nl
+  #     dockerfile: docker/Dockerfile.scheduler
+  #     # During building the images, we use an internal RUG caching server for getting the raw docker images
+  #     # Without this, we will hit a rate limit of Docker. This only works on the RUG Drones building setup
+  #     build_args:
+  #       - DOCKER_CACHE=registry.webhosting.rug.nl/cache/library/
+  #     tags:
+  #       - ${DRONE_SOURCE_BRANCH/\//-}
+  #       - ${DRONE_SOURCE_BRANCH/\//-}-${DRONE_COMMIT_SHA:0:8}
+  #   when:
+  #     event:
+  #       exclude:
+  #         - tag
 
 ---
 kind: pipeline

VRE/apps/vre_apps/fixtures/vre_apps_initial_data.json

@@ -16,11 +16,18 @@
                 5,
                 6,
                 7,
-                12,
                 8,
-                9,
+                10,
                 11,
-                10
+                13,
+                14,
+                15,
+                16,
+                17,
+                18,
+                19,
+                20,
+                21
             ]
         }
     },
@@ -41,11 +48,18 @@
                 5,
                 6,
                 7,
-                12,
                 8,
-                9,
+                10,
                 11,
-                10
+                13,
+                14,
+                15,
+                16,
+                17,
+                18,
+                19,
+                20,
+                21
             ]
         }
     },

docker-compose.yaml

@@ -4,7 +4,7 @@ services:
   # This is the database server that is used by Django
   db:
     container_name: postgresdb
-    image: postgres:latest
+    image: postgres:13.5-bullseye
     restart: always
     env_file:
       - docker/project.env
@@ -16,7 +16,7 @@ services:
   # This is the redis server for HUEY (background processes)
   redis:
     container_name: redisdb
-    image: redis
+    image: redis:6.2-bullseye
     env_file:
       - docker/project.env
     # We need the 'bash' command in order to get the environment variables
@@ -38,7 +38,7 @@ services:
       # But while building with Drones, you can. So Drones has this argument active!
       # args:
       #   DOCKER_CACHE: registry.webhosting.rug.nl/cache/library/
-      dockerfile: ./docker/Dockerfile.api
+      dockerfile: ./docker/Dockerfile
       args:
         DEBUG: "False"
     entrypoint: /opt/VRE/entrypoint.api.sh
@@ -49,8 +49,8 @@ services:
       - db
       - redis
     volumes:
-      - staticfiles:/home/app/web/staticfiles
-      - mediafiles:/home/app/web/mediafiles
+      - staticfiles:/opt/VRE/staticfiles
+      - mediafiles:/opt/VRE/mediafiles
 
   # The standard NGINX server in front of the API. This will filter out all non API calls (/api/)
   # And will also server as a static file server for the API documentation
@@ -87,7 +87,10 @@ services:
       # But while building with Drones, you can. So Drones has this argument active!
       # args:
       #   DOCKER_CACHE: registry.webhosting.rug.nl/cache/library/
-      dockerfile: ./docker/Dockerfile.scheduler
+      # dockerfile: ./docker/Dockerfile.scheduler
+
+      # Reuse the API docker image.....
+      dockerfile: ./docker/Dockerfile
       args:
         DEBUG: "False"
     command: python manage.py run_huey

docker/Dockerfile

@@ -15,7 +15,7 @@ RUN python3 -m venv $VIRTUAL_ENV
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 # Update packages and install nc for database up detection
-RUN rm -rf /var/lib/apt/lists/* && apt-get update && apt-get -y full-upgrade && apt-get install -y --no-install-recommends netcat
+RUN rm -rf /var/lib/apt/lists/* && apt-get update && apt-get -y full-upgrade && apt-get install -y --no-install-recommends netcat && apt-get --purge autoremove -y && rm -rf /var/lib/apt/lists/*
 
 ARG DEBUG="False"
 ENV DEBUG="${DEBUG}"
@@ -42,7 +42,7 @@ COPY ./VRE/. ${APP_HOME}
 RUN pip install --upgrade pip wheel
 
 # Install the requirements.
-RUN pip install --no-cache-dir -r requirements.txt
+RUN pip install --no-cache-dir -r requirements.txt && find ${VIRTUAL_ENV} -type d -name  "__pycache__" -exec rm -r {} +
 RUN if [ "$DEBUG" = "True" ] ; then pip install --no-cache-dir -r requirements-dev.txt; fi
 
 # Startup script when the image is starting up with

docker/Dockerfile.scheduler

@@ -1,40 +0,0 @@
-# Use the official Python image from the Docker Hub. But cached by the RUG registry to not to hit rate limits
-# Use build args: DOCKER_CACHE=registry.webhosting.rug.nl/cache/library/
-ARG DOCKER_CACHE
-FROM ${DOCKER_CACHE}python:3.8
-
-# These two environment variables prevent __pycache__/ files.
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONDONTWRITEBYTECODE 1
-
-ENV DEBIAN_FRONTEND=noninteractive
-
-ENV APP_HOME=/opt/VRE
-ENV VIRTUAL_ENV=/opt/venv
-RUN python3 -m venv $VIRTUAL_ENV
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-# Update packages and install nc for database up detection
-RUN rm -rf /var/lib/apt/lists/* && apt-get update && apt-get -y full-upgrade
-
-ARG DEBUG="False"
-ENV DEBUG="${DEBUG}"
-RUN if [ "$DEBUG" = "True" ] ; then apt-get install -y graphviz libgraphviz-dev; fi
-
-RUN apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
-
-# Create a logfile dir... not sure if this is the right way
-RUN mkdir -p ${APP_HOME}/../log
-
-# Change the workdir.
-WORKDIR ${APP_HOME}
-
-# Copy the code.
-COPY ./VRE/. ${APP_HOME}
-
-# Upgrade pip
-RUN pip install --upgrade pip wheel
-
-# Install the requirements.
-RUN pip install --no-cache-dir -r requirements.txt
-RUN if [ "$DEBUG" = "True" ] ; then pip install --no-cache-dir -r requirements-dev.txt; fi