Speedup docker builds

master
Joshua Rubingh 10 months ago
parent 14e74382ba
commit 9fc4af5915
  1. 13
      .dockerignore
  2. 50
      .drone.yml
  3. 26
      VRE/apps/vre_apps/fixtures/vre_apps_initial_data.json
  4. 15
      docker-compose.yaml
  5. 4
      docker/Dockerfile
  6. 40
      docker/Dockerfile.scheduler

@ -6,6 +6,13 @@ __pycache__
**/__pycache__
**/.env
**/db.sqlite3
**/db.sqlite3*
docker-compose.yaml
db.sqlite3*
docker-compose.yaml
sent_emails
media
VRE/env.example
clouds.yaml*
VRE/db.sqlite3*
.idea
docker/project.env
media/

@ -30,7 +30,7 @@ name: build-docker-image
type: kubernetes
steps:
- name: build-docker-image-branch
- name: scheduler-and-api
image: plugins/docker
settings:
cache_from:
@ -41,7 +41,7 @@ steps:
from_secret: HARBOR_PASSWORD
repo: registry.webhosting.rug.nl/${DRONE_REPO,,}
registry: registry.webhosting.rug.nl
dockerfile: docker/Dockerfile.api
dockerfile: docker/Dockerfile
# During building the images, we use an internal RUG caching server for getting the raw docker images
# Without this, we will hit a rate limit of Docker. This only works on the RUG Drones building setup
build_args:
@ -79,29 +79,29 @@ steps:
exclude:
- tag
- name: scheduler
image: plugins/docker
settings:
cache_from:
- ${DRONE_REPO,,}:${DRONE_SOURCE_BRANCH/\//-}
username:
from_secret: HARBOR_USER
password:
from_secret: HARBOR_PASSWORD
repo: registry.webhosting.rug.nl/${DRONE_REPO,,}-scheduler
registry: registry.webhosting.rug.nl
dockerfile: docker/Dockerfile.scheduler
# During building the images, we use an internal RUG caching server for getting the raw docker images
# Without this, we will hit a rate limit of Docker. This only works on the RUG Drones building setup
build_args:
- DOCKER_CACHE=registry.webhosting.rug.nl/cache/library/
tags:
- ${DRONE_SOURCE_BRANCH/\//-}
- ${DRONE_SOURCE_BRANCH/\//-}-${DRONE_COMMIT_SHA:0:8}
when:
event:
exclude:
- tag
# - name: scheduler
# image: plugins/docker
# settings:
# cache_from:
# - ${DRONE_REPO,,}:${DRONE_SOURCE_BRANCH/\//-}
# username:
# from_secret: HARBOR_USER
# password:
# from_secret: HARBOR_PASSWORD
# repo: registry.webhosting.rug.nl/${DRONE_REPO,,}-scheduler
# registry: registry.webhosting.rug.nl
# dockerfile: docker/Dockerfile.scheduler
# # During building the images, we use an internal RUG caching server for getting the raw docker images
# # Without this, we will hit a rate limit of Docker. This only works on the RUG Drones building setup
# build_args:
# - DOCKER_CACHE=registry.webhosting.rug.nl/cache/library/
# tags:
# - ${DRONE_SOURCE_BRANCH/\//-}
# - ${DRONE_SOURCE_BRANCH/\//-}-${DRONE_COMMIT_SHA:0:8}
# when:
# event:
# exclude:
# - tag
---
kind: pipeline

@ -16,11 +16,18 @@
5,
6,
7,
12,
8,
9,
10,
11,
10
13,
14,
15,
16,
17,
18,
19,
20,
21
]
}
},
@ -41,11 +48,18 @@
5,
6,
7,
12,
8,
9,
10,
11,
10
13,
14,
15,
16,
17,
18,
19,
20,
21
]
}
},

@ -4,7 +4,7 @@ services:
# This is the database server that is used by Django
db:
container_name: postgresdb
image: postgres:latest
image: postgres:13.5-bullseye
restart: always
env_file:
- docker/project.env
@ -16,7 +16,7 @@ services:
# This is the redis server for HUEY (background processes)
redis:
container_name: redisdb
image: redis
image: redis:6.2-bullseye
env_file:
- docker/project.env
# We need the 'bash' command in order to get the environment variables
@ -38,7 +38,7 @@ services:
# But while building with Drones, you can. So Drones has this argument active!
# args:
# DOCKER_CACHE: registry.webhosting.rug.nl/cache/library/
dockerfile: ./docker/Dockerfile.api
dockerfile: ./docker/Dockerfile
args:
DEBUG: "False"
entrypoint: /opt/VRE/entrypoint.api.sh
@ -49,8 +49,8 @@ services:
- db
- redis
volumes:
- staticfiles:/home/app/web/staticfiles
- mediafiles:/home/app/web/mediafiles
- staticfiles:/opt/VRE/staticfiles
- mediafiles:/opt/VRE/mediafiles
# The standard NGINX server in front of the API. This will filter out all non API calls (/api/)
# And will also server as a static file server for the API documentation
@ -87,7 +87,10 @@ services:
# But while building with Drones, you can. So Drones has this argument active!
# args:
# DOCKER_CACHE: registry.webhosting.rug.nl/cache/library/
dockerfile: ./docker/Dockerfile.scheduler
# dockerfile: ./docker/Dockerfile.scheduler
# Reuse the API docker image.....
dockerfile: ./docker/Dockerfile
args:
DEBUG: "False"
command: python manage.py run_huey

@ -15,7 +15,7 @@ RUN python3 -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
# Update packages and install nc for database up detection
RUN rm -rf /var/lib/apt/lists/* && apt-get update && apt-get -y full-upgrade && apt-get install -y --no-install-recommends netcat
RUN rm -rf /var/lib/apt/lists/* && apt-get update && apt-get -y full-upgrade && apt-get install -y --no-install-recommends netcat && apt-get --purge autoremove -y && rm -rf /var/lib/apt/lists/*
ARG DEBUG="False"
ENV DEBUG="${DEBUG}"
@ -42,7 +42,7 @@ COPY ./VRE/. ${APP_HOME}
RUN pip install --upgrade pip wheel
# Install the requirements.
RUN pip install --no-cache-dir -r requirements.txt
RUN pip install --no-cache-dir -r requirements.txt && find ${VIRTUAL_ENV} -type d -name "__pycache__" -exec rm -r {} +
RUN if [ "$DEBUG" = "True" ] ; then pip install --no-cache-dir -r requirements-dev.txt; fi
# Startup script when the image is starting up with

@ -1,40 +0,0 @@
# Use the official Python image from the Docker Hub. But cached by the RUG registry to not to hit rate limits
# Use build args: DOCKER_CACHE=registry.webhosting.rug.nl/cache/library/
ARG DOCKER_CACHE
FROM ${DOCKER_CACHE}python:3.8
# These two environment variables prevent __pycache__/ files.
ENV PYTHONUNBUFFERED 1
ENV PYTHONDONTWRITEBYTECODE 1
ENV DEBIAN_FRONTEND=noninteractive
ENV APP_HOME=/opt/VRE
ENV VIRTUAL_ENV=/opt/venv
RUN python3 -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
# Update packages and install nc for database up detection
RUN rm -rf /var/lib/apt/lists/* && apt-get update && apt-get -y full-upgrade
ARG DEBUG="False"
ENV DEBUG="${DEBUG}"
RUN if [ "$DEBUG" = "True" ] ; then apt-get install -y graphviz libgraphviz-dev; fi
RUN apt-get autoremove -y && rm -rf /var/lib/apt/lists/*
# Create a logfile dir... not sure if this is the right way
RUN mkdir -p ${APP_HOME}/../log
# Change the workdir.
WORKDIR ${APP_HOME}
# Copy the code.
COPY ./VRE/. ${APP_HOME}
# Upgrade pip
RUN pip install --upgrade pip wheel
# Install the requirements.
RUN pip install --no-cache-dir -r requirements.txt
RUN if [ "$DEBUG" = "True" ] ; then pip install --no-cache-dir -r requirements-dev.txt; fi
Loading…
Cancel
Save