VRE
/
Broker
4
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Split up code

Browse Source
V1
Joshua Rubingh 2 years ago
parent 76a20f73bc
commit d552185e5d
157 changed files with 10835 additions and 2 deletions
Show Stats Download Patch File Download Diff File

11
.dockerignore
Unescape Escape View File

@ -0,0 +1,11 @@
.git
.vscode
venv
log/*
__pycache__
**/__pycache__
**/.env
**/db.sqlite3
clouds.yaml
docker-compose.yaml

120
.drone.yml
Unescape Escape View File

@ -0,0 +1,120 @@
---
kind: pipeline
name: notify-pipeline-start
type: kubernetes
steps:
- name: slack
image: plugins/slack
settings:
webhook:
from_secret: SLACK_WEBHOOK
link_names: true
template: >
{{#if build.pull }}
*Build started*: {{ repo.owner }}/{{ repo.name }} - <https://git.webhosting.rug.nl/{{ repo.owner }}/{{ repo.name }}/pull/{{ build.pull }}|Pull Request #{{ build.pull }}>
{{else}}
*Build started: {{ repo.owner }}/{{ repo.name }} - Build #{{ build.number }}* (type: `{{ build.event }}`)
{{/if}}
Commit: <https://git.webhosting.rug.nl/{{ repo.owner }}/{{ repo.name }}/commit/{{ build.commit }}|{{ truncate build.commit 8 }}>
Branch: <https://git.webhosting.rug.nl/{{ repo.owner }}/{{ repo.name }}/commits/{{ build.branch }}|{{ build.branch }}>
Author: {{ build.author }}
<{{ build.link }}|Visit build page ↗>
---
kind: pipeline
name: build-docker-image
type: kubernetes
steps:
- name: build-docker-image-branch
image: plugins/docker
settings:
cache_from:
- ${DRONE_REPO,,}:${DRONE_SOURCE_BRANCH/\//-}
username:
from_secret: rug_docker_repo_user
password:
from_secret: rug_docker_repo_password
repo: registry.webhosting.rug.nl/${DRONE_REPO,,}
registry: registry.webhosting.rug.nl
dockerfile: docker/Dockerfile.api
tags:
- ${DRONE_SOURCE_BRANCH/\//-}
- ${DRONE_SOURCE_BRANCH/\//-}-${DRONE_COMMIT_SHA:0:8}
when:
event:
exclude:
- tag
- name: nginx-frontend-proxy
image: plugins/docker
settings:
cache_from:
- ${DRONE_REPO,,}:${DRONE_SOURCE_BRANCH/\//-}
username:
from_secret: rug_docker_repo_user
password:
from_secret: rug_docker_repo_password
repo: registry.webhosting.rug.nl/${DRONE_REPO,,}-ngx
registry: registry.webhosting.rug.nl
dockerfile: docker/Dockerfile.nginx
tags:
- ${DRONE_SOURCE_BRANCH/\//-}
- ${DRONE_SOURCE_BRANCH/\//-}-${DRONE_COMMIT_SHA:0:8}
when:
event:
exclude:
- tag
- name: scheduler
image: plugins/docker
settings:
cache_from:
- ${DRONE_REPO,,}:${DRONE_SOURCE_BRANCH/\//-}
username:
from_secret: rug_docker_repo_user
password:
from_secret: rug_docker_repo_password
repo: registry.webhosting.rug.nl/${DRONE_REPO,,}-ngx
registry: registry.webhosting.rug.nl
dockerfile: docker/Dockerfile.scheduler
tags:
- ${DRONE_SOURCE_BRANCH/\//-}
- ${DRONE_SOURCE_BRANCH/\//-}-${DRONE_COMMIT_SHA:0:8}
when:
event:
exclude:
- tag
---
kind: pipeline
name: notify-pipeline-end
type: kubernetes
steps:
- name: slack
image: plugins/slack
settings:
webhook:
from_secret: SLACK_WEBHOOK
link_names: true
template: >
{{#if build.pull }}
*{{#success build.status}}✔{{ else }}✘{{/success}} {{ uppercasefirst build.status }}*: {{ repo.owner }}/{{ repo.name }} - <https://git.webhosting.rug.nl/{{ repo.owner }}/{{ repo.name }}/pull/{{ build.pull }}|Pull Request #{{ build.pull }}>
{{else}}
*{{#success build.status}}✔{{ else }}✘{{/success}} {{ uppercasefirst build.status }}: {{ repo.owner }}/{{ repo.name }} - Build #{{ build.number }}* (type: `{{ build.event }}`)
{{/if}}
Commit: <https://git.webhosting.rug.nl/{{ repo.owner }}/{{ repo.name }}/commit/{{ build.commit }}|{{ truncate build.commit 8 }}>
Branch: <https://git.webhosting.rug.nl/{{ repo.owner }}/{{ repo.name }}/commits/{{ build.branch }}|{{ build.branch }}>
Author: {{ build.author }}
Duration: {{ since build.created }}
<{{ build.link }}|Visit build page ↗>
depends_on:
- build-docker-image
trigger:
status:
- success
- failure

4
.gitignore vendored
Unescape Escape View File

@ -1,3 +1,7 @@
clouds.yaml
doc/_build/
doc/output/
# ---> Python
# Byte-compiled / optimized / DLL files
__pycache__/

19
.vscode/launch.json vendored
Unescape Escape View File

@ -0,0 +1,19 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Django VRE Broker API",
"type": "python",
"request": "launch",
"program": "${workspaceFolder}/VRE/manage.py",
"args": [
"runserver",
"0.0.0.0:8080"
],
"django": true
}
]
}

11
.vscode/settings.json vendored
Unescape Escape View File

@ -0,0 +1,11 @@
{
"python.pythonPath": "venv/bin/python",
"files.watcherExclude": {
"**/.git/objects/**": true,
"**/.git/subtree-cache/**": true,
"**/node_modules/*/**": true,
"**/venv/*/**": true,
},
"restructuredtext.languageServer.disabled": true,
"restructuredtext.confPath": "${workspaceFolder}/doc"
}

19
LICENSE
Unescape Escape View File

@ -0,0 +1,19 @@
MIT License Copyright (c) 2020 Joshua Rubingh, Elwin Buisman
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is furnished
to do so, subject to the following conditions:
The above copyright notice and this permission notice (including the next
paragraph) shall be included in all copies or substantial portions of the
Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS
OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

311
README.md
Unescape Escape View File

@ -1,3 +1,310 @@
# Broker
# Virtual Research Environment
VRE Backend API and Scheduler
Secure data drop-off & routing software.
With this software it is possible to safely upload private and sensitive data like WeTransfer or Dropbox. It is possible to upload single or multiple files at once though a web interface or through an API.
## Installation
In order to install this Data drop off project, we need the following packages / software.
- Django
- TUS (The Upload Server)
- NGINX
## Django
We install Django with standard settings. We could run it in Aync way, but then you need some more steps: https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ So for now, we keep it simple.
### Install
Clone the code on `/opt/deploy/data_drop-off`
```sh
git clone https://git.web.rug.nl/VRE/data_drop-off.git /opt/deploy/data_drop-off
```
Then create a virtual environment
```sh
cd /opt/deploy/data_drop-off
python3 -m venv .
source bin/activate
```
Finally we install the required Python modules
```python
pip install -r requirements
```
This will install all the needed Python modules we need to run this Django project.
### External libraries:
#### Production
https://gitlab.com/eeriksp/django-model-choices
https://github.com/georgemarshall/django-cryptography
https://github.com/jacobian/dj-database-url
https://github.com/ierror/django-js-reverse
https://github.com/henriquebastos/python-decouple
https://github.com/ezhov-evgeny/webdav-client-python-3
https://github.com/dblueai/giteapy
https://pypi.org/project/PyGithub/
#### Development
https://github.com/jazzband/django-debug-toolbar
### Settings
The settings for Django are set in an `.env` file so that you can easily change the environment from production to testing. There is an `.env.example` file that could be used as a template.
```ini
# A uniquely secret key
SECRET_KEY=@wb=#(f4uc0l%e!5*eo+aoflnxb(@!l9!=c5w=4b+x$=!8&vy%a
# Disable debug in production
DEBUG=False
# Allowed hosts that Django does server. Take care when NGINX is proxying infront of Django
ALLOWED_HOSTS=127.0.0.1,localhost
# Enter the database url connection: https://github.com/jacobian/dj-database-url
DATABASE_URL=sqlite:////opt/deploy/data_drop-off/db.sqlite3
# Email settings
# Mail host
EMAIL_HOST=
# Email user name
EMAIL_HOST_USER=
# Email password
EMAIL_HOST_PASSWORD=
# Email server port number to use
EMAIL_PORT=25
# Does the email server supports TLS?
EMAIL_USE_TLS=
```
Next we have to make the database structure. If you are using SQLite3 as a backend, make sure the database file **DOES** exist on disk.
```sh
touch /opt/deploy/data_drop-off/db.sqlite3
```
Then in the Python virtual environment we run the following commands:
```sh
./manage.py migrate
./manage.py loaddata virtual_machine_initial_data
./manage.py createsuperuser
./manage.py collectstatic
```
And finally you should be able to start the Django application
```sh
./manage.py runserver
```
### TUS
TUS = [The Upload Server](https://tus.io/). This is a resumable upload server that speaks HTTP. This server is a stand-alone server that is running behind the NGINX server.
It is even possible to run a TUS instance on a different location (Amsterdam). As long as the TUS is reachable by the NGINX frontend server, and the TUS server can post webhooks back to the frontend server.
#### Setup
If needs the package ecnfs* so install that first: `sudo apt install encfs`
The setup is quit simple. This works the same way as Django by using .env file. So start by creating a new settings files based on the example.
`cp .env.example .env`
```ini
# TUS Daemon settings
# Change the variable below to your needs. You can also add more variables that are used in the startup.sh script
WEBHOOK_URL="http://localhost:8000/datadrops/webhook/"
DROPOFF_API_HAWK_KEY="[ENTER_HAWK_KEY]"
DROPOFF_API_HAWK_SECRET="[ENTER_HAWK_SECRET]"
```
You need to create an API user in Django that is allowed to communicatie between the TUS daemon and Django. This can be done by creating a new usre in the Django admin. This will also generate a new token, which is needed. This token can be found at the API -> Tokens page.
The default webhook url is: /datadrops/webhook/
Then you can start the upload server by starting with the 'start.sh' script: `./start.sh`
This will start the TUS server running on TCP port 1050.
#### Data storage
The upload data is stored at a folder that is configured in the TUS startup command. This should be folder that is writable by the user that is running the TUS instance. Make sure that the upload folder is not directly accessible by the webserver. Else files can be downloaded.
#### Hooks
The TUS is capable of handling hooks based on uploaded files. There are two types of hooks. 'Normal' hooks and webhooks. It is not possible to run both hook systems at the same time due to the blocking nature of the pre-create hook. So we use the 'normal' hook system. That means that custom scripts are run. Those scripts can then post the data to a webserver in order to get a Webhook functionality with the 'normal' hooks.
At the moment, there is only a HTTP webcall done in the hook system. There is no actual file movement yet.
For now we have used the following hooks:
- **pre-create**: This hook will run when a new upload starts. This will trigger the Django server to store the upload in the database, and check if the upload is allowed based on an unique upload url and unique upload code.
- **post-finish**: This hook will run when an upload is finished. And will update the Database/Django with the file size and actual filename (unique) on disk.
An example of a hook as used in this project. The only changes that should be done is:
- **WEBHOOK_URL**: This is the full url to the Django webhook
Do not change the **HTTP_HOOK_NAME** as this will give errors with Django.
```python
#!/usr/bin/env python
import sys
import json
import requests
# Tus webhook name
HTTP_HOOK_NAME='pre-create'
# Django webserver with hook url path
WEBHOOK_URL='http://localhost:8000/webhook/'
# Read stdin input data from the TUS daemon
data = ''.join(sys.stdin.readlines())
# Test if data is valid JSON... just to be sure...
try:
json.loads(data)
except Exception as ex:
print(ex)
# Send exit code higher then 0 to stop the upload process on the Tus server
sys.exit(1)
# We know for sure that JSON input data is 'valid'. So we post to the webhook for further checking
try:
# Create a webhook POST request with the needed headers and data. The data is the RAW data from the input.
webhook = requests.post(WEBHOOK_URL, headers={'HOOK-NAME':HTTP_HOOK_NAME}, data=data)
# If the POST is ok, and we get a 200 status back, so the upload can continue
if webhook.status_code == requests.codes.ok:
# This will make the Tus server continue the upload
sys.exit(0)
except requests.exceptions.RequestException as ex:
# Webhook post failed
print(ex)
# We had some errors, so upload has to be stopped
sys.exit(1)
```
This hook uses the same data payload as when TUS would use the Webhook system. So using 'Normal' hooks or using Webhooks with DJANGO should both work out of the box.
### NGINX
Install NGINX with LUA support through the package manager. For Ubuntu this would be
```sh
apt install nginx libnginx-mod-http-lua
```
Also configure SSL to make the connections secure. This is outside this installation scope.
#### LUA
There is usage of LUA in NGINX so we can handle some dynamic data on the server side. All LUA code should be placed in the folder `/etc/nginx/lua`.
#### Setup
After installation of the packages, create a symbolic link in the `/etc/nginx/sites-enabled` so that a new VHost is created.
Important parts of the VHost configuration:
```nginx
lua_package_path "/etc/nginx/lua/?.lua;;";
server {
listen 80 default_server;
listen [::]:80 default_server;
# SSL configuration
#
# listen 443 ssl default_server;
# listen [::]:443 ssl default_server;
#
# Note: You should disable gzip for SSL traffic.
# See: https://bugs.debian.org/773332
#
# Read up on ssl_ciphers to ensure a secure configuration.
# See: https://bugs.debian.org/765782
#
# Self signed certs generated by the ssl-cert package
# Don't use them in a production server!
#
# include snippets/snakeoil.conf;
root /var/www/html;
# Add index.php to the list if you are using PHP
index index.html;
server_name localhost;
# This location is hit when the Tus upload is starting and providing meta data for the upload.
# The actual upload is done with the /files location below
location ~ /files/([0-9a-f]+\-[0-9a-f]+\-[1-5][0-9a-f]+\-[89ab][0-9a-f]+\-[0-9a-f]+)?/ {
set $project_id $1; # Here we capture the UUIDv4 value to use in the Tus metadata manipulation
set $tusmetadata '';
# Here we manipulate the metadata from the TUS upload server.
# Now we are able to store some extra meta data based on the upload url.
access_by_lua_block {
local dropoff_tus = require('dropoff_tus');
local project_metadata = ngx.req.get_headers()['Upload-Metadata'];
if project_metadata ~= nill then
ngx.var.tusmetadata = dropoff_tus.updateTusMetadata(project_metadata,ngx.var.project_id);
end
}
# Here we update the Tus server metadata so we can add the project uuid to it for further processing
proxy_set_header Upload-Metadata $tusmetadata;
# Rewrite the url so that the project UUIDv4 is stripped from the url to the Tus server
rewrite ^.*$ /files/ break;
# Disable request and response buffering
proxy_request_buffering off;
proxy_buffering off;
client_max_body_size 0;
# Forward incoming requests to local tusd instance.
# This can also be a remote server on a different location.
proxy_pass http://localhost:1080;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
proxy_set_header X-Forwarded-Proto $scheme;
}
location ~ /files {
# Disable request and response buffering
proxy_request_buffering off;
proxy_buffering off;
client_max_body_size 0;
# Forward incoming requests to local tusd instance
proxy_pass http://localhost:1080;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $server_name;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
```
And there should be a `lua` folder in the `/etc/nginx` folder.
In order to test if NGINX is configured correctly run `nginx -t` and it should give an OK message:
```sh
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful
```
## Security (not yet inplemented)
It is possible to secure the upload files with PGP encryption. This is done automatically in the Web interface. When you want PGP encryption though API upload, the encryption has to be done before the upload is started. This is a manual action done by the uploader.
So automatic encryption is only available through the Web upload.

76
VRE/VRE/.env.example
Unescape Escape View File

@ -0,0 +1,76 @@
# A uniquely secret key
# https://docs.djangoproject.com/en/dev/ref/settings/#secret-key
SECRET_KEY=@wb=#(f4uc0l%e!5*eo+aoflnxb(@!l9!=c5w=4b+x$=!8&vy%'
# Disable debug in production
# https://docs.djangoproject.com/en/dev/ref/settings/#debug
DEBUG=False
# Allowed hosts that Django does server. Use comma separated list Take care when NGINX is proxying in front of Django
# https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
ALLOWED_HOSTS=127.0.0.1,localhost
# All internal IPS for Django. Use comma separated list
# https://docs.djangoproject.com/en/dev/ref/settings/#internal-ips
INTERNAL_IPS=127.0.0.1
# Enter the database url connection. Enter all parts even the port numbers: https://github.com/jacobian/dj-database-url
# By default a local sqlite3 database is used.
DATABASE_URL=sqlite:///db.sqlite3
# The location on disk where the static files will be placed during deployment. Setting is required
# https://docs.djangoproject.com/en/dev/ref/settings/#static-root
STATIC_ROOT=
# Enter the default timezone for the visitors when it is not known.
# https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-TIME_ZONE
TIME_ZONE=Europe/Amsterdam
# Email settings
# https://docs.djangoproject.com/en/dev/ref/settings/#email-host
# EMAIL_HOST=
# Email user name
# https://docs.djangoproject.com/en/dev/ref/settings/#email-host-user
# EMAIL_HOST_USER=
# Email password
# https://docs.djangoproject.com/en/dev/ref/settings/#email-host-password
# EMAIL_HOST_PASSWORD=
# Email server port number to use. Default is 25
# https://docs.djangoproject.com/en/dev/ref/settings/#email-port
# EMAIL_PORT=
# Does the email server supports TLS?
# https://docs.djangoproject.com/en/dev/ref/settings/#email-use-tls
# EMAIL_USE_TLS=
https://docs.djangoproject.com/en/dev/ref/settings/#default-from-email
DEFAULT_FROM_EMAIL=Do not reply<no-reply@rug.nl>
# The sender address. This needs to be one of the allowed domains due to SPF checks
# The code will use a reply-to header to make sure that replies goes to the researcher and not this address
EMAIL_FROM_ADDRESS=Do not reply<no-reply@rug.nl>
# The Redis server is used for background tasks. Enter the variables below. Leave password empty if authentication is not enabled.
# The hostname or IP where the Redis server is running. Default is localhost
REDIS_HOST=localhost
# The Redis port number on which the server is running. Default is 6379
REDIS_PORT=6379
# The Redis password when authentication is enabled
# REDIS_PASSWORD=
# The amount of connections to be made inside a connection pool. Default is 10
REDIS_CONNECTIONS=10
# Enter the full path to the Webbased file uploading without the Study ID part. The Study ID will be added to this url based on the visitor.
DROPOFF_BASE_URL=http://localhost:8000/dropoffs/
# Enter the full url to the NGINX service that is in front of the TUSD service. By default that is http://localhost:1090
DROPOFF_UPLOAD_HOST=http://localhost:1090
# Which file extensions are **NOT** allowed to be uploaded. By default the extensions exe,com,bat,lnk,sh are not allowed
DROPOFF_NOT_ALLOWED_EXTENSIONS=exe,com,bat,lnk,sh

0
VRE/VRE/__init__.py
Unescape Escape View File

16
VRE/VRE/asgi.py
Unescape Escape View File

@ -0,0 +1,16 @@
"""
ASGI config for VRE project.
It exposes the ASGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/
"""
import os
from django.core.asgi import get_asgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VRE.settings')
application = get_asgi_application()

234
VRE/VRE/settings.py
Unescape Escape View File

@ -0,0 +1,234 @@
"""
Django settings for VRE project.
Generated by 'django-admin startproject' using Django 3.1.5.
For more information on this file, see
https://docs.djangoproject.com/en/3.1/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.1/ref/settings/
"""
from redis import ConnectionPool
from pathlib import Path
from decouple import config, Csv
from dj_database_url import parse as db_url
from django.utils.translation import ugettext_lazy as _
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = config('SECRET_KEY')
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = config('DEBUG', default=False, cast=bool)
ALLOWED_HOSTS = config('ALLOWED_HOSTS', default='localhost,127.0.0.1', cast=Csv())
# Application definition
# We load the application in steps, based on which are available on disk
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'apps.api',
'apps.dropoff',
'apps.invitation',
'apps.researcher',
'apps.storage',
'apps.study',
'apps.virtual_machine',
'djoser',
'rest_framework',
'drf_yasg',
'hawkrest',
'huey.contrib.djhuey',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'hawkrest.middleware.HawkResponseMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'django.middleware.locale.LocaleMiddleware',
]
ROOT_URLCONF = 'VRE.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
],
},
},
]
WSGI_APPLICATION = 'VRE.wsgi.application'
# Database
# https://docs.djangoproject.com/en/3.1/ref/settings/#databases
DATABASES = {
'default': config(
'DATABASE_URL',
default=f'sqlite:///{BASE_DIR / "db.sqlite3"}', # + os.path.join(BASE_DIR, 'db.sqlite3')
cast=db_url
)
}
# Password validation
# https://docs.djangoproject.com/en/3.1/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/3.1/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = config('TIME_ZONE', default='UTC')
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.1/howto/static-files/
STATIC_URL = '/static/'
STATICFILES_DIRS = [
BASE_DIR / 'static',
]
STATIC_ROOT = config('STATIC_ROOT',None)
INTERNAL_IPS = config('INTERNAL_IPS',default='127.0.0.1',cast=Csv())
# SSL Checks / Setup
# This will tell Django if the request is trough SSL (proxy). This is needed for Hawk authentication
SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
# settings.py
HUEY = {
'huey_class': 'huey.RedisHuey', # Huey implementation to use.
'name': DATABASES['default']['NAME'].split('/')[-1], # Use db name for huey.
'results': True, # Store return values of tasks.
'store_none': False, # If a task returns None, do not save to results.
'immediate': False, # If DEBUG=True, run synchronously.
'utc': True, # Use UTC for all times internally.
'blocking': True, # Perform blocking pop rather than poll Redis.
'connection': {
# 'host': config('REDIS_HOST'),
# 'port': 6379,
# 'db': 0,
'connection_pool': ConnectionPool(
host=config('REDIS_HOST', 'localhost'),
password=config('REDIS_PASSWORD', None),
port=config('REDIS_PORT', default=6379, cast=int),
max_connections=config('REDIS_CONNECTIONS', default=10, cast=int)), # Definitely you should use pooling!
# ... tons of other options, see redis-py for details.
# huey-specific connection parameters.
'read_timeout': 1, # If not polling (blocking pop), use timeout.
'url': None, # Allow Redis config via a DSN.
},
'consumer': {
'workers': 1,
'worker_type': 'thread',
'initial_delay': 0.1, # Smallest polling interval, same as -d.
'backoff': 1.15, # Exponential backoff using this rate, -b.
'max_delay': 10.0, # Max possible polling interval, -m.
'scheduler_interval': 1, # Check schedule every second, -s.
'periodic': True, # Enable crontab feature.
'check_worker_health': True, # Enable worker health checks.
'health_check_interval': 1, # Check worker health every second.
},
}
# Email settings for sending out upload invitations.
DEFAULT_FROM_EMAIL = config('DEFAULT_FROM_EMAIL', default='Do not reply<no-reply@rug.nl>')
EMAIL_HOST = config('EMAIL_HOST', default='')
EMAIL_HOST_USER = config('EMAIL_HOST_USER', default='')
EMAIL_HOST_PASSWORD = config('EMAIL_HOST_PASSWORD', default='')
EMAIL_PORT = config('EMAIL_PORT', default=25, cast=int)
EMAIL_USE_TLS = config('EMAIL_USE_TLS', default=False, cast=bool)
# The sender address. This needs to be one of the allowed domains due to SPF checks
# The code will use a reply-to header to make sure that replies goes to the researcher and not this address
EMAIL_FROM_ADDRESS = config('EMAIL_FROM_ADDRESS', default='Do not reply<no-reply@rug.nl>')
if DEBUG:
EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend'
EMAIL_FILE_PATH = BASE_DIR / 'sent_emails'
# Dropoff settings.
# Enter the full path to the Webbased file uploading without the Study ID part. The Study ID will be added to this url based on the visitor.
DROPOFF_BASE_URL = config('DROPOFF_BASE_URL', default='http://localhost:8000/dropoffs/',)
# Enter the full url to the NGINX service that is in front of the TUSD service. By default that is http://localhost:1090
DROPOFF_UPLOAD_HOST = config('DROPOFF_UPLOAD_HOST', default='http://localhost:1090',)
# Which file extensions are **NOT** allowed to be uploaded. By default the extensions exe,com,bat,lnk,sh are not allowed
DROPOFF_NOT_ALLOWED_EXTENSIONS = config('DROPOFF_NOT_ALLOWED_EXTENSIONS',default='exe,com,bat,lnk,sh',cast=Csv())
# LOGGING = {
# 'version': 1,
# 'disable_existing_loggers': False,
# 'handlers': {
# 'file': {
# 'class': 'logging.FileHandler',
# 'filename': f'{BASE_DIR}/../log/debug.log',
# },
# },
# 'loggers': {
# 'django': {
# 'handlers': ['file'],
# 'level': 'DEBUG' if DEBUG else 'INFO',
# 'propagate': True,
# },
# 'hawkrest': {
# 'handlers': ['file'],
# 'level': 'DEBUG' if DEBUG else 'INFO',
# }
# },
# }

22
VRE/VRE/urls.py
Unescape Escape View File

@ -0,0 +1,22 @@
"""VRE URL Configuration
The `urlpatterns` list routes URLs to views. For more information please see:
https://docs.djangoproject.com/en/3.1/topics/http/urls/
Examples:
Function views
1. Add an import: from my_app import views
2. Add a URL to urlpatterns: path('', views.home, name='home')
Class-based views
1. Add an import: from other_app.views import Home
2. Add a URL to urlpatterns: path('', Home.as_view(), name='home')
Including another URLconf
1. Import the include() function: from django.urls import include, path
2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
"""
from django.contrib import admin
from django.urls import path, include, re_path
urlpatterns = [
path('api/', include('apps.api.urls')),
path('admin/', admin.site.urls),
]

16
VRE/VRE/wsgi.py
Unescape Escape View File

@ -0,0 +1,16 @@
"""
WSGI config for VRE project.
It exposes the WSGI callable as a module-level variable named ``application``.
For more information on this file, see
https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/
"""
import os
from django.core.wsgi import get_wsgi_application
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VRE.settings')
application = get_wsgi_application()

1
VRE/apps/api/__init__.py
Unescape Escape View File

@ -0,0 +1 @@
default_app_config = 'apps.api.apps.ApiConfig'

9
VRE/apps/api/admin.py
Unescape Escape View File

@ -0,0 +1,9 @@
from django.contrib import admin
from .models import Token
@admin.register(Token)
class TokenAdmin(admin.ModelAdmin):
list_display = ('key', 'user','is_supertoken', 'last_access')
ordering = ('-last_access', 'user', )
search_fields = ('key', 'user__username',)
readonly_fields = ('created_at', 'updated_at')

90
VRE/apps/api/apps.py
Unescape Escape View File

@ -0,0 +1,90 @@
from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _
from django.conf import settings
class ApiConfig(AppConfig):
name = 'apps.api'
label = 'api'
verbose_name = _('API')
verbose_name_plural = _('APIs')
try:
assert settings.SWAGGER_SETTINGS
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.SWAGGER_SETTINGS = {
'SECURITY_DEFINITIONS': {
'Hawk': {
'type': 'apiKey',
'description': 'HTTP Holder-Of-Key Authentication Scheme, https://github.com/hapijs/hawk, https://hawkrest.readthedocs.io/en/latest/<br /><strong>Ex header:</strong><br />\'Authorization\': \'Hawk mac="F4+S9cu7yZiZEgdtqzMpOOdudvqcV2V2Yzk2WcphECc=", hash="+7fKUX+djeQolvnLTxr0X47e//UHKbkRlajwMw3tx3w=", id="7FI5JET4", ts="1592905433", nonce="DlV-fL"\'',
'name': 'Authorization',
'in': 'header'
}
}
}
try:
assert settings.REST_FRAMEWORK
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
# To protect all API views with Hawk by default, put this in your settings:
# https://hawkrest.readthedocs.io/en/latest/usage.html#protecting-api-views-with-hawk
settings.REST_FRAMEWORK = {
'DEFAULT_AUTHENTICATION_CLASSES': (
'apps.api.authentication.APIHawk',
),
'DEFAULT_PERMISSION_CLASSES': (
'rest_framework.permissions.IsAuthenticated',
),
# 'DEFAULT_AUTHENTICATION_CLASSES': (
# 'rest_framework.authentication.TokenAuthentication',
# ),
# 'DEFAULT_PERMISSION_CLASSES': (
# 'rest_framework.permissions.IsAuthenticated', ),
# Use Django's standard `django.contrib.auth` permissions,
# or allow read-only access for unauthenticated users.
#'DEFAULT_PERMISSION_CLASSES': [
# 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
#],
'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.PageNumberPagination',
'PAGE_SIZE': 10
}
# try:
# assert settings.HAWK_CREDENTIALS_LOOKUP
# except AttributeError:
# settings.HAWK_CREDENTIALS_LOOKUP = 'apps.api.authentication.hawk_credentials_lookup'
# try:
# assert settings.HAWK_USER_LOOKUP
# except AttributeError:
# settings.HAWK_USER_LOOKUP = 'apps.api.authentication.hawk_user_lookup'
try:
assert settings.HAWK_MESSAGE_EXPIRATION
except AttributeError:
# We only load this setting, if it is not available in the overall settings.py file
settings.HAWK_MESSAGE_EXPIRATION = 60
try:
assert settings.DJOSER
except AttributeError:
settings.DJOSER = {
# 'PASSWORD_RESET_CONFIRM_URL': '#/password/reset/confirm/{uid}/{token}',
# 'USERNAME_RESET_CONFIRM_URL': '#/username/reset/confirm/{uid}/{token}',
'ACTIVATION_URL': '#/activate/{uid}/{token}',
'SEND_ACTIVATION_EMAIL': False,
'SEND_CONFIRMATION_EMAIL' : True,
'HIDE_USERS': True,
'SERIALIZERS': {},
}
def ready(self):
from . import signals

73
VRE/apps/api/authentication.py
Unescape Escape View File

@ -0,0 +1,73 @@
# import the logging library
import logging
# Get an instance of a logger
logger = logging.getLogger(__name__)
import django.utils
from rest_framework import exceptions
from hawkrest import HawkAuthentication
from .models import Token
class APIHawk(HawkAuthentication):
"""This is the API authentication that is using the HAWK authentication mechanism.
This class will implement a custom credentials and user lookups so that we can dynamically add new users and update tokens.
"""
def hawk_credentials_lookup(self, id):
"""This method will perform the check if the used token is an existing/known token in the database. This will not lookup a user. Only an existing token.
Args:
id (string): The token key to lookup in the database for existing token.
Raises:
exceptions.AuthenticationFailed: If the given token does not exists.
Returns:
dict: The dictionary holds the token id, the token secret and the used hashing algoritem that is used.
"""
try:
token = Token.objects.get(key=id)
except Token.DoesNotExist:
logger.warning(f'Requested to validate with invalid/non existing token: {id}')
raise exceptions.AuthenticationFailed(f'No such token: {id}')
return {
'id' : id,
'key' : token.secret,
'algorithm' : 'sha256'
}
def hawk_user_lookup(self, request, credentials):
"""Return the user account that is connected to the used token.
Args:
request ([type]): The incoming HTTP/API request
credentials (dict): The credentials from ~hawk_credentials_lookup
Raises:
exceptions.AuthenticationFailed: If the given token does not exists to an existing user
Returns:
tuple: Returns a tuple holding the user as first item
"""
user = None
try:
user = Token.objects.get(key=credentials['id']).user
except Token.DoesNotExist:
logger.warning(f'Requested to validate non existing user: {id}')
raise exceptions.AuthenticationFailed(f'No user for token: {id}')
# Update the date time stamp to now for last access data
user.token.last_access = django.utils.timezone.now()
user.token.save()
return (user,None)
def __str__(self):
"""Authentication identifier.
Returns:
string: Returns the name of the used authentication mechanism.
"""
return 'Hawk authenticator'

67
VRE/apps/api/locale/en/LC_MESSAGES/django.po
Unescape Escape View File

@ -0,0 +1,67 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
#, fuzzy
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2021-02-19 11:19+0100\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
"Language: \n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
#: apps/api/apps.py:9
msgid "API"
msgstr ""
#: apps/api/apps.py:10
msgid "APIs"
msgstr ""
#: apps/api/models.py:43
msgid "token"
msgstr ""
#: apps/api/models.py:44
msgid "tokens"
msgstr ""
#: apps/api/models.py:46
msgid "Select the user for this token"
msgstr ""
#: apps/api/models.py:47
msgid "Key"
msgstr ""
#: apps/api/models.py:47
msgid "The key for this token. This is used for Hawk verification."
msgstr ""
#: apps/api/models.py:48
msgid "Secret"
msgstr ""
#: apps/api/models.py:48
msgid "The secret for this token. This is used for Hawk signing."
msgstr ""
#: apps/api/models.py:49
msgid "Last access"
msgstr ""
#: apps/api/models.py:49
msgid "The date and time when this token is last used."
msgstr ""
#: apps/api/models.py:64
msgid "Super token"
msgstr ""

67
VRE/apps/api/locale/nl/LC_MESSAGES/django.po
Unescape Escape View File

@ -0,0 +1,67 @@
# SOME DESCRIPTIVE TITLE.
# Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
# This file is distributed under the same license as the PACKAGE package.
# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
#
msgid ""
msgstr ""
"Project-Id-Version: \n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2021-02-19 11:19+0100\n"
"PO-Revision-Date: 2020-05-27 16:25+0200\n"
"Last-Translator: Joshua Rubingh <j.g.rubingh@rug.nl>\n"
"Language-Team: \n"
"Language: nl\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Plural-Forms: nplurals=2; plural=(n != 1);\n"
"X-Generator: Poedit 2.0.6\n"
#: apps/api/apps.py:9
msgid "API"
msgstr "API"
#: apps/api/apps.py:10
msgid "APIs"
msgstr "APIs"
#: apps/api/models.py:43
msgid "token"
msgstr ""
#: apps/api/models.py:44
msgid "tokens"
msgstr ""
#: apps/api/models.py:46
msgid "Select the user for this token"
msgstr ""
#: apps/api/models.py:47
msgid "Key"
msgstr ""
#: apps/api/models.py:47
msgid "The key for this token. This is used for Hawk verification."
msgstr ""
#: apps/api/models.py:48
msgid "Secret"
msgstr ""
#: apps/api/models.py:48
msgid "The secret for this token. This is used for Hawk signing."
msgstr ""
#: apps/api/models.py:49
msgid "Last access"
msgstr ""
#: apps/api/models.py:49
msgid "The date and time when this token is last used."
msgstr ""
#: apps/api/models.py:64
msgid "Super token"
msgstr ""

40
VRE/apps/api/management/commands/dockersetup.py
Unescape Escape View File

@ -0,0 +1,40 @@
from django.core.management.base import BaseCommand, CommandError
from django.contrib.auth.models import User
from django.db.utils import IntegrityError
#from polls.models import Question as Poll
class Command(BaseCommand):
help = 'Setting up admin and tusd users for VRE'
def add_arguments(self, parser):
parser.add_argument('username', help='Username')
parser.add_argument('password', help='Password')
parser.add_argument('email', help='Email address')
parser.add_argument('--key', help='Token key')
parser.add_argument('--secret', help='Token secret')
def handle(self, *args, **options):
try:
user = User.objects.create_superuser(username=options['username'], password=options['password'], email=options['email'])
self.stdout.write(self.style.SUCCESS('Successfully created user "%s"' % options['username']))
if options['key'] is not None and options['secret'] is not None:
user.token.key=options['key']
user.token.secret=options['secret']
user.token.save()
self.stdout.write(self.style.SUCCESS('Successfully created token for user "%s"' % options['username']))
else:
# We do not want an token for the admin
user.token.delete()
except IntegrityError as ex:
if 'unique constraint' in str(ex).lower():
self.stdout.write(self.style.WARNING('User "%s" already exists' % options['username']))
else:
raise CommandError('Could not create user "%s": %s' % (options['username'],ex))

35
VRE/apps/api/migrations/0001_initial.py
Unescape Escape View File

@ -0,0 +1,35 @@
# Generated by Django 3.1.7 on 2021-02-23 14:37
import apps.api.models
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import django_cryptography.fields
class Migration(migrations.Migration):
initial = True
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='Token',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True, help_text='The date and time this model has been created', verbose_name='Date created')),
('updated_at', models.DateTimeField(auto_now=True, help_text='The date and time this model has been updated', verbose_name='Date updated')),
('key', models.CharField(default=apps.api.models.get_random_key, help_text='The key for this token. This is used for Hawk verification.', max_length=16, unique=True, verbose_name='Key')),
('secret', django_cryptography.fields.encrypt(models.CharField(default=apps.api.models.get_random_secret, help_text='The secret for this token. This is used for Hawk signing.', max_length=64, verbose_name='Secret'))),
('last_access', models.DateTimeField(auto_now_add=True, help_text='The date and time when this token is last used.', verbose_name='Last access')),
('user', models.OneToOneField(help_text='Select the user for this token', on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
],
options={
'verbose_name': 'token',
'verbose_name_plural': 'tokens',
},
),
]

0
VRE/apps/api/migrations/__init__.py
Unescape Escape View File

70
VRE/apps/api/models.py
Unescape Escape View File

@ -0,0 +1,70 @@
from django.contrib.auth.models import User
from django.db import models
from django.utils.translation import gettext_lazy as _
from django_cryptography.fields import encrypt
from lib.utils.general import get_random_string
from lib.models.base import MetaDataModel
def get_random_key():
return get_random_string(8)
def get_random_secret():
return get_random_string(32)
class TokenManager(models.Manager):
"""
Custom queryset which will prefetch related user table data when requesting a token from the database as the user is mostly needed every time the token is requested.
"""
def get_queryset(self):
return super(TokenManager, self).get_queryset().select_related('user')
class Token(MetaDataModel):
"""Token model that holds all the tokens that are used for the API authentication.
A new token is generated every time when a new user is created. So there is no need for manual token creating. This is done through a signal :attr:`~apps.api.signals.create_user_token`
Attributes
----------
user : :class:`~django.contrib.auth.models.User`
The user to which this token belongs too
key : str
The key value that is used for token lookups
secret : str
The secret that is used for encrypting/signing the API messages
last_access : datetime
The date and time when the token is last used (logged in)
"""
class Meta:
verbose_name = _('token')
verbose_name_plural = _('tokens')
user = models.OneToOneField(User, on_delete=models.CASCADE, help_text=_('Select the user for this token'))
key = models.CharField(_('Key') , unique=True, default=get_random_key, max_length=16, help_text=_('The key for this token. This is used for Hawk verification.'))
secret = encrypt(models.CharField(_('Secret') ,max_length=64, default=get_random_secret, help_text=_('The secret for this token. This is used for Hawk signing.')))
last_access = models.DateTimeField(_('Last access'),auto_now_add=True, help_text=_('The date and time when this token is last used.'))
# Custom manager that will retrieve the related user table as well.
objects = TokenManager()
def is_supertoken(self):
"""Boolean check if the token is belonging to a user with super user rights. Then this token is a super token.
Returns:
bool: Returns true when the token belongs to a super user.
"""
# TODO: Is it allowed to be a super user and researcher? Could give conflict of interests. With the API token you can read other researchers data...
return self.user.is_superuser == True
is_supertoken.boolean = True
is_supertoken.short_description = _('Super token')
def __str__(self):
"""
Print the full name of the researcher based on the first and last name fields of the User model.
"""
return f'{self.key} ({self.user.get_full_name()})'

13
VRE/apps/api/serializers.py
Unescape Escape View File

@ -0,0 +1,13 @@
from django.utils.translation import gettext_lazy as _
from rest_framework import serializers
from .models import Token
class TokenSerializer(serializers.ModelSerializer):
class Meta:
model = Token
fields = ['key','secret']
class TokenLoginSerializer(serializers.Serializer):
username = serializers.CharField(max_length=200, help_text=_('Your username to login'))
password = serializers.CharField(max_length=200, help_text=_('Your password to login'))

24
VRE/apps/api/signals.py
Unescape Escape View File

@ -0,0 +1,24 @@
from django.conf import settings
from django.db.models.signals import post_save
from django.dispatch import receiver
from .models import Token
@receiver(post_save, sender=settings.AUTH_USER_MODEL)
def create_user_token(sender, instance=None, created=False, **kwargs):
"""
When a new user is created, this signal will also create a new API token for this user. So every user will have an API token.
Arguments
----------
sender : sender
The model that has triggered the signal
instance: :attr:`~django.contrib.auth.models.User`
The newly created user model data
created : boolean
Wether the object was created (True) or updated (False).
"""
if created:
Token.objects.create(user=instance)

57
VRE/apps/api/tests.py
Unescape Escape View File

@ -0,0 +1,57 @@
from django.test import TestCase, override_settings
from rest_framework.test import RequestsClient
from requests_hawk import HawkAuth
from django.contrib.auth.models import User
from django.urls import reverse
import json
from lib.api.client import VRE_API_Client
# Create your tests here.
class UserLoginTest(TestCase):
@classmethod
def setUpTestData(cls):
cls.email = 'dummy@rug.nl'
cls.username = 'dummy@rug.nl'
cls.password = 'doemaarwat'
User.objects.create_user(username=cls.username,password=cls.password,email=cls.email)
def setUp(self):
# We want to use the API REST Framework request client, as this enables us to use HAWK authentication during testing
self.client = RequestsClient()
def test_missing_credentials(self):
endpoint = 'http://testserver/api/auth/users/me/'
response = self.client.get(endpoint)
self.assertEqual(response.status_code, 401)
def test_valid_login(self):
login_data = {'username' : self.username, 'password' : self.password}
endpoint = 'http://testserver' + reverse('api:api-login')
response = self.client.post(endpoint, json=login_data)
self.assertEqual(response.status_code, 200)
self.assertIn('key', response.json())
self.assertIn('secret', response.json())
self.key = response.json()['key']
self.secret = response.json()['secret']
# Add HAWK Authentication to make sure key and secret are correct.
self.client.auth = HawkAuth(id=self.key , key=self.secret, always_hash_content=False)
endpoint = 'http://testserver/api/auth/users/me/'
response = self.client.get(endpoint)
self.assertEqual(response.status_code, 200)
self.assertIn('email', response.json())
self.assertIn('id', response.json())
self.assertIn('username', response.json())
self.assertEqual(response.json()['email'], self.email)
self.assertEqual(response.json()['username'], self.username)

77
VRE/apps/api/urls.py
Unescape Escape View File

@ -0,0 +1,77 @@
from django.urls import path, re_path, include
from rest_framework import permissions, routers
from drf_yasg.views import get_schema_view
from drf_yasg import openapi
from . import views
from apps.dropoff.views import DatadropViewSet
from apps.invitation.views import InvitationViewSet
from apps.researcher.views import ResearcherViewSet
from apps.storage.views import StorageEngineViewSet, StorageLocationViewSet
from apps.study.views import StudyViewSet
from apps.virtual_machine.views import (VirtualMachineViewSet,
VirtualMachineOperatingSystemViewSet,
VirtualMachineProfileViewSet,
VirtualMachineMemoryViewSet,
VirtualMachineNetworkViewSet,
VirtualMachineStorageViewSet,
VirtualMachineGPUViewSet,
VirtualMachineAccessViewSet)
schema_view = get_schema_view(
openapi.Info(
title="Virtual Research Environment API",
default_version='v1',
description="Here you can see a list of API endpoints and actions that are available to communicate with the VRE API",
terms_of_service="https://www.rug.nl",
contact=openapi.Contact(email="vre_team@rug.nl"),
license=openapi.License(name="MIT License"),
),
public=True,
permission_classes=(permissions.AllowAny,),
)
api_router_v1 = routers.DefaultRouter()
api_router_v1.register(r'researchers', ResearcherViewSet)
api_router_v1.register(r'studies', StudyViewSet)
api_router_v1.register(r'dropoffs', DatadropViewSet)
api_router_v1.register(r'invitations', InvitationViewSet)
api_router_v1.register(r'storageengines', StorageEngineViewSet)
api_router_v1.register(r'storagelocations', StorageLocationViewSet)
# Order is important for virtual machines. Longest match first
api_router_v1.register(r'virtualmachines/profiles', VirtualMachineProfileViewSet)
api_router_v1.register(r'virtualmachines/storage', VirtualMachineStorageViewSet)
api_router_v1.register(r'virtualmachines/access', VirtualMachineAccessViewSet)
api_router_v1.register(r'virtualmachines/memory', VirtualMachineMemoryViewSet)
api_router_v1.register(r'virtualmachines/network', VirtualMachineNetworkViewSet)
api_router_v1.register(r'virtualmachines/gpu', VirtualMachineGPUViewSet)
api_router_v1.register(r'virtualmachines/os', VirtualMachineOperatingSystemViewSet)
api_router_v1.register(r'virtualmachines', VirtualMachineViewSet)
# Main namespace for the API urls
app_name = 'api'
urlpatterns = [
re_path(r'^swagger(?P<format>\.json|\.yaml)$', schema_view.without_ui(cache_timeout=0), name='schema-json'),
path('swagger/', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'),
path('redoc/', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'),
# Authentication urls
path('auth/', include('djoser.urls')),
path('auth/login/', views.Login.as_view(), name='api-login'),
# Extra /api/info path for checking if the Hawk authentication is working.
# Also this will give the full url to the OpenAPI documentation
path('info/', views.Info.as_view(), name='api-info'),
# Add extra namespace for versioning the API
path('v1/', include((api_router_v1.urls,'api'),namespace='v1')),
]

111
VRE/apps/api/views.py
Unescape Escape View File

@ -0,0 +1,111 @@
from rest_framework.views import APIView
from rest_framework.response import Response
from rest_framework.decorators import schema
from rest_framework.permissions import AllowAny
from django.urls import reverse
from django.contrib.auth.models import User
from rest_framework import exceptions
from hawkrest import HawkAuthentication
from lib.utils.general import get_ip_address, generate_encryption_key