Browse Source

Cleanup

master
Joshua Rubingh 12 months ago
parent
commit
f4f8b84fbe
  1. 2
      VRE/VRE/settings.py
  2. 1
      VRE/apps/openstack/__init__.py
  3. 26
      VRE/apps/openstack/admin.py
  4. 30
      VRE/apps/openstack/apps.py
  5. 50
      VRE/apps/openstack/fixtures/openstack_initial_data.json
  6. 48
      VRE/apps/openstack/migrations/0001_initial.py
  7. 23
      VRE/apps/openstack/migrations/0002_auto_20210625_0900.py
  8. 0
      VRE/apps/openstack/migrations/__init__.py
  9. 129
      VRE/apps/openstack/models.py
  10. 57
      VRE/apps/openstack/signals.py
  11. 166
      VRE/apps/openstack/tasks.py
  12. 3
      VRE/apps/openstack/tests.py

2
VRE/VRE/settings.py

@ -68,8 +68,6 @@ INSTALLED_APPS = [ @@ -68,8 +68,6 @@ INSTALLED_APPS = [
'apps.virtual_machine',
'apps.virtual_machine.providers.vrw',
'apps.virtual_machine.providers.openstack',
# 'apps.openstack',
'apps.university',
'djoser',

1
VRE/apps/openstack/__init__.py

@ -1 +0,0 @@ @@ -1 +0,0 @@
default_app_config = 'apps.openstack.apps.OpenstackConfig'

26
VRE/apps/openstack/admin.py

@ -1,26 +0,0 @@ @@ -1,26 +0,0 @@
from django.contrib import admin
from .models import WorkspacePart, Workspace
from lib.models.admin import VMRelatedSelectWidget
# Register your models here.
@admin.register(WorkspacePart)
class OpenstackPartAdmin(admin.ModelAdmin):
list_display = ('name', 'created_at')
search_fields = ('name', )
ordering = ('-created_at', 'name',)
readonly_fields = ('created_at', 'updated_at')
def formfield_for_dbfield(self, db_field, **kwargs):
if db_field.name == 'content_type':
kwargs['widget'] = VMRelatedSelectWidget
return super().formfield_for_dbfield(db_field, **kwargs)
@admin.register(Workspace)
class OpenstackWorkspaceAdmin(admin.ModelAdmin):
list_display = ('virtual_machine', 'created_at')
ordering = ('-created_at',)
readonly_fields = ('created_at', 'updated_at')

30
VRE/apps/openstack/apps.py

@ -1,30 +0,0 @@ @@ -1,30 +0,0 @@
from django.apps import AppConfig
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from datetime import timedelta
class OpenstackConfig(AppConfig):
"""This is the default configuration for the OpenStack application.
The settings can be configured in the main settings.py file of the project.
Args:
settings.OPENSTACK_MACHINE_ACTIVE_DURATION (timedelta): This is the duration of which a :term:`VPS` is available. Default 1 year.
"""
name = 'apps.openstack'
label = 'openstack'
verbose_name = _('OpenStack')
verbose_name_plural = _('OpenStack')
if not hasattr(settings, 'OPENSTACK_MACHINE_ACTIVE_DURATION'):
# We only load this setting, if it is not available in the overall settings.py file
settings.OPENSTACK_MACHINE_ACTIVE_DURATION = timedelta(days=365)
def ready(self):
"""
Load custom :attr:`~apps.openstack.signals` for creating :term:`VPS` models
"""
import apps.openstack.signals

50
VRE/apps/openstack/fixtures/openstack_initial_data.json

@ -1,50 +0,0 @@ @@ -1,50 +0,0 @@
[
{
"model": "openstack.workspacepart",
"pk": 1,
"fields": {
"created_at": "2021-04-15T08:07:32.636Z",
"updated_at": "2021-05-26T08:25:13.844Z",
"cloud_id": "637562ae-25ab-4a7a-8a6c-ed48c924be26",
"name": "Private network (internal)",
"content_type": 18,
"object_id": 1
}
},
{
"model": "openstack.workspacepart",
"pk": 2,
"fields": {
"created_at": "2021-04-15T08:07:57.293Z",
"updated_at": "2021-05-26T08:25:01.483Z",
"cloud_id": "afd0a5a6-a4ce-415a-9a28-9325a6857dfd",
"name": "Public network (vlan16)",
"content_type": 18,
"object_id": 2
}
},
{
"model": "openstack.workspacepart",
"pk": 3,
"fields": {
"created_at": "2021-04-15T08:09:00.238Z",
"updated_at": "2021-05-26T08:24:40.012Z",
"cloud_id": "fd0e4ec4-8df3-4e42-adbb-394a16e900fc",
"name": "Ubuntu 18.04 LTS",
"content_type": 19,
"object_id": 1
}
},
{
"model": "openstack.workspacepart",
"pk": 4,
"fields": {
"created_at": "2021-04-15T08:10:01.328Z",
"updated_at": "2021-05-26T08:24:50.813Z",
"cloud_id": "3",
"name": "Basic profile",
"content_type": 21,
"object_id": 1
}
}
]

48
VRE/apps/openstack/migrations/0001_initial.py

@ -1,48 +0,0 @@ @@ -1,48 +0,0 @@
# Generated by Django 3.2 on 2021-05-25 08:54
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
('virtual_machine', '0001_initial'),
('contenttypes', '0002_remove_content_type_name'),
]
operations = [
migrations.CreateModel(
name='Workspace',
fields=[
('created_at', models.DateTimeField(auto_now_add=True, help_text='The date and time this model has been created', verbose_name='Date created')),
('updated_at', models.DateTimeField(auto_now=True, help_text='The date and time this model has been updated', verbose_name='Date updated')),
('cloud_id', models.CharField(blank=True, help_text='The ID on the cloud platform', max_length=50, null=True)),
('virtual_machine', models.OneToOneField(help_text='The virtual machine configuration.', on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='workspace_openstack', serialize=False, to='virtual_machine.virtualmachine')),
('starting_at', models.DateTimeField(help_text='Enter the start date when this workspace should be created', verbose_name='Starting at')),
('ending_at', models.DateTimeField(help_text='Enter the end date when this workspace should be terminated', verbose_name='Ending at')),
('status', models.CharField(choices=[('NEW', 'New'), ('UPDATING', 'Updating'), ('DONE', 'Done'), ('ERROR', 'Error'), ('OFFLINE', 'Offline'), ('DELETE', 'Delete'), ('TERMINATED', 'Terminated')], default='NEW', help_text='The status of the workspace.', max_length=10, verbose_name='Status')),
],
options={
'verbose_name': 'virtual workspace',
'verbose_name_plural': 'virtual workspaces',
},
),
migrations.CreateModel(
name='WorkspacePart',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('created_at', models.DateTimeField(auto_now_add=True, help_text='The date and time this model has been created', verbose_name='Date created')),
('updated_at', models.DateTimeField(auto_now=True, help_text='The date and time this model has been updated', verbose_name='Date updated')),
('cloud_id', models.CharField(blank=True, help_text='The ID on the cloud platform', max_length=50, null=True)),
('name', models.CharField(help_text='Human readable name for this workspace part.', max_length=100, verbose_name='Name')),
('object_id', models.PositiveIntegerField()),
('content_type', models.ForeignKey(help_text='Select the Virtual Machine part that correspondents with this workspace part.', limit_choices_to={'app_label': 'virtual_machine'}, on_delete=django.db.models.deletion.CASCADE, related_name='openstack_ct', to='contenttypes.contenttype')),
],
options={
'abstract': False,
},
),
]

23
VRE/apps/openstack/migrations/0002_auto_20210625_0900.py

@ -1,23 +0,0 @@ @@ -1,23 +0,0 @@
# Generated by Django 3.2 on 2021-06-25 09:00
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('openstack', '0001_initial'),
]
operations = [
migrations.AlterField(
model_name='workspace',
name='cloud_id',
field=models.CharField(blank=True, help_text='The ID on the cloud platform', max_length=255, null=True),
),
migrations.AlterField(
model_name='workspacepart',
name='cloud_id',
field=models.CharField(blank=True, help_text='The ID on the cloud platform', max_length=255, null=True),
),
]

0
VRE/apps/openstack/migrations/__init__.py

129
VRE/apps/openstack/models.py

@ -1,129 +0,0 @@ @@ -1,129 +0,0 @@
from django.db import models
from django.utils.translation import gettext_lazy as _
from django.contrib.contenttypes.fields import GenericForeignKey
from django.contrib.contenttypes.models import ContentType
from lib.models.base import MetaDataModel
from lib.models.cloud import CloudBasicDataModel
from apps.virtual_machine.models import VirtualMachine
# Create your models here.
class WorkspacePart(MetaDataModel, CloudBasicDataModel):
"""
A class for creating :term:`VPS` parts as counterparts for the Virtual Machine parts (:class:`~apps.virtual_machine.models.VirtualMachinePart`). This means that we can make translations between the general Virtual Machine Part model and this specific :term:`VPS` Part model.
In order to create :term:`VPS` parts you need to select the Virtual machine part type and item to make a Virtual Machine part translation. Together with a cloud_id you can then make virtual machines with the right configurations.
It will inherit the attributes :attr:`~lib.models.base.MetaDataModel.created_at` and :attr:`~lib.models.base.MetaDataModel.updated_at` from the Abstract model :class:`~lib.models.base.MetaDataModel`
It will inherit the attributes :attr:`~lib.models.cloud.CloudBasicDataModel.cloud_id` from the Abstract model :class:`~lib.models.cloud.CloudBasicDataModel`
Attributes
----------
name : str
The name of :term:`VPS` part. Use a easy to remember name for humans.
content_type : ContentType
Specify the virtual machine part type.
object_id : int
Specify the virtual machine part id
"""
name = models.CharField(_('Name'), max_length=100, help_text=_('Human readable name for this workspace part.'))
content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE, limit_choices_to={'app_label': 'virtual_machine'}, help_text=_('Select the Virtual Machine part that correspondents with this workspace part.'), related_name='openstack_ct')
object_id = models.PositiveIntegerField()
content_object = GenericForeignKey('content_type', 'object_id')
def __str__(self):
return f'OpenStack Workspace part: {self.name}'
class WorkspaceStatus(models.TextChoices):
"""
A class for defining :term:`VPS` status as choices. Currently the following statusses are supported:
.. data:: NEW
This is the state where every :term:`VPS` starts with. This means that the :term:`VPS` config is created, but it has not been created yet by the cloud provider
.. data:: UPDATING
The :term:`VPS` is being created on the cloud provider infrastructure.
.. data:: DONE
The :term:`VPS` is created by the cloud provider and can be used by the customer
.. data:: ERROR
The :term:`VPS` could not be created by the cloud provider. There was an (unknown) error.
.. data:: OFFLINE
The :term:`VPS` is offline and not reachable. Reasons are not known. But this can be used for trigger an investigation
.. data:: DELETE
The :term:`VPS` is marked for deleting fromt the cloud platform
.. data:: TERMINATED
The :term:`VPS` is closed/deleted by the cloud provider wihtout reason.
"""
NEW = ('NEW', _('New'))
UPDATING = ('UPDATING', _('Updating'))
DONE = ('DONE', _('Done'))
ERROR = ('ERROR', _('Error'))
OFFLINE = ('OFFLINE', _('Offline'))
DELETE = ('DELETE', _('Delete'))
TERMINATED = ('TERMINATED', _('Terminated'))
class Workspace(MetaDataModel, CloudBasicDataModel):
"""
A class for creating :term:`VPS` from Virtual Machines. When creating a new Workspace with the status :attr:`~WorkspaceStatus.NEW`, it should be picked up by the Workspace system in order to create the new :term:`VPS`.
By default the :term:`VPS` is created by a signal process where it uses the variable ':attr:`~settings.OPENSTACK_MACHINE_ACTIVE_DURATION`' to create :term:`VPS` for 1 year.
It will inherit the attributes :attr:`~lib.models.base.MetaDataModel.created_at` and :attr:`~lib.models.base.MetaDataModel.updated_at` from the Abstract model :class:`~lib.models.base.MetaDataModel`
It will inherit the attributes :attr:`~lib.models.cloud.CloudBasicDataModel.cloud_id` from the Abstract model :class:`~lib.models.cloud.CloudBasicDataModel`
Attributes
----------
virtual_machine : :class:`~apps.virtual_machine.models.VirtualMachine`
The Virtual Machine that needs to be created as a :term:`VPS`
starting_at : Datetime
The date and timestamp when this :term:`VPS` should be created.
ending_at : int
The date and timestamp when this :term:`VPS` should be removed.
status : string
The status of the :term:`VPS`. When created. Default is :attr:`~WorkspaceStatus.NEW`
"""
class Meta:
verbose_name = _('virtual workspace')
verbose_name_plural = _('virtual workspaces')
virtual_machine = models.OneToOneField(VirtualMachine, on_delete=models.CASCADE, primary_key=True, help_text=_('The virtual machine configuration.'), related_name='workspace_openstack')
starting_at = models.DateTimeField(_('Starting at'), help_text=_('Enter the start date when this workspace should be created'))
ending_at = models.DateTimeField(_('Ending at'), help_text=_('Enter the end date when this workspace should be terminated'))
status = models.CharField(_('Status'), default=WorkspaceStatus.NEW, max_length=10, choices=WorkspaceStatus.choices, help_text=_('The status of the workspace.'))
@property
def type(self):
"""Return the translated :term:`VPS` profile name.
Returns:
string: The :attr:`~lib.models.cloud.CloudBasicDataModel.cloud_id` value of the :term:`VPS` part
"""
try:
openstack_part = WorkspacePart.objects.filter(content_type=ContentType.objects.get_for_model(self.virtual_machine.profile),
object_id=self.virtual_machine.profile.pk).get()
return openstack_part.cloud_id
except WorkspacePart.DoesNotExist:
return self.virtual_machine.profile.name

57
VRE/apps/openstack/signals.py

@ -1,57 +0,0 @@ @@ -1,57 +0,0 @@
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.db.models.signals import post_save, pre_delete
from django.dispatch import receiver
from django.utils import timezone
from apps.virtual_machine.models import VirtualMachine
from .models import Workspace as OpenStackWorkspace, WorkspacePart as OpenStackWorkspacePart
from .tasks import create_virtual_machine_task, delete_virtual_machine_task
@receiver(post_save, sender=VirtualMachine)
def create_virtual_machine_openstack(sender, instance, created, **kwargs):
"""When a new virtual machine is created, this signal will be fired in order to check if a :term:`VPS` needs te be created.
When a VirtualMachine is created for the first time, and does not have a OpenStack Workspace attached, the software will check on the operating system what to do.
At this point the check if a :term:`VPS` needs te be created is done on the **operating system**. If the selected operating system has a :term:`VPS` Part configured, it is assumed that we need to create the :term:`VPS`.
Args:
sender (class): The modelclass VirtualMachine
instance (VirtualMachine): The Virtual machine that is either created or updated.
created (bool): Is the Virtual Machine created. If false, it is an update.
"""
# When a new virtual machine is created and does not have a OpenStack Workspace attached to it, it is a candidate for the OpenStack
if created and not instance.has_workspace and instance.operating_system is not None:
# Here we check if the Operating system is known as OpenStack part. If so, we create a OpenStack machine that is active for 1 year.
try:
_ = OpenStackWorkspacePart.objects.filter(content_type=ContentType.objects.get_for_model(instance.operating_system),
object_id=instance.operating_system.pk).get()
new_workspace = OpenStackWorkspace(virtual_machine=instance,
starting_at=timezone.now(),
ending_at=timezone.now() + settings.OPENSTACK_MACHINE_ACTIVE_DURATION)
new_workspace.save()
# We need a delay to make sure the database has stored the data....
create_virtual_machine_task.schedule((new_workspace.pk,), delay=3)
except OpenStackWorkspacePart.DoesNotExist:
# Not a Linux Operating system, so ignore here
pass
@receiver(pre_delete, sender=VirtualMachine)
def terminate_virtual_machine(sender, instance, **kwargs):
if hasattr(instance, 'workspace_openstack'):
try:
workspace = OpenStackWorkspace.objects.get(pk=instance.workspace_openstack.pk)
# When the workspace is not a OpenStackWorkspace, we get a Does Not Exist exception.
if workspace.cloud_id:
delete_virtual_machine_task(workspace.cloud_id)
except OpenStackWorkspace.DoesNotExist:
# This workspace is not an Openstack Workspace
pass

166
VRE/apps/openstack/tasks.py

@ -1,166 +0,0 @@ @@ -1,166 +0,0 @@
from django.contrib.contenttypes.models import ContentType
from huey.contrib.djhuey import task
from lib.cloud.openstack_client import VRE_OpenStackClient
from Crypto.PublicKey import RSA
import crypt
import string
import random
from apps.storage.models import StorageLocation, StorageEngine
from apps.virtual_machine.models import VirtualNetworkType, VirtualMachineAccess
from .models import Workspace as OpenStackWorkspace, WorkspacePart as OpenStackWorkspacePart
from datetime import datetime
# Create the virtual machine in the background with a 30 sec delay, so that the model is fully saved (M2M)
@task()
def create_virtual_machine_task(workspace_id):
"""
This task will create a new :term:`VPS` with a 30 seconds delay. It will use the requested resources to create a new :term:`VPS`.
The actions that are done here are:
- Generate a new password for authentication on the :term:`VPS`
- Generate a public and private SSH key (RSA 4096 bits) for authentication on the :term:`VPS`
- Create a mount point for the research data
- Add auto mounting of the research data
- Setting up the right network connections
- Store the IPv4 of the new :term:`VPS` for later use.
Args:
workspace_id (int): The ID of the newly created workspace
"""
print(f'[{datetime.now()}] New workspace ID: {workspace_id} is being created')
workspace = OpenStackWorkspace.objects.get(pk=int(workspace_id))
virtual_machine = workspace.virtual_machine
# Create a SHA512 password for the password file.
ssh_password = "".join(random.sample(string.ascii_lowercase + string.ascii_uppercase + string.digits + string.punctuation, 16))
hashed_password = crypt.crypt(ssh_password, crypt.mksalt(crypt.METHOD_SHA512))
key = RSA.generate(4096)
private_key = key.export_key().decode('utf8')
public_key = key.publickey().export_key(format='OpenSSH').decode('utf8')
mount_share = ''
try:
mount_share = f' - echo "{virtual_machine.study.storagelocation.storageengine.location}{virtual_machine.study.storagelocation.path} {virtual_machine.study.storagelocation.storageengine.username} {virtual_machine.study.storagelocation.storageengine.password}" >> /etc/davfs2/secrets\n'
mount_share += f' - echo "{virtual_machine.study.storagelocation.storageengine.location}{virtual_machine.study.storagelocation.path} /opt/research_data davfs user,rw,auto,uid={virtual_machine.researcher.user.username},gid={virtual_machine.researcher.user.username} 0 0" >> /etc/fstab\n'
mount_share += ' - [mount, /opt/research_data]\n'
except StorageLocation.DoesNotExist:
# No storagelocation connected to the study, therefore no 'mounted' drives
pass
except StorageEngine.DoesNotExist:
# No storageengine connected to the study, therefore no 'mounted' drives
pass
user_data = f'''#cloud-config
users:
- default
- name: {virtual_machine.researcher.user.username}
gecos: {virtual_machine.researcher.user.get_full_name()}
sudo: ALL=(ALL) ALL
groups: davfs2
shell: /bin/bash
lock_passwd: false
passwd: {hashed_password}
ssh_authorized_keys:
- {public_key}
ssh_pwauth: True
apt:
preserve_sources_list: true
sources:
x2go.ppa:
source: "ppa:x2go/stable"
package_update: true
# package_upgrade: true
packages:
- davfs2
- ubuntu-mate-core
- x2goserver
- x2goserver-xsession
- x2gomatebindings
groups:
- davfs2: [root,ubuntu]
# We add 2FA to the SSH by using a private key AND password
runcmd:
- echo 'AuthenticationMethods publickey,password' >> /etc/ssh/sshd_config
- service ssh restart
- [mkdir, /opt/research_data]
{mount_share}
'''
# Openstack values:
name = f'VRE_{virtual_machine.study.name}_({virtual_machine.researcher.user.get_full_name()})'
flavour = OpenStackWorkspacePart.objects.filter(content_type=ContentType.objects.get_for_model(virtual_machine.profile),
object_id=virtual_machine.profile.pk).get()
flavour = flavour.cloud_id
image = OpenStackWorkspacePart.objects.filter(content_type=ContentType.objects.get_for_model(virtual_machine.operating_system),
object_id=virtual_machine.operating_system.pk).get()
image = image.cloud_id
networks = {
'private': [],
'float': []
}
# Only add private networks now, the public network should be added as a floating IP
for network in virtual_machine.networks.all():
openstack_network = OpenStackWorkspacePart.objects.filter(content_type=ContentType.objects.get_for_model(network),
object_id=network.pk).get()
if network.network_type == VirtualNetworkType.PRIVATE:
networks['private'].append(openstack_network.cloud_id)
elif network.network_type == VirtualNetworkType.PUBLIC:
networks['float'].append(openstack_network.cloud_id)
opts = {
'memory': virtual_machine.total_memory,
'storage': virtual_machine.total_storage,
'user_data': user_data
}
client = VRE_OpenStackClient('hpc')
result = client.create_vps(name, image, flavour, networks, opts)
workspace.cloud_id = result['id']
workspace.save()
# # TODO: figure this out.... why update_or_create does not work... For now, we work around it.
access, created = VirtualMachineAccess.objects.get_or_create(
researcher=virtual_machine.researcher,
virtual_machine=virtual_machine
)
access.virtual_machine_ip = result['ipv4']
access.login_key = private_key
access.password = ssh_password
access.save()
@task()
def delete_virtual_machine_task(cloud_id):
"""
This task will delete a :term:`VPS` with a 30 seconds delay.
Args:
cloud_id (int): The remote ID of the :term:`VPS` that needs to be deleted.
"""
client = VRE_OpenStackClient('hpc')
client.remove_vps(cloud_id)

3
VRE/apps/openstack/tests.py

@ -1,3 +0,0 @@ @@ -1,3 +0,0 @@
from django.test import TestCase
# Create your tests here.
Loading…
Cancel
Save