Browse Source

Add missing libs

V1
Joshua Rubingh 1 year ago
parent
commit
0991fa3a06
  1. 1
      .gitignore
  2. 51
      VRE/lib/api/base.py
  3. 242
      VRE/lib/api/client.py
  4. 72
      VRE/lib/cloud/openstack_client.py
  5. 80
      VRE/lib/cloud/user_data.txt
  6. 20
      VRE/lib/models/base.py
  7. 84
      VRE/lib/utils/emails.py
  8. 26
      VRE/lib/utils/general.py

1
.gitignore vendored

@ -19,7 +19,6 @@ dist/ @@ -19,7 +19,6 @@ dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/

51
VRE/lib/api/base.py

@ -0,0 +1,51 @@ @@ -0,0 +1,51 @@
from rest_framework import viewsets, permissions, serializers
from rest_framework.permissions import BasePermission
class IsOwner(BasePermission):
def has_object_permission (self, request, view, obj ):
"""Return 'True' if permission is granted, 'False' otherwise."""
# TODO: If this is the 'way to go', we should consider adding the researcher reference to all models and save actions
return request.user.is_authenticated and (obj.researcher == request.user.researcher or obj.study.researcher == request.user.researcher)
class BaseReadOnlyViewSet(viewsets.ReadOnlyModelViewSet):
permission_classes = [permissions.IsAuthenticated, IsOwner]
# TODO: If this is the 'way to go', we should consider adding the researcher reference to all models and save actions
def get_queryset(self):
if getattr(self, 'swagger_fake_view', False):
return self.queryset.all()
try:
qs = self.queryset.filter(researcher = self.request.user.researcher)
except:
qs = self.queryset.filter(study__researcher = self.request.user.researcher)
return qs
class BaseViewSet(viewsets.ModelViewSet):
permission_classes = [permissions.IsAuthenticated, IsOwner]
# TODO: If this is the 'way to go', we should consider adding the researcher reference to all models and save actions
def get_queryset(self):
if getattr(self, 'swagger_fake_view', False):
return self.queryset.all()
try:
qs = self.queryset.filter(researcher = self.request.user.researcher)
except:
qs = self.queryset.filter(study__researcher = self.request.user.researcher)
return qs
class BaseHyperlinkedModelSerializer(serializers.HyperlinkedModelSerializer):
# This ID field is handy to have.... Due to HyperlinkedModelSerializer we do not have this field by default
id = serializers.ReadOnlyField()
# Only show the researcher full name
researcher = serializers.StringRelatedField()
# Only show link to full researcher data
#researcher = serializers.HyperlinkedRelatedField(view_name= 'api:v1:researcher-detail', read_only=True)
# Show the full researcher information
#researcher = ResearcherSerializer(read_only=True)

242
VRE/lib/api/client.py

@ -0,0 +1,242 @@ @@ -0,0 +1,242 @@
import requests
from requests_hawk import HawkAuth
from datetime import datetime
import copy
import json
from diskcache import Cache
class VRE_API_Exception(Exception):
def __init__(self, message, errors):
# Call Exception.__init__(message)
# to use the same Message header as the parent class
super().__init__(message)
self.errors = errors
# Display the errors
print('Printing Errors:')
print(errors)
class VRE_API_Exception_Factory(VRE_API_Exception):
def __new__(self, message, error_code):
print(f'VRE_API_Exception_Factory {error_code} -> {message}')
if error_code == 400:
return VRE_API_400(message)
elif error_code == 401:
return VRE_API_401(message)
elif error_code == 403:
return VRE_API_403(message)
elif error_code == 404:
return VRE_API_404(message)
class VRE_API_400(VRE_API_Exception):
def __init__(self, message):
# Call the base class constructor with the parameters it needs
super().__init__(message, 400)
class VRE_API_401(VRE_API_Exception):
def __init__(self, message):
# Call the base class constructor with the parameters it needs
super().__init__(message, 401)
class VRE_API_403(VRE_API_Exception):
def __init__(self, message):
# Call the base class constructor with the parameters it needs
super().__init__(message, 403)
class VRE_API_404(VRE_API_Exception):
def __init__(self, message):
# Call the base class constructor with the parameters it needs
super().__init__(message, 404)
class VRE_API_Client():
DATE_TIME_FIELDS = 'created_at,updated_at,mail_sent'.split(',')
HEADERS = {
'Content-Type': 'application/json',
'cache-control': 'no-cache'
}
SWAGGER_CACHE_TIMEOUT = 60 * 60 * 24
def __init__(self, host, url = None, token = None, secret = None):
self.authentication = None
# Create a disk based caching for getting the swagger data only once an hour
self.cache = Cache('/tmp/vre_client/')
self.host = host.strip('/') + '/'
self.token = token
self.secret = secret
self.data = {}
self.set_token(token)
self.set_secret(secret)
self.__get_urls_from_swagger_data()
self.set_url(url)
def __get_full_url(self):
return f'{self.host}{self.url}'
def __parse_date_time_fields(self, data):
for item in data:
# TODO: Should provide better solution for this try/catch. For now it works
try:
if isinstance(item,list) or isinstance(item,dict):
self.__parse_date_time_fields(item)
elif isinstance(data[item],list) or isinstance(data[item],dict):
self.__parse_date_time_fields(data[item])
elif item in self.DATE_TIME_FIELDS and isinstance(data[item],str):
try:
data[item] = datetime.strptime(data[item],'%Y-%m-%dT%H:%M:%S.%fZ')
except Exception:
data[item] = datetime.strptime(data[item][::-1].replace(':','',1)[::-1].replace(' ','T'),'%Y-%m-%dT%H:%M:%S.%f%z')
except Exception:
pass
def __parse_data(self, start = None):
if len(self.DATE_TIME_FIELDS) > 0:
self.__parse_date_time_fields(self.data)
def __get_urls_from_swagger_data(self):
endpoints = self.cache.get('swagger')
if endpoints is None:
self.set_url('swagger.json')
result = requests.get(self.__get_full_url(), headers=self.HEADERS)
if result.status_code in [200,201]:
data = result.json()
endpoints = {}
for endpoint_url in data['paths']:
for http_action in data['paths'][endpoint_url]:
if http_action not in ['get','post','put','patch','delete']:
continue
endpoint_id = data['paths'][endpoint_url][http_action]['operationId']
endpoints[endpoint_id] = endpoint_url
self.cache.set('swagger',endpoints,expire=self.SWAGGER_CACHE_TIMEOUT)
else:
error_detail = ''
try:
error_detail = result.json()['detail']
except json.decoder.JSONDecodeError:
if 404 == result.status_code:
error_detail = 'not found'
raise VRE_API_Exception_Factory(f'Error({result.status_code}) with url {self.url}: {error_detail}',result.status_code)
self.endpoints = endpoints
def set_url(self, url):
if url is None:
return
if url.startswith(':'):
url = self.endpoints.get(url[1:])
if url is None:
return
if url.startswith('/'):
url = url[1:]
self.url = url
def set_token(self, token):
if token is None:
return
self.token = token
self.authentication = HawkAuth(id=self.token , key=self.secret)
def set_secret(self, secret):
if secret is None:
return
self.secret = secret
self.authentication = HawkAuth(id=self.token , key=self.secret)
def get_data(self):
# For get requests, we have to adjust the headers and HAWK authentication
# For get requests there is no content, so make sure the 'Content-Type' header is '' (empty)
# And that you instruct the HawkAtuh not to hash the content (as there is none)
headers = copy.copy(self.HEADERS)
headers['Content-Type'] = ''
authentication = HawkAuth(id=self.token , key=self.secret, always_hash_content=False)
result = requests.get(self.__get_full_url(), auth=authentication, headers=headers)
if result.status_code in [200,201]:
self.data = result.json()
self.__parse_data()
else:
error_detail = ''
try:
error_detail = result.json()['detail']
except json.decoder.JSONDecodeError:
if 404 == result.status_code:
error_detail = 'not found'
raise VRE_API_Exception_Factory(f'Error with url {self.url}: {error_detail}',result.status_code)
return self.data
def post_data(self, payload):
result = requests.post(self.__get_full_url(), json=payload, auth=self.authentication, headers=self.HEADERS)
self.data['status_code'] = result.status_code
if result.status_code in [200,201]:
self.data = result.json()
self.__parse_data()
else:
error_detail = ''
try:
error_detail = result.json()['detail']
except json.decoder.JSONDecodeError:
if 404 == result.status_code:
error_detail = 'not found'
raise VRE_API_Exception_Factory(f'Error with url {self.url}: {error_detail}',result.status_code)
return self.data
def put_data(self, payload):
result = requests.put(self.__get_full_url(), json=payload, auth=self.authentication, headers=self.HEADERS)
self.data['status_code'] = result.status_code
if result.status_code in [200,201]:
self.data = result.json()
self.__parse_data()
else:
error_detail = ''
try:
error_detail = result.json()['detail']
except json.decoder.JSONDecodeError:
if 404 == result.status_code:
error_detail = 'not found'
raise VRE_API_Exception_Factory(f'Error with url {self.url}: {error_detail}',result.status_code)
return self.data
def delete_data(self):
try:
# Django HAWK has issues with a delete action. It needs/wants a content-type header, but there is no content.....
# https://github.com/kumar303/hawkrest/issues/46
result = requests.delete(self.__get_full_url(), json={}, auth=self.authentication, headers=self.HEADERS)
return result.status_code in [200,201,204]
except Exception as ex:
raise VRE_API_Exception_Factory(f'Error with url {self.url}: {ex}',500)
return False
if __name__ == "__main__":
client = VRE_API_Client('http://localhost:8000/api/')
client.set_url(':auth_login_create')
access_token = client.post_data({'username' : 'j.g.rubingh@rug.nl', 'password' : 'jos24kin'})
print(access_token)

72
VRE/lib/cloud/openstack_client.py

@ -0,0 +1,72 @@ @@ -0,0 +1,72 @@
# https://docs.openstack.org/openstacksdk/latest/user/index.html#user-guides
# https://docs.openstack.org/openstacksdk/latest/user/proxies/compute.html
import openstack
import base64
from Crypto.PublicKey import RSA
from django.template.defaultfilters import slugify
class VRE_OpenStackClient():
def __init__(self, cloud = None, debug = False):
self.cloud = cloud
self.debug = debug
if self.debug:
openstack.enable_logging(debug=True)
self.connect()
def connect(self):
# Initialize and turn on debug logging
openstack.enable_logging(debug=self.debug)
self.conn = openstack.connect()
def list_vps(self, vps = None):
for server in self.conn.compute.servers():
pass
# #print(server)
# print('Server:')
# print(dir(server))
# print(server.user_data)
# #print(server.to_dict())
def create_vps(self, name, image, flavor, network, opts):
networks = [{'uuid': id} for id in network['private']]
#print(f'Netowrk: {network}')
# Here we create a SSH key pair on the openstack cloud. But that is used for the default ubuntu user... Leave it here for future use
# keypair = self.conn.compute.create_keypair(name=slugify(f'Keys_{name}'))
user_data = None
if 'user_data' in opts:
user_data = opts['user_data']
user_data = base64.b64encode(user_data.encode()).decode('utf8')
server = self.conn.compute.create_server(
name=name, image_id=image, flavor_id=flavor,
networks=networks, user_data=user_data) #key_name=keypair.name,
server = self.conn.compute.wait_for_server(server)
ip = server.access_ipv4
if 'float' in network and len (network['float']) > 0:
floating_ip = self.conn.available_floating_ip(network=network['float'])
ip = floating_ip.floating_ip_address
self.conn.compute.add_floating_ip_to_server(server,ip)
return {
'id' : server.id,
'ipv4' : ip
}
def remove_vps(self, id):
try:
self.conn.compute.delete_server(id)
return True
except Exception as ex:
print('Need to handle remove_vps exception')
print(ex)

80
VRE/lib/cloud/user_data.txt

@ -0,0 +1,80 @@ @@ -0,0 +1,80 @@
#cloud-config
# vim: syntax=yaml
#
# ***********************
# ---- for more examples look at: ------
# ---> https://cloudinit.readthedocs.io/en/latest/topics/examples.html
# ******************************
#
# This is the configuration syntax that the write_files module
# will know how to understand. encoding can be given b64 or gzip or (gz+b64).
# The content will be decoded accordingly and then written to the path that is
# provided.
#
# Note: Content strings here are truncated for example purposes.
users:
- default
- name: P300021
groups: davfs2
shell: /bin/bash
lock_passwd: false
passwd: $6$XHFFmWR/P773$Vq5Qf8uzb0XCsGBwDPQ20e9EsoyriCxffZXK0EBwvkeLaxyrJtJ7i5pANLZ1mNnA6NehHgYa.w2DazsTHrE1T1
# passwd: meismaster
ssh_pwauth: True
chpasswd:
list: |
root:meismaster
expire: False
#power_state:
# delay: "+5"
# mode: poweroff
# message: Bye Bye
# timeout: 30
# condition: True
package_update: true
#package_upgrade: true
packages:
- davfs2
groups:
- davfs2: [root,ubuntu]
runcmd:
- echo "https://unishare.nl/remote.php/dav/files/_8016/ P300021 watXc-Sejif-mD2Mp-BdkZM-JQ783" >> /etc/davfs2/secrets
- echo "https://unishare.nl/remote.php/dav/files/_8016/ /opt/research_data davfs user,rw,auto,uid=P300021,gid=P300021 0 0" >> /etc/fstab
- [mkdir, /opt/research_data]
- [mount, /opt/research_data]
final_message: "The system is finally up, after $UPTIME seconds"
#cloud-config
users:
- default
- name: j.g.rubingh@rug.nl
groups: davfs2
shell: /bin/bash
lock_passwd: false
passwd: $6$8Thtkoi1IWtWUaNC$0.DSfRI8ROE79jt7zlN4vSDNkt9HxdDA.Vbi1UV06Zq1t9wPadhNhTf.uf3w11OdXGljQXWoACusmN7gs2Sb61
ssh_pwauth: True
package_update: true
#package_upgrade: true
packages:
- davfs2
groups:
- davfs2: [root,ubuntu]
runcmd:
- echo "https://unishare.nl/remote.php/dav/files/_8016//onderzoek-naar-iets-met-spaties/03-raw-data P300021 watXc-Sejif-mD2Mp-BdkZM-JQ783" >> /etc/davfs2/secrets
- echo "https://unishare.nl/remote.php/dav/files/_8016//onderzoek-naar-iets-met-spaties/03-raw-data /opt/research_data davfs user,rw,auto,uid=j.g.rubingh@rug.nl,gid=j.g.rubingh@rug.nl 0 0" >> /etc/fstab
- [mkdir, /opt/research_data]
- [mount, /opt/research_data]

20
VRE/lib/models/base.py

@ -0,0 +1,20 @@ @@ -0,0 +1,20 @@
from django.db import models
from django.utils.translation import gettext_lazy as _
class MetaDataModel(models.Model):
"""
This is an abstract Django model with some general meta fields that can be used for other models.
Attributes
----------
created_at : datetime
The date and time when the model has been created. This will be automatically set once during creating.
updated_at : datetime
The date and time when the model has been updated. This will be automatically updated when the model is updated.
"""
created_at = models.DateTimeField(_('Date created'),auto_now_add=True, help_text=_('The date and time this model has been created'))
updated_at = models.DateTimeField(_('Date updated'),auto_now=True, help_text=_('The date and time this model has been updated'))
class Meta:
abstract = True

84
VRE/lib/utils/emails.py

@ -0,0 +1,84 @@ @@ -0,0 +1,84 @@
import os.path
import re
import mimetypes
from email.mime.base import MIMEBase
from django.core.mail import EmailMultiAlternatives, SafeMIMEMultipart
from django.conf import settings
# Source: https://djangosnippets.org/snippets/2215/
class EmailMultiRelated(EmailMultiAlternatives):
"""
A version of EmailMessage that makes it easy to send multipart/related
messages. For example, including text and HTML versions with inline images.
"""
related_subtype = 'related'
def __init__(self, subject='', body='', from_email=None, to=None, bcc=None,
connection=None, attachments=None, headers=None, alternatives=None):
# self.related_ids = []
self.related_attachments = []
super(EmailMultiRelated, self).__init__(subject, body, from_email, to, bcc, connection, attachments, headers, alternatives)
def attach_related(self, filename=None, content=None, mimetype=None):
"""
Attaches a file with the given filename and content. The filename can
be omitted and the mimetype is guessed, if not provided.
If the first parameter is a MIMEBase subclass it is inserted directly
into the resulting message attachments.
"""
if isinstance(filename, MIMEBase):
assert content == mimetype == None
self.related_attachments.append(filename)
else:
assert content is not None
self.related_attachments.append((filename, content, mimetype))
def attach_related_file(self, path, mimetype=None):
"""Attaches a file from the filesystem."""
filename = os.path.basename(path)
content = open(path, 'rb').read()
if mimetype is None:
mimetypes.init()
mimetype = mimetypes.guess_type(filename)[0]
self.attach_related(filename, content, mimetype)
def _create_message(self, msg):
return self._create_attachments(self._create_related_attachments(self._create_alternatives(msg)))
def _create_alternatives(self, msg):
for i, (content, mimetype) in enumerate(self.alternatives):
if mimetype == 'text/html':
for filename, _, _ in self.related_attachments:
content = re.sub(r'(?<!cid:)%s' % re.escape(filename), 'cid:%s' % filename, content)
self.alternatives[i] = (content, mimetype)
return super(EmailMultiRelated, self)._create_alternatives(msg)
def _create_related_attachments(self, msg):
encoding = self.encoding or settings.DEFAULT_CHARSET
if self.related_attachments:
body_msg = msg
msg = SafeMIMEMultipart(_subtype=self.related_subtype, encoding=encoding)
if self.body:
msg.attach(body_msg)
for related in self.related_attachments:
msg.attach(self._create_related_attachment(*related))
return msg
def _create_related_attachment(self, filename, content, mimetype=None):
"""
Convert the filename, content, mimetype triple into a MIME attachment
object. Adjust headers to use Content-ID where applicable.
Taken from http://code.djangostudy.com/ticket/4771
"""
attachment = super(EmailMultiRelated, self)._create_attachment(filename, content, mimetype)
if filename:
mimetype = attachment['Content-Type']
del(attachment['Content-Type'])
del(attachment['Content-Disposition'])
attachment.add_header('Content-Disposition', 'inline', filename=filename)
attachment.add_header('Content-Type', mimetype, name=filename)
attachment.add_header('Content-ID', '<%s>' % filename)
return attachment

26
VRE/lib/utils/general.py

@ -0,0 +1,26 @@ @@ -0,0 +1,26 @@
import re
import random
import string
def remove_html_tags(text):
"""Remove html tags from a string"""
clean = re.compile('<.*?>')
return re.sub(clean, '', text)
def get_random_int_value(length = 6):
return ''.join(list(map(lambda x: str(random.randint(1,9)), list(range(length)))))
def get_random_string(length = 8):
return ''.join(random.choices(string.ascii_uppercase + string.digits + string.ascii_lowercase, k=length))
def generate_encryption_key(length = 32):
return get_random_string(length)
def get_ip_address(request):
""" use requestobject to fetch client machine's IP Address """
x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR', None)
if x_forwarded_for:
ip = x_forwarded_for.split(',')[0]
else:
ip = request.META.get('REMOTE_ADDR') ### Real IP address of client Machine
return ip
Loading…
Cancel
Save