Add missing libs

.gitignore

@@ -19,7 +19,6 @@ dist/
 downloads/
 eggs/
 .eggs/
-lib/
 lib64/
 parts/
 sdist/

VRE/lib/api/base.py

@@ -0,0 +1,51 @@
+from rest_framework import viewsets, permissions, serializers
+from rest_framework.permissions import BasePermission
+
+class IsOwner(BasePermission):
+    def has_object_permission (self, request, view, obj ):
+        """Return 'True' if permission is granted, 'False' otherwise."""
+        # TODO: If this is the 'way to go', we should consider adding the researcher reference to all models and save actions
+        return request.user.is_authenticated and (obj.researcher == request.user.researcher or obj.study.researcher == request.user.researcher)
+
+class BaseReadOnlyViewSet(viewsets.ReadOnlyModelViewSet):
+    permission_classes = [permissions.IsAuthenticated, IsOwner]
+
+    # TODO: If this is the 'way to go', we should consider adding the researcher reference to all models and save actions
+    def get_queryset(self):
+        if getattr(self, 'swagger_fake_view', False):
+            return self.queryset.all()
+
+        try:
+            qs = self.queryset.filter(researcher = self.request.user.researcher)
+        except:
+            qs = self.queryset.filter(study__researcher = self.request.user.researcher)
+
+        return qs
+
+class BaseViewSet(viewsets.ModelViewSet):
+    permission_classes = [permissions.IsAuthenticated, IsOwner]
+
+    # TODO: If this is the 'way to go', we should consider adding the researcher reference to all models and save actions
+    def get_queryset(self):
+        if getattr(self, 'swagger_fake_view', False):
+            return self.queryset.all()
+
+        try:
+            qs = self.queryset.filter(researcher = self.request.user.researcher)
+        except:
+            qs = self.queryset.filter(study__researcher = self.request.user.researcher)
+
+        return qs
+
+class BaseHyperlinkedModelSerializer(serializers.HyperlinkedModelSerializer):
+    # This ID field is handy to have.... Due to HyperlinkedModelSerializer we do not have this field by default
+    id = serializers.ReadOnlyField()
+
+    # Only show the researcher full name
+    researcher = serializers.StringRelatedField()
+
+    # Only show link to full researcher data
+    #researcher = serializers.HyperlinkedRelatedField(view_name= 'api:v1:researcher-detail', read_only=True)
+
+    # Show the full researcher information
+    #researcher = ResearcherSerializer(read_only=True)

VRE/lib/api/client.py

@@ -0,0 +1,242 @@
+import requests
+from requests_hawk import HawkAuth
+from datetime import datetime
+import copy
+import json
+
+from diskcache import Cache
+
+class VRE_API_Exception(Exception):
+    def __init__(self, message, errors):
+        # Call Exception.__init__(message)
+        # to use the same Message header as the parent class
+        super().__init__(message)
+        self.errors = errors
+        # Display the errors
+        print('Printing Errors:')
+        print(errors)
+
+class VRE_API_Exception_Factory(VRE_API_Exception):
+
+    def __new__(self, message, error_code):
+        print(f'VRE_API_Exception_Factory {error_code} -> {message}')
+        if error_code == 400:
+            return VRE_API_400(message)
+        elif error_code == 401:
+            return VRE_API_401(message)
+        elif error_code == 403:
+            return VRE_API_403(message)
+        elif error_code == 404:
+            return VRE_API_404(message)
+
+class VRE_API_400(VRE_API_Exception):
+    def __init__(self, message):
+        # Call the base class constructor with the parameters it needs
+        super().__init__(message, 400)
+
+class VRE_API_401(VRE_API_Exception):
+    def __init__(self, message):
+        # Call the base class constructor with the parameters it needs
+        super().__init__(message, 401)
+
+class VRE_API_403(VRE_API_Exception):
+    def __init__(self, message):
+        # Call the base class constructor with the parameters it needs
+        super().__init__(message, 403)
+
+class VRE_API_404(VRE_API_Exception):
+    def __init__(self, message):
+        # Call the base class constructor with the parameters it needs
+        super().__init__(message, 404)
+
+class VRE_API_Client():
+
+    DATE_TIME_FIELDS = 'created_at,updated_at,mail_sent'.split(',')
+
+    HEADERS = {
+        'Content-Type': 'application/json',
+        'cache-control': 'no-cache'
+    }
+
+    SWAGGER_CACHE_TIMEOUT = 60 * 60 * 24
+
+    def __init__(self, host, url = None, token = None, secret = None):
+        self.authentication = None
+        # Create a disk based caching for getting the swagger data only once an hour
+        self.cache = Cache('/tmp/vre_client/')
+
+        self.host   = host.strip('/') + '/'
+        self.token  = token
+        self.secret = secret
+
+        self.data = {}
+        self.set_token(token)
+        self.set_secret(secret)
+        
+        self.__get_urls_from_swagger_data()
+        self.set_url(url)
+
+    def __get_full_url(self):
+        return f'{self.host}{self.url}'
+
+    def __parse_date_time_fields(self, data):
+        for item in data:
+            # TODO: Should provide better solution for this try/catch. For now it works
+            try:
+                if isinstance(item,list) or isinstance(item,dict):
+                    self.__parse_date_time_fields(item)
+
+                elif isinstance(data[item],list) or isinstance(data[item],dict):
+                    self.__parse_date_time_fields(data[item])
+
+                elif item in self.DATE_TIME_FIELDS and isinstance(data[item],str):
+                    try:
+                        data[item] = datetime.strptime(data[item],'%Y-%m-%dT%H:%M:%S.%fZ')
+                    except Exception:
+                        data[item] = datetime.strptime(data[item][::-1].replace(':','',1)[::-1].replace(' ','T'),'%Y-%m-%dT%H:%M:%S.%f%z')
+            except Exception:
+                pass
+
+    def __parse_data(self, start = None):
+        if len(self.DATE_TIME_FIELDS) > 0:
+            self.__parse_date_time_fields(self.data)
+
+    def __get_urls_from_swagger_data(self):
+        endpoints = self.cache.get('swagger')
+        if endpoints is None:
+            self.set_url('swagger.json')
+            result = requests.get(self.__get_full_url(), headers=self.HEADERS)
+
+            if result.status_code in [200,201]:
+                data = result.json()
+                endpoints = {}
+                for endpoint_url in data['paths']:
+                    for http_action in data['paths'][endpoint_url]:
+                        if http_action not in ['get','post','put','patch','delete']:
+                            continue
+                    
+                        endpoint_id = data['paths'][endpoint_url][http_action]['operationId']
+                        endpoints[endpoint_id] = endpoint_url
+
+                self.cache.set('swagger',endpoints,expire=self.SWAGGER_CACHE_TIMEOUT)
+            else:
+                error_detail = ''
+                try:
+                    error_detail = result.json()['detail']
+                except json.decoder.JSONDecodeError:
+                    if 404 == result.status_code:
+                        error_detail = 'not found'
+
+                raise VRE_API_Exception_Factory(f'Error({result.status_code}) with url {self.url}: {error_detail}',result.status_code)
+
+        
+        self.endpoints = endpoints
+
+        
+
+    def set_url(self, url):
+        if url is None:
+            return
+
+        if url.startswith(':'):
+            url = self.endpoints.get(url[1:])
+            if url is None:
+                return
+
+        if url.startswith('/'):
+            url = url[1:]
+
+        self.url = url
+
+    def set_token(self, token):
+        if token is None:
+            return 
+
+        self.token = token
+        self.authentication = HawkAuth(id=self.token , key=self.secret)
+
+    def set_secret(self, secret):
+        if secret is None:
+            return
+
+        self.secret = secret
+        self.authentication = HawkAuth(id=self.token , key=self.secret)
+
+    def get_data(self):
+        # For get requests, we have to adjust the headers and HAWK authentication
+        # For get requests there is no content, so make sure the 'Content-Type' header is '' (empty)
+        # And that you instruct the HawkAtuh not to hash the content (as there is none)
+        headers = copy.copy(self.HEADERS)
+        headers['Content-Type'] = ''
+        authentication = HawkAuth(id=self.token , key=self.secret, always_hash_content=False)
+
+        result = requests.get(self.__get_full_url(), auth=authentication, headers=headers)
+
+        if result.status_code in [200,201]:
+            self.data = result.json()
+            self.__parse_data()
+        else:
+            error_detail = ''
+            try:
+                error_detail = result.json()['detail']
+            except json.decoder.JSONDecodeError:
+                if 404 == result.status_code:
+                    error_detail = 'not found'
+            raise VRE_API_Exception_Factory(f'Error with url {self.url}: {error_detail}',result.status_code)
+
+        return self.data
+
+    def post_data(self, payload):
+        result = requests.post(self.__get_full_url(), json=payload, auth=self.authentication, headers=self.HEADERS)
+
+        self.data['status_code'] = result.status_code
+
+        if result.status_code in [200,201]:
+            self.data = result.json()
+            self.__parse_data()
+        else:
+            error_detail = ''
+            try:
+                error_detail = result.json()['detail']
+            except json.decoder.JSONDecodeError:
+                if 404 == result.status_code:
+                    error_detail = 'not found'
+            raise VRE_API_Exception_Factory(f'Error with url {self.url}: {error_detail}',result.status_code)
+
+        return self.data
+
+    def put_data(self, payload):
+        result = requests.put(self.__get_full_url(), json=payload, auth=self.authentication, headers=self.HEADERS)
+        self.data['status_code'] = result.status_code
+
+        if result.status_code in [200,201]:
+            self.data = result.json()
+            self.__parse_data()
+        else:
+            error_detail = ''
+            try:
+                error_detail = result.json()['detail']
+            except json.decoder.JSONDecodeError:
+                if 404 == result.status_code:
+                    error_detail = 'not found'
+            raise VRE_API_Exception_Factory(f'Error with url {self.url}: {error_detail}',result.status_code)
+
+        return self.data
+
+    def delete_data(self):
+        try:
+            # Django HAWK has issues with a delete action. It needs/wants a content-type header, but there is no content.....
+            # https://github.com/kumar303/hawkrest/issues/46
+            result = requests.delete(self.__get_full_url(), json={}, auth=self.authentication, headers=self.HEADERS)
+            return result.status_code in [200,201,204]
+        except Exception as ex:
+            raise VRE_API_Exception_Factory(f'Error with url {self.url}: {ex}',500)
+
+        return False
+
+if __name__ == "__main__":
+    client = VRE_API_Client('http://localhost:8000/api/')
+    client.set_url(':auth_login_create')
+    
+    access_token = client.post_data({'username' : 'j.g.rubingh@rug.nl', 'password' : 'jos24kin'})
+    print(access_token)

VRE/lib/cloud/openstack_client.py

@@ -0,0 +1,72 @@
+# https://docs.openstack.org/openstacksdk/latest/user/index.html#user-guides
+# https://docs.openstack.org/openstacksdk/latest/user/proxies/compute.html
+
+import openstack
+import base64
+
+from Crypto.PublicKey import RSA
+
+from django.template.defaultfilters import slugify
+
+class VRE_OpenStackClient():
+
+    def __init__(self, cloud = None, debug = False):
+        self.cloud = cloud
+        self.debug = debug
+        if self.debug:
+            openstack.enable_logging(debug=True)
+
+        self.connect()
+    
+    def connect(self):
+        # Initialize and turn on debug logging
+        openstack.enable_logging(debug=self.debug)
+        self.conn = openstack.connect()
+
+    def list_vps(self, vps = None):
+        for server in self.conn.compute.servers():
+            pass
+
+            # #print(server)
+            # print('Server:')
+            # print(dir(server))
+            # print(server.user_data)
+            # #print(server.to_dict())
+
+    def create_vps(self, name, image, flavor, network, opts):
+        networks = [{'uuid': id} for id in network['private']]
+
+        #print(f'Netowrk: {network}')
+
+        # Here we create a SSH key pair on the openstack cloud. But that is used for the default ubuntu user... Leave it here for future use
+        # keypair = self.conn.compute.create_keypair(name=slugify(f'Keys_{name}'))
+
+        user_data = None
+        if 'user_data' in opts:
+            user_data = opts['user_data']
+            user_data = base64.b64encode(user_data.encode()).decode('utf8')
+
+        server = self.conn.compute.create_server(
+            name=name, image_id=image, flavor_id=flavor,
+            networks=networks, user_data=user_data) #key_name=keypair.name, 
+
+        server = self.conn.compute.wait_for_server(server)
+        ip = server.access_ipv4
+
+        if 'float' in network and len (network['float']) > 0:
+            floating_ip = self.conn.available_floating_ip(network=network['float'])
+            ip = floating_ip.floating_ip_address
+            self.conn.compute.add_floating_ip_to_server(server,ip)
+
+        return {
+            'id'   : server.id,
+            'ipv4' : ip
+        }
+
+    def remove_vps(self, id):
+        try:
+            self.conn.compute.delete_server(id)
+            return True
+        except Exception as ex:
+            print('Need to handle remove_vps exception')
+            print(ex)

VRE/lib/cloud/user_data.txt

@@ -0,0 +1,80 @@
+#cloud-config
+# vim: syntax=yaml
+#
+# ***********************
+# 	---- for more examples look at: ------
+# ---> https://cloudinit.readthedocs.io/en/latest/topics/examples.html
+# ******************************
+#
+# This is the configuration syntax that the write_files module
+# will know how to understand. encoding can be given b64 or gzip or (gz+b64).
+# The content will be decoded accordingly and then written to the path that is
+# provided.
+#
+# Note: Content strings here are truncated for example purposes.
+
+users:
+  - default
+  - name: P300021
+    groups: davfs2
+    shell: /bin/bash
+    lock_passwd: false
+    passwd: $6$XHFFmWR/P773$Vq5Qf8uzb0XCsGBwDPQ20e9EsoyriCxffZXK0EBwvkeLaxyrJtJ7i5pANLZ1mNnA6NehHgYa.w2DazsTHrE1T1
+#    passwd: meismaster
+
+ssh_pwauth: True
+
+chpasswd:
+  list: |
+     root:meismaster
+  expire: False
+
+#power_state:
+#  delay: "+5"
+#  mode: poweroff
+#  message: Bye Bye
+#  timeout: 30
+#  condition: True
+
+package_update: true
+#package_upgrade: true
+packages:
+  - davfs2
+
+groups:
+  - davfs2: [root,ubuntu]
+
+runcmd:
+  - echo "https://unishare.nl/remote.php/dav/files/_8016/ P300021 watXc-Sejif-mD2Mp-BdkZM-JQ783" >> /etc/davfs2/secrets
+  - echo "https://unishare.nl/remote.php/dav/files/_8016/ /opt/research_data davfs user,rw,auto,uid=P300021,gid=P300021 0 0" >> /etc/fstab
+  - [mkdir, /opt/research_data]
+  - [mount, /opt/research_data]
+
+final_message: "The system is finally up, after $UPTIME seconds"
+
+
+
+#cloud-config
+users:
+  - default
+  - name: j.g.rubingh@rug.nl
+    groups: davfs2
+    shell: /bin/bash
+    lock_passwd: false
+    passwd: $6$8Thtkoi1IWtWUaNC$0.DSfRI8ROE79jt7zlN4vSDNkt9HxdDA.Vbi1UV06Zq1t9wPadhNhTf.uf3w11OdXGljQXWoACusmN7gs2Sb61
+
+ssh_pwauth: True
+
+package_update: true
+#package_upgrade: true
+packages:
+  - davfs2
+
+groups:
+  - davfs2: [root,ubuntu]
+
+runcmd:
+  - echo "https://unishare.nl/remote.php/dav/files/_8016//onderzoek-naar-iets-met-spaties/03-raw-data P300021 watXc-Sejif-mD2Mp-BdkZM-JQ783" >> /etc/davfs2/secrets
+  - echo "https://unishare.nl/remote.php/dav/files/_8016//onderzoek-naar-iets-met-spaties/03-raw-data /opt/research_data davfs user,rw,auto,uid=j.g.rubingh@rug.nl,gid=j.g.rubingh@rug.nl 0 0" >> /etc/fstab
+  - [mkdir, /opt/research_data]
+  - [mount, /opt/research_data]

VRE/lib/models/base.py

@@ -0,0 +1,20 @@
+
+from django.db import models
+from django.utils.translation import gettext_lazy as _
+
+class MetaDataModel(models.Model):
+    """
+    This is an abstract Django model with some general meta fields that can be used for other models.
+
+    Attributes
+    ----------
+    created_at : datetime
+        The date and time when the model has been created. This will be automatically set once during creating.
+    updated_at : datetime
+        The date and time when the model has been updated. This will be automatically updated when the model is updated.
+    """
+    created_at = models.DateTimeField(_('Date created'),auto_now_add=True, help_text=_('The date and time this model has been created'))
+    updated_at = models.DateTimeField(_('Date updated'),auto_now=True, help_text=_('The date and time this model has been updated'))
+
+    class Meta:
+        abstract = True

VRE/lib/utils/emails.py

@@ -0,0 +1,84 @@
+import os.path
+import re
+import mimetypes
+
+from email.mime.base import MIMEBase
+from django.core.mail import EmailMultiAlternatives, SafeMIMEMultipart
+from django.conf import settings
+
+# Source: https://djangosnippets.org/snippets/2215/
+class EmailMultiRelated(EmailMultiAlternatives):
+    """
+    A version of EmailMessage that makes it easy to send multipart/related
+    messages. For example, including text and HTML versions with inline images.
+    """
+    related_subtype = 'related'
+
+    def __init__(self, subject='', body='', from_email=None, to=None, bcc=None,
+            connection=None, attachments=None, headers=None, alternatives=None):
+        # self.related_ids = []
+        self.related_attachments = []
+        super(EmailMultiRelated, self).__init__(subject, body, from_email, to, bcc, connection, attachments, headers, alternatives)
+
+    def attach_related(self, filename=None, content=None, mimetype=None):
+        """
+        Attaches a file with the given filename and content. The filename can
+        be omitted and the mimetype is guessed, if not provided.
+
+        If the first parameter is a MIMEBase subclass it is inserted directly
+        into the resulting message attachments.
+        """
+        if isinstance(filename, MIMEBase):
+            assert content == mimetype == None
+            self.related_attachments.append(filename)
+        else:
+            assert content is not None
+            self.related_attachments.append((filename, content, mimetype))
+
+    def attach_related_file(self, path, mimetype=None):
+        """Attaches a file from the filesystem."""
+        filename = os.path.basename(path)
+        content = open(path, 'rb').read()
+        if mimetype is None:
+            mimetypes.init()
+            mimetype = mimetypes.guess_type(filename)[0]
+        self.attach_related(filename, content, mimetype)
+
+    def _create_message(self, msg):
+        return self._create_attachments(self._create_related_attachments(self._create_alternatives(msg)))
+
+    def _create_alternatives(self, msg):
+        for i, (content, mimetype) in enumerate(self.alternatives):
+            if mimetype == 'text/html':
+                for filename, _, _ in self.related_attachments:
+                    content = re.sub(r'(?<!cid:)%s' % re.escape(filename), 'cid:%s' % filename, content)
+                self.alternatives[i] = (content, mimetype)
+
+        return super(EmailMultiRelated, self)._create_alternatives(msg)
+
+    def _create_related_attachments(self, msg):
+        encoding = self.encoding or settings.DEFAULT_CHARSET
+        if self.related_attachments:
+            body_msg = msg
+            msg = SafeMIMEMultipart(_subtype=self.related_subtype, encoding=encoding)
+            if self.body:
+                msg.attach(body_msg)
+            for related in self.related_attachments:
+                msg.attach(self._create_related_attachment(*related))
+        return msg
+
+    def _create_related_attachment(self, filename, content, mimetype=None):
+        """
+        Convert the filename, content, mimetype triple into a MIME attachment
+        object. Adjust headers to use Content-ID where applicable.
+        Taken from http://code.djangostudy.com/ticket/4771
+        """
+        attachment = super(EmailMultiRelated, self)._create_attachment(filename, content, mimetype)
+        if filename:
+            mimetype = attachment['Content-Type']
+            del(attachment['Content-Type'])
+            del(attachment['Content-Disposition'])
+            attachment.add_header('Content-Disposition', 'inline', filename=filename)
+            attachment.add_header('Content-Type', mimetype, name=filename)
+            attachment.add_header('Content-ID', '<%s>' % filename)
+        return attachment

VRE/lib/utils/general.py

@@ -0,0 +1,26 @@
+import re
+import random
+import string
+
+def remove_html_tags(text):
+    """Remove html tags from a string"""
+    clean = re.compile('<.*?>')
+    return re.sub(clean, '', text)
+
+def get_random_int_value(length = 6):
+    return ''.join(list(map(lambda x: str(random.randint(1,9)), list(range(length)))))
+
+def get_random_string(length = 8):
+    return ''.join(random.choices(string.ascii_uppercase + string.digits + string.ascii_lowercase, k=length))
+
+def generate_encryption_key(length = 32):
+    return get_random_string(length)
+
+def get_ip_address(request):
+    """ use requestobject to fetch client machine's IP Address """
+    x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR', None)
+    if x_forwarded_for:
+        ip = x_forwarded_for.split(',')[0]
+    else:
+        ip = request.META.get('REMOTE_ADDR')    ### Real IP address of client Machine
+    return ip