Browse Source

Add researcher and study import

master
Joshua Rubingh 6 months ago
parent
commit
647fee80c1
  1. 5
      VRE/apps/researcher/admin.py
  2. 74
      VRE/apps/researcher/management/commands/import_researchers.py
  3. 7
      VRE/apps/study/admin.py
  4. 86
      VRE/apps/study/management/commands/import_studies.py
  5. 46
      VRE/lib/models/admin.py
  6. 39
      VRE/templates/admin/import_change_list.html
  7. 21
      VRE/templates/admin/json_form.html

5
VRE/apps/researcher/admin.py

@ -1,14 +1,17 @@ @@ -1,14 +1,17 @@
from django.contrib import admin
from .models import Researcher
from lib.models.admin import JSONImportMixin
@admin.register(Researcher)
class ResearcherAdmin(admin.ModelAdmin):
class ResearcherAdmin(JSONImportMixin, admin.ModelAdmin):
list_display = ('user', 'last_name', 'faculty')
ordering = ('user', )
search_fields = ('user__username', 'user__first_name', 'user__last_name')
readonly_fields = ('created_at', 'updated_at', 'last_name')
change_list_template = 'admin/import_change_list.html'
import_type = 'researcher'
def last_name(self, obj):
return obj.user.last_name

74
VRE/apps/researcher/management/commands/import_researchers.py

@ -0,0 +1,74 @@ @@ -0,0 +1,74 @@
from django.core.management.base import BaseCommand, CommandError, no_translations
from django.contrib.auth.models import User
import json
from pathlib import Path
class Command(BaseCommand):
help = '''Import researchers from external data source
Valid JSON schema is:
{
'email_address' :'',
'first_name': '',
'last_name' : '',
'mobile': '',
'pmunber' :''
}
'''
def add_arguments(self, parser):
parser.add_argument('import_file', help='JSON Import file with researchers')
@no_translations
def handle(self, *args, **options):
if options.get('import_file'):
import_file = Path(options.get("import_file"))
if not import_file.exists():
raise CommandError('Import file "%s" does not exists' % (import_file,))
try:
researchers_list = json.loads(import_file.read_text())
except json.JSONDecodeError:
raise CommandError('Could not read file "%s" data is not a valid JSON file' % (import_file,))
self.stdout.write(f'Start importing file: {import_file}')
import_counter = 0
for researcher_data in researchers_list:
if 'email_address' not in researcher_data or 'first_name' not in researcher_data or 'last_name' not in researcher_data:
self.stdout.write(self.style.ERROR('Skip import line due to wrong data...'))
continue
user, created = User.objects.get_or_create(email=researcher_data['email_address'],
defaults={
'username': researcher_data['email_address'],
'first_name': researcher_data['first_name'],
'last_name': researcher_data['last_name'],
'is_active': True}
)
if created:
user.set_unusable_password()
user.save()
update_researcher = False
if '' == user.researcher.mobilephone and 'mobile' in researcher_data and '' != researcher_data['mobile']:
user.researcher.mobilephone = researcher_data['mobile']
update_researcher = True
if '' == user.researcher.idnumber and 'pnumber' in researcher_data and '' != researcher_data['pnumber']:
user.researcher.idnumber = researcher_data['pnumber']
update_researcher = True
if update_researcher:
user.researcher.save()
import_counter += 1
self.stdout.write(self.style.SUCCESS(f'Researcher {user.researcher.display_name} {"is created" if created else "does already exists"}'))
self.stdout.write(f'Done importing file: {import_file}, {import_counter} entries ok!')

7
VRE/apps/study/admin.py

@ -2,6 +2,8 @@ from django.contrib import admin @@ -2,6 +2,8 @@ from django.contrib import admin
from django.template.defaultfilters import filesizeformat
from .models import Study, StudyRole
from lib.models.admin import JSONImportMixin
# Register your models here.
admin.site.register(StudyRole)
@ -13,7 +15,7 @@ class role_inline(admin.TabularInline): @@ -13,7 +15,7 @@ class role_inline(admin.TabularInline):
@admin.register(Study)
class StudyAdmin(admin.ModelAdmin):
class StudyAdmin(JSONImportMixin, admin.ModelAdmin):
inlines = (role_inline,)
list_display = ('name', 'total_files', 'total_file_size', 'total_invitations', 'created_at')
@ -21,5 +23,8 @@ class StudyAdmin(admin.ModelAdmin): @@ -21,5 +23,8 @@ class StudyAdmin(admin.ModelAdmin):
search_fields = ('name', )
readonly_fields = ('upload_uuid', 'api_upload_url', 'total_files', 'total_file_size', 'total_invitations', 'created_at', 'updated_at')
change_list_template = 'admin/import_change_list.html'
import_type = 'study'
def total_file_size(self, obj):
return filesizeformat(obj.total_file_size)

86
VRE/apps/study/management/commands/import_studies.py

@ -0,0 +1,86 @@ @@ -0,0 +1,86 @@
from django.core.management.base import BaseCommand, CommandError, no_translations
from django.contrib.auth.models import User
from apps.researcher.models import Researcher
from apps.study.models import Study, StudyRole
import json
from pathlib import Path
class Command(BaseCommand):
help = '''Import studies from external data source
Valid JSON schema is:
{
'email_address' :'',
'first_name': '',
'last_name' : '',
'mobile': '',
'pmunber' :''
}
'''
def add_arguments(self, parser):
parser.add_argument('import_file', help='JSON Import file with researchers')
@no_translations
def handle(self, *args, **options):
if options.get('import_file'):
import_file = Path(options.get("import_file"))
if not import_file.exists():
raise CommandError('Import file "%s" does not exists' % (import_file,))
self.stdout.write(f'Start importing file: {import_file}')
try:
studies_list = json.loads(import_file.read_text())
except json.JSONDecodeError:
raise CommandError('Could not read file "%s" data is not a valid JSON file' % (import_file,))
import_counter = 0
for study_data in studies_list:
if 'name' not in study_data or 'description' not in study_data or 'code' not in study_data or 'contributors' not in study_data:
self.stdout.write(self.style.ERROR('Skip import line due to wrong data...'))
continue
# Determen the owner.... For now, the first admin role will do
Owner = None
for role in study_data['contributors']:
if 'Administrator' == role['role']:
Owner = User.objects.get(email=role['uid']).researcher
if Owner is None:
self.stdout.write(self.style.ERROR(f'Study {study.name} cannot be created as there is no owner defined (Administrator)'))
continue
# TODO: Or do we match on study code field??? Not sure if that data is available and correct in the LDAP
study, created = Study.objects.get_or_create(name=study_data['name'],
defaults={
'description': study_data['description'],
'code': study_data['code'],
'human_subject': False,
'field_id': 1,
'owner_id': Owner.pk}
)
# Add the contributors to the study
for role in study_data['contributors']:
try:
researcher = Researcher.objects.get(user__email=role['uid'])
except Researcher.DoesNotExist:
# The researcher is not imported first, so we have to ignore the role here
continue
role, created = StudyRole.objects.get_or_create(study=study, researcher=researcher,
defaults={
'role': role['role'].upper(),
'active': True
})
# Create the workspace also here (vre_app)
import_counter += 1
self.stdout.write(self.style.SUCCESS(f'Study {study.name} {"is created" if created else "does already exists"}'))
self.stdout.write(f'Done importing file: {import_file}, {import_counter} entries')

46
VRE/lib/models/admin.py

@ -1,6 +1,13 @@ @@ -1,6 +1,13 @@
from django import forms
from django.contrib.contenttypes.models import ContentType
from django.core.management import call_command
from django.shortcuts import redirect, render
from django.utils.safestring import mark_safe
from django.urls import path
from django.utils.translation import gettext_lazy as _
from pathlib import Path
class VMRelatedSelectWidget(forms.Select):
@ -58,3 +65,42 @@ class VMRelatedSelectWidget(forms.Select): @@ -58,3 +65,42 @@ class VMRelatedSelectWidget(forms.Select):
html = super().render(name, value, attrs, renderer) + '<script>' + javascript_data + '</script>'
return mark_safe(html)
class JSONImportForm(forms.Form):
json_file = forms.FileField( label='JSON Import file', allow_empty_file=False, required=True, help_text=_('Select a valid JSON file for uploading'))
class JSONImportMixin():
def get_urls(self):
urls = super().get_urls()
my_urls = [
path('import-json/', self.import_json, name=f'{self.import_type}_import_json'),
]
return my_urls + urls
def import_json(self, request):
if request.method == "POST":
import_file = '/tmp/' + 'study' if 'study' == self.import_type else 'researcher' + '.upload.json'
import_file = Path(import_file)
if import_file.exists():
import_file.unlink()
with request.FILES["json_file"].open('r') as source:
with open(import_file, mode='ab') as destination:
for line in source:
destination.write(line)
import_cmd = 'import_studies' if 'study' == self.import_type else 'import_researchers'
call_command(import_cmd, import_file)
import_file.unlink()
self.message_user(request, 'Your JSON file has been imported. Check if the results are ok below.')
return redirect("..")
form = JSONImportForm()
payload = {"form": form}
return render(
request, "admin/json_form.html", payload
)

39
VRE/templates/admin/import_change_list.html

@ -0,0 +1,39 @@ @@ -0,0 +1,39 @@
{% extends 'admin/change_list.html' %}
{% load i18n %}
{% block object-tools %}
{{ block.super }}
<style>
div#admin_json_import_popup {
position: absolute;
top: 150px;
right: 50px;
border: solid 1px black;
width: 600px;
background-color: white;
}
</style>
<script>
function showForm(url) {
window.django.jQuery.get(url.target.href, function (data, ok) {
let popup_div = window.django.jQuery('div#admin_json_import_popup');
if (popup_div.length == 0) {
popup_div = window.django.jQuery('<div>').attr('id', 'admin_json_import_popup').css('display', 'none');
window.django.jQuery('div#content').append(popup_div);
}
data = data.replace('<div id="container">', '<div id="container" style="display: inline">')
popup_div.html(data).show();
return false;
});
return false;
}
window.django.jQuery(function () {
let import_link = window.django.jQuery('<a>').attr('href', 'import-json/').addClass('addlink').text('{% translate "Import JSON" %}').on('click', showForm);
window.django.jQuery('#content-main ul.object-tools').append(
window.django.jQuery('<li>').append(import_link)
);
});
</script>
{% endblock %}

21
VRE/templates/admin/json_form.html

@ -0,0 +1,21 @@ @@ -0,0 +1,21 @@
{% extends "admin/base_site.html" %}
{% load i18n admin_urls static admin_modify %}
{% block content %}
<h2>JSON Import</h2>
<form action="import-json/" method="POST" enctype="multipart/form-data">
{% csrf_token %}
<div>
<fieldset class="module aligned ">
<div class="form-row">
{{ form.json_file.label_tag }}
{{ form.json_file }}
<div class="help">{{ form.json_file.help_text }}</div>
</div>
</fieldset>
<div class="submit-row">
<input type="submit" value="Upload JSON File" class="default" name="_save">
</div>
</div>
</form>
{% endblock %}
Loading…
Cancel
Save