Skip to content
Snippets Groups Projects
Commit 7a4a29bd authored by Jan Maximilian Michal's avatar Jan Maximilian Michal
Browse files

Added converter script, made population script more reliable

parent e07e9e13
Branches
Tags
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 408 additions and 108 deletions
.gitignore
+ 1
− 0
View file @ 7a4a29bd
......@@ -21,3 +21,4 @@ env-grady/
.DS_Store
reinit.sh
data/
raw/
convert.py 0 → 100755
+ 125
− 0
View file @ 7a4a29bd
#!/usr/local/bin/python3
""" a simple script that converts ilias exam output to readable json
The json output will look like this:
{
"AF12345678": { <<--- all uppercase letter of the name + username/matrikel_no
"matrikel_no": "12345678",
"name": "Mustermann, Max",
"task_list": {
"[task_id_1]": "print Hello World!",
....,
"[task_id_n]": "#include <stdio.h> etc."
}
},
... ans so on
}
usage: convert.py [-h] [-n NUMBER_OF_TASKS] INFILE OUTFILE
positional arguments:
INFILE Ilias exam data
OUTFILE Where to write the final file
optional arguments:
-h, --help show this help message and exit
-n NUMBER_OF_TASKS, --NUMBER_OF_TASKS NUMBER_OF_TASKS
Where to write the final file
Author: Jan Maximilian Michal
Date: 30 March 2017
"""
import json
import os
import re
import argparse
import urllib.parse
from collections import namedtuple
from xlrd import open_workbook
parser = argparse.ArgumentParser()
parser.add_argument('INFILE', help='Ilias exam data')
parser.add_argument('OUTFILE', help='Where to write the final file')
parser.add_argument(
'-n', '--NUMBER_OF_TASKS',
default=0, # don't check
metavar='NUMBER_OF_TASKS',
type=int,
help='Where to write the final file')
args = parser.parse_args()
# meta sheet contains ilias evaluation names usernames etc - data contains code
meta, *data = open_workbook(args.INFILE, open(os.devnull, 'w')).sheets()
# one user has one submission (code) per task
# yes, I know it is possible to name match groups via (?P<name>) but
# I like this solution better since it gets the job done nicely
user_head = namedtuple('user_head', 'kohorte, name')
user_head_re = re.compile(r'^Ergebnisse von Testdurchlauf (?P<kohorte>\d+) für (?P<name>[\w\s\.,-]+)$')
# one task has a title and id and hpfly code
task_head = namedtuple('task_head', 'id, title')
task_head_re = re.compile(r'^Quellcode Frage\[(?P<id>[a-z]\d{2})\] (?P<title>.*) \d{8}$')
# nor parsing the weird mat no
matno_re = re.compile(r'^(?P<matrikel_no>\d{8})-(\d{3})-(\d{3})$')
# Modify these iterators in order to change extraction behaviour
def sheet_iter_meta(sheet):
""" yield first and second col entry as tuple of (name, matnr) """
for row in (sheet.row(i) for i in range(1, sheet.nrows)):
m = re.search(matno_re, row[1].value)
yield row[0].value, m.group('matrikel_no') if m else row[1].value
def sheet_iter_data(sheet):
""" yields all rows that are not of empty type as one string """
for row in (sheet.row(i) for i in range(sheet.nrows)):
if any(map(lambda c: c.ctype, row)):
yield row[0].value + row[1].value
# nice!
name2mat = dict(sheet_iter_meta(meta))
# from xls to lists and namedtuples
# [ [user0, task0_h, code0, ..., taskn, coden ], ..., [...] ]
root = []
for sheet in data:
for row in sheet_iter_data(sheet):
user = re.search(user_head_re, row)
task = re.search(task_head_re, row)
if user:
root.append([user_head(*user.groups())])
elif task:
root[-1].append(task_head(*task.groups()))
else: # should be code
root[-1].append(urllib.parse.unquote(row).strip())
if args.NUMBER_OF_TASKS:
for (user, *task_list) in sorted(root, key=lambda u: u[0].name):
assert len(task_list) == args.NUMBER_OF_TASKS * 2
# form list to json_like via comprehension
# the format {userinitials + matrikel_no : {name:, matrikel_no:, tasklist: {id:, ..., id:}}}
json_dict = {
''.join(filter(str.isupper, user.name)) + name2mat[user.name] : {
'name' : user.name,
'matrikel_no' : name2mat[user.name],
'submissions' : {
f"[{task.id}] {task.title}" : code
for task, code in zip(task_list[::2], task_list[1::2])
}
} for (user, *task_list) in sorted(root, key=lambda u: u[0].name)
}
# just encode python style
with open(args.OUTFILE, "w") as out:
out.write(json.JSONEncoder().encode(json_dict))
print(f"Wrote data to {args.OUTFILE}. Done.")
core/migrations/0004_auto_20170331_1233.py 0 → 100644
+ 20
− 0
View file @ 7a4a29bd
# -*- coding: utf-8 -*-
# Generated by Django 1.10.6 on 2017-03-31 12:33
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0003_auto_20170330_1709'),
]
operations = [
migrations.AlterField(
model_name='submissiontype',
name='name',
field=models.CharField(max_length=50, unique=True),
),
]
core/migrations/0005_student_name.py 0 → 100644
+ 20
− 0
View file @ 7a4a29bd
# -*- coding: utf-8 -*-
# Generated by Django 1.10.6 on 2017-03-31 12:40
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0004_auto_20170331_1233'),
]
operations = [
migrations.AddField(
model_name='student',
name='name',
field=models.CharField(default='__no_name__', max_length=50),
),
]
core/migrations/0006_auto_20170331_1246.py 0 → 100644
+ 21
− 0
View file @ 7a4a29bd
# -*- coding: utf-8 -*-
# Generated by Django 1.10.6 on 2017-03-31 12:46
from __future__ import unicode_literals
import core.models
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0005_student_name'),
]
operations = [
migrations.AlterField(
model_name='student',
name='matrikel_no',
field=models.PositiveIntegerField(default=core.models.random_matrikel_no, unique=True),
),
]
core/migrations/0007_auto_20170331_1249.py 0 → 100644
+ 21
− 0
View file @ 7a4a29bd
# -*- coding: utf-8 -*-
# Generated by Django 1.10.6 on 2017-03-31 12:49
from __future__ import unicode_literals
import core.models
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('core', '0006_auto_20170331_1246'),
]
operations = [
migrations.AlterField(
model_name='student',
name='matrikel_no',
field=models.CharField(default=core.models.random_matrikel_no, max_length=8, unique=True),
),
]
core/models.py
+ 8
− 5
View file @ 7a4a29bd
from random import sample
from random import sample, randrange
from string import ascii_lowercase
from django.contrib.auth.models import User
......@@ -12,11 +12,13 @@ SLUG_LENGTH = 16
def random_slug():
return ''.join(sample(ascii_lowercase, SLUG_LENGTH))
def random_matrikel_no():
return str(2e7 + randrange(1e8))
class SubmissionType(models.Model):
# Fields
name = models.CharField(max_length=50)
name = models.CharField(max_length=50, unique=True)
slug = models.SlugField(editable=False, unique=True, default=random_slug)
full_score = models.PositiveIntegerField(default=0)
task_description = models.TextField()
......@@ -34,7 +36,8 @@ class SubmissionType(models.Model):
class Student(models.Model):
# Fields
matrikel_no = models.PositiveIntegerField(unique=True, default=0)
matrikel_no = models.CharField(unique=True, max_length=8, default=random_matrikel_no)
name = models.CharField(max_length=50, default="__no_name__")
user = models.OneToOneField(
User, on_delete=models.CASCADE,
limit_choices_to={'groups__name': 'Students'},
......@@ -45,7 +48,7 @@ class Student(models.Model):
verbose_name_plural = "Student Set"
def __str__(self):
return "{} ({})".format(self.user.username, self.matrikel_no)
return self.user.username
class Submission(models.Model):
......@@ -62,7 +65,7 @@ class Submission(models.Model):
text = models.TextField(blank=True)
pre_corrections = models.TextField(blank=True)
final_feedback = models.OneToOneField('Feedback', null=True, blank=True)
student = models.ForeignKey(Student)
student = models.ForeignKey(Student, on_delete=models.CASCADE)
class Meta:
verbose_name = "Submission"
......
core/templates/core/reviewer_startpage.html
+ 94
− 44
View file @ 7a4a29bd
......@@ -7,7 +7,7 @@
{% block body_block %}
<div class="row my-3">
<div class="col-4">
<div class="col-4 nopadding-right">
{# This is a control panel where new work can be requested #}
<div class="card">
......@@ -21,7 +21,7 @@
{% for submission_type in submission_type_list %}
<tr>
<td class="align-middle">{{ submission_type }}</td>
<td class="align-middle"><code>{{ submission_type.submissions__feedback_list__count }} / {{submission_type.submissions__count}}</code></td>
<td class="align-middle"><code>{{ submission_type.feedback_count }} / {{submission_type.submission_count}}</code></td>
</tr>
{% endfor %}
</tbody>
......@@ -53,56 +53,106 @@
</div>
<div class="col-8">
<div class="row page-header">
<div class="row-auto page-header">
{% if feedback_list|length == 0 %}
<h2>Nobody has provided any feedback, yet. Maybe you have to kick some ass?</h2>
{% else %}
{% if feedback_list|length == 1 %}
<h2>Nobody has provided any feedback, yet. <br>Maybe you have to kick some ass?</h2>
{% elif feedback_list|length == 1 %}
<h2>Well, one is better than nothing</h2>
{% else %}
<h2>So far {{feedback_list|length}} contributions were provided</h2>
{% endif %}
</div>
<div class="row my-2">
<table class="table">
<thead>
<tr>
<th></th>
<th>Submission Type</th>
<th>Student</th>
<th>Score</th>
<th>Tutor</th>
<th></th>
</tr>
</thead>
<tbody>
{% for feedback in feedback_list %}
<tr>
<td class="align-middle"> {% if feedback.final %}
<span class="badge badge-success">Final</span>
{% endif %}</td>
<td class="align-middle"> {{ feedback.of_submission.type }} </td>
<td class="align-middle"> {{ feedback.of_submission.student }} </td>
<td class="align-middle"> <code> {{ feedback.score }} / {{ feedback.of_submission.type.full_score }} </code> </td>
<td class="align-middle"> {{ feedback.of_tutor}} </td>
<td class="align-middle">
<a href="{% url 'FeedbackEdit' feedback.slug %}" class="btn btn-outline-primary mb-1" name="edit" value="View">View</a>
<a href="{% url 'FeedbackDelete' feedback.slug %}" class="btn btn-outline-danger mb-1" name="delete" value="Delete">Delete</a>
{% if not feedback.final %}
<a class="btn btn-outline-success mb-1" href="{% url 'FeedbackMarkFinal' feedback.slug %}"> Mark final </a>
{% else %}
<a class="btn btn-outline-secondary mb-1" href="{% url 'FeedbackMarkNotFinal' feedback.slug %}"> Undo </a>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
<div class="row-auto my-1">
<div class="card">
<h5 class="card-header">This feedback was hard work</h5>
<div class="card-block m-2">
<table class="table">
<thead>
<tr>
<th></th>
<th>Submission Type</th>
<th>Student</th>
<th>Score</th>
<th>Tutor</th>
<th></th>
</tr>
</thead>
<tbody>
{% for feedback in feedback_list %}
<tr>
<td class="align-middle"> {% if feedback.final %}
<span class="badge badge-success">Final</span>
{% endif %}
</td>
<td class="align-middle"> {{ feedback.of_submission.type }} </td>
<td class="align-middle"> {{ feedback.of_submission.student }} </td>
<td class="align-middle"> <code> {{ feedback.score }} / {{ feedback.of_submission.type.full_score }} </code> </td>
<td class="align-middle"> {{ feedback.of_tutor}} </td>
<td class="align-middle">
<a href="{% url 'FeedbackEdit' feedback.slug %}" class="btn btn-outline-primary mb-1" name="edit" value="View">View</a>
<a href="{% url 'FeedbackDelete' feedback.slug %}" class="btn btn-outline-danger mb-1" name="delete" value="Delete">Delete</a>
{% if not feedback.final %}
<a class="btn btn-outline-success mb-1" href="{% url 'FeedbackMarkFinal' feedback.slug %}"> Mark final </a>
{% else %}
<a class="btn btn-outline-secondary mb-1" href="{% url 'FeedbackMarkNotFinal' feedback.slug %}"> Undo </a>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
{% endif %}
{# This is card for empty feedback for the sake of completeness #}
<div class="row-auto my-1">
<div class="card">
<a data-toggle="collapse" href="#collapseEmpty">
<h5 class="card-header">Empty feedback</h5>
</a>
<div id="collapseEmpty" class="collapse hide" role="tabpanel">
<div class="card-block m-2">
<table class="table">
<thead>
<tr>
<th></th>
<th>Submission Type</th>
<th>Student</th>
<th>Score</th>
<th>Auto feedback</th>
<th></th>
</tr>
</thead>
<tbody>
{% for feedback in feedback_list_auto %}
<tr>
<td class="align-middle">
{% if feedback.final %}
<span class="badge badge-success">Final</span>
{% endif %}
</td>
<td class="align-middle"> {{ feedback.of_submission.type }} </td>
<td class="align-middle"> {{ feedback.of_submission.student }} </td>
<td class="align-middle"> <code> {{ feedback.score }} / {{ feedback.of_submission.type.full_score }} </code> </td>
<td class="align-middle"> {{ feedback.get_origin_display }} </td>
<td class="align-middle">
{% if not feedback.origin == feedback.WAS_EMPTY %}
<a href="{% url 'FeedbackEdit' feedback.slug %}" class="btn btn-outline-primary mb-1" name="edit" value="View">View</a>
<a href="{% url 'FeedbackDelete' feedback.slug %}" class="btn btn-outline-danger mb-1" name="delete" value="Delete">Delete</a>
<a class="btn btn-outline-secondary mb-1" href="{% url 'FeedbackMarkNotFinal' feedback.slug %}"> Undo </a>
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
</div>
{% endblock body_block %}
{% endblock body_block %}
core/templates/core/tutor_startpage.html
+ 1
− 1
View file @ 7a4a29bd
......@@ -22,7 +22,7 @@
{% for submission_type in submission_type_list %}
<tr>
<td class="align-middle">{{ submission_type }}</td>
<td class="align-middle"><code>{{ submission_type.submissions__feedback_list__count }} / {{submission_type.submissions__count}}</code></td>
<td class="align-middle"><code>{{ submission_type.feedback_count }} / {{submission_type.submission_count}}</code></td>
<td class="align-middle"><a role="button" class="btn btn-secondary" href="{% url 'CreateFeedbackForType' submission_type.slug %}">Get</a></td>
</tr>
{% endfor %}
......
core/views/feedback.py
+ 1
− 0
View file @ 7a4a29bd
......@@ -50,6 +50,7 @@ def markfinal_feedback(request, feedback_slug):
def markunfinal_feedback(request, feedback_slug):
instance = Feedback.objects.get(slug=feedback_slug)
instance.unfinalize_feedback()
instance.origin = Feedback.MANUAL
return HttpResponseRedirect(reverse('start'))
......
core/views/user_startpages.py
+ 10
− 3
View file @ 7a4a29bd
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.db.models import Count
from django.db.models import Count, Case, When, IntegerField, Value
from django.http import HttpResponseRedirect
from django.shortcuts import render
from django.urls import reverse
......@@ -26,13 +26,19 @@ def get_annotated_feedback_count():
count both
* number of submission per submission type
* count of received feedback per submission type
*
Alternative with case
Count(Case(
When(submissions__feedback_list__origin=Feedback.MANUAL,
then=Value(1)), output_field=IntegerField())
)
Returns:
annotated queryset
"""
return SubmissionType.objects.annotate( # to display only manual
Count('submissions__feedback_list')).annotate(
Count('submissions')
feedback_count=Count('submissions__feedback_list')).annotate(
submission_count=Count('submissions')
).all()
@group_required('Tutors')
......@@ -60,5 +66,6 @@ def reviewer_view(request):
'tutor_list': User.objects.annotate(Count('corrected_submissions')).filter(groups__name='Tutors'),
'submission_list': Submission.objects.all(),
'feedback_list': Feedback.objects.filter(origin=Feedback.MANUAL),
'feedback_list_auto': Feedback.objects.exclude(origin=Feedback.MANUAL),
}
return render(request, 'core/reviewer_startpage.html', context)
populatedb.py
+ 86
− 55
View file @ 7a4a29bd
import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'grady.settings')
import random
import django
import getpass
import json
......@@ -15,21 +14,68 @@ from django.contrib.auth.models import Group, User
from core.models import Student, Submission, SubmissionType, Feedback
def parseme():
parser = argparse.ArgumentParser()
parser.add_argument(
'--superuser',
help='Superuser will be created users be created',
action='store_true')
parser.add_argument(
'DATA',
help='a folder containing a predefined set of files with information',
default='data',
metavar='DATA')
parser.add_argument(
'-s', '--submissions',
help='A file with submission code and student user names',
default='submissions.json',
metavar='FILE')
parser.add_argument(
'-t', '--tutors',
help='A list of tutor names',
default='tutors',
metavar="FILE")
parser.add_argument(
'-r', '--reviewers',
help='A list of reviewer names',
default='reviewers',
metavar="FILE")
parser.add_argument(
'-st', '--submission_types',
help='some kind of descriptions for all the submission types',
default='submission_types.json',
metavar="FILE")
args = parser.parse_args()
args.tutors = os.path.join(args.DATA, args.tutors)
args.reviewers = os.path.join(args.DATA, args.reviewers)
args.submissions = os.path.join(args.DATA, args.submissions)
args.submission_types = os.path.join(args.DATA, args.submission_types)
return args
def add_submission_type(name,
score,
task_description="__task_description: what the student saw",
possible_solution="__possible_solution: a sample solution",
correction_guideline="__possible_solution: a way to correct the task",):
task = SubmissionType()
task.name = name
task, created = SubmissionType.objects.get_or_create(name=name)
task.full_score = score
task.task_description = task_description
task.possible_solution = possible_solution
task.correction_guideline = correction_guideline
task.save()
print(f"- Created Task {task.name}")
if created:
print(f"- Created Task {task.name}")
else:
print(f"- Got Task {task.name}")
return task
def student_has_all_submissions(student):
return Submission.objects.filter(student=student).count() \
== SubmissionType.objects.all().count()
def add_submission(type, text, student, pre="Vorgabe"):
def get_compiler_output():
......@@ -38,10 +84,13 @@ def add_submission(type, text, student, pre="Vorgabe"):
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
input=text,
encoding='latin-1',
encoding='utf-8',
)
return command.stderr # and returncode
if student_has_all_submissions(student):
return None
sub = Submission()
sub.type = type
sub.text = text
......@@ -67,6 +116,10 @@ def add_empty_feedback(submission):
def add_empty_submission(type, student):
if student_has_all_submissions(student):
return None
sub = Submission()
sub.type = type
sub.student = student
......@@ -77,12 +130,16 @@ def add_empty_submission(type, student):
return sub
def add_student(name):
student_group = Group.objects.get(name='Students')
student_user = add_user(name, student_group)
student = Student(
user=student_user, matrikel_no=20000000 + random.randrange(10000000))
student.save()
def add_student(username, name, matrikel_no):
student_group, _ = Group.objects.get_or_create(name='Students')
student, created = Student.objects.get_or_create(
matrikel_no=matrikel_no,
user=add_user(username, student_group)
)
if created:
student.name = name
student.matrikel_no = matrikel_no
student.save()
return student
......@@ -126,69 +183,42 @@ class PopulateDatabase:
'reviewer_group',
)
def __init__(self):
self.parseme()
def __init__(self, args):
self.args = args
self.create_groups()
self.create_admins()
self.create_user_accounts()
self.populate_submissions()
def parseme(self):
parser = argparse.ArgumentParser()
parser.add_argument(
'--superuser',
help='Superuser will be created users be created',
action='store_true')
parser.add_argument(
'-s', '--submissions',
help='A file with submission code and student user names',
default='data/submissions.json',
type=argparse.FileType('r'),
metavar='FILE')
parser.add_argument(
'-a', '--admins',
help='A list of tutor and reviewer names',
default='data/admins.json',
type=argparse.FileType('r'),
metavar="FILE")
parser.add_argument(
'-t', '--submission_types',
help='some kind of descriptions for all the sumbission types',
default='data/submission_types.json',
type=argparse.FileType('r'),
metavar="FILE")
self.args = parser.parse_args()
def create_groups(self):
self.student_group = add_group('Students')
self.tutor_group = add_group('Tutors')
self.reviewer_group = add_group('Reviewers')
def create_admins(self):
with self.args.admins as admins:
admin_data = json.JSONDecoder().decode(admins.read())
for tutor in admin_data['tutors']:
add_user(tutor, self.tutor_group)
def create_user_accounts(self):
with open(self.args.tutors) as tutors:
for tutor in tutors:
add_user(tutor.strip(), self.tutor_group)
for reviewer in admin_data['reviewers']:
add_user(reviewer, self.reviewer_group)
with open(self.args.reviewers) as reviewers:
for reviewer in reviewers:
add_user(reviewer.strip(), self.reviewer_group)
def populate_submissions(self):
with self.args.submissions as data:
with open(self.args.submissions) as data:
stud_data = json.JSONDecoder().decode(data.read())
# dirty
type_dict = {}
for submission in stud_data.values():
for submission_type in submission:
for submission_type in submission['submissions']:
type_dict[submission_type] = add_submission_type(
submission_type, 15)
break
for user, submissions in stud_data.items():
student = add_student(user)
for s, code in submissions.items():
for user, userdata in stud_data.items():
student = add_student(
user, userdata['name'], userdata['matrikel_no'])
for s, code in userdata['submissions'].items():
if code:
add_submission(type_dict[s], code, student)
else:
......@@ -197,6 +227,7 @@ class PopulateDatabase:
# Start execution here!
if __name__ == '__main__':
args = parseme()
print("Starting population script...")
PopulateDatabase(args)
create_superuser()
PopulateDatabase()
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment