From 910add55b84172e6cb092bf7a219a19ed669bdad Mon Sep 17 00:00:00 2001
From: janmax <mail-github@jmx.io>
Date: Fri, 7 Jul 2017 18:55:20 +0200
Subject: [PATCH] Created a first version of the importer script
---
core/models.py | 20 +++----
grady/database_router.py | 40 -------------
grady/settings/default.py | 3 -
grrr.py | 8 ++-
util/importer.py | 116 ++++++++++++++++++++++++++++++++++++++
5 files changed, 133 insertions(+), 54 deletions(-)
delete mode 100644 grady/database_router.py
create mode 100644 util/importer.py
diff --git a/core/models.py b/core/models.py
index 6507451a..c3dcbd4a 100644
--- a/core/models.py
+++ b/core/models.py
@@ -62,18 +62,18 @@ def random_matrikel_no():
class SubmissionType(models.Model):
# Fields
- name = models.CharField(max_length=50, unique=True)
- full_score = models.PositiveIntegerField(default=0)
- task_description = models.TextField()
- possible_solution = models.TextField()
- slug = models.SlugField(
+ name = models.CharField(max_length=50, unique=True)
+ full_score = models.PositiveIntegerField(default=0)
+ description = models.TextField()
+ solution = models.TextField()
+ slug = models.SlugField(
editable=False, unique=True, default=random_slug)
def __str__(self):
return self.name
class Meta:
- verbose_name = "SubmissionType"
+ verbose_name = "SubmissionType"
verbose_name_plural = "SubmissionType Set"
@@ -111,7 +111,7 @@ class Student(models.Model):
return self.user.username
class Meta:
- verbose_name = "Student"
+ verbose_name = "Student"
verbose_name_plural = "Student Set"
@@ -134,9 +134,9 @@ class Submission(models.Model):
related_name='submissions')
class Meta:
- verbose_name = "Submission"
+ verbose_name = "Submission"
verbose_name_plural = "Submission Set"
- unique_together = (('type', 'student'),)
+ unique_together = (('type', 'student'),)
def __str__(self):
return "Submission of type '{}' from Student '{}'".format(
@@ -254,7 +254,7 @@ class Feedback(models.Model):
)
class Meta:
- verbose_name = "Feedback"
+ verbose_name = "Feedback"
verbose_name_plural = "Feedback Set"
def __str__(self):
diff --git a/grady/database_router.py b/grady/database_router.py
deleted file mode 100644
index 03721181..00000000
--- a/grady/database_router.py
+++ /dev/null
@@ -1,40 +0,0 @@
-
-class AuthRouter(object):
- """
- A router to control all database operations on models in the
- auth application.
- """
- def db_for_read(self, model, **hints):
- """
- Attempts to read auth models go to auth_db.
- """
- if model._meta.app_label == 'auth':
- return 'auth_db'
- return None
-
- def db_for_write(self, model, **hints):
- """
- Attempts to write auth models go to auth_db.
- """
- if model._meta.app_label == 'auth':
- return 'auth_db'
- return None
-
- def allow_relation(self, obj1, obj2, **hints):
- """
- Allow relations if a model in the auth app is involved.
- """
- if obj1._meta.app_label == 'auth' or \
- obj2._meta.app_label == 'auth':
- return True
- return True
-
- def allow_migrate(self, db, app_label, model_name=None, **hints):
- """
- Make sure the auth app only appears in the 'auth_db'
- database.
- """
- if app_label == 'auth':
- return db == 'auth_db'
- return None
-
diff --git a/grady/settings/default.py b/grady/settings/default.py
index beabda38..646bf784 100644
--- a/grady/settings/default.py
+++ b/grady/settings/default.py
@@ -91,9 +91,6 @@ AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
- {
- 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
- },
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
diff --git a/grrr.py b/grrr.py
index 4294b4fe..d97add4c 100644
--- a/grrr.py
+++ b/grrr.py
@@ -12,7 +12,7 @@ django.setup()
from django.contrib.auth.models import User
from core.models import Student, Submission
-
+import util.importer
def parseme():
parser = argparse.ArgumentParser()
@@ -75,6 +75,9 @@ def parseme():
### parser for extracting submissions ###
subparsers.add_parser('extractsubmissions')
+ ### parser for extracting submissions ###
+ subparsers.add_parser('importer')
+
return parser.parse_args()
@@ -122,6 +125,9 @@ def handle_extractsubmissions(output, **kwargs):
print(submission.feedback.score, repr(submission.text), file=open(str(submission.type).replace(' ', '_'), 'a'))
+def handle_importer(**kwargs):
+ util.importer.start()
+
def main():
args = parseme()
if args.command:
diff --git a/util/importer.py b/util/importer.py
new file mode 100644
index 00000000..99e11908
--- /dev/null
+++ b/util/importer.py
@@ -0,0 +1,116 @@
+import csv
+import os
+import readline
+import secrets
+
+from django.contrib.auth.models import Group, User
+from core.models import Student, Submission, SubmissionType, Feedback
+
+
+STUDENTS = Group.objects.get(name='Students')
+TUTORS = Group.objects.get(name='Tutors')
+REVIEWERS = Group.objects.get(name='Reviewers')
+
+
+def get_xkcd_password(k=2):
+ with open('/usr/share/dict/words') as words:
+ choose_from = list({word.strip().lower()
+ for word in words if 5 < len(word) < 8})
+
+ return ''.join(secrets.choice(choose_from) for _ in range(k))
+
+
+def i(prompt, default=''):
+ if default:
+ return input(f'[Q] {prompt} ({default}): ') or default
+ return input(f'[Q] {prompt}: ')
+
+
+def make_submission_type_objects(csvfilename, lsg_dir, desc_dir):
+ with open(csvfilename, encoding='utf-8') as tfile:
+ reader = csv.reader(tfile)
+
+ for row in reader:
+ tid, name, score = row
+ with open(os.path.join(lsg_dir, tid + '-lsg.c'), encoding='utf-8') as lsg, open(os.path.join(desc_dir, tid + '.html'), encoding='utf-8') as desc:
+ yield {
+ 'name' : name,
+ 'description' : desc.read(),
+ 'solution' : lsg.read(),
+ 'score' : int(score),
+ }
+
+
+def add_user(username, group):
+ user = User(username=username.strip())
+
+ password = get_xkcd_password()
+ user.set_password(password)
+ user.save()
+
+ group.user_set.add(user)
+
+ return user
+
+def add_user_list(lst, group):
+ for name in lst:
+ add_user(name, group)
+
+print('''Welcome to the Grady importer!
+
+This script aims at making to setup of the database as easy as
+possible. It at the same time serves as a documentation on how data is imported
+in Grady. Let\'s dive right in.\n''')
+
+
+def main_loop():
+ path = i('location of data files', '.')
+ os.chdir(path)
+
+ print('''Please provide a .csv file with
+
+ id, name, score
+
+ The id should correspond to the names of description files if you want to
+ include them now.''')
+
+ submission_types_csv = i('CSV file', 'submission_types.csv')
+ lsg_dir = i('solution dir prefix', 'code-lsg')
+ desc_dir = i('html descriptions dir prefix', 'html')
+
+ submission_types = [d for d in make_submission_type_objects(
+ submission_types_csv, lsg_dir, desc_dir)]
+
+ print('Now please provide files where you list usernames for reviewer and tutors')
+ tutors = i('list of tutors', 'tutors')
+ reviewers = i('list of reviewer', 'reviewers')
+
+
+ with open(tutors) as tutors_f:
+ add_user_list(tutors_f, TUTORS)
+
+ with open(reviewers) as reviewers_f:
+ add_user_list(reviewers_f, REVIEWERS)
+
+def start():
+ if User.objects.filter(is_superuser=False) :
+ print('Warning database is not clean. Aborting')
+ exit(0)
+
+ while True:
+ try:
+ main_loop()
+ except FileNotFoundError as err:
+ print(err)
+ except (EOFError, KeyboardInterrupt) as err:
+ print()
+ exit(0)
+
+
+# Note to myself: split the module in single tests that perform some action
+# on the database. save success in a .importer history along with readline
+# completition.
+#
+# on importer load it can be determined which imports have already been done
+# and which are still due. it also saves us from repeating all over again
+# if a some file not found error occured.
--
GitLab