Created a first version of the importer script

core/models.py

@@ -62,18 +62,18 @@ def random_matrikel_no():
 
 class SubmissionType(models.Model):
     # Fields
-    name              = models.CharField(max_length=50, unique=True)
-    full_score        = models.PositiveIntegerField(default=0)
-    task_description  = models.TextField()
-    possible_solution = models.TextField()
-    slug              = models.SlugField(
+    name        = models.CharField(max_length=50, unique=True)
+    full_score  = models.PositiveIntegerField(default=0)
+    description = models.TextField()
+    solution    = models.TextField()
+    slug        = models.SlugField(
         editable=False, unique=True, default=random_slug)
 
     def __str__(self):
         return self.name
 
     class Meta:
-        verbose_name = "SubmissionType"
+        verbose_name        = "SubmissionType"
         verbose_name_plural = "SubmissionType Set"
 
 
@@ -111,7 +111,7 @@ class Student(models.Model):
         return self.user.username
 
     class Meta:
-        verbose_name = "Student"
+        verbose_name        = "Student"
         verbose_name_plural = "Student Set"
 
 
@@ -134,9 +134,9 @@ class Submission(models.Model):
         related_name='submissions')
 
     class Meta:
-        verbose_name = "Submission"
+        verbose_name        = "Submission"
         verbose_name_plural = "Submission Set"
-        unique_together = (('type', 'student'),)
+        unique_together     = (('type', 'student'),)
 
     def __str__(self):
         return "Submission of type '{}' from Student '{}'".format(
@@ -254,7 +254,7 @@ class Feedback(models.Model):
     )
 
     class Meta:
-        verbose_name = "Feedback"
+        verbose_name        = "Feedback"
         verbose_name_plural = "Feedback Set"
 
     def __str__(self):

grady/database_router.py

-
-class AuthRouter(object):
-    """
-    A router to control all database operations on models in the
-    auth application.
-    """
-    def db_for_read(self, model, **hints):
-        """
-        Attempts to read auth models go to auth_db.
-        """
-        if model._meta.app_label == 'auth':
-            return 'auth_db'
-        return None
-
-    def db_for_write(self, model, **hints):
-        """
-        Attempts to write auth models go to auth_db.
-        """
-        if model._meta.app_label == 'auth':
-            return 'auth_db'
-        return None
-
-    def allow_relation(self, obj1, obj2, **hints):
-        """
-        Allow relations if a model in the auth app is involved.
-        """
-        if obj1._meta.app_label == 'auth' or \
-           obj2._meta.app_label == 'auth':
-           return True
-        return True
-
-    def allow_migrate(self, db, app_label, model_name=None, **hints):
-        """
-        Make sure the auth app only appears in the 'auth_db'
-        database.
-        """
-        if app_label == 'auth':
-            return db == 'auth_db'
-        return None
-

grady/settings/default.py

@@ -91,9 +91,6 @@ AUTH_PASSWORD_VALIDATORS = [
     {
         'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
     },
-    {
-        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
-    },
     {
         'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
     },

grrr.py

@@ -12,7 +12,7 @@ django.setup()
 from django.contrib.auth.models import User
 
 from core.models import Student, Submission
-
+import util.importer
 
 def parseme():
     parser      = argparse.ArgumentParser()
@@ -75,6 +75,9 @@ def parseme():
     ### parser for extracting submissions ###
     subparsers.add_parser('extractsubmissions')
 
+    ### parser for extracting submissions ###
+    subparsers.add_parser('importer')
+
     return parser.parse_args()
 
 
@@ -122,6 +125,9 @@ def handle_extractsubmissions(output, **kwargs):
         print(submission.feedback.score, repr(submission.text), file=open(str(submission.type).replace(' ', '_'), 'a'))
 
 
+def handle_importer(**kwargs):
+    util.importer.start()
+
 def main():
     args = parseme()
     if args.command:

util/importer.py

+import csv
+import os
+import readline
+import secrets
+
+from django.contrib.auth.models import Group, User
+from core.models import Student, Submission, SubmissionType, Feedback
+
+
+STUDENTS  = Group.objects.get(name='Students')
+TUTORS    = Group.objects.get(name='Tutors')
+REVIEWERS = Group.objects.get(name='Reviewers')
+
+
+def get_xkcd_password(k=2):
+    with open('/usr/share/dict/words') as words:
+        choose_from = list({word.strip().lower()
+                            for word in words if 5 < len(word) < 8})
+
+    return ''.join(secrets.choice(choose_from) for _ in range(k))
+
+
+def i(prompt, default=''):
+    if default:
+        return input(f'[Q] {prompt} ({default}): ') or default
+    return input(f'[Q] {prompt}: ')
+
+
+def make_submission_type_objects(csvfilename, lsg_dir, desc_dir):
+    with open(csvfilename, encoding='utf-8') as tfile:
+        reader = csv.reader(tfile)
+
+        for row in reader:
+            tid, name, score = row
+            with open(os.path.join(lsg_dir, tid + '-lsg.c'), encoding='utf-8') as lsg, open(os.path.join(desc_dir, tid + '.html'), encoding='utf-8') as desc:
+                yield {
+                    'name' : name,
+                    'description' : desc.read(),
+                    'solution' : lsg.read(),
+                    'score' : int(score),
+                }
+
+
+def add_user(username, group):
+    user = User(username=username.strip())
+
+    password = get_xkcd_password()
+    user.set_password(password)
+    user.save()
+
+    group.user_set.add(user)
+
+    return user
+
+def add_user_list(lst, group):
+    for name in lst:
+        add_user(name, group)
+
+print('''Welcome to the Grady importer!
+
+This script aims at making to setup of the database as easy as
+possible. It at the same time serves as a documentation on how data is imported
+in Grady. Let\'s dive right in.\n''')
+
+
+def main_loop():
+    path = i('location of data files', '.')
+    os.chdir(path)
+
+    print('''Please provide a .csv file with
+
+        id, name, score
+
+    The id should correspond to the names of description files if you want to
+    include them now.''')
+
+    submission_types_csv    = i('CSV file', 'submission_types.csv')
+    lsg_dir                 = i('solution dir prefix', 'code-lsg')
+    desc_dir                = i('html descriptions dir prefix', 'html')
+
+    submission_types = [d for d in make_submission_type_objects(
+        submission_types_csv, lsg_dir, desc_dir)]
+
+    print('Now please provide files where you list usernames for reviewer and tutors')
+    tutors    = i('list of tutors', 'tutors')
+    reviewers = i('list of reviewer', 'reviewers')
+
+
+    with open(tutors) as tutors_f:
+        add_user_list(tutors_f, TUTORS)
+
+    with open(reviewers) as reviewers_f:
+        add_user_list(reviewers_f, REVIEWERS)
+
+def start():
+    if User.objects.filter(is_superuser=False) :
+        print('Warning database is not clean. Aborting')
+        exit(0)
+
+    while True:
+        try:
+            main_loop()
+        except FileNotFoundError as err:
+            print(err)
+        except (EOFError, KeyboardInterrupt) as err:
+            print()
+            exit(0)
+
+
+# Note to myself: split the module in single tests that perform some action
+# on the database. save success in a .importer history along with readline
+# completition.
+#
+# on importer load it can be determined which imports have already been done
+# and which are still due. it also saves us from repeating all over again
+# if a some file not found error occured.