Introduction some files that will be useful in deployment

* Also restructured the converter script so it handles
  mixed (gap and source code) export files as well
* Added a sad hack that enables using a base url

.gitlab-ci.yml

@@ -6,7 +6,7 @@ stages:
   - staging
 
 variables:
-  IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA
 
 
 # ========================== Build Testing section =========================== #
@@ -118,6 +118,8 @@ staging:
     on_stop: staging_stop
   script:
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+    - docker-compose stop
+    - docker-compose pull
     - docker-compose up -d --force-recreate
 
 staging_stop:

.pre-commit-config.yaml

@@ -6,7 +6,7 @@
   - id: debug-statements
   - id: flake8
     args:
-    - --exclude=*/migrations/*,docs/*
+    - --exclude=*/migrations/*,docs/*,grady/*
   - id: check-added-large-files
   - id: requirements-txt-fixer
     args:

Dockerfile

@@ -24,5 +24,6 @@ COPY --from=node /app/dist /code/frontend/dist
 COPY --from=node /app/dist/index.html /code/core/templates/index.html
 
 RUN pip install -r requirements.txt && rm -rf /root/.cache
+RUN python util/format_index.py
 RUN python manage.py collectstatic --noinput
 RUN apk del build-deps

core/urls.py

-from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 from django.urls import path
 from rest_framework.routers import DefaultRouter
 
@@ -33,5 +32,4 @@ regular_views_urlpatterns = [
 urlpatterns = [
     *router.urls,
     *regular_views_urlpatterns,
-    *staticfiles_urlpatterns()
 ]

deploy.sh

+#!/bin/sh
+sleep 1
+python manage.py migrate --noinput
+gunicorn \
+  --bind 0.0.0.0:8000 \
+  --workers=2 \
+  --worker-class=gevent \
+  --log-level debug \
+  grady.wsgi:application

grady/settings/__init__.py

 import os
+from .default import *
 
 dev = os.environ.get('DJANGO_DEV', False)
 
-from .default import *
-
 if not dev:
-    from .live import *
+    from .live import *  # noqa
+    from .url_hack import *  # noqa

grady/settings/default.py

@@ -52,6 +52,7 @@ INSTALLED_APPS = [
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.messages',
+    'whitenoise.runserver_nostatic',
     'django.contrib.staticfiles',
     'rest_framework',
     'corsheaders',
@@ -132,10 +133,6 @@ STATICFILES_DIRS = (
     'frontend/dist/static',
 )
 
-GRAPH_MODELS = {
-    'all_applications': True,
-    'group_models': True,
-}
 
 LOGIN_REDIRECT_URL = '/'
 LOGIN_URL = '/'

grady/settings/url_hack.py

+""" Ok, what the hell? This is especially ugly, hence I keep it hidden in
+this file. We have the requirement that the application instances should
+run under http://$host/$instancename/. And therefore the frontend, whitenoise,
+django, gunicorn and the top http proxy all have to handle this stuff.
+
+Usage: Just set the SCRIPT_NAME env variable to /<name of your instance>
+       and things will work. """
+
+import os
+
+FORCE_SCRIPT_NAME = os.environ.get('SCRIPT_NAME', '')
+if FORCE_SCRIPT_NAME:
+    FORCE_SCRIPT_NAME += '/'
+
+STATIC_URL_BASE = '/static/'
+STATIC_URL = os.path.join(FORCE_SCRIPT_NAME + STATIC_URL_BASE)
+WHITENOISE_STATIC_PREFIX = STATIC_URL_BASE

grady/urls.py

@@ -10,6 +10,5 @@ urlpatterns = [
                               namespace='rest_framework')),
     path('api-token-auth/', obtain_jwt_token),
     path('api-token-refresh/', refresh_jwt_token),
-    path('', TemplateView.as_view(template_name='index.html'))
-
+    path('', TemplateView.as_view(template_name='index.html')),
 ]

requirements.txt

@@ -2,10 +2,10 @@ django-cors-headers~=2.1.0
 django-extensions~=1.7.7
 djangorestframework-jwt~=1.11.0
 djangorestframework~=3.7.7
-drf-dynamic-fields~=0.2.0
 Django~=2.0
+drf-dynamic-fields~=0.2.0
 gevent~=1.2.2
 gunicorn~=19.7.0
-psycopg2~=2.7.1
+psycopg2-binary~=2.7.4
 whitenoise~=3.3.1
 xlrd~=1.0.0

util/convert.py

@@ -57,15 +57,13 @@ parser.add_argument(
 # one user has one submission (code) per task
 # yes, I know it is possible to name match groups via (?P<name>) but
 # I like this solution better since it gets the job done nicely
-user_head = namedtuple('user_head', 'kohorte, name')
-user_head_re = re.compile(r'^Ergebnisse von Testdurchlauf '
-                          '(?P<kohorte>\d+) für (?P<name>[\w\s\.,-]+)$')
+user_t = namedtuple('user_head', 'name matrikel_no')
 
 # one task has a title and id and hpfly code
-task_head_re = re.compile(r'^Quellcode Frage(?P<title>.*) \d{8}$')
+task_head_re = re.compile(r'^Quellcode Frage (?P<title>.*) ?(\d{8})?$')
 
 # nor parsing the weird mat no
-matno_re = re.compile(r'^(?P<matrikel_no>\d{8})-(\d{3})-(\d{3})$')
+matno_re = re.compile(r'^(?P<matrikel_no>\d{8})-(\d+)-(\d+)$')
 
 
 def converter(infile, usernames=None, number_of_tasks=0,):
@@ -79,13 +77,15 @@ def converter(infile, usernames=None, number_of_tasks=0,):
             yield row[0].value, m.group('matrikel_no') if m else row[1].value
 
     def sheet_iter_data(sheet):
-        """ yields all rows that are not of empty type as one string """
-        for row in (sheet.row(i) for i in range(sheet.nrows)):
-            if any(map(lambda c: c.ctype, row)):
-                yield ''.join(c.value for c in row)
-
-    # meta sheet contains ilias evaluation names usernames etc - data contains
-    # code
+        """ yields all source code titel and code tuples """
+        def row(i):
+            return sheet.row(i)
+        for top, low in ((row(i), row(i + 1)) for i in range(sheet.nrows - 1)):
+            if any(map(lambda c: c.ctype, top)) and 'Quell' in top[0].value:
+                yield (' '.join(c.value for c in top),
+                       ' '.join(c.value for c in low))
+
+    # meta sheet contains ilias names usernames etc - data contains code
     meta, *data = open_workbook(infile, open(os.devnull, 'w')).sheets()
 
     # nice!
@@ -95,16 +95,12 @@ def converter(infile, usernames=None, number_of_tasks=0,):
     # from xls to lists and namedtuples
     # [ [user0, task0_h, code0, ..., taskn, coden ], ..., [...] ]
     root = []
-    for sheet in data:
-        for row in sheet_iter_data(sheet):
-            user = re.search(user_head_re, row)
-            task = re.search(task_head_re, row)
-            if user:
-                root.append([user_head(*user.groups())])
-            elif task:
-                root[-1].append(task.group('title'))
-            else:  # should be code
-                root[-1].append(urllib.parse.unquote(row).strip())
+    for user, sheet in zip(sheet_iter_meta(meta), data):
+        root.append([user_t(*user)])
+        for task, code in sheet_iter_data(sheet):
+            task = re.search(task_head_re, task)
+            root[-1].append(task.group('title'))
+            root[-1].append(urllib.parse.unquote(code).strip())
 
     if number_of_tasks:
         for (user, *task_list) in sorted(root, key=lambda u: u[0].name):
@@ -127,7 +123,7 @@ def converter(infile, usernames=None, number_of_tasks=0,):
     # {id:, ..., id:}}}
     return {
         usernames[user.name]: {
-            'name': user.name,
+            'fullname': user.name,
             'email': mat_to_email[name2mat[user.name]],
             'matrikel_no': name2mat[user.name],
             'submissions': [
@@ -144,7 +140,7 @@ def converter(infile, usernames=None, number_of_tasks=0,):
 def write_to_file(json_dict, outfile):
     # just encode python style
     with open(outfile, "w") as out:
-        out.write(json.JSONEncoder().encode(json_dict))
+        json.dump(json_dict, out, indent=2)
 
     print(f"Wrote data to {outfile}. Done.")
 

util/format_index.py

+import sys
+import fileinput
+
+file = 'core/templates/index.html'
+
+with open(file, "r+") as f:
+    s = f.read()
+    f.seek(0)
+    f.write("{% load staticfiles %}\n" + s)
+
+for i, line in enumerate(fileinput.input(file, inplace=1)):
+    sys.stdout.write(line.replace('/static/', "{% static '"))
+for i, line in enumerate(fileinput.input(file, inplace=1)):
+    sys.stdout.write(line.replace('.css', ".css' %}"))
+for i, line in enumerate(fileinput.input(file, inplace=1)):
+    sys.stdout.write(line.replace('.js', ".js' %}"))

util/importer.py

@@ -110,7 +110,7 @@ def add_tests(submission_obj, tests):
 
     for name, test_data in ((name, tests[name]) for name in TEST_ORDER):
         test_obj, created = Test.objects.update_or_create(
-            name=test_data['name'],
+            name=test_data['fullname'],
             submission=submission_obj,
             defaults={
                 'label': test_data['label'],