From ec9f0d012f0545decd84408ac6170fef1de4786a Mon Sep 17 00:00:00 2001
From: janmax <j.michal@stud.uni-goettingen.de>
Date: Thu, 15 Feb 2018 15:18:19 +0100
Subject: [PATCH] Introduction some files that will be useful in deployment
* Also restructured the converter script so it handles
mixed (gap and source code) export files as well
* Added a sad hack that enables using a base url
---
.gitlab-ci.yml | 4 +++-
.pre-commit-config.yaml | 2 +-
Dockerfile | 1 +
core/urls.py | 2 --
deploy.sh | 9 ++++++++
grady/__init__.py | 0
grady/settings/__init__.py | 6 +++---
grady/settings/default.py | 5 +----
grady/settings/url_hack.py | 17 +++++++++++++++
grady/urls.py | 3 +--
requirements.txt | 4 ++--
util/convert.py | 44 +++++++++++++++++---------------------
util/format_index.py | 16 ++++++++++++++
util/importer.py | 2 +-
14 files changed, 75 insertions(+), 40 deletions(-)
create mode 100644 deploy.sh
delete mode 100644 grady/__init__.py
create mode 100644 grady/settings/url_hack.py
create mode 100644 util/format_index.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 241d7579..b5ef16b9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -6,7 +6,7 @@ stages:
- staging
variables:
- IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+ IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME-$CI_COMMIT_SHA
# ========================== Build Testing section =========================== #
@@ -118,6 +118,8 @@ staging:
on_stop: staging_stop
script:
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
+ - docker-compose stop
+ - docker-compose pull
- docker-compose up -d --force-recreate
staging_stop:
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 053bd657..7bf469f1 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@
- id: debug-statements
- id: flake8
args:
- - --exclude=*/migrations/*,docs/*
+ - --exclude=*/migrations/*,docs/*,grady/*
- id: check-added-large-files
- id: requirements-txt-fixer
args:
diff --git a/Dockerfile b/Dockerfile
index 3a33f19c..2a790dbb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,5 +24,6 @@ COPY --from=node /app/dist /code/frontend/dist
COPY --from=node /app/dist/index.html /code/core/templates/index.html
RUN pip install -r requirements.txt && rm -rf /root/.cache
+RUN python util/format_index.py
RUN python manage.py collectstatic --noinput
RUN apk del build-deps
diff --git a/core/urls.py b/core/urls.py
index 3eaaa62d..4853a567 100644
--- a/core/urls.py
+++ b/core/urls.py
@@ -1,4 +1,3 @@
-from django.contrib.staticfiles.urls import staticfiles_urlpatterns
from django.urls import path
from rest_framework.routers import DefaultRouter
@@ -33,5 +32,4 @@ regular_views_urlpatterns = [
urlpatterns = [
*router.urls,
*regular_views_urlpatterns,
- *staticfiles_urlpatterns()
]
diff --git a/deploy.sh b/deploy.sh
new file mode 100644
index 00000000..fd384da7
--- /dev/null
+++ b/deploy.sh
@@ -0,0 +1,9 @@
+#!/bin/sh
+sleep 1
+python manage.py migrate --noinput
+gunicorn \
+ --bind 0.0.0.0:8000 \
+ --workers=2 \
+ --worker-class=gevent \
+ --log-level debug \
+ grady.wsgi:application
diff --git a/grady/__init__.py b/grady/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/grady/settings/__init__.py b/grady/settings/__init__.py
index f3fadcf6..a8e95f7c 100644
--- a/grady/settings/__init__.py
+++ b/grady/settings/__init__.py
@@ -1,8 +1,8 @@
import os
+from .default import *
dev = os.environ.get('DJANGO_DEV', False)
-from .default import *
-
if not dev:
- from .live import *
+ from .live import * # noqa
+ from .url_hack import * # noqa
diff --git a/grady/settings/default.py b/grady/settings/default.py
index 0f18f2e7..5cda30d1 100644
--- a/grady/settings/default.py
+++ b/grady/settings/default.py
@@ -52,6 +52,7 @@ INSTALLED_APPS = [
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
+ 'whitenoise.runserver_nostatic',
'django.contrib.staticfiles',
'rest_framework',
'corsheaders',
@@ -132,10 +133,6 @@ STATICFILES_DIRS = (
'frontend/dist/static',
)
-GRAPH_MODELS = {
- 'all_applications': True,
- 'group_models': True,
-}
LOGIN_REDIRECT_URL = '/'
LOGIN_URL = '/'
diff --git a/grady/settings/url_hack.py b/grady/settings/url_hack.py
new file mode 100644
index 00000000..a4443f3e
--- /dev/null
+++ b/grady/settings/url_hack.py
@@ -0,0 +1,17 @@
+""" Ok, what the hell? This is especially ugly, hence I keep it hidden in
+this file. We have the requirement that the application instances should
+run under http://$host/$instancename/. And therefore the frontend, whitenoise,
+django, gunicorn and the top http proxy all have to handle this stuff.
+
+Usage: Just set the SCRIPT_NAME env variable to /<name of your instance>
+ and things will work. """
+
+import os
+
+FORCE_SCRIPT_NAME = os.environ.get('SCRIPT_NAME', '')
+if FORCE_SCRIPT_NAME:
+ FORCE_SCRIPT_NAME += '/'
+
+STATIC_URL_BASE = '/static/'
+STATIC_URL = os.path.join(FORCE_SCRIPT_NAME + STATIC_URL_BASE)
+WHITENOISE_STATIC_PREFIX = STATIC_URL_BASE
diff --git a/grady/urls.py b/grady/urls.py
index e36863f6..6d84db55 100644
--- a/grady/urls.py
+++ b/grady/urls.py
@@ -10,6 +10,5 @@ urlpatterns = [
namespace='rest_framework')),
path('api-token-auth/', obtain_jwt_token),
path('api-token-refresh/', refresh_jwt_token),
- path('', TemplateView.as_view(template_name='index.html'))
-
+ path('', TemplateView.as_view(template_name='index.html')),
]
diff --git a/requirements.txt b/requirements.txt
index c6db510e..9881225c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,10 +2,10 @@ django-cors-headers~=2.1.0
django-extensions~=1.7.7
djangorestframework-jwt~=1.11.0
djangorestframework~=3.7.7
-drf-dynamic-fields~=0.2.0
Django~=2.0
+drf-dynamic-fields~=0.2.0
gevent~=1.2.2
gunicorn~=19.7.0
-psycopg2~=2.7.1
+psycopg2-binary~=2.7.4
whitenoise~=3.3.1
xlrd~=1.0.0
diff --git a/util/convert.py b/util/convert.py
index dcfc3e79..82041e19 100755
--- a/util/convert.py
+++ b/util/convert.py
@@ -57,15 +57,13 @@ parser.add_argument(
# one user has one submission (code) per task
# yes, I know it is possible to name match groups via (?P<name>) but
# I like this solution better since it gets the job done nicely
-user_head = namedtuple('user_head', 'kohorte, name')
-user_head_re = re.compile(r'^Ergebnisse von Testdurchlauf '
- '(?P<kohorte>\d+) für (?P<name>[\w\s\.,-]+)$')
+user_t = namedtuple('user_head', 'name matrikel_no')
# one task has a title and id and hpfly code
-task_head_re = re.compile(r'^Quellcode Frage(?P<title>.*) \d{8}$')
+task_head_re = re.compile(r'^Quellcode Frage (?P<title>.*) ?(\d{8})?$')
# nor parsing the weird mat no
-matno_re = re.compile(r'^(?P<matrikel_no>\d{8})-(\d{3})-(\d{3})$')
+matno_re = re.compile(r'^(?P<matrikel_no>\d{8})-(\d+)-(\d+)$')
def converter(infile, usernames=None, number_of_tasks=0,):
@@ -79,13 +77,15 @@ def converter(infile, usernames=None, number_of_tasks=0,):
yield row[0].value, m.group('matrikel_no') if m else row[1].value
def sheet_iter_data(sheet):
- """ yields all rows that are not of empty type as one string """
- for row in (sheet.row(i) for i in range(sheet.nrows)):
- if any(map(lambda c: c.ctype, row)):
- yield ''.join(c.value for c in row)
-
- # meta sheet contains ilias evaluation names usernames etc - data contains
- # code
+ """ yields all source code titel and code tuples """
+ def row(i):
+ return sheet.row(i)
+ for top, low in ((row(i), row(i + 1)) for i in range(sheet.nrows - 1)):
+ if any(map(lambda c: c.ctype, top)) and 'Quell' in top[0].value:
+ yield (' '.join(c.value for c in top),
+ ' '.join(c.value for c in low))
+
+ # meta sheet contains ilias names usernames etc - data contains code
meta, *data = open_workbook(infile, open(os.devnull, 'w')).sheets()
# nice!
@@ -95,16 +95,12 @@ def converter(infile, usernames=None, number_of_tasks=0,):
# from xls to lists and namedtuples
# [ [user0, task0_h, code0, ..., taskn, coden ], ..., [...] ]
root = []
- for sheet in data:
- for row in sheet_iter_data(sheet):
- user = re.search(user_head_re, row)
- task = re.search(task_head_re, row)
- if user:
- root.append([user_head(*user.groups())])
- elif task:
- root[-1].append(task.group('title'))
- else: # should be code
- root[-1].append(urllib.parse.unquote(row).strip())
+ for user, sheet in zip(sheet_iter_meta(meta), data):
+ root.append([user_t(*user)])
+ for task, code in sheet_iter_data(sheet):
+ task = re.search(task_head_re, task)
+ root[-1].append(task.group('title'))
+ root[-1].append(urllib.parse.unquote(code).strip())
if number_of_tasks:
for (user, *task_list) in sorted(root, key=lambda u: u[0].name):
@@ -127,7 +123,7 @@ def converter(infile, usernames=None, number_of_tasks=0,):
# {id:, ..., id:}}}
return {
usernames[user.name]: {
- 'name': user.name,
+ 'fullname': user.name,
'email': mat_to_email[name2mat[user.name]],
'matrikel_no': name2mat[user.name],
'submissions': [
@@ -144,7 +140,7 @@ def converter(infile, usernames=None, number_of_tasks=0,):
def write_to_file(json_dict, outfile):
# just encode python style
with open(outfile, "w") as out:
- out.write(json.JSONEncoder().encode(json_dict))
+ json.dump(json_dict, out, indent=2)
print(f"Wrote data to {outfile}. Done.")
diff --git a/util/format_index.py b/util/format_index.py
new file mode 100644
index 00000000..c3f1f7d6
--- /dev/null
+++ b/util/format_index.py
@@ -0,0 +1,16 @@
+import sys
+import fileinput
+
+file = 'core/templates/index.html'
+
+with open(file, "r+") as f:
+ s = f.read()
+ f.seek(0)
+ f.write("{% load staticfiles %}\n" + s)
+
+for i, line in enumerate(fileinput.input(file, inplace=1)):
+ sys.stdout.write(line.replace('/static/', "{% static '"))
+for i, line in enumerate(fileinput.input(file, inplace=1)):
+ sys.stdout.write(line.replace('.css', ".css' %}"))
+for i, line in enumerate(fileinput.input(file, inplace=1)):
+ sys.stdout.write(line.replace('.js', ".js' %}"))
diff --git a/util/importer.py b/util/importer.py
index 9f60042b..683dccc7 100644
--- a/util/importer.py
+++ b/util/importer.py
@@ -110,7 +110,7 @@ def add_tests(submission_obj, tests):
for name, test_data in ((name, tests[name]) for name in TEST_ORDER):
test_obj, created = Test.objects.update_or_create(
- name=test_data['name'],
+ name=test_data['fullname'],
submission=submission_obj,
defaults={
'label': test_data['label'],
--
GitLab