From 128988aa401b0f6cb62b317df6c712fe15725b52 Mon Sep 17 00:00:00 2001
From: janmax <j.michal@stud.uni-goettingen.de>
Date: Thu, 30 Nov 2017 12:37:02 +0100
Subject: [PATCH] Ran autopep8 on util and removed old script folder
---
.gitlab-ci.yml | 17 ++++----
.pylintrc | 20 +++++++++
delbert.py | 14 +++---
scripts/README.rst | 11 -----
scripts/compile.py | 104 ---------------------------------------------
util/convert.py | 30 +++++++------
util/factories.py | 2 +-
util/importer.py | 18 ++++----
util/messages.py | 5 +++
util/processing.py | 48 ++++++++++-----------
util/testcases.py | 29 +++++++------
11 files changed, 107 insertions(+), 191 deletions(-)
create mode 100644 .pylintrc
delete mode 100644 scripts/README.rst
delete mode 100755 scripts/compile.py
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2d62fe51..7519a7a0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -21,8 +21,6 @@ build_backend:
.test_template_backend: &test_definition_backend
stage: test
image: $IMAGE_TAG
- before_script:
- - cd backend/
test_pytest:
<<: *test_definition_backend
@@ -32,7 +30,7 @@ test_pytest:
- DJANGO_SETTINGS_MODULE=grady.settings pytest --cov
artifacts:
paths:
- - backend/.coverage
+ - .coverage
test_prospector:
<<: *test_definition_backend
@@ -54,16 +52,18 @@ test_frontend:
# =========================== Gitlab pages section =========================== #
test_coverage:
- <<: *test_definition_backend
+ image: $IMAGE_TAG
stage:
pages
script:
- - coverage html -d ../public
+ - coverage html -d public
dependencies:
- test_pytest
artifacts:
paths:
- public
+ only:
+ - master
# ============================== Staging section ============================= #
.staging_template: &staging_definition
@@ -71,6 +71,8 @@ test_coverage:
image: docker:latest
only:
- master
+ before_script:
+ - apk add --update py-pip && pip install docker-compose
staging:
<<: *staging_definition
@@ -79,15 +81,12 @@ staging:
url: https://staging.grady.janmax.org
on_stop: staging_stop
script:
- - apk add --update py-pip && pip install docker-compose
- docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
- - docker-compose pull
- - docker-compose up -d --build
+ - docker-compose up -d --force-recreate
staging_stop:
<<: *staging_definition
script:
- - apk add --update py-pip && pip install docker-compose
- docker-compose rm --force --stop
when: manual
environment:
diff --git a/.pylintrc b/.pylintrc
new file mode 100644
index 00000000..2bfbcef1
--- /dev/null
+++ b/.pylintrc
@@ -0,0 +1,20 @@
+[MASTER]
+
+# Add files or directories to the blacklist. They should be base names, not
+# paths.
+ignore=CVS, migrations, static, env, docs, manage.py
+
+# Use multiple processes to speed up Pylint.
+jobs=4
+
+# Pickle collected data for later comparisons.
+persistent=yes
+
+# When enabled, pylint would attempt to guess common misconfiguration and emit
+# user-friendly hints instead of false-positive error messages
+suggestion-mode=yes
+
+# List of plugins (as comma separated values of python modules names) to load,
+# usually to register additional checkers.
+load-plugins=pylint_django
+
diff --git a/delbert.py b/delbert.py
index 4e05dabd..d3526487 100755
--- a/delbert.py
+++ b/delbert.py
@@ -14,13 +14,12 @@ import util.importer
from core.models import Student, Submission
+unused_variable = []
-unused_variable = []
-
def parseme():
- parser = argparse.ArgumentParser()
- subparsers = parser.add_subparsers(dest="command")
+ parser = argparse.ArgumentParser()
+ subparsers = parser.add_subparsers(dest="command")
parser.add_argument(
'-o', '--output',
@@ -109,7 +108,7 @@ def handle_enableusers(switch, exclude, include, **kwargs):
for user in User.objects.filter(username__in=include):
user.is_active = switch == 'on'
user.save()
- else: # this includes nothing set
+ else: # this includes nothing set
for user in User.objects.exclude(username__in=exclude):
user.is_active = switch == 'on'
user.save()
@@ -126,16 +125,19 @@ def handle_replaceusernames(matno2username_dict, **kwargs):
def handle_extractsubmissions(output, **kwargs):
for submission in Submission.objects.filter(feedback__isnull=False).order_by('type'):
- print(submission.feedback.score, repr(submission.text), file=open(str(submission.type).replace(' ', '_'), 'a'))
+ print(submission.feedback.score, repr(submission.text),
+ file=open(str(submission.type).replace(' ', '_'), 'a'))
def handle_importer(**kwargs):
util.importer.start()
+
def main():
args = parseme()
if args.command:
globals()['handle_' + args.command](**vars(args))
+
if __name__ == '__main__':
main()
diff --git a/scripts/README.rst b/scripts/README.rst
deleted file mode 100644
index 46af791e..00000000
--- a/scripts/README.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-What is this directory about?
-=============================
-
-Well, it just serves as a collection of files that currently live in folders
-not part of the git repository, since they contain volatile or test data. I
-include them here for the sake of completeness, but they will be removed in
-later versions, since their work has to be encapsulated in the overall process.
-When documentations becomes more accurate a detailed explanation on how to use
-them will be added.
-
-.. note:: Please keep in mind: These file are of poor quality and are likely to fail if not used in a correct manner.
diff --git a/scripts/compile.py b/scripts/compile.py
deleted file mode 100755
index 091669a7..00000000
--- a/scripts/compile.py
+++ /dev/null
@@ -1,104 +0,0 @@
-#!/usr/local/bin/python3
-"""This script adds compiler output to the json output of the convert script
-
-[description]
-"""
-import subprocess
-import json
-import os
-
-TEMP_DIR = 'temp_code'
-OUTFILE = 'submissions_compiled.json'
-INFILE = 'submissions.json'
-OBJECT_DIR = 'klausur_tag_01/objects'
-
-ALL_OK = 0
-COMPILE_ERROR = 1
-LINKER_ERROR = 2
-WARNINGS = 3
-
-
-def get_compiler_output(task_id, text):
- dst = open(os.path.join(TEMP_DIR, task_id + '.o'), 'w')
- try:
- compile_cmd = subprocess.run(
- [
- "gcc-6", "-Wall", "-std=c11", "-c", "-xc",
- "-o", dst.name,
- f"-I{TEMP_DIR}",
- "-"
- ],
- stderr=subprocess.PIPE,
- input=text,
- encoding='utf-8',
- )
-
- if compile_cmd.returncode:
- return compile_cmd, None, COMPILE_ERROR # it fucking failed
-
- object_code = os.path.join(OBJECT_DIR, f"{task_id}-testing.o")
- if os.path.exists(object_code): # this is ok
- link_cmd = subprocess.run(
- [
- "gcc-6",
- "-o", "/dev/null",
- dst.name,
- object_code
-
- ],
- stderr=subprocess.PIPE,
- encoding='utf-8',
- )
- else: # this case is weird
- if task_id == 'a05': # for day 2 task a05 its ok to just compile
- return compile_cmd, None, compile_cmd.returncode
- elif task_id == 'a06':
- link_cmd = subprocess.run(
- [
- "gcc-6",
- "-o", "/dev/null",
- dst.name,
- os.path.join(TEMP_DIR, "a05.o"),
- ],
- stderr=subprocess.PIPE,
- encoding='utf-8',
- )
-
- if link_cmd.returncode:
- return compile_cmd, link_cmd, LINKER_ERROR
-
- return compile_cmd, link_cmd, ALL_OK
- finally:
- dst.close()
-
-
-def main():
- with open(INFILE, 'r', encoding='utf-8') as submissions:
- data = json.JSONDecoder().decode(submissions.read())
-
- total = len(data)
- for i, (username, userinfo) in enumerate(data.items()):
- print(f"\r- {i+1}/{total} done. processing submissions of {username}\t\t\t\t", end='')
-
- # create new entry
- co = userinfo['compiler_output'] = {}
-
- for task_id, submission in userinfo['submissions'].items():
- if submission:
- _compile, _link, err = get_compiler_output(task_id, submission)
- msg = {
- COMPILE_ERROR: "\n\n[FAIL] DID NOT COMPILE",
- LINKER_ERROR: "\n\n[FAIL] COULD NOT LINK",
- ALL_OK: "",
- }[err]
- co[task_id] = _compile.stderr + (_link.stderr if _link else "") + msg
- else:
- co[task_id] = ""
-
- print()
-
- with open(OUTFILE, 'w', encoding='utf-8') as submissions:
- submissions.write(json.JSONEncoder().encode(data))
-
-if __name__ == '__main__':
- main()
diff --git a/util/convert.py b/util/convert.py
index e038b7f3..44995cbb 100755
--- a/util/convert.py
+++ b/util/convert.py
@@ -48,18 +48,18 @@ parser.add_argument('OUTFILE', help='Where to write the final file')
parser.add_argument('-u', '--usernames', help='a json dict matno -> email')
parser.add_argument(
'-n', '--NUMBER_OF_TASKS',
- default=0, # don't check
+ default=0, # don't check
metavar='NUMBER_OF_TASKS',
type=int,
help='Where to write the final file')
-
# one user has one submission (code) per task
# yes, I know it is possible to name match groups via (?P<name>) but
# I like this solution better since it gets the job done nicely
user_head = namedtuple('user_head', 'kohorte, name')
-user_head_re = re.compile(r'^Ergebnisse von Testdurchlauf (?P<kohorte>\d+) für (?P<name>[\w\s\.,-]+)$')
+user_head_re = re.compile(
+ r'^Ergebnisse von Testdurchlauf (?P<kohorte>\d+) für (?P<name>[\w\s\.,-]+)$')
# one task has a title and id and hpfly code
task_head_re = re.compile(r'^Quellcode Frage(?P<title>.*) \d{8}$')
@@ -67,6 +67,7 @@ task_head_re = re.compile(r'^Quellcode Frage(?P<title>.*) \d{8}$')
# nor parsing the weird mat no
matno_re = re.compile(r'^(?P<matrikel_no>\d{8})-(\d{3})-(\d{3})$')
+
def converter(infile, usernames=None, number_of_tasks=0,):
# Modify these iterators in order to change extraction behaviour
@@ -77,7 +78,6 @@ def converter(infile, usernames=None, number_of_tasks=0,):
m = re.search(matno_re, row[1].value)
yield row[0].value, m.group('matrikel_no') if m else row[1].value
-
def sheet_iter_data(sheet):
""" yields all rows that are not of empty type as one string """
for row in (sheet.row(i) for i in range(sheet.nrows)):
@@ -102,7 +102,7 @@ def converter(infile, usernames=None, number_of_tasks=0,):
root.append([user_head(*user.groups())])
elif task:
root[-1].append(task.group('title'))
- else: # should be code
+ else: # should be code
root[-1].append(urllib.parse.unquote(row).strip())
if number_of_tasks:
@@ -119,25 +119,26 @@ def converter(infile, usernames=None, number_of_tasks=0,):
return mat_to_email[name2mat[user.name]].split('@')[0]
return ''.join(filter(str.isupper, user.name)) + name2mat[user.name]
- usernames = {user.name : get_username(user) for (user, *_) in root}
+ usernames = {user.name: get_username(user) for (user, *_) in root}
# form list to json_like via comprehension
# the format {userinitials + matrikel_no : {name:, matrikel_no:, tasklist: {id:, ..., id:}}}
return {
- usernames[user.name] : {
- 'name' : user.name,
- 'email' : mat_to_email[name2mat[user.name]],
- 'matrikel_no' : name2mat[user.name],
- 'submissions' : [
+ usernames[user.name]: {
+ 'name': user.name,
+ 'email': mat_to_email[name2mat[user.name]],
+ 'matrikel_no': name2mat[user.name],
+ 'submissions': [
{
- "type" : task,
- "code" : code,
- "tests" : {},
+ "type": task,
+ "code": code,
+ "tests": {},
} for task, code in zip(task_list[::2], task_list[1::2])
]
} for (user, *task_list) in sorted(root, key=lambda u: u[0].name)
}
+
def write_to_file(json_dict, outfile):
# just encode python style
with open(outfile, "w") as out:
@@ -151,6 +152,7 @@ def main():
json_dict = converter(args.INFILE, args.usernames, args.NUMBER_OF_TASKS)
write_to_file(json_dict, args.OUTFILE)
+
if __name__ == '__main__':
SCRIPT = True
main()
diff --git a/util/factories.py b/util/factories.py
index 32c2d67f..3d2edef2 100644
--- a/util/factories.py
+++ b/util/factories.py
@@ -22,7 +22,7 @@ def store_password(username, groupname, password):
storage = configparser.ConfigParser()
storage.read(PASSWORDS)
- if not groupname in storage:
+ if groupname not in storage:
storage[groupname] = {}
storage[groupname][username] = password
diff --git a/util/importer.py b/util/importer.py
index f9fed844..1fbbbb77 100644
--- a/util/importer.py
+++ b/util/importer.py
@@ -7,8 +7,8 @@ from typing import Callable
import util.convert
import util.processing
from core.models import UserAccount as User
-from core.models import (ExamType, Feedback, Reviewer, Student, Submission,
- SubmissionType, Test, Tutor)
+from core.models import (ExamType, Feedback, Student, Submission,
+ SubmissionType, Test)
from util.messages import info, warn
from util.processing import EmptyTest
@@ -60,7 +60,8 @@ class chdir_context(object):
def i(prompt: str, default: str='', is_path: bool=False, is_file: bool=False):
if default is YES or default is NO:
- answer = valid[input(f'[Q] {prompt} ({default}): ').lower() or ('y' if YES == default else 'n')]
+ answer = valid[input(f'[Q] {prompt} ({default}): ').lower() or (
+ 'y' if YES == default else 'n')]
elif default:
answer = input(f'[Q] {prompt} ({default}): ') or default
else:
@@ -72,7 +73,6 @@ def i(prompt: str, default: str='', is_path: bool=False, is_file: bool=False):
return answer
-# TODO more factories
def add_user(username, group, **kwargs):
user = GradyUserFactory()._make_base_user(
@@ -277,8 +277,8 @@ def do_preprocess_submissions():
print('''
Preprocessing might take some time depending on the amount of data
- and the complexity of the programs and the corresponding unit tests. You can
- specify what test you want to run.
+ and the complexity of the programs and the corresponding unit tests. You
+ can specify what test you want to run.
Tests do depend on each other. Therefore specifying a test will also
result in running all its dependencies\n''')
@@ -360,9 +360,9 @@ def start():
print('''Welcome to the Grady importer!
- This script aims at making the setup of the database as easy as possible. It
- at the same time serves as a documentation on how data is imported in Grady.
- Let\'s dive right in.\n''')
+ This script aims at making the setup of the database as easy as possible.
+ It at the same time serves as a documentation on how data is imported in
+ Grady. Let\'s dive right in.\n''')
try:
print('The following importers are available:\n')
diff --git a/util/messages.py b/util/messages.py
index 8f236313..7f1aeac4 100644
--- a/util/messages.py
+++ b/util/messages.py
@@ -4,18 +4,23 @@ import sys
def warn(*message):
print('[W]', *message)
+
def debug(*message):
print('[DEBUG]', *message)
+
def info(*message):
print('[I]', *message)
+
def error(*message):
print('[E]', *message)
+
def abort(*message):
print('[FATAL]', *message)
sys.exit('exiting...')
+
def exit(message='exiting...'):
sys.exit(*message)
diff --git a/util/processing.py b/util/processing.py
index e83a44eb..26671595 100644
--- a/util/processing.py
+++ b/util/processing.py
@@ -12,11 +12,11 @@ try:
except ModuleNotFoundError:
from util import testcases
-DESCFILE = '../data/descfile.txt'
-BINARIES = '../data/klausur_zweittermin/bin'
-OBJECTS = '../data/klausur_zweittermin/objects'
+DESCFILE = '../data/descfile.txt'
+BINARIES = '../data/klausur_zweittermin/bin'
+OBJECTS = '../data/klausur_zweittermin/objects'
SUBMISSIONS = '../data/binf1801_pre.json'
-HEADER = '../data/klausur_zweittermin/code-testing'
+HEADER = '../data/klausur_zweittermin/code-testing'
def run_cmd(cmd, stdin=None, check=False, timeout=1):
@@ -46,18 +46,18 @@ class Test(metaclass=abc.ABCMeta):
@classmethod
def available_tests(cls):
- return {sub.__name__ : sub for sub in all_subclasses(cls)}
+ return {sub.__name__: sub for sub in all_subclasses(cls)}
def __new__(cls, *args, **kwargs):
- assert hasattr(cls, 'depends'), "depends not defined"
- assert hasattr(cls, 'label_success'), "label_success not defined"
- assert hasattr(cls, 'label_failure'), "label_failure not defined"
+ assert hasattr(cls, 'depends'), "depends not defined"
+ assert hasattr(cls, 'label_success'), "label_success not defined"
+ assert hasattr(cls, 'label_failure'), "label_failure not defined"
return super().__new__(cls)
def __init__(self, submission_obj, **kwargs):
if not self.dependencies_satisfied(submission_obj):
- self.result = False
+ self.result = False
self.annotation = "TEST DEPENDENCY NOT MET"
self.serialize(submission_obj)
@@ -79,13 +79,13 @@ class Test(metaclass=abc.ABCMeta):
return all(dep(submission_obj).result for dep in self.depends)
def deserialize(self, test):
- self.result = test['label'] == self.label_success
+ self.result = test['label'] == self.label_success
self.annotation = test['annotation']
def serialize(self, submission_obj):
as_dict = {
- 'name' : str(self),
- 'annotation' : self.annotation
+ 'name': str(self),
+ 'annotation': self.annotation
}
if self.result:
@@ -103,9 +103,9 @@ class Test(metaclass=abc.ABCMeta):
class EmptyTest(Test):
"""docstring for EmptyTest"""
- depends = ()
- label_success = 'NOT_EMPTY'
- label_failure = 'EMPTY'
+ depends = ()
+ label_success = 'NOT_EMPTY'
+ label_failure = 'EMPTY'
def run_test(self, submission_obj):
return bool(submission_obj['code'].strip()), ""
@@ -113,9 +113,9 @@ class EmptyTest(Test):
class CompileTest(Test):
- depends = (EmptyTest, )
- label_success = 'COMPILATION_SUCCESSFUL'
- label_failure = 'COMPILATION_FAILED'
+ depends = (EmptyTest, )
+ label_success = 'COMPILATION_SUCCESSFUL'
+ label_failure = 'COMPILATION_FAILED'
def run_test(self, submission_obj):
@@ -126,9 +126,9 @@ class CompileTest(Test):
class LinkTest(Test):
- depends = (CompileTest, )
- label_success = 'LINKING_SUCCESSFUL'
- label_failure = 'LINKING_FAILED'
+ depends = (CompileTest, )
+ label_success = 'LINKING_SUCCESSFUL'
+ label_failure = 'LINKING_FAILED'
def run_test(self, submission_obj):
@@ -142,9 +142,9 @@ class LinkTest(Test):
class UnitTestTest(Test):
"""docstring for UnitTestTest"""
- depends = (LinkTest, )
- label_success = 'UNITTEST_SUCCSESSFUL'
- label_failure = 'UNITTEST_FAILED'
+ depends = (LinkTest, )
+ label_success = 'UNITTEST_SUCCSESSFUL'
+ label_failure = 'UNITTEST_FAILED'
@staticmethod
def testcase(i, args, stdout):
diff --git a/util/testcases.py b/util/testcases.py
index b94fab23..a95e657d 100644
--- a/util/testcases.py
+++ b/util/testcases.py
@@ -12,8 +12,7 @@ except ModuleNotFoundError:
types = ('integer', 'unsigned_integer', 'character', 'string')
list_sep = '...'
-re_task = re.compile(
- r'^-- (?P<title>.*)\n(USAGE: (?P<cmd>[\./\w]+) (?P<syntax>.*)|NO EXECUTABLE)', re.MULTILINE)
+re_task = re.compile(r'^-- (?P<title>.*)\n(USAGE: (?P<cmd>[\./\w]+) (?P<syntax>.*)|NO EXECUTABLE)', re.MULTILINE)
re_args = re.compile(rf"<({'|'.join(types)}|{'|'.join(t + '_list' for t in types)})>")
@@ -30,7 +29,7 @@ def unsigned_integer(upper=50):
def character():
- return random.choice(10*ascii_letters + 2*digits + '%*+,-./:?@[]^_{}~')
+ return random.choice(10 * ascii_letters + 2 * digits + '%*+,-./:?@[]^_{}~')
def string(lenght=31):
@@ -48,8 +47,7 @@ def rubbish():
def argument_generator(syntax):
- syntax, _ = re.subn(
- r'<([\w\s]+)> <\1> \.\.\. <\1> <\1>', r'<\1_list>', syntax)
+ syntax, _ = re.subn(r'<([\w\s]+)> <\1> \.\.\. <\1> <\1>', r'<\1_list>', syntax)
syntax, _ = re.subn(r'<(\w+)\s(\w+)>', r'<\1_\2>', syntax)
return ' '.join(str(call_function(arg)) for arg in re.findall(re_args, syntax))
@@ -64,7 +62,7 @@ def testcases_generator(task, n=10):
yield ''
yield '0'
- for i in range(n//2):
+ for i in range(n // 2):
yield rubbish()
for i in range(n):
@@ -73,26 +71,31 @@ def testcases_generator(task, n=10):
def testcases(description_path):
for t in types:
- globals()[t + '_list'] = type_list(t) # I fucking love it
+ globals()[t + '_list'] = type_list(t) # I fucking love it
with open(description_path) as description_file:
description = description_file.read()
return {
- task['title'] : {
- 'cmd' : task['cmd'],
- 'cases' : [t for t in testcases_generator(task)]
+ task['title']: {
+ 'cmd': task['cmd'],
+ 'cases': [t for t in testcases_generator(task)]
} for task in re.finditer(re_task, description)
}
+
def evaluated_testcases(description_path):
task_testcases = testcases(description_path)
for task in filter(lambda t: t['cmd'], task_testcases.values()):
- path_to_binary = os.path.join(os.path.join(processing.BINARIES, os.path.basename(task['cmd'])))
- task['results'] = [processing.run_cmd(f"{path_to_binary} {case}").stdout for case in task['cases']]
+ path_to_binary = os.path.join(os.path.join(
+ processing.BINARIES, os.path.basename(task['cmd'])))
+ task['results'] = [processing.run_cmd(
+ f"{path_to_binary} {case}").stdout for case in task['cases']]
return task_testcases
+
if __name__ == '__main__':
- print(json.dumps(evaluated_testcases(processing.DESCFILE), sort_keys=True, indent=4))
+ print(json.dumps(evaluated_testcases(
+ processing.DESCFILE), sort_keys=True, indent=4))
--
GitLab