First version of test runner

This script has been extracted from the proper gray project

.gitignore

+.venv/
\ No newline at end of file

README.md

-# grady-test-runner
+# Grady Test Runner
 
-Run different tests on submissions contained in export transformed by Rusty Hektor.
+Run different tests on submissions contained in export transformed by Rusty Hektor.
\ No newline at end of file
+
+Currently supports empty and C compile tests.
\ No newline at end of file

requirements.txt

+tqdm~=4.31
\ No newline at end of file

test.py

+import abc
+import json
+import logging
+import os
+import shutil
+import subprocess
+import tempfile
+
+from tqdm import tqdm
+
+log = logging.getLogger(__name__)
+
+
+def run_cmd(cmd, stdin=None, check=False, timeout=1):
+    return subprocess.run(
+        'timeout 1 ' + cmd,
+        stderr=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        input=stdin,
+        shell=True,
+        check=check,
+        encoding='utf-8',
+        timeout=timeout
+    )
+
+
+def all_subclasses(cls):
+    return cls.__subclasses__() \
+        + [g for s in cls.__subclasses__() for g in all_subclasses(s)]
+
+
+class Test(metaclass=abc.ABCMeta):
+    """docstring for IliasQuestion"""
+
+    @classmethod
+    def available_tests(cls):
+        return {sub.__name__: sub for sub in all_subclasses(cls)}
+
+    def __new__(cls, *args, **kwargs):
+        assert hasattr(cls, 'depends'), "depends not defined"
+        assert hasattr(cls, 'label_success'), "label_success not defined"
+        assert hasattr(cls, 'label_failure'), "label_failure not defined"
+        return super().__new__(cls)
+
+    def __init__(self, submission_obj, **kwargs):
+        if not self.dependencies_satisfied(submission_obj):
+            self.result = False
+            self.annotation = "TEST DEPENDENCY NOT MET"
+            self.serialize(submission_obj)
+
+        elif str(self) in submission_obj['tests']:
+            self.deserialize(submission_obj['tests'][str(self)])
+
+        else:
+            self.result, self.annotation = self.run_test(submission_obj, **kwargs)
+            self.serialize(submission_obj)
+
+    def __bool__(self):
+        return self.result
+
+    def __str__(self):
+        return self.__class__.__name__
+
+    def dependencies_satisfied(self, submission_obj):
+        return all(dep(submission_obj).result for dep in self.depends)
+
+    def deserialize(self, test):
+        self.result = test['label'] == self.label_success
+        self.annotation = test['annotation']
+
+    def serialize(self, submission_obj):
+        as_dict = {
+            'name': str(self),
+            'annotation': self.annotation
+        }
+
+        if self.result:
+            as_dict['label'] = self.label_success
+        else:
+            as_dict['label'] = self.label_failure
+
+        submission_obj['tests'][str(self)] = as_dict
+
+    @abc.abstractmethod
+    def run_test(self, submission_obj, **kwargs) -> (bool, str):
+        return NotImplemented
+
+
+class EmptyTest(Test):
+    """docstring for EmptyTest"""
+
+    depends = ()
+    label_success = 'NOT_EMPTY'
+    label_failure = 'EMPTY'
+
+    def run_test(self, submission_obj, **kwargs):
+        return bool(submission_obj['code'].strip()), ""
+
+
+class CompileTest(Test):
+
+    depends = (EmptyTest, )
+    label_success = 'COMPILATION_SUCCESSFUL'
+    label_failure = 'COMPILATION_FAILED'
+
+    def run_test(self, submission_obj, **kwargs):
+
+        ret = run_cmd(
+            f"gcc -Wall -c -x c -std=c11 -I {kwargs['header']} -o code.o -",
+            submission_obj['code'])
+        return not ret.returncode, ret.stderr
+
+
+def process(submissions, header, highest_test):
+    highest_test_class = Test.available_tests()[highest_test]
+
+    with open(submissions) as submission_file:
+        submissions_json = json.JSONDecoder().decode(
+            submission_file.read())
+
+    # Get something disposable
+    if highest_test != EmptyTest.__name__:
+        path = tempfile.mkdtemp()
+        run_cmd(f'cp -r {header} {path}')
+        os.chdir(path)
+
+    def iterate_submissions():
+        yield from (obj
+                    for student in tqdm(submissions_json['students'])
+                    for obj in student['submissions'])
+
+    for submission_obj in tqdm(iterate_submissions()):
+        highest_test_class(submission_obj, header=header)
+        if highest_test != EmptyTest.__name__:
+            run_cmd('rm code*')
+    print()  # line after progress bar
+    if highest_test != EmptyTest.__name__:
+        shutil.rmtree(path)
+    return submissions_json
+
+
+def parseme():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument('submissions')
+    parser.add_argument('header')
+    parser.add_argument('test')
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    args = parseme()
+    print(json.dumps(process(args.submissions,
+                             args.header,
+                             args.test),
+                     sort_keys=True,
+                     indent=4))