diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..0cafc1cde1985c69113a5b2ae7ba42299aa7ebc2 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.venv/ \ No newline at end of file diff --git a/README.md b/README.md index cbe5b1838b7d1c998f05cf6b8b7a74f0361eb7a8..850f3b40aca2673b3025f8c8b25e30e29814fee9 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ -# grady-test-runner +# Grady Test Runner -Run different tests on submissions contained in export transformed by Rusty Hektor. \ No newline at end of file +Run different tests on submissions contained in export transformed by Rusty Hektor. + +Currently supports empty and C compile tests. \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..93a1e7b63b6f8b4fa150314a3329c4c4492d2b36 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +tqdm~=4.31 \ No newline at end of file diff --git a/test.py b/test.py new file mode 100755 index 0000000000000000000000000000000000000000..6f66a9cf084286e98cd508fd12bb652a312ef8ba --- /dev/null +++ b/test.py @@ -0,0 +1,157 @@ +import abc +import json +import logging +import os +import shutil +import subprocess +import tempfile + +from tqdm import tqdm + +log = logging.getLogger(__name__) + + +def run_cmd(cmd, stdin=None, check=False, timeout=1): + return subprocess.run( + 'timeout 1 ' + cmd, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + input=stdin, + shell=True, + check=check, + encoding='utf-8', + timeout=timeout + ) + + +def all_subclasses(cls): + return cls.__subclasses__() \ + + [g for s in cls.__subclasses__() for g in all_subclasses(s)] + + +class Test(metaclass=abc.ABCMeta): + """docstring for IliasQuestion""" + + @classmethod + def available_tests(cls): + return {sub.__name__: sub for sub in all_subclasses(cls)} + + def __new__(cls, *args, **kwargs): + assert hasattr(cls, 'depends'), "depends not defined" + assert hasattr(cls, 'label_success'), "label_success not defined" + assert hasattr(cls, 'label_failure'), "label_failure not defined" + return super().__new__(cls) + + def __init__(self, submission_obj, **kwargs): + if not self.dependencies_satisfied(submission_obj): + self.result = False + self.annotation = "TEST DEPENDENCY NOT MET" + self.serialize(submission_obj) + + elif str(self) in submission_obj['tests']: + self.deserialize(submission_obj['tests'][str(self)]) + + else: + self.result, self.annotation = self.run_test(submission_obj, **kwargs) + self.serialize(submission_obj) + + def __bool__(self): + return self.result + + def __str__(self): + return self.__class__.__name__ + + def dependencies_satisfied(self, submission_obj): + return all(dep(submission_obj).result for dep in self.depends) + + def deserialize(self, test): + self.result = test['label'] == self.label_success + self.annotation = test['annotation'] + + def serialize(self, submission_obj): + as_dict = { + 'name': str(self), + 'annotation': self.annotation + } + + if self.result: + as_dict['label'] = self.label_success + else: + as_dict['label'] = self.label_failure + + submission_obj['tests'][str(self)] = as_dict + + @abc.abstractmethod + def run_test(self, submission_obj, **kwargs) -> (bool, str): + return NotImplemented + + +class EmptyTest(Test): + """docstring for EmptyTest""" + + depends = () + label_success = 'NOT_EMPTY' + label_failure = 'EMPTY' + + def run_test(self, submission_obj, **kwargs): + return bool(submission_obj['code'].strip()), "" + + +class CompileTest(Test): + + depends = (EmptyTest, ) + label_success = 'COMPILATION_SUCCESSFUL' + label_failure = 'COMPILATION_FAILED' + + def run_test(self, submission_obj, **kwargs): + + ret = run_cmd( + f"gcc -Wall -c -x c -std=c11 -I {kwargs['header']} -o code.o -", + submission_obj['code']) + return not ret.returncode, ret.stderr + + +def process(submissions, header, highest_test): + highest_test_class = Test.available_tests()[highest_test] + + with open(submissions) as submission_file: + submissions_json = json.JSONDecoder().decode( + submission_file.read()) + + # Get something disposable + if highest_test != EmptyTest.__name__: + path = tempfile.mkdtemp() + run_cmd(f'cp -r {header} {path}') + os.chdir(path) + + def iterate_submissions(): + yield from (obj + for student in tqdm(submissions_json['students']) + for obj in student['submissions']) + + for submission_obj in tqdm(iterate_submissions()): + highest_test_class(submission_obj, header=header) + if highest_test != EmptyTest.__name__: + run_cmd('rm code*') + print() # line after progress bar + if highest_test != EmptyTest.__name__: + shutil.rmtree(path) + return submissions_json + + +def parseme(): + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('submissions') + parser.add_argument('header') + parser.add_argument('test') + return parser.parse_args() + + +if __name__ == '__main__': + args = parseme() + print(json.dumps(process(args.submissions, + args.header, + args.test), + sort_keys=True, + indent=4))