diff --git a/util/processing.py b/util/processing.py index bf0b60baa78d7bcdbe2f587c0a1086546e2d2f19..d67d3c4a70fde90b6c0e63493612bb7d049a1005 100644 --- a/util/processing.py +++ b/util/processing.py @@ -15,6 +15,7 @@ OBJECTS = '../data/klausur20170627/objects' SUBMISSIONS = '../data/ok.json' HEADER = '../data/klausur20170627/code-testing' + def run_cmd(cmd, stdin=None, check=False): return subprocess.run( 'ulimit -v 1024;' + cmd, @@ -27,20 +28,19 @@ def run_cmd(cmd, stdin=None, check=False): timeout=0.5 ) -'submission' def testcase(i, args, stdout): try: ret = run_cmd("./code %s" % args, check=True) assert ret.stdout == stdout except AssertionError: - return f"Case #{i}: [ASSERT FAILED] ./program {args} WAS '{ret.stdout}' SHOULD '{stdout}'" - except subprocess.CalledProcessError: - return f"Case #{i}: [FAILED] ./program {args}" + return False, f"Case #{i}: [ASSERT FAILED] ./program {args} WAS '{ret.stdout}' SHOULD '{stdout}'" + except subprocess.CalledProcessError as err: + return False, f"Case #{i}: [FAILED] ./program {args} WITH ERROR '{err.stderr}'" except subprocess.TimeoutExpired: - return f"Case #{i}: [TIMEOUT] ./program {args}" + return False, f"Case #{i}: [TIMEOUT] ./program {args}" else: - return f"Case #{i}: [SUCCESS] ./program {args}" + return True, f"Case #{i}: [SUCCESS] ./program {args}" def all_subclasses(cls): @@ -76,7 +76,8 @@ class Test(metaclass=abc.ABCMeta): self.deserialize(submission_obj['tests'][str(self)]) else: - self.result, self.annotation = self.run_test(submission_obj, **kwargs) + self.result, self.annotation = self.run_test( + submission_obj, **kwargs) self.serialize(submission_obj) def __bool__(self): @@ -129,7 +130,8 @@ class CompileTest(Test): def run_test(self, submission_obj): - ret = run_cmd("gcc-7 -c -xc -Icode-testing -o code.o -", submission_obj['code']) + ret = run_cmd("gcc-7 -c -xc -Icode-testing -o code.o -", + submission_obj['code']) return not ret.returncode, ret.stderr @@ -158,11 +160,13 @@ class UnitTestTest(Test): def run_test(self, submission_obj): task = testcases_dict[submission_obj['type']] + results, messages = zip(*list(testcase(i, case, result) + for i, (case, result) in enumerate(zip(task['cases'], task['results'])))) - return 1, '\n'.join(testcase(i, case, run_cmd(f"{task['cmd']} {case}").stdout) for i, case in enumerate(task['cases'])) + return all(results), '\n'.join(messages) -def main(): +def processing(): with open(SUBMISSIONS) as submission_file: submissions = json.JSONDecoder().decode(submission_file.read()) @@ -179,15 +183,10 @@ def main(): UnitTestTest(submission_obj) run_cmd('rm code*') - print(json.dumps(submissions, sort_keys=True, indent=4)) - -if __name__ == '__main__': - testcases_dict = testcases.testcases(DESCFILE) - - from pprint import pprint - main() - + shutil.rmtree(path) + return submissions -# x = "#include \"a06-testing.h\"\n#include <stdio.h>\nvoid main() {\nchar *a;\nint *b;\nint i=0;\nint j=0;\nint k=0;\nwhile (scanf(\"%s\", &a)) {\nb[i]=ahoi(a);\ni++;\n}\nwhile(i-j) {\nprintf(\"%i\", b[j++]);\n}\nint c=1;\nint *t=b+i;\nint recSort(b, t, c)\nwhile (i-k) {\nprintf(\"%i\", b[k++]);\n}" -# print(x) +if __name__ == '__main__': + testcases_dict = testcases.evaluated_testcases(DESCFILE) + print(json.dumps(processing(), sort_keys=True, indent=4)) diff --git a/util/testcases.py b/util/testcases.py index e96efc26b21a1bfa131125bea0711468bae8ddd1..dab44d1733b3488e5192f63c874793893e153530 100644 --- a/util/testcases.py +++ b/util/testcases.py @@ -1,8 +1,11 @@ import re +import os import json import random from string import ascii_letters, digits +import processing + types = ('integer', 'unsigned_integer', 'character', 'string') list_sep = '...' @@ -15,11 +18,11 @@ def call_function(name: str, *args, **kwargs): return globals()[name](*args, **kwargs) -def integer(bounds=1000): +def integer(bounds=50): return random.randint(-bounds, bounds) -def unsigned_integer(upper=1000): +def unsigned_integer(upper=50): return random.randint(0, upper) @@ -33,7 +36,7 @@ def string(lenght=31): def type_list(_type): def generic_list(): - return ' '.join(str(call_function(_type)) for i in range(unsigned_integer(30))) + return ' '.join(str(call_function(_type)) for i in range(unsigned_integer(10))) return generic_list @@ -79,6 +82,14 @@ def testcases(description_path): } for task in re.finditer(re_task, description) } +def evaluated_testcases(description_path): + task_testcases = testcases(description_path) + + for task in filter(lambda t: t['cmd'], task_testcases.values()): + path_to_binary = os.path.join(os.path.join(processing.BINARIES, os.path.basename(task['cmd']))) + task['results'] = [processing.run_cmd(f"{path_to_binary} {case}").stdout for case in task['cases']] + + return task_testcases if __name__ == '__main__': - print(json.JSONEncoder().encode(testcases())) + print(json.dumps(evaluated_testcases(processing.DESCFILE), sort_keys=True, indent=4))