From f26049b983bf0b71faf633c5c8ed999742a4b776 Mon Sep 17 00:00:00 2001
From: janmax <mail-github@jmx.io>
Date: Thu, 6 Jul 2017 14:24:41 +0200
Subject: [PATCH] Added results of sample program to testcases
---
util/processing.py | 39 +++++++++++++++++++--------------------
util/testcases.py | 19 +++++++++++++++----
2 files changed, 34 insertions(+), 24 deletions(-)
diff --git a/util/processing.py b/util/processing.py
index bf0b60ba..d67d3c4a 100644
--- a/util/processing.py
+++ b/util/processing.py
@@ -15,6 +15,7 @@ OBJECTS = '../data/klausur20170627/objects'
SUBMISSIONS = '../data/ok.json'
HEADER = '../data/klausur20170627/code-testing'
+
def run_cmd(cmd, stdin=None, check=False):
return subprocess.run(
'ulimit -v 1024;' + cmd,
@@ -27,20 +28,19 @@ def run_cmd(cmd, stdin=None, check=False):
timeout=0.5
)
-'submission'
def testcase(i, args, stdout):
try:
ret = run_cmd("./code %s" % args, check=True)
assert ret.stdout == stdout
except AssertionError:
- return f"Case #{i}: [ASSERT FAILED] ./program {args} WAS '{ret.stdout}' SHOULD '{stdout}'"
- except subprocess.CalledProcessError:
- return f"Case #{i}: [FAILED] ./program {args}"
+ return False, f"Case #{i}: [ASSERT FAILED] ./program {args} WAS '{ret.stdout}' SHOULD '{stdout}'"
+ except subprocess.CalledProcessError as err:
+ return False, f"Case #{i}: [FAILED] ./program {args} WITH ERROR '{err.stderr}'"
except subprocess.TimeoutExpired:
- return f"Case #{i}: [TIMEOUT] ./program {args}"
+ return False, f"Case #{i}: [TIMEOUT] ./program {args}"
else:
- return f"Case #{i}: [SUCCESS] ./program {args}"
+ return True, f"Case #{i}: [SUCCESS] ./program {args}"
def all_subclasses(cls):
@@ -76,7 +76,8 @@ class Test(metaclass=abc.ABCMeta):
self.deserialize(submission_obj['tests'][str(self)])
else:
- self.result, self.annotation = self.run_test(submission_obj, **kwargs)
+ self.result, self.annotation = self.run_test(
+ submission_obj, **kwargs)
self.serialize(submission_obj)
def __bool__(self):
@@ -129,7 +130,8 @@ class CompileTest(Test):
def run_test(self, submission_obj):
- ret = run_cmd("gcc-7 -c -xc -Icode-testing -o code.o -", submission_obj['code'])
+ ret = run_cmd("gcc-7 -c -xc -Icode-testing -o code.o -",
+ submission_obj['code'])
return not ret.returncode, ret.stderr
@@ -158,11 +160,13 @@ class UnitTestTest(Test):
def run_test(self, submission_obj):
task = testcases_dict[submission_obj['type']]
+ results, messages = zip(*list(testcase(i, case, result)
+ for i, (case, result) in enumerate(zip(task['cases'], task['results']))))
- return 1, '\n'.join(testcase(i, case, run_cmd(f"{task['cmd']} {case}").stdout) for i, case in enumerate(task['cases']))
+ return all(results), '\n'.join(messages)
-def main():
+def processing():
with open(SUBMISSIONS) as submission_file:
submissions = json.JSONDecoder().decode(submission_file.read())
@@ -179,15 +183,10 @@ def main():
UnitTestTest(submission_obj)
run_cmd('rm code*')
- print(json.dumps(submissions, sort_keys=True, indent=4))
-
-if __name__ == '__main__':
- testcases_dict = testcases.testcases(DESCFILE)
-
- from pprint import pprint
- main()
-
+ shutil.rmtree(path)
+ return submissions
-# x = "#include \"a06-testing.h\"\n#include <stdio.h>\nvoid main() {\nchar *a;\nint *b;\nint i=0;\nint j=0;\nint k=0;\nwhile (scanf(\"%s\", &a)) {\nb[i]=ahoi(a);\ni++;\n}\nwhile(i-j) {\nprintf(\"%i\", b[j++]);\n}\nint c=1;\nint *t=b+i;\nint recSort(b, t, c)\nwhile (i-k) {\nprintf(\"%i\", b[k++]);\n}"
-# print(x)
+if __name__ == '__main__':
+ testcases_dict = testcases.evaluated_testcases(DESCFILE)
+ print(json.dumps(processing(), sort_keys=True, indent=4))
diff --git a/util/testcases.py b/util/testcases.py
index e96efc26..dab44d17 100644
--- a/util/testcases.py
+++ b/util/testcases.py
@@ -1,8 +1,11 @@
import re
+import os
import json
import random
from string import ascii_letters, digits
+import processing
+
types = ('integer', 'unsigned_integer', 'character', 'string')
list_sep = '...'
@@ -15,11 +18,11 @@ def call_function(name: str, *args, **kwargs):
return globals()[name](*args, **kwargs)
-def integer(bounds=1000):
+def integer(bounds=50):
return random.randint(-bounds, bounds)
-def unsigned_integer(upper=1000):
+def unsigned_integer(upper=50):
return random.randint(0, upper)
@@ -33,7 +36,7 @@ def string(lenght=31):
def type_list(_type):
def generic_list():
- return ' '.join(str(call_function(_type)) for i in range(unsigned_integer(30)))
+ return ' '.join(str(call_function(_type)) for i in range(unsigned_integer(10)))
return generic_list
@@ -79,6 +82,14 @@ def testcases(description_path):
} for task in re.finditer(re_task, description)
}
+def evaluated_testcases(description_path):
+ task_testcases = testcases(description_path)
+
+ for task in filter(lambda t: t['cmd'], task_testcases.values()):
+ path_to_binary = os.path.join(os.path.join(processing.BINARIES, os.path.basename(task['cmd'])))
+ task['results'] = [processing.run_cmd(f"{path_to_binary} {case}").stdout for case in task['cases']]
+
+ return task_testcases
if __name__ == '__main__':
- print(json.JSONEncoder().encode(testcases()))
+ print(json.dumps(evaluated_testcases(processing.DESCFILE), sort_keys=True, indent=4))
--
GitLab