Added results of sample program to testcases

util/processing.py

@@ -15,6 +15,7 @@ OBJECTS     = '../data/klausur20170627/objects'
 SUBMISSIONS = '../data/ok.json'
 HEADER      = '../data/klausur20170627/code-testing'
 
+
 def run_cmd(cmd, stdin=None, check=False):
     return subprocess.run(
         'ulimit -v 1024;' + cmd,
@@ -27,20 +28,19 @@ def run_cmd(cmd, stdin=None, check=False):
         timeout=0.5
     )
 
-'submission'
 
 def testcase(i, args, stdout):
     try:
         ret = run_cmd("./code %s" % args, check=True)
         assert ret.stdout == stdout
     except AssertionError:
-        return f"Case #{i}: [ASSERT FAILED] ./program {args} WAS '{ret.stdout}' SHOULD '{stdout}'"
-    except subprocess.CalledProcessError:
-        return f"Case #{i}: [FAILED] ./program {args}"
+        return False, f"Case #{i}: [ASSERT FAILED] ./program {args} WAS '{ret.stdout}' SHOULD '{stdout}'"
+    except subprocess.CalledProcessError as err:
+        return False, f"Case #{i}: [FAILED] ./program {args} WITH ERROR '{err.stderr}'"
     except subprocess.TimeoutExpired:
-        return f"Case #{i}: [TIMEOUT] ./program {args}"
+        return False, f"Case #{i}: [TIMEOUT] ./program {args}"
     else:
-        return f"Case #{i}: [SUCCESS] ./program {args}"
+        return True,  f"Case #{i}: [SUCCESS] ./program {args}"
 
 
 def all_subclasses(cls):
@@ -76,7 +76,8 @@ class Test(metaclass=abc.ABCMeta):
             self.deserialize(submission_obj['tests'][str(self)])
 
         else:
-            self.result, self.annotation = self.run_test(submission_obj, **kwargs)
+            self.result, self.annotation = self.run_test(
+                submission_obj, **kwargs)
             self.serialize(submission_obj)
 
     def __bool__(self):
@@ -129,7 +130,8 @@ class CompileTest(Test):
 
     def run_test(self, submission_obj):
 
-        ret = run_cmd("gcc-7 -c -xc -Icode-testing -o code.o -", submission_obj['code'])
+        ret = run_cmd("gcc-7 -c -xc -Icode-testing -o code.o -",
+                      submission_obj['code'])
         return not ret.returncode, ret.stderr
 
 
@@ -158,11 +160,13 @@ class UnitTestTest(Test):
     def run_test(self, submission_obj):
 
         task = testcases_dict[submission_obj['type']]
+        results, messages = zip(*list(testcase(i, case, result)
+                                      for i, (case, result) in enumerate(zip(task['cases'], task['results']))))
 
-        return 1, '\n'.join(testcase(i, case, run_cmd(f"{task['cmd']} {case}").stdout) for i, case in enumerate(task['cases']))
+        return all(results), '\n'.join(messages)
 
 
-def main():
+def processing():
 
     with open(SUBMISSIONS) as submission_file:
         submissions = json.JSONDecoder().decode(submission_file.read())
@@ -179,15 +183,10 @@ def main():
             UnitTestTest(submission_obj)
             run_cmd('rm code*')
 
-    print(json.dumps(submissions, sort_keys=True, indent=4))
-
-if __name__ == '__main__':
-    testcases_dict = testcases.testcases(DESCFILE)
-
-    from pprint import pprint
-    main()
-
+    shutil.rmtree(path)
+    return submissions
 
-# x = "#include \"a06-testing.h\"\n#include <stdio.h>\nvoid main() {\nchar *a;\nint *b;\nint i=0;\nint j=0;\nint k=0;\nwhile (scanf(\"%s\", &a)) {\nb[i]=ahoi(a);\ni++;\n}\nwhile(i-j) {\nprintf(\"%i\", b[j++]);\n}\nint c=1;\nint *t=b+i;\nint recSort(b, t, c)\nwhile (i-k) {\nprintf(\"%i\", b[k++]);\n}"
 
-# print(x)
+if __name__ == '__main__':
+    testcases_dict = testcases.evaluated_testcases(DESCFILE)
+    print(json.dumps(processing(), sort_keys=True, indent=4))

util/testcases.py

 import re
+import os
 import json
 import random
 from string import ascii_letters, digits
 
+import processing
+
 types = ('integer', 'unsigned_integer', 'character', 'string')
 list_sep = '...'
 
@@ -15,11 +18,11 @@ def call_function(name: str, *args, **kwargs):
     return globals()[name](*args, **kwargs)
 
 
-def integer(bounds=1000):
+def integer(bounds=50):
     return random.randint(-bounds, bounds)
 
 
-def unsigned_integer(upper=1000):
+def unsigned_integer(upper=50):
     return random.randint(0, upper)
 
 
@@ -33,7 +36,7 @@ def string(lenght=31):
 
 def type_list(_type):
     def generic_list():
-        return ' '.join(str(call_function(_type)) for i in range(unsigned_integer(30)))
+        return ' '.join(str(call_function(_type)) for i in range(unsigned_integer(10)))
     return generic_list
 
 
@@ -79,6 +82,14 @@ def testcases(description_path):
         } for task in re.finditer(re_task, description)
     }
 
+def evaluated_testcases(description_path):
+    task_testcases = testcases(description_path)
+
+    for task in filter(lambda t: t['cmd'], task_testcases.values()):
+        path_to_binary = os.path.join(os.path.join(processing.BINARIES, os.path.basename(task['cmd'])))
+        task['results'] = [processing.run_cmd(f"{path_to_binary} {case}").stdout for case in task['cases']]
+
+    return task_testcases
 
 if __name__ == '__main__':
-    print(json.JSONEncoder().encode(testcases()))
+    print(json.dumps(evaluated_testcases(processing.DESCFILE), sort_keys=True, indent=4))