Added ability to convert multiple .zip exports at the same time

Cargo.lock

@@ -1067,7 +1067,7 @@ dependencies = [
 
 [[package]]
 name = "rusty-hektor"
-version = "5.0.0"
+version = "5.1.0"
 dependencies = [
  "base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)",
  "calamine 0.15.5 (registry+https://github.com/rust-lang/crates.io-index)",

Cargo.toml

 [package]
 name = "rusty-hektor"
-version = "5.0.0"
+version = "5.1.0"
 authors = ["robinwilliam.hundt <robinwilliam.hundt@stud.uni-goettingen.de>"]
 license = "MIT OR Apache-2.0"
 description = "A tool to convert ILIAS exam output"

README.md

@@ -18,12 +18,12 @@ rusty-hektor -h
 
 Convert an ILIAS `.zip` (can be generated on ILIAS for an exam under `Export -> Erstelle Exportdatei (inkl. Teilnehmerergebnisse)`) containing `.xml` into `.json`:
 ```
-rusty-hektor -o <out_file_path> <path_to_zip>
+rusty-hektor -o <out_file_path> [<path_to_zip>]
 ```
 
 Converting an ILIAS Output into an **anonymised** json representation, storing the mapping file at `map.csv` (DO NOT LOOSE THIS FILE!).
 ```
-rusty-hektor -a --map-file map.csv -o <out_file_path> <path_to_(xls|zip)> 
+rusty-hektor -a --map-file map.csv -o <out_file_path> [<path_to_zip>] 
 ```
 
 If `-o` is omitted, the output is printed to stdout.  
@@ -31,6 +31,9 @@ If `-o` is omitted, the output is printed to stdout.
 By default, the program parses the free text questions and submissions of the `.xml` input. Should you
 wish to skip those questions simply call the program with the `--skip-text` flag.
 
+It is possible to submit multiple `.zip` exports. In case the exercise types contained in the provided exports are differing, a warning is issued and the first set of exercise types is used. If another set should be used, the order of the export files has to be changed when calling the program.  
+I a user with the same identifier is present in multiple exports, the program aborts and the user needs to fix the data manually (in practice this should not occur).
+
 ### Word List
 
 The anonymised names are generated by selecting random words from the EFF long list, intended for password generation.

src/main.rs → src/bin/main.rs

@@ -10,7 +10,7 @@ use semver::Version;
 use serde_derive::Deserialize;
 use structopt::StructOpt;
 
-use rusty_hektor::exam::{Exam, ExamSerializable};
+use rusty_hektor::exam::{Exam, ExamSerializable, merge_exams};
 use rusty_hektor::module::Module;
 use rusty_hektor::parser::ipynb_parser::notebook::Notebook;
 use rusty_hektor::parser::xml_parser::XMLParser;
@@ -20,14 +20,15 @@ use rusty_hektor::submission_type::{ProgrammingLang, SubmissionType};
 use rusty_hektor::testrunner::empty_test::EmptyTest;
 use rusty_hektor::testrunner::Test;
 use rusty_hektor::{anonymizer, run_test, yes_or_no};
-use ParsedData::*;
 
 /// Parse ILIAS exam export for importing into Grady
 #[derive(Debug, StructOpt)]
 struct Opt {
-    /// Provide the path to the zipped xml, in the future conversion and merge of multiple exports
-    /// of the same exam will be supported
-    #[structopt(name = "IN_FILE", required = true, max_values = 1, parse(from_os_str))]
+    /// Provide the path to the zipped xml, if the exam was split into multiple
+    /// cohorts with the same module and exercises, you can provide multiple .zip exports.
+    /// If provided exercises are not identical, a warning is issued and the program falls back
+    /// to the exercises of the first export file
+    #[structopt(name = "IN_FILE", required = true, parse(from_os_str))]
     in_files: Vec<PathBuf>,
 
     /// Where to store the parsed data, prints to stdout if not provided
@@ -87,10 +88,6 @@ impl From<&OsStr> for TestEnum {
     }
 }
 
-enum ParsedData {
-    XML(Exam),
-}
-
 fn main() -> Result<(), Box<dyn Error>> {
     setup_logging();
     if let Err(_) = check_for_new_version() {
@@ -107,13 +104,13 @@ fn main() -> Result<(), Box<dyn Error>> {
     let parse_text = opt.skip_text.not();
     let render_latex = opt.no_latex.not();
 
-    let parsed_data: Result<Vec<ParsedData>, Box<dyn Error>> = opt
+    let parsed_data: Result<Vec<Exam>, Box<dyn Error>> = opt
         .in_files
         .iter()
-        .map(|path| -> Result<ParsedData, Box<dyn Error>> {
+        .map(|path| {
             let extension = path
                 .extension()
-                .expect("You need to provide a file with an .xls or .zip extension");
+                .expect("You need to provide a file with a .zip extension");
             if extension == "xls" {
                 println!(
                     ".xls parsing has been removed due to the instability of the format.\
@@ -121,7 +118,7 @@ fn main() -> Result<(), Box<dyn Error>> {
                 );
                 std::process::exit(1)
             } else if extension == "zip" {
-                Ok(XML(XMLParser::parse(path, parse_text, render_latex)?))
+                Ok(XMLParser::parse(path, parse_text, render_latex)?)
             } else {
                 Err(ParserError::new(format!(
                     "Unsupported filetype: {:?}",
@@ -131,15 +128,10 @@ fn main() -> Result<(), Box<dyn Error>> {
         })
         .collect();
 
-    let mut parsed_data = parsed_data?;
+    let parsed_data = parsed_data?;
+    let parsed_data= merge_exams(&parsed_data)?;
 
-    let mut serializable = if parsed_data.len() == 1 {
-        match parsed_data.pop().unwrap() {
-            XML(data) => data.into_serializable()?,
-        }
-    } else {
-        unreachable!("test")
-    };
+    let mut serializable = parsed_data.into_serializable()?;
 
     let mapping = match opt.anon {
         true => Some(anonymizer::replace_students(&mut serializable.students)),
@@ -158,7 +150,10 @@ fn main() -> Result<(), Box<dyn Error>> {
     let json = serde_json::to_string_pretty(&serializable)?;
     match opt.out_file {
         Some(path) => fs::write(path, json)?,
-        None => println!("{}", json),
+        None => {
+            eprintln!("Output (use -o <path> flag to write to file system)\n");
+            println!("{}", json);
+        }
     }
 
     if let Some(map) = mapping {

src/exam.rs

-use crate::errors::{EmptyFieldError, MergeError};
-use crate::student::StudentSerializable;
-use std::collections::BTreeMap;
 use std::collections::BTreeSet;
+use std::collections::{BTreeMap, HashSet};
+use std::error::Error;
 
+use crate::errors::{EmptyFieldError, MergeError};
 use crate::meta_information::MetaInformation;
 use crate::module::Module;
 use crate::student::Student;
+use crate::student::StudentSerializable;
 use crate::submission_type::SubmissionType;
+use crate::yes_or_no;
+use itertools::Itertools;
 
 #[derive(Debug, Default)]
 pub struct Exam {
@@ -50,29 +53,75 @@ impl Exam {
             student.username = None;
         }
     }
+}
 
-    pub fn validate_and_merge_student_information(
-        &mut self,
-        other: &mut Exam,
-    ) -> Result<(), MergeError> {
-        let mut other_students: BTreeMap<String, &mut Student> = BTreeMap::new();
-        for student in other.students.values_mut() {
-            let name = student
-                .fullname
-                .clone()
-                .expect("Students need names for merging");
-            if other_students.contains_key(&name) {
-                panic!("Name: '{}' appears twice. Unable to merge.", name);
+pub fn merge_exams(exams: &[Exam]) -> Result<Exam, Box<dyn Error>> {
+    let submission_types = merge_submission_types(
+        &exams
+            .iter()
+            .map(|exam| &exam.submission_types)
+            .collect_vec(),
+    )?;
+    let mut students: BTreeMap<String, Student> = BTreeMap::new();
+    for exam in exams {
+        for (name, student) in &exam.students {
+            match students.insert(name.clone(), student.clone()) {
+                Some(student) => Err(format!("Student {:#?} seems to appear twice in input data. Please resolve this manually in the data.", student))?,
+                None => ()
             }
-            other_students.insert(name, student);
         }
+    }
+    Ok(Exam {
+        module: None,
+        submission_types,
+        students
+    })
+}
 
-        for (name, student) in self.students.iter_mut() {
-            student.merge(other_students.get_mut(name).expect(&format!(
-                "Differing students in exports. {} contained in one but not the other.",
-                name
-            )))?
-        }
-        Ok(())
+pub fn merge_submission_types(
+    submission_types: &[&BTreeSet<SubmissionType>],
+) -> Result<BTreeSet<SubmissionType>, Box<dyn Error>> {
+    if submission_types.len() == 0 {
+        Err("Called merge_submission_types with no exams!")?;
     }
+    let submission_type = if submission_types
+        .windows(2)
+        .all(|sub_types| sub_types[0] == sub_types[1])
+    {
+        submission_types[0]
+    } else {
+        log::warn!(
+            "The parsed submission types are differing. \
+             This might be due to insignificant formatting differences. \
+             Falling back to first set of submission types:\n{:#?}",
+            &submission_types[0]
+        );
+        submission_types[0]
+    };
+    Ok(submission_type.clone())
 }
+
+//pub fn validate_and_merge_student_information(
+//    &mut self,
+//    other: &mut Exam,
+//) -> Result<(), MergeError> {
+//    let mut other_students: BTreeMap<String, &mut Student> = BTreeMap::new();
+//    for student in other.students.values_mut() {
+//        let name = student
+//            .fullname
+//            .clone()
+//            .expect("Students need names for merging");
+//        if other_students.contains_key(&name) {
+//            panic!("Name: '{}' appears twice. Unable to merge.", name);
+//        }
+//        other_students.insert(name, student);
+//    }
+//
+//    for (name, student) in self.students.iter_mut() {
+//        student.merge(other_students.get_mut(name).expect(&format!(
+//            "Differing students in exports. {} contained in one but not the other.",
+//            name
+//        )))?
+//    }
+//    Ok(())
+//}

src/student.rs

@@ -6,7 +6,7 @@ use std::hash::{Hash, Hasher};
 
 use crate::{errors::MergeError, submission::Submission, MergeOption};
 
-#[derive(Debug, Eq, PartialEq, Default, Serialize, PartialOrd, Ord)]
+#[derive(Debug, Eq, PartialEq, Default, Serialize, PartialOrd, Ord, Clone)]
 pub struct Student {
     pub fullname: Option<String>,
     pub identifier: Option<String>,