Student passwords can now be set when exporting

The former student data endpoint /export/csv/ has been replaces by /export/json/ .
This new endpoint exports the data as normal json. It also allows the client to send setPasswords: true as an option which will results in random passwords beign generated for all students and included in the export data.

core/models.py

@@ -301,7 +301,7 @@ class StudentInfo(models.Model):
         """ TODO: get rid of it and use an annotation. """
         if self.submissions.all():
             return OrderedDict({
-                s.type: s.feedback.score if hasattr(s, 'feedback') else 0
+                s.type.name: s.feedback.score if hasattr(s, 'feedback') else 0
                 for s in self.submissions.order_by('type__name')
             })
 

core/tests/test_export.py

 from django.test import Client, TestCase
 from rest_framework import status
+from rest_framework.utils import json
 
 from util.factories import make_test_data
 
 
-class ExportCSVTest(TestCase):
+class ExportJSONTest(TestCase):
 
     @classmethod
     def setUpTestData(cls):
@@ -78,14 +79,37 @@ class ExportCSVTest(TestCase):
     def setUp(self):
         client = Client()
         client.force_login(user=self.data['reviewers'][0])
-
-        self.response = client.get('/api/export/csv/')
+        self.response = client.post('/api/export/json/', content_type='application/json')
 
     def test_can_access(self):
         self.assertEqual(status.HTTP_200_OK, self.response.status_code)
 
     def test_data_is_correct(self):
-        head, student1, student2, _ = self.response.content.split(b'\r\n')
-        self.assertIn(b'Matrikel;Name;Exam;Sum;01. Sort;02. Shuffle', head)
-        self.assertIn(b';;Test Exam 01;5;5;0', student1)
-        self.assertIn(b';;Test Exam 01;0;0;0', student2)
+        # for some reason the data is not automatically parsed...
+        student1, student2 = json.loads(self.response.content)
+        self.assertIn('Matrikel', student1)
+        self.assertIn('Matrikel', student2)
+
+        self.assertEqual('', student1['Name'])
+        self.assertEqual('', student2['Name'])
+
+        self.assertEqual('Test Exam 01', student1['Exam'])
+        self.assertEqual('Test Exam 01', student2['Exam'])
+
+        self.assertEqual(5, student1['Sum'])
+        self.assertEqual(0, student2['Sum'])
+
+        self.assertEqual('student01', student1['Username'])
+        self.assertEqual('student02', student2['Username'])
+
+        self.assertEqual('01. Sort', student1['Scores'][0]['type'])
+        self.assertEqual('01. Sort', student2['Scores'][0]['type'])
+
+        self.assertEqual('02. Shuffle', student1['Scores'][1]['type'])
+        self.assertEqual('02. Shuffle', student2['Scores'][1]['type'])
+
+        self.assertEqual(5, student1['Scores'][0]['score'])
+        self.assertEqual(0, student2['Scores'][0]['score'])
+
+        self.assertEqual(0, student1['Scores'][1]['score'])
+        self.assertEqual(0, student2['Scores'][1]['score'])

core/urls.py

@@ -46,7 +46,7 @@ regular_views_urlpatterns = [
     path('jwt-time-delta/',
          views.get_jwt_expiration_delta,
          name='jwt-time-delta'),
-    path('export/csv/', views.StudentCSVExport.as_view(), name='export-csv'),
+    path('export/json/', views.StudentJSONExport.as_view(), name='export-json'),
     re_path(r'swagger(?P<format>\.json|\.yaml)$',
             schema_view.without_ui(cache_timeout=0), name='schema-json'),
     re_path(r'swagger/$', schema_view.with_ui('swagger', cache_timeout=0),

core/views/__init__.py

 from .feedback import FeedbackApiView, FeedbackCommentApiView  # noqa
 from .subscription import SubscriptionApiViewSet, AssignmentApiViewSet  # noqa
 from .common_views import *  # noqa
-from .export import StudentCSVExport  # noqa
+from .export import StudentJSONExport  # noqa

core/views/export.py

 from rest_framework.response import Response
 from rest_framework.views import APIView
-from rest_framework_csv import renderers
 
-from core.models import StudentInfo, SubmissionType
+import xkcdpass.xkcd_password as xp
+
+from core.models import StudentInfo, UserAccount
 from core.permissions import IsReviewer
 
+words = xp.generate_wordlist(wordfile=xp.locate_wordfile(), min_length=5, max_length=8)
 
-class SemicolonCSVRenderer(renderers.CSVRenderer):
-    writer_opts = {
-        'delimiter': ';',
-    }
 
+def _set_student_passwords():
+    student_password_dict = {}
+    for student in UserAccount.get_students():
+        password = xp.generate_xkcdpassword(words, numwords=3, delimiter='-')
+        student.set_password(password)
+        student.save()
+        student_password_dict[student.pk] = password
 
-class StudentCSVExport(APIView):
-    renderer_classes = (SemicolonCSVRenderer, )
-    permission_classes = (IsReviewer, )
+    return student_password_dict
 
-    def get_renderer_context(self):
-        context = super().get_renderer_context()
-        context['header'] = ('Matrikel', 'Name', 'Exam', 'Sum',
-                             *SubmissionType.objects.values_list('name',
-                                                                 flat=True))
-        context['delimiter'] = ';'
-        return context
-
-    def finalize_response(self, request, response, *args, **kwargs):
-        response['Content-Disposition'] = \
-            "attachment; filename=%s" % 'results.csv'
-        return super().finalize_response(request, response, *args, **kwargs)
-
-    def get(self, request, format=None):
-        content = [{'Matrikel': student.matrikel_no,
-                    'Name': student.user.fullname,
-                    'Sum': student.overall_score,
-                    'Exam': student.exam.module_reference,
-                    **student.score_per_submission()
-                    } for student
-                   in StudentInfo.get_annotated_score_submission_list()]
 
+class StudentJSONExport(APIView):
+    permission_classes = (IsReviewer, )
+
+    def post(self, request, format=None):
+        set_passwords = request.data.get('set_passwords')
+        passwords = _set_student_passwords() if set_passwords else None
+
+        content = [
+            {'Matrikel': student.matrikel_no,
+             'Name': student.user.fullname,
+             'Username': student.user.username,
+             'Sum': student.overall_score,
+             'Exam': student.exam.module_reference,
+             'Password': passwords[student.user.pk] if set_passwords else '********',
+             'Scores': [
+                 {
+                     'type': submission_type,
+                     'score': score
+                 } for submission_type, score in student.score_per_submission().items()]
+             } for student
+            in StudentInfo.get_annotated_score_submission_list()]
         return Response(content)

frontend/src/api.ts

@@ -15,7 +15,7 @@ import {
 
 function getInstanceBaseUrl (): string {
   if (process.env.NODE_ENV === 'production') {
-    return `https://${window.location.host}${window.location.pathname}`
+    return `https://${window.location.host}${window.location.pathname}`.replace(/\/+$/, '')
   } else {
     return 'http://localhost:8000/'
   }
@@ -197,4 +197,18 @@ export async function changeActiveForUser (userPk: string, active: boolean): Pro
   return (await ax.patch(`/api/user/${userPk}/change_active/`, { 'is_active': active })).data
 }
 
+export interface StudentExportOptions { setPasswords?: boolean }
+export interface StudentExportItem {
+  Matrikel: string,
+  Name: string,
+  Username: string,
+  Sum: number,
+  Exam: string,
+  Password: string,
+  Scores: { type: string, score: number }[]
+}
+export async function fetchStudentExportData (options: StudentExportOptions): Promise<StudentExportItem[]> {
+  return (await ax.post('/api/export/json/', options)).data
+}
+
 export default ax

frontend/src/components/DataExport.vue

@@ -3,7 +3,7 @@
     <v-tooltip bottom>
       <v-btn :color="exportColor"
              slot="activator"
-             @click="getCSVFileAndProcess"
+             @click="showDialog"
       >
         export
         <v-icon>file_download</v-icon>
@@ -11,28 +11,51 @@
       <span v-if="corrected">All submissions have been corrected!</span>
       <span v-else>UNCORRECTED submissions left! Export will be incomplete.</span>
     </v-tooltip>
-    <v-dialog v-model="mapFileDialog" max-width="30vw">
+    <v-dialog v-model="exportDialog" max-width="30vw">
       <v-card>
         <v-card-title class="title">
-          Currently no student mapping file is selected
+          Student Data Export
         </v-card-title>
         <v-card-text>
-          If you select a mapping file, the anonymized data
-          will be mapped back automatically and locally on your machine.
+          <div v-if="!mapFileLoaded">
+            If you select a mapping file, the anonymized data
+            will be mapped back automatically and locally on your machine.
 
-          <v-layout row align-center>
-            <file-select v-model="mapFile" display-text="Select map file" class="ma-3"/>
-            <span>Without the mapping, the data will still be obfuscated.</span>
+            <v-layout row align-center>
+              <file-select v-model="mapFile" display-text="Select map file" class="ma-3"/>
+              <span>Without the mapping, the data will still be obfuscated.</span>
+          </v-layout>
+          </div>
+          <v-layout row>
+            <v-flex xs4>
+              <v-tooltip top>
+                <v-checkbox
+                  label="Set passwords"
+                  v-model="setPasswords"
+                  slot="activator"
+                />
+                <span>Setting this will cause all student passwords
+                  to be reset upon export. The new passwords will be contained in the
+                  export file.
+                </span>
+              </v-tooltip>
+            </v-flex>
+            <v-flex xs3 offset-xs1>
+              <v-select
+                label="Export file format"
+                :items="availableExportTypes"
+                v-model="exportType"
+              />
+            </v-flex>
           </v-layout>
-
           <v-card-actions>
             <v-btn
               flat color="blue lighten-2"
-              @click="mapFileDialog = false"
+              @click="exportDialog = false"
             >close</v-btn>
             <v-spacer/>
-            <v-btn flat outline @click="getCSVFileAndProcess"
-            >{{mapFile ? 'Download and apply mapping' : 'Download without mapping'}}</v-btn>
+            <v-btn flat outline @click="getExportFile"
+            >{{mapFile || mapFileLoaded ? 'Download and apply mapping' : 'Download without mapping'}}</v-btn>
           </v-card-actions>
         </v-card-text>
       </v-card>
@@ -40,123 +63,151 @@
   </div>
 </template>
 
-<script>
+<script lang="ts">
+import {Vue, Component, Mixins} from 'vue-property-decorator'
 import { getters } from '@/store/getters'
-import ax from '@/api'
-import FileSelect from '@/components/util/FileSelect'
+import ax, { StudentExportItem, fetchStudentExportData } from '@/api'
+import FileSelect from '@/components/util/FileSelect.vue'
 import { mutations as mut } from '@/store/mutations'
 import { parseCSVMapMixin } from '@/components/mixins/mixins'
 
-export default {
-  components: { FileSelect },
-  name: 'data-export',
-  mixins: [parseCSVMapMixin],
-  data () {
-    return {
-      fileReader: new FileReader(),
-      mapFile: null,
-      mapFileDialog: false
-    }
-  },
-  computed: {
-    corrected () { return getters.corrected },
-    exportColor () {
-      return this.corrected ? 'green darken-1' : 'red lighten-1'
-    },
-    downloadUrl () {
-      let url = ''
-      if (process.env.NODE_ENV === 'production') {
-        const baseUrl = `https://${window.location.host}${window.location.pathname}`.replace(/\/+$/, '')
-        url = `${baseUrl}/api/export/csv`
-      } else {
-        url = 'http://localhost:8000/api/export/csv/'
-      }
-      return url
-    },
-    studentMap () {
-      return this.$store.state.studentMap
-    },
-    mapFileLoaded () {
-      return Object.keys(this.studentMap).length > 0
-    }
-  },
-  methods: {
-    readMapFileAndCommit (callback) {
-      this.fileReader.onload = event => {
+
+enum ExportType {
+  JSON = 'JSON',
+  CSV = 'CSV'
+}
+
+
+@Component({
+  components: { FileSelect }
+})
+export default class DataExport extends Mixins(parseCSVMapMixin) {
+  exportDialog = false
+  mapFile: File | null = null
+  setPasswords = false
+  exportType = ExportType.CSV
+
+  get corrected () { return getters.corrected }
+  get studentMap () { return getters.state.studentMap }
+  get mapFileLoaded () { 
+    return Object.keys(getters.state.studentMap).length > 0 
+  }
+  get exportColor () {
+    return this.corrected ? 'green darken-1' : 'red lighten-1'
+  }
+  get availableExportTypes (): ExportType[] {
+    return Object.values(ExportType)
+  }
+
+  showDialog () {
+    this.exportDialog = true
+  }
+
+  readMapFileAndCommit () {
+    const fileReader = new FileReader()
+    return new Promise((resolve, reject) => {
+      fileReader.onload = event => {
+        // @ts-ignore typings of EventTarget seem to be wrong
         const studentMap = this.parseCSVMap(event.target.result)
         mut.SET_STUDENT_MAP(studentMap)
-        callback()
+        resolve()
+      },
+      fileReader.onerror = () => {
+        fileReader.abort();
+        reject(new DOMException("Problem parsing input file."));
       }
-      this.fileReader.readAsText(this.mapFile)
-    },
-    async download () {
-      const response = await ax.get(this.downloadUrl, { responseType: 'blob' })
-      return new Blob([response.data], { type: 'text/csv' })
-    },
-    CSVToJson (csvString) {
-      const lines = csvString.split('\n')
-      const headers = lines.shift().split(';')
-      return lines
-        .filter(line => !!line) // remove empty strings
-        .map(line => {
-          const lineItems = line.split(';')
-          return headers.reduce((acc, curr, i) => {
-            acc[headers[i]] = lineItems[i]
-            return acc
-          }, {})
-        })
-    },
-    jsonToCSV (data) {
-      const headerLine = Object.keys(data[0]).reduce((acc, curr) => {
-        return acc ? `${acc};${curr}` : `${curr}`
-      }, '')
-      const lines = data.map(studentData => {
-        return Object.values(studentData).reduce((acc, curr) => {
-          return acc ? `${acc};${curr}` : `${curr}`
-        }, '')
-      })
-      return headerLine + lines.reduce((acc, curr) => {
-        return `${acc}\n${curr}`
-      }, '') + '\n' // add trailing newline
-    },
-    mapStudentData (students) {
-      return students.map(studentData => {
-        return {
-          ...studentData,
-          Matrikel: this.$store.state.studentMap[studentData.Matrikel].matrikelNo,
-          Name: this.$store.state.studentMap[studentData.Matrikel].name
-        }
-      })
-    },
-    getMappedCSV () {
-      this.download().then(blobData => {
-        if (this.mapFileLoaded) {
-          this.fileReader.onload = event => {
-            const jsonData = this.CSVToJson(event.target.result)
-            const mappedData = this.mapStudentData(jsonData)
-            const csvData = this.jsonToCSV(mappedData)
-            const mappedBlobData = new Blob([csvData], { type: 'text/csv' })
-            window.open(window.URL.createObjectURL(mappedBlobData))
-          }
-          this.fileReader.readAsText(blobData)
-        } else {
-          window.open(window.URL.createObjectURL(blobData))
-        }
-      })
-    },
-    getCSVFileAndProcess () {
-      if (!this.mapFileLoaded && !this.mapFileDialog) {
-        this.mapFileDialog = true
+
+      if (!this.mapFile) {
+        reject(new Error("Can only call" + 
+          " readMapFileAndCommit when mapFile is not undefined"))
       } else {
-        if (this.mapFile) {
-          this.readMapFileAndCommit(this.getMappedCSV)
-        } else {
-          this.getMappedCSV()
-        }
+        fileReader.readAsText(this.mapFile)
+      }
+    })
+  }
+
+  applyMapping (studentExport: StudentExportItem[]) {
+    return studentExport.map(student => {
+      return { 
+        ...student,
+        Matrikel: this.studentMap[student.Matrikel].matrikelNo,
+        Name: this.studentMap[student.Matrikel].name
+      }
+    })
+  }
+
+  jsonToCSV (studentExport: StudentExportItem[], delimeter = ';') {
+    let headerLine = Object.keys(studentExport[0]).reduce((acc: string, curr) => {
+      if (curr === 'Scores') {
+        return acc
       }
+      return acc ? `${acc};${curr}` : `${curr}`
+    }, '')
+    headerLine += Object.values(studentExport[0].Scores)
+      .reduce((acc: string, curr) => {
+      return `${acc};${curr.type}`
+    }, '')
+
+    const lines = studentExport.map(student => {
+      const normalFields = Object.values(student).reduce((acc: string, curr): string => {
+        // skip  the Scores field
+        if (typeof curr === 'object') {
+          return acc
+        }  
+        return acc ? `${acc};${curr}` : `${curr}`
+      }, '')
+
+      const scoreFields = Object.values(student.Scores).reduce((acc: string, curr) => {
+        return `${acc};${curr.score}`
+      }, '')
+      return normalFields + scoreFields
+    })
+
+    return headerLine + lines.reduce((acc, curr) => {
+      return `${acc}\n${curr}`
+    }, '') + '\n' // add trailing newline
+  }
+
+  createDownloadPopup (content: string | StudentExportItem[], fileType: ExportType) {
+    const blobProperties: BlobPropertyBag = {}
+    if (fileType === ExportType.JSON) {
+      blobProperties.type = 'application/json'
+      content = JSON.stringify(content)
+    } else {
+      blobProperties.type = 'text/csv'
     }
+    const blobData = new Blob([<string> content], blobProperties)
+    window.open(window.URL.createObjectURL(blobData))
+  }
+
+  optionalConvertAndCreatePopup (studentData: StudentExportItem[]) {
+    const convertedData = this.exportType === ExportType.CSV ? 
+      this.jsonToCSV(studentData) : studentData
+
+    this.createDownloadPopup(convertedData, this.exportType)
   }
 
+  async getMappedExportFile (studentData: StudentExportItem[]) {
+    if (!this.mapFile && !this.mapFileLoaded) {
+      throw new Error("Either mapFile must be selected or already loaded "+
+                      "to call getMappedExportFile")
+    }
+    if (this.mapFile) {
+      await this.readMapFileAndCommit()
+    }
+    const mappedData = this.applyMapping(studentData)
+    this.optionalConvertAndCreatePopup(mappedData)
+  }
+
+  async getExportFile () {
+    const studentData = await fetchStudentExportData({setPasswords: this.setPasswords})
+
+    if (this.mapFile || this.mapFileLoaded) {
+      this.getMappedExportFile(studentData)
+    } else {
+      this.optionalConvertAndCreatePopup(studentData)
+    }
+  }
 }
 </script>
 

frontend/src/components/mixins/mixins.js

-export var parseCSVMapMixin = {
-  methods: {
-    parseCSVMap (csvMap) {
-      let lines = csvMap.split('\n')
-      lines.shift() // drop the first line since it contains only headings
-      return lines.reduce((acc, curr) => {
-        if (curr) {
-          let [key, matrikelNo, name] = curr.split(';')
-          acc[key] = { matrikelNo: matrikelNo, name }
-        }
-        return acc
-      }, {})
-    }
-  }
-}

frontend/src/components/mixins/mixins.ts

+import {Vue, Component} from 'vue-property-decorator'
+
+@Component
+export class parseCSVMapMixin extends Vue {
+  parseCSVMap (csvMap: string) {
+    let lines = csvMap.split('\n')
+    lines.shift() // drop the first line since it contains only headings
+    // TODO remove any type
+    return lines.reduce((acc: any, curr) => {
+      if (curr) {
+        let [key, matrikelNo, name] = curr.split(';')
+        acc[key] = { matrikelNo: matrikelNo, name }
+      }
+      return acc
+    }, {})
+  }
+}

frontend/src/store/modules/subscriptions.ts

@@ -44,9 +44,9 @@ const availableStagesGetter = mb.read(function availableStages (state, getters)
   return stages
 })
 const availableStagesReadableGetter = mb.read(function availableStagesReadable (state, getters) {
-  let stages = ['create', 'validate']
+  let stages = ['initial', 'validate']
   if (Authentication.isReviewer) {
-    stages.push('resolve')
+    stages.push('conflict')
   }
   return stages
 })

frontend/src/store/mutations.ts

@@ -25,7 +25,13 @@ function SET_STUDENT (state: RootState, student: StudentInfoForListView) {
   }, state.studentMap))
 }
 // TODO proper types for student map
-function SET_STUDENT_MAP (state: RootState, map: object) {
+export interface StudentMap {
+  [pseudoMatrikelNo: string]: {
+    matrikelNo: string,
+    name: string
+  }
+}
+function SET_STUDENT_MAP (state: RootState, map: StudentMap) {
   state.studentMap = map
 }
 function SET_TUTORS (state: RootState, tutors: Array<Tutor>) {

frontend/src/store/store.ts

@@ -42,7 +42,11 @@ export interface RootInitialState {
     submissionTypes: {[pk: string]: SubmissionType}
     submissions: {[pk: string]: SubmissionNoType}
     students: {[pk: string]: StudentInfoForListView}
-    studentMap: {} // is used to map obfuscated student data back to the original
+    studentMap: {
+      [matrikel: string]: {
+        matrikelNo: string, name: string
+      }
+    } // is used to map obfuscated student data back to the original
     statistics: Statistics
     tutors: Array<Tutor>
 }

requirements.txt

 django-cors-headers~=2.1.0
 django-extensions~=2.1
-djangorestframework-csv~=2.0.0
 djangorestframework-jwt~=1.11.0
 djangorestframework~=3.8
 git+https://github.com/robinhundt/djangorestframework-camel-case
@@ -14,3 +13,4 @@ python-json-logger~=0.1.9
 tqdm~=4.19.5
 whitenoise~=3.3.1
 xlrd~=1.0.0
+xkcdpass==1.16.5