From c2c45ecc83d3fa5d364b3a7196758cace31eed6f Mon Sep 17 00:00:00 2001
From: "robinwilliam.hundt" <robinwilliam.hundt@stud.uni-goettingen.de>
Date: Sun, 30 Sep 2018 18:34:02 +0200
Subject: [PATCH] Student passwords can now be set when exporting
The former student data endpoint /export/csv/ has been replaces by /export/json/ .
This new endpoint exports the data as normal json. It also allows the client to send setPasswords: true as an option which will results in random passwords beign generated for all students and included in the export data.
---
core/models.py | 2 +-
core/tests/test_export.py | 38 ++-
core/urls.py | 2 +-
core/views/__init__.py | 2 +-
core/views/export.py | 63 ++---
frontend/src/api.ts | 16 +-
frontend/src/components/DataExport.vue | 285 ++++++++++++--------
frontend/src/components/mixins/mixins.js | 15 --
frontend/src/components/mixins/mixins.ts | 17 ++
frontend/src/store/modules/subscriptions.ts | 4 +-
frontend/src/store/mutations.ts | 8 +-
frontend/src/store/store.ts | 6 +-
requirements.txt | 2 +-
13 files changed, 282 insertions(+), 178 deletions(-)
delete mode 100644 frontend/src/components/mixins/mixins.js
create mode 100644 frontend/src/components/mixins/mixins.ts
diff --git a/core/models.py b/core/models.py
index bc33deec..499faf4c 100644
--- a/core/models.py
+++ b/core/models.py
@@ -301,7 +301,7 @@ class StudentInfo(models.Model):
""" TODO: get rid of it and use an annotation. """
if self.submissions.all():
return OrderedDict({
- s.type: s.feedback.score if hasattr(s, 'feedback') else 0
+ s.type.name: s.feedback.score if hasattr(s, 'feedback') else 0
for s in self.submissions.order_by('type__name')
})
diff --git a/core/tests/test_export.py b/core/tests/test_export.py
index c7422190..72609c90 100644
--- a/core/tests/test_export.py
+++ b/core/tests/test_export.py
@@ -1,10 +1,11 @@
from django.test import Client, TestCase
from rest_framework import status
+from rest_framework.utils import json
from util.factories import make_test_data
-class ExportCSVTest(TestCase):
+class ExportJSONTest(TestCase):
@classmethod
def setUpTestData(cls):
@@ -78,14 +79,37 @@ class ExportCSVTest(TestCase):
def setUp(self):
client = Client()
client.force_login(user=self.data['reviewers'][0])
-
- self.response = client.get('/api/export/csv/')
+ self.response = client.post('/api/export/json/', content_type='application/json')
def test_can_access(self):
self.assertEqual(status.HTTP_200_OK, self.response.status_code)
def test_data_is_correct(self):
- head, student1, student2, _ = self.response.content.split(b'\r\n')
- self.assertIn(b'Matrikel;Name;Exam;Sum;01. Sort;02. Shuffle', head)
- self.assertIn(b';;Test Exam 01;5;5;0', student1)
- self.assertIn(b';;Test Exam 01;0;0;0', student2)
+ # for some reason the data is not automatically parsed...
+ student1, student2 = json.loads(self.response.content)
+ self.assertIn('Matrikel', student1)
+ self.assertIn('Matrikel', student2)
+
+ self.assertEqual('', student1['Name'])
+ self.assertEqual('', student2['Name'])
+
+ self.assertEqual('Test Exam 01', student1['Exam'])
+ self.assertEqual('Test Exam 01', student2['Exam'])
+
+ self.assertEqual(5, student1['Sum'])
+ self.assertEqual(0, student2['Sum'])
+
+ self.assertEqual('student01', student1['Username'])
+ self.assertEqual('student02', student2['Username'])
+
+ self.assertEqual('01. Sort', student1['Scores'][0]['type'])
+ self.assertEqual('01. Sort', student2['Scores'][0]['type'])
+
+ self.assertEqual('02. Shuffle', student1['Scores'][1]['type'])
+ self.assertEqual('02. Shuffle', student2['Scores'][1]['type'])
+
+ self.assertEqual(5, student1['Scores'][0]['score'])
+ self.assertEqual(0, student2['Scores'][0]['score'])
+
+ self.assertEqual(0, student1['Scores'][1]['score'])
+ self.assertEqual(0, student2['Scores'][1]['score'])
diff --git a/core/urls.py b/core/urls.py
index a042f5f9..2450f5f2 100644
--- a/core/urls.py
+++ b/core/urls.py
@@ -46,7 +46,7 @@ regular_views_urlpatterns = [
path('jwt-time-delta/',
views.get_jwt_expiration_delta,
name='jwt-time-delta'),
- path('export/csv/', views.StudentCSVExport.as_view(), name='export-csv'),
+ path('export/json/', views.StudentJSONExport.as_view(), name='export-json'),
re_path(r'swagger(?P<format>\.json|\.yaml)$',
schema_view.without_ui(cache_timeout=0), name='schema-json'),
re_path(r'swagger/$', schema_view.with_ui('swagger', cache_timeout=0),
diff --git a/core/views/__init__.py b/core/views/__init__.py
index e653604f..da92a342 100644
--- a/core/views/__init__.py
+++ b/core/views/__init__.py
@@ -1,4 +1,4 @@
from .feedback import FeedbackApiView, FeedbackCommentApiView # noqa
from .subscription import SubscriptionApiViewSet, AssignmentApiViewSet # noqa
from .common_views import * # noqa
-from .export import StudentCSVExport # noqa
+from .export import StudentJSONExport # noqa
diff --git a/core/views/export.py b/core/views/export.py
index f31e3d4b..1ac84e7f 100644
--- a/core/views/export.py
+++ b/core/views/export.py
@@ -1,41 +1,44 @@
from rest_framework.response import Response
from rest_framework.views import APIView
-from rest_framework_csv import renderers
-from core.models import StudentInfo, SubmissionType
+import xkcdpass.xkcd_password as xp
+
+from core.models import StudentInfo, UserAccount
from core.permissions import IsReviewer
+words = xp.generate_wordlist(wordfile=xp.locate_wordfile(), min_length=5, max_length=8)
-class SemicolonCSVRenderer(renderers.CSVRenderer):
- writer_opts = {
- 'delimiter': ';',
- }
+def _set_student_passwords():
+ student_password_dict = {}
+ for student in UserAccount.get_students():
+ password = xp.generate_xkcdpassword(words, numwords=3, delimiter='-')
+ student.set_password(password)
+ student.save()
+ student_password_dict[student.pk] = password
-class StudentCSVExport(APIView):
- renderer_classes = (SemicolonCSVRenderer, )
- permission_classes = (IsReviewer, )
+ return student_password_dict
- def get_renderer_context(self):
- context = super().get_renderer_context()
- context['header'] = ('Matrikel', 'Name', 'Exam', 'Sum',
- *SubmissionType.objects.values_list('name',
- flat=True))
- context['delimiter'] = ';'
- return context
-
- def finalize_response(self, request, response, *args, **kwargs):
- response['Content-Disposition'] = \
- "attachment; filename=%s" % 'results.csv'
- return super().finalize_response(request, response, *args, **kwargs)
-
- def get(self, request, format=None):
- content = [{'Matrikel': student.matrikel_no,
- 'Name': student.user.fullname,
- 'Sum': student.overall_score,
- 'Exam': student.exam.module_reference,
- **student.score_per_submission()
- } for student
- in StudentInfo.get_annotated_score_submission_list()]
+class StudentJSONExport(APIView):
+ permission_classes = (IsReviewer, )
+
+ def post(self, request, format=None):
+ set_passwords = request.data.get('set_passwords')
+ passwords = _set_student_passwords() if set_passwords else None
+
+ content = [
+ {'Matrikel': student.matrikel_no,
+ 'Name': student.user.fullname,
+ 'Username': student.user.username,
+ 'Sum': student.overall_score,
+ 'Exam': student.exam.module_reference,
+ 'Password': passwords[student.user.pk] if set_passwords else '********',
+ 'Scores': [
+ {
+ 'type': submission_type,
+ 'score': score
+ } for submission_type, score in student.score_per_submission().items()]
+ } for student
+ in StudentInfo.get_annotated_score_submission_list()]
return Response(content)
diff --git a/frontend/src/api.ts b/frontend/src/api.ts
index 3c4569d3..ec02ebdd 100644
--- a/frontend/src/api.ts
+++ b/frontend/src/api.ts
@@ -15,7 +15,7 @@ import {
function getInstanceBaseUrl (): string {
if (process.env.NODE_ENV === 'production') {
- return `https://${window.location.host}${window.location.pathname}`
+ return `https://${window.location.host}${window.location.pathname}`.replace(/\/+$/, '')
} else {
return 'http://localhost:8000/'
}
@@ -197,4 +197,18 @@ export async function changeActiveForUser (userPk: string, active: boolean): Pro
return (await ax.patch(`/api/user/${userPk}/change_active/`, { 'is_active': active })).data
}
+export interface StudentExportOptions { setPasswords?: boolean }
+export interface StudentExportItem {
+ Matrikel: string,
+ Name: string,
+ Username: string,
+ Sum: number,
+ Exam: string,
+ Password: string,
+ Scores: { type: string, score: number }[]
+}
+export async function fetchStudentExportData (options: StudentExportOptions): Promise<StudentExportItem[]> {
+ return (await ax.post('/api/export/json/', options)).data
+}
+
export default ax
diff --git a/frontend/src/components/DataExport.vue b/frontend/src/components/DataExport.vue
index cf4b35cc..4e327c37 100644
--- a/frontend/src/components/DataExport.vue
+++ b/frontend/src/components/DataExport.vue
@@ -3,7 +3,7 @@
<v-tooltip bottom>
<v-btn :color="exportColor"
slot="activator"
- @click="getCSVFileAndProcess"
+ @click="showDialog"
>
export
<v-icon>file_download</v-icon>
@@ -11,28 +11,51 @@
<span v-if="corrected">All submissions have been corrected!</span>
<span v-else>UNCORRECTED submissions left! Export will be incomplete.</span>
</v-tooltip>
- <v-dialog v-model="mapFileDialog" max-width="30vw">
+ <v-dialog v-model="exportDialog" max-width="30vw">
<v-card>
<v-card-title class="title">
- Currently no student mapping file is selected
+ Student Data Export
</v-card-title>
<v-card-text>
- If you select a mapping file, the anonymized data
- will be mapped back automatically and locally on your machine.
+ <div v-if="!mapFileLoaded">
+ If you select a mapping file, the anonymized data
+ will be mapped back automatically and locally on your machine.
- <v-layout row align-center>
- <file-select v-model="mapFile" display-text="Select map file" class="ma-3"/>
- <span>Without the mapping, the data will still be obfuscated.</span>
+ <v-layout row align-center>
+ <file-select v-model="mapFile" display-text="Select map file" class="ma-3"/>
+ <span>Without the mapping, the data will still be obfuscated.</span>
+ </v-layout>
+ </div>
+ <v-layout row>
+ <v-flex xs4>
+ <v-tooltip top>
+ <v-checkbox
+ label="Set passwords"
+ v-model="setPasswords"
+ slot="activator"
+ />
+ <span>Setting this will cause all student passwords
+ to be reset upon export. The new passwords will be contained in the
+ export file.
+ </span>
+ </v-tooltip>
+ </v-flex>
+ <v-flex xs3 offset-xs1>
+ <v-select
+ label="Export file format"
+ :items="availableExportTypes"
+ v-model="exportType"
+ />
+ </v-flex>
</v-layout>
-
<v-card-actions>
<v-btn
flat color="blue lighten-2"
- @click="mapFileDialog = false"
+ @click="exportDialog = false"
>close</v-btn>
<v-spacer/>
- <v-btn flat outline @click="getCSVFileAndProcess"
- >{{mapFile ? 'Download and apply mapping' : 'Download without mapping'}}</v-btn>
+ <v-btn flat outline @click="getExportFile"
+ >{{mapFile || mapFileLoaded ? 'Download and apply mapping' : 'Download without mapping'}}</v-btn>
</v-card-actions>
</v-card-text>
</v-card>
@@ -40,123 +63,151 @@
</div>
</template>
-<script>
+<script lang="ts">
+import {Vue, Component, Mixins} from 'vue-property-decorator'
import { getters } from '@/store/getters'
-import ax from '@/api'
-import FileSelect from '@/components/util/FileSelect'
+import ax, { StudentExportItem, fetchStudentExportData } from '@/api'
+import FileSelect from '@/components/util/FileSelect.vue'
import { mutations as mut } from '@/store/mutations'
import { parseCSVMapMixin } from '@/components/mixins/mixins'
-export default {
- components: { FileSelect },
- name: 'data-export',
- mixins: [parseCSVMapMixin],
- data () {
- return {
- fileReader: new FileReader(),
- mapFile: null,
- mapFileDialog: false
- }
- },
- computed: {
- corrected () { return getters.corrected },
- exportColor () {
- return this.corrected ? 'green darken-1' : 'red lighten-1'
- },
- downloadUrl () {
- let url = ''
- if (process.env.NODE_ENV === 'production') {
- const baseUrl = `https://${window.location.host}${window.location.pathname}`.replace(/\/+$/, '')
- url = `${baseUrl}/api/export/csv`
- } else {
- url = 'http://localhost:8000/api/export/csv/'
- }
- return url
- },
- studentMap () {
- return this.$store.state.studentMap
- },
- mapFileLoaded () {
- return Object.keys(this.studentMap).length > 0
- }
- },
- methods: {
- readMapFileAndCommit (callback) {
- this.fileReader.onload = event => {
+
+enum ExportType {
+ JSON = 'JSON',
+ CSV = 'CSV'
+}
+
+
+@Component({
+ components: { FileSelect }
+})
+export default class DataExport extends Mixins(parseCSVMapMixin) {
+ exportDialog = false
+ mapFile: File | null = null
+ setPasswords = false
+ exportType = ExportType.CSV
+
+ get corrected () { return getters.corrected }
+ get studentMap () { return getters.state.studentMap }
+ get mapFileLoaded () {
+ return Object.keys(getters.state.studentMap).length > 0
+ }
+ get exportColor () {
+ return this.corrected ? 'green darken-1' : 'red lighten-1'
+ }
+ get availableExportTypes (): ExportType[] {
+ return Object.values(ExportType)
+ }
+
+ showDialog () {
+ this.exportDialog = true
+ }
+
+ readMapFileAndCommit () {
+ const fileReader = new FileReader()
+ return new Promise((resolve, reject) => {
+ fileReader.onload = event => {
+ // @ts-ignore typings of EventTarget seem to be wrong
const studentMap = this.parseCSVMap(event.target.result)
mut.SET_STUDENT_MAP(studentMap)
- callback()
+ resolve()
+ },
+ fileReader.onerror = () => {
+ fileReader.abort();
+ reject(new DOMException("Problem parsing input file."));
}
- this.fileReader.readAsText(this.mapFile)
- },
- async download () {
- const response = await ax.get(this.downloadUrl, { responseType: 'blob' })
- return new Blob([response.data], { type: 'text/csv' })
- },
- CSVToJson (csvString) {
- const lines = csvString.split('\n')
- const headers = lines.shift().split(';')
- return lines
- .filter(line => !!line) // remove empty strings
- .map(line => {
- const lineItems = line.split(';')
- return headers.reduce((acc, curr, i) => {
- acc[headers[i]] = lineItems[i]
- return acc
- }, {})
- })
- },
- jsonToCSV (data) {
- const headerLine = Object.keys(data[0]).reduce((acc, curr) => {
- return acc ? `${acc};${curr}` : `${curr}`
- }, '')
- const lines = data.map(studentData => {
- return Object.values(studentData).reduce((acc, curr) => {
- return acc ? `${acc};${curr}` : `${curr}`
- }, '')
- })
- return headerLine + lines.reduce((acc, curr) => {
- return `${acc}\n${curr}`
- }, '') + '\n' // add trailing newline
- },
- mapStudentData (students) {
- return students.map(studentData => {
- return {
- ...studentData,
- Matrikel: this.$store.state.studentMap[studentData.Matrikel].matrikelNo,
- Name: this.$store.state.studentMap[studentData.Matrikel].name
- }
- })
- },
- getMappedCSV () {
- this.download().then(blobData => {
- if (this.mapFileLoaded) {
- this.fileReader.onload = event => {
- const jsonData = this.CSVToJson(event.target.result)
- const mappedData = this.mapStudentData(jsonData)
- const csvData = this.jsonToCSV(mappedData)
- const mappedBlobData = new Blob([csvData], { type: 'text/csv' })
- window.open(window.URL.createObjectURL(mappedBlobData))
- }
- this.fileReader.readAsText(blobData)
- } else {
- window.open(window.URL.createObjectURL(blobData))
- }
- })
- },
- getCSVFileAndProcess () {
- if (!this.mapFileLoaded && !this.mapFileDialog) {
- this.mapFileDialog = true
+
+ if (!this.mapFile) {
+ reject(new Error("Can only call" +
+ " readMapFileAndCommit when mapFile is not undefined"))
} else {
- if (this.mapFile) {
- this.readMapFileAndCommit(this.getMappedCSV)
- } else {
- this.getMappedCSV()
- }
+ fileReader.readAsText(this.mapFile)
+ }
+ })
+ }
+
+ applyMapping (studentExport: StudentExportItem[]) {
+ return studentExport.map(student => {
+ return {
+ ...student,
+ Matrikel: this.studentMap[student.Matrikel].matrikelNo,
+ Name: this.studentMap[student.Matrikel].name
+ }
+ })
+ }
+
+ jsonToCSV (studentExport: StudentExportItem[], delimeter = ';') {
+ let headerLine = Object.keys(studentExport[0]).reduce((acc: string, curr) => {
+ if (curr === 'Scores') {
+ return acc
}
+ return acc ? `${acc};${curr}` : `${curr}`
+ }, '')
+ headerLine += Object.values(studentExport[0].Scores)
+ .reduce((acc: string, curr) => {
+ return `${acc};${curr.type}`
+ }, '')
+
+ const lines = studentExport.map(student => {
+ const normalFields = Object.values(student).reduce((acc: string, curr): string => {
+ // skip the Scores field
+ if (typeof curr === 'object') {
+ return acc
+ }
+ return acc ? `${acc};${curr}` : `${curr}`
+ }, '')
+
+ const scoreFields = Object.values(student.Scores).reduce((acc: string, curr) => {
+ return `${acc};${curr.score}`
+ }, '')
+ return normalFields + scoreFields
+ })
+
+ return headerLine + lines.reduce((acc, curr) => {
+ return `${acc}\n${curr}`
+ }, '') + '\n' // add trailing newline
+ }
+
+ createDownloadPopup (content: string | StudentExportItem[], fileType: ExportType) {
+ const blobProperties: BlobPropertyBag = {}
+ if (fileType === ExportType.JSON) {
+ blobProperties.type = 'application/json'
+ content = JSON.stringify(content)
+ } else {
+ blobProperties.type = 'text/csv'
}
+ const blobData = new Blob([<string> content], blobProperties)
+ window.open(window.URL.createObjectURL(blobData))
+ }
+
+ optionalConvertAndCreatePopup (studentData: StudentExportItem[]) {
+ const convertedData = this.exportType === ExportType.CSV ?
+ this.jsonToCSV(studentData) : studentData
+
+ this.createDownloadPopup(convertedData, this.exportType)
}
+ async getMappedExportFile (studentData: StudentExportItem[]) {
+ if (!this.mapFile && !this.mapFileLoaded) {
+ throw new Error("Either mapFile must be selected or already loaded "+
+ "to call getMappedExportFile")
+ }
+ if (this.mapFile) {
+ await this.readMapFileAndCommit()
+ }
+ const mappedData = this.applyMapping(studentData)
+ this.optionalConvertAndCreatePopup(mappedData)
+ }
+
+ async getExportFile () {
+ const studentData = await fetchStudentExportData({setPasswords: this.setPasswords})
+
+ if (this.mapFile || this.mapFileLoaded) {
+ this.getMappedExportFile(studentData)
+ } else {
+ this.optionalConvertAndCreatePopup(studentData)
+ }
+ }
}
</script>
diff --git a/frontend/src/components/mixins/mixins.js b/frontend/src/components/mixins/mixins.js
deleted file mode 100644
index 4018b4aa..00000000
--- a/frontend/src/components/mixins/mixins.js
+++ /dev/null
@@ -1,15 +0,0 @@
-export var parseCSVMapMixin = {
- methods: {
- parseCSVMap (csvMap) {
- let lines = csvMap.split('\n')
- lines.shift() // drop the first line since it contains only headings
- return lines.reduce((acc, curr) => {
- if (curr) {
- let [key, matrikelNo, name] = curr.split(';')
- acc[key] = { matrikelNo: matrikelNo, name }
- }
- return acc
- }, {})
- }
- }
-}
diff --git a/frontend/src/components/mixins/mixins.ts b/frontend/src/components/mixins/mixins.ts
new file mode 100644
index 00000000..ba6ab61f
--- /dev/null
+++ b/frontend/src/components/mixins/mixins.ts
@@ -0,0 +1,17 @@
+import {Vue, Component} from 'vue-property-decorator'
+
+@Component
+export class parseCSVMapMixin extends Vue {
+ parseCSVMap (csvMap: string) {
+ let lines = csvMap.split('\n')
+ lines.shift() // drop the first line since it contains only headings
+ // TODO remove any type
+ return lines.reduce((acc: any, curr) => {
+ if (curr) {
+ let [key, matrikelNo, name] = curr.split(';')
+ acc[key] = { matrikelNo: matrikelNo, name }
+ }
+ return acc
+ }, {})
+ }
+}
diff --git a/frontend/src/store/modules/subscriptions.ts b/frontend/src/store/modules/subscriptions.ts
index bb399b1a..e7f181e5 100644
--- a/frontend/src/store/modules/subscriptions.ts
+++ b/frontend/src/store/modules/subscriptions.ts
@@ -44,9 +44,9 @@ const availableStagesGetter = mb.read(function availableStages (state, getters)
return stages
})
const availableStagesReadableGetter = mb.read(function availableStagesReadable (state, getters) {
- let stages = ['create', 'validate']
+ let stages = ['initial', 'validate']
if (Authentication.isReviewer) {
- stages.push('resolve')
+ stages.push('conflict')
}
return stages
})
diff --git a/frontend/src/store/mutations.ts b/frontend/src/store/mutations.ts
index 7dacc5de..c2a79d5f 100644
--- a/frontend/src/store/mutations.ts
+++ b/frontend/src/store/mutations.ts
@@ -25,7 +25,13 @@ function SET_STUDENT (state: RootState, student: StudentInfoForListView) {
}, state.studentMap))
}
// TODO proper types for student map
-function SET_STUDENT_MAP (state: RootState, map: object) {
+export interface StudentMap {
+ [pseudoMatrikelNo: string]: {
+ matrikelNo: string,
+ name: string
+ }
+}
+function SET_STUDENT_MAP (state: RootState, map: StudentMap) {
state.studentMap = map
}
function SET_TUTORS (state: RootState, tutors: Array<Tutor>) {
diff --git a/frontend/src/store/store.ts b/frontend/src/store/store.ts
index b27d7a96..6689ec63 100644
--- a/frontend/src/store/store.ts
+++ b/frontend/src/store/store.ts
@@ -42,7 +42,11 @@ export interface RootInitialState {
submissionTypes: {[pk: string]: SubmissionType}
submissions: {[pk: string]: SubmissionNoType}
students: {[pk: string]: StudentInfoForListView}
- studentMap: {} // is used to map obfuscated student data back to the original
+ studentMap: {
+ [matrikel: string]: {
+ matrikelNo: string, name: string
+ }
+ } // is used to map obfuscated student data back to the original
statistics: Statistics
tutors: Array<Tutor>
}
diff --git a/requirements.txt b/requirements.txt
index 3a910671..5f6f7175 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,5 @@
django-cors-headers~=2.1.0
django-extensions~=2.1
-djangorestframework-csv~=2.0.0
djangorestframework-jwt~=1.11.0
djangorestframework~=3.8
git+https://github.com/robinhundt/djangorestframework-camel-case
@@ -14,3 +13,4 @@ python-json-logger~=0.1.9
tqdm~=4.19.5
whitenoise~=3.3.1
xlrd~=1.0.0
+xkcdpass==1.16.5
--
GitLab