diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8caa909beb82b5de4699aa49c2889685be5862b6..a92fb09d3e995643ea80fbb46a09a657d3509308 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,7 +2,6 @@ stages: - build - test - build_image - - test_build - pages - staging @@ -32,6 +31,8 @@ build_test_env: - .venv tags: - docker + interruptible: true + build_frontend: image: node:carbon @@ -54,6 +55,29 @@ build_frontend: - frontend/node_modules/ tags: - docker + interruptible: true + +build_test_image: + image: docker:latest + stage: build + only: + - branches + services: + - docker:dind + variables: + DOCKER_HOST: tcp://docker:2375/ + DOCKER_DRIVER: overlay2 + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker pull $DEV_IMAGE_BASE || true + - docker build --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE_BASE --target node . + - docker pull $DEV_IMAGE || true + - docker build --cache-from $DEV_IMAGE --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE . + - docker push $DEV_IMAGE_BASE + - docker push $DEV_IMAGE + tags: + - docker + interruptible: true # ============================== Testing section ============================= # # ----------------------------- Backend subsection --------------------------- # @@ -65,6 +89,9 @@ build_frontend: - build_test_env tags: - docker + interruptible: true + needs: + - build_test_env test_pytest: <<: *test_definition_virtualenv @@ -97,6 +124,10 @@ test_flake8: - build_frontend tags: - docker + interruptible: true + needs: + - build_frontend + - build_test_env test_frontend: <<: *test_definition_frontend @@ -124,37 +155,17 @@ test_frontend_unit: script: - cd frontend/ - yarn test:unit - + interruptible: true + needs: + - build_frontend # =========================== Build Image section ============================ # -build_dev_image: - image: docker:latest - stage: build_image - only: - - branches - services: - - docker:dind - variables: - DOCKER_HOST: tcp://docker:2375/ - DOCKER_DRIVER: overlay2 - script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - docker pull $DEV_IMAGE_BASE || true - - docker build --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE_BASE --target node . - - docker pull $DEV_IMAGE || true - - docker build --cache-from $DEV_IMAGE --cache-from $DEV_IMAGE_BASE -t $DEV_IMAGE . - - docker push $DEV_IMAGE_BASE - - docker push $DEV_IMAGE - tags: - - docker build_release_image: image: docker:latest stage: build_image only: - tags - except: - - branches services: - docker:dind variables: @@ -184,7 +195,10 @@ pages: - public only: - master - + interruptible: true + needs: + - test_pytest + - build_test_env # ============================== Staging section ============================= # .staging_template: &staging_definition diff --git a/core/tests/test_import_views.py b/core/tests/test_import_views.py new file mode 100644 index 0000000000000000000000000000000000000000..3784b85f27567166169ff71cb546728e17ed53b2 --- /dev/null +++ b/core/tests/test_import_views.py @@ -0,0 +1,80 @@ +from rest_framework import status +from rest_framework.test import APIClient, APITestCase +from core.models import UserAccount, SubmissionType + +from util.factories import GradyUserFactory + +test_data = { + "meta": { + "version": "4.0.0" + }, + "module": { + "module_reference": "test", + "pass_only": True, + "pass_score": 1, + "total_score": 99 + }, + "students": [ + { + "fullname": "test", + "identifier": "test-test", + "submissions": [ + { + "code": "some messy, perhaps incorrect stuff", + "tests": {}, + "type": "[a0] coding stuff" + }, + { + "code": "i don't know man", + "tests": {}, + "type": "[a1] improvise" + } + ], + } + ], + "submission_types": [ + { + "description": "code some 1337 stuff", + "full_score": 99, + "name": "[a0] coding stuff", + "programming_language": "c", + "solution": "how dare u" + }, + { + "description": "now this one's hard", + "full_score": 1, + "name": "[a1] improvise", + "programming_language": "haskell", + "solution": "nope" + }, + ] +} + + +class ImportViewTest(APITestCase): + + factory = GradyUserFactory() + + def setUp(self): + self.url = '/api/import/' + self.client = APIClient() + self.client.force_login(user=self.factory.make_reviewer()) + + def test_can_not_submit_nothing(self): + res = self.client.post(self.url) + self.assertEqual(status.HTTP_400_BAD_REQUEST, res.status_code) + + def test_will_fail_on_wrong_importer_version(self): + data = {"meta": {"version": "0.0.0"}} + res = self.client.post(self.url, data) + self.assertEqual(status.HTTP_409_CONFLICT, res.status_code) + + def test_data_is_imported_correctly(self): + res = self.client.post(self.url, test_data) + + sub_types = SubmissionType.objects.all() + students = UserAccount.objects.all().filter(role='Student') + + self.assertEqual(2, len(sub_types)) + self.assertEqual(1, len(students)) + self.assertEqual(status.HTTP_201_CREATED, res.status_code) diff --git a/core/urls.py b/core/urls.py index 25fad50d86185a3e09ee8d545a513f0b8b0ca597..66dea3ff2820b65335cb146626d4ac9c7f0287ec 100644 --- a/core/urls.py +++ b/core/urls.py @@ -52,6 +52,7 @@ regular_views_urlpatterns = [ name='jwt-time-delta'), path('instance/export/', views.InstanceExport.as_view(), name="instance-export"), path('export/json/', views.StudentJSONExport.as_view(), name='export-json'), + path('import/', views.ImportApiViewSet.as_view(), name='import-json'), re_path(r'swagger(?P<format>\.json|\.yaml)$', schema_view.without_ui(cache_timeout=0), name='schema-json'), re_path(r'swagger/$', schema_view.with_ui('swagger', cache_timeout=0), diff --git a/core/views/__init__.py b/core/views/__init__.py index 64f568df3ea9115cdb646beec0cefb017d6d9855..c455fbd93bacba1f16ae9449167d3fda0e16b0bd 100644 --- a/core/views/__init__.py +++ b/core/views/__init__.py @@ -3,3 +3,4 @@ from .subscription import SubscriptionApiViewSet, AssignmentApiViewSet # noqa from .common_views import * # noqa from .export import StudentJSONExport, InstanceExport # noqa from .label import LabelApiViewSet, LabelStatistics # noqa +from .importer import ImportApiViewSet # noqa diff --git a/core/views/importer.py b/core/views/importer.py new file mode 100644 index 0000000000000000000000000000000000000000..ee5c19756b11d956ea61c9a2f877257f3f92c348 --- /dev/null +++ b/core/views/importer.py @@ -0,0 +1,25 @@ +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import APIView +from rest_framework.exceptions import ValidationError +from core.permissions import IsReviewer +from util.importer import parse_and_import_hektor_json + + +class ImportApiViewSet(APIView): + permission_classes = (IsReviewer, ) + + def post(self, request): + exam_data = request.data + + if not exam_data: + return Response({"Error": "You need to submit the exam data to be imported"}, + status.HTTP_400_BAD_REQUEST) + + try: + parse_and_import_hektor_json(exam_data) + except ValidationError as err: + return Response({"ValidationError": err.detail}, + status.HTTP_409_CONFLICT) + + return Response({}, status.HTTP_201_CREATED) diff --git a/frontend/src/api.ts b/frontend/src/api.ts index c13a4fefdeb76d842942c9feb2c83aa4b294359d..c616cf07f811536b07aac284a1417843c03425a9 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -256,6 +256,10 @@ export async function fetchStudentExportData (options: StudentExportOptions): Pr return (await ax.post('/api/export/json/', options)).data } +export async function importData (data: Object): Promise<AxiosResponse<void>> { + return ax.post('/api/import/', data) +} + // Note, this interface does not represent all of the returned data, // but only the fields which have to be transformed for deanonymisation export interface InstanceExportData { diff --git a/frontend/src/components/BaseLayout.vue b/frontend/src/components/BaseLayout.vue index ae32a6af46cbccbbbde8429a8d25b7cd0f03d559..789b03e5c0b15e024a6758b15c86ee0b68c630fa 100644 --- a/frontend/src/components/BaseLayout.vue +++ b/frontend/src/components/BaseLayout.vue @@ -71,7 +71,7 @@ <slot name="toolbar-center"/> <div class="toolbar-content"> <v-menu bottom offset-y v-if="!isStudent"> - <v-btn slot="activator" color="cyan" style="text-transform: none"> + <v-btn id="user-options" slot="activator" color="cyan" style="text-transform: none"> {{ userRole }} | {{ username }} <v-icon>arrow_drop_down</v-icon> </v-btn> <user-options class="mt-1" v-if="!isStudent"/> diff --git a/frontend/src/components/ImportDialog.vue b/frontend/src/components/ImportDialog.vue new file mode 100644 index 0000000000000000000000000000000000000000..1f88e5cdc74f1e8bda0d600168f51cb543538530 --- /dev/null +++ b/frontend/src/components/ImportDialog.vue @@ -0,0 +1,91 @@ +<template> + <v-dialog v-model="show" width="30%"> + <v-card> + <v-card-title class="title">Import data</v-card-title> + <v-card-text> + <p> + You can use this component to import data into Grady. + You can use + <a + href="https://gitlab.gwdg.de/grady-corp/rusty-hektor" + target="_blank" + >rusty-hektor</a> to convert + and pseudonomize ILIAS output. + </p> + <file-select v-model="hektorFile" display-text="Select json file" /> + </v-card-text> + <v-card-actions> + <v-btn @click="submitData" :loading="loading" id="submit-import">Import</v-btn> + <v-btn @click="$emit('hide')" color="red">Cancel</v-btn> + </v-card-actions> + </v-card> + </v-dialog> +</template> + +<script> + import FileSelect from "@/components/util/FileSelect.vue"; + import { importData } from "@/api"; + + export default { + name: "ImportDialog", + components: { + FileSelect + }, + data: () => { + return { + show: true, + loading: false, + hektorFile: null + }; + }, + methods: { + async submitData() { + this.loading = true + let data; + try { + data = await this.readFile(); + data = JSON.parse(data) + } catch (error) { + this.$notify({ + type: 'error', + title: 'Error reading import file', + text: error.message + }) + this.loading = false + return + } + + try { + await importData(data) + this.$emit('imported') + this.$notify({ + title: 'Successfully imported data. Please log out and in again.', + type: 'success' + }) + } finally { + this.loading = false + } + }, + readFile() { + const fileReader = new FileReader(); + return new Promise((resolve, reject) => { + fileReader.onload = event => { + resolve(event.target.result); + }; + fileReader.onerror = () => { + fileReader.abort(); + reject(new Error("Problem parsing input file.")); + }; + fileReader.readAsText(this.hektorFile) + }); + } + }, + watch: { + show(val) { + if (!val) { + this.$emit("hide"); + } + } + } + }; +</script> diff --git a/frontend/src/components/UserOptions.vue b/frontend/src/components/UserOptions.vue index 465288e3113d22da164e6368496fcacf04283e54..db59c4900df645f0752f01f9c3c0d5716d2699e4 100644 --- a/frontend/src/components/UserOptions.vue +++ b/frontend/src/components/UserOptions.vue @@ -5,6 +5,7 @@ <v-list-tile v-if="opt.condition()" @click="opt.action" + :id="opt.id" :key="i" > {{opt.display}} @@ -17,11 +18,12 @@ <script> import PasswordChangeDialog from '@/components/PasswordChangeDialog' +import ImportDialog from '@/components/ImportDialog' import { Authentication } from '@/store/modules/authentication' import { deleteAllActiveAssignments } from '@/api' export default { name: 'UserOptions', - components: { PasswordChangeDialog }, + components: { PasswordChangeDialog, ImportDialog }, data () { return { displayComponent: null, @@ -29,12 +31,20 @@ export default { { display: 'Change password', action: () => { this.displayComponent = PasswordChangeDialog }, - condition: () => !Authentication.isStudent + condition: () => !Authentication.isStudent, + id: "change-password-list-tile" }, { display: 'Free all reserved submissions', action: deleteAllActiveAssignments, - condition: () => Authentication.isReviewer + condition: () => Authentication.isReviewer, + id: "free-assignments-list-tile" + }, + { + display: 'Import data', + action: () => { this.displayComponent = ImportDialog }, + condition: () => Authentication.isReviewer, + id: "import-data-list-tile" } ] } diff --git a/frontend/src/components/util/FileSelect.vue b/frontend/src/components/util/FileSelect.vue index 1bf8b055fc73aa7b268ed046107ca09794f770af..8e5334d903c9e425694523fc8de0a9e4fa47e5d8 100644 --- a/frontend/src/components/util/FileSelect.vue +++ b/frontend/src/components/util/FileSelect.vue @@ -4,7 +4,7 @@ <span v-if="value">Selected: {{value.name}}</span> <span v-else>{{displayText}}</span> </div> - <input type="file" @change="handleFileChange"/> + <input id="file-input" type="file" @change="handleFileChange"/> </label> </template> @@ -17,7 +17,6 @@ export default { default: 'Select File' } }, - methods: { handleFileChange (e) { this.$emit('input', e.target.files[0]) diff --git a/frontend/src/pages/StartPageSelector.vue b/frontend/src/pages/StartPageSelector.vue index 4cb6a3c2ea1d4781d23efb16b15949339a259e84..397111a8ed0319d3bad6ebd2b5c446c7be477ee6 100644 --- a/frontend/src/pages/StartPageSelector.vue +++ b/frontend/src/pages/StartPageSelector.vue @@ -1,6 +1,5 @@ <template> - <component :is="startPage"> - </component> + <component :is="startPage"/> </template> <script> diff --git a/frontend/src/pages/reviewer/ReviewerStartPage.vue b/frontend/src/pages/reviewer/ReviewerStartPage.vue index 860c602f5b55b316184d779a49649257c67b7642..ca74f4e88511cabcb3e701a24427cdb47772646c 100644 --- a/frontend/src/pages/reviewer/ReviewerStartPage.vue +++ b/frontend/src/pages/reviewer/ReviewerStartPage.vue @@ -1,5 +1,5 @@ <template> - <div> + <div v-if="dataLoaded"> <v-layout row wrap> <v-flex lg5 md9 xs12> <correction-statistics class="ma-4"></correction-statistics> @@ -10,22 +10,66 @@ </v-layout> <SubmissionTypesOverview class="ma-4"/> </div> + <v-layout v-else justify-center class="mt-4 pt-4"> + <import-dialog + v-if="showImportDialog" + @hide="showImportDialog = false" + @imported="importDone" + /> + <v-card class="import-card"> + <v-card-title class="title"> + Import data + </v-card-title> + <v-card-text> + It looks like this instance doesn't contain any data. + Would you like to import some? + </v-card-text> + <v-card-actions class="justify-center"> + <v-btn @click="showImportDialog = true" class="info">Import data</v-btn> + </v-card-actions> + </v-card> + </v-layout> </template> <script> import CorrectionStatistics from '@/components/CorrectionStatistics' +import ImportDialog from '@/components/ImportDialog' import SubscriptionList from '@/components/subscriptions/SubscriptionList' import SubmissionTypesOverview from '@/components/submission_type/SubmissionTypesOverview' +import { getters } from '../../store/getters' +import { Subscriptions } from '../../store/modules/subscriptions' export default { components: { + ImportDialog, SubmissionTypesOverview, SubscriptionList, CorrectionStatistics }, - name: 'reviewer-start-page' + name: 'reviewer-start-page', + data: () => { + return { + showImportDialog: false, + dataImported: false + } + }, + computed: { + dataLoaded () { + return Object.keys(getters.state.submissionTypes).length !== 0 || this.dataImported + } + }, + methods: { + importDone() { + this.dataImported = true + Subscriptions.RESET_STATE() + } + } } </script> <style scoped> +.import-card { + width: 30%; +} + </style> diff --git a/functional_tests/data/hektor.json b/functional_tests/data/hektor.json new file mode 100644 index 0000000000000000000000000000000000000000..571ab05fe8718217a9fcfc90919fa75c987007a7 --- /dev/null +++ b/functional_tests/data/hektor.json @@ -0,0 +1,34 @@ +{ + "meta": { + "version": "4.0.0" + }, + "module": { + "module_reference": "B.Inf.1801", + "total_score": 50, + "pass_score": 25, + "pass_only": false + }, + "submission_types": [ + { + "name": "Eine Bibliothek für Permutationen (I1-ID: l120mlc005h0)", + "full_score": 50, + "description": "A <b>description</b>!", + "solution": "Blub", + "programming_language": "java" + } + ], + "students": [ + { + "fullname": "Test, User", + "identifier": "20000000", + "username": "TU20000000", + "submissions": [ + { + "code": "234;", + "type": "Eine Bibliothek für Permutationen (I1-ID: l120mlc005h0)", + "tests": {} + } + ] + } + ] +} diff --git a/functional_tests/test_import.py b/functional_tests/test_import.py new file mode 100644 index 0000000000000000000000000000000000000000..791d9a9e4a66d5d2d8eb6cf27d8fcefba5454079 --- /dev/null +++ b/functional_tests/test_import.py @@ -0,0 +1,57 @@ +import os +from django.test import LiveServerTestCase +from selenium import webdriver +from selenium.webdriver.support.ui import WebDriverWait + +from core import models +from functional_tests.util import (login, create_browser, query_returns_object, + reset_browser_after_test) +from util import factory_boys as fact + + +JSON_EXPORT_FILE = os.path.join(os.path.dirname(__file__), 'data/hektor.json') + + +class TestImport(LiveServerTestCase): + browser: webdriver.Firefox = None + username = None + password = None + role = None + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.browser = create_browser() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + cls.browser.quit() + + def setUp(self): + super().setUp() + self.username = 'rev' + self.password = 'p' + fact.UserAccountFactory( + username=self.username, + password=self.password, + role=models.UserAccount.REVIEWER + ) + + def tearDown(self): + reset_browser_after_test(self.browser, self.live_server_url) + + def _login(self): + login(self.browser, self.live_server_url, self.username, self.password) + + def test_reviewer_can_import_data(self): + self._login() + self.browser.find_element_by_id("user-options").click() + self.browser.find_element_by_id("import-data-list-tile").click() + self.browser.execute_script( + "document.getElementById('file-input').style.display='block';" + ) + file_input = self.browser.find_element_by_id("file-input") + file_input.send_keys(JSON_EXPORT_FILE) + self.browser.find_element_by_id("submit-import").click() + WebDriverWait(self.browser, 20).until(query_returns_object(models.SubmissionType)) diff --git a/functional_tests/util.py b/functional_tests/util.py index b83992ae927852896e1e1af925cd5f3721e045fc..42911a3e87d49d007d4c652c551c3cae5e99b694 100644 --- a/functional_tests/util.py +++ b/functional_tests/util.py @@ -18,6 +18,7 @@ def create_browser() -> webdriver.Firefox: options = Options() options.headless = bool(os.environ.get('HEADLESS_TESTS', False)) options.set_capability('unhandledPromptBehavior', 'accept') + options.set_capability('strictFileInteractability', False) profile = FirefoxProfile() profile.set_preference("dom.disable_beforeunload", True) profile.set_preference("browser.download.folderList", 2) diff --git a/util/importer.py b/util/importer.py index 6441ba5297cd4c2396c366c9223507ac326fff0e..21fa74e77ee1d15a45784108ca4fe4b356f3c10a 100644 --- a/util/importer.py +++ b/util/importer.py @@ -3,6 +3,7 @@ import os import readline import logging +from rest_framework.exceptions import ValidationError from util.messages import warn from core.models import ExamType, Feedback, Submission, SubmissionType, Test, FeedbackLabel from core.models import UserAccount as User @@ -28,8 +29,8 @@ PASSWORDS = '.importer_passwords' YES = 'Y/n' NO = 'y/N' -RUSTY_HEKTOR_MIN_VER = ">=3.0.0" -RUSTY_HEKTOR_MAX_VER = "<4.0.0" +RUSTY_HEKTOR_MIN_VER = ">=4.0.0" +RUSTY_HEKTOR_MAX_VER = "<5.0.0" valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} @@ -119,11 +120,17 @@ def load_hektor_json(): with open(file, 'r') as f: exam_data = json.JSONDecoder().decode(f.read()) + parse_and_import_hektor_json(exam_data) + + +def parse_and_import_hektor_json(exam_data): hektor_version = exam_data['meta']['version'] if not (semver.match(hektor_version, RUSTY_HEKTOR_MIN_VER) and semver.match(hektor_version, RUSTY_HEKTOR_MAX_VER)): - warn(f'The data you\'re trying to import has the wrong version {hektor_version}\n' - f'Requirements: {RUSTY_HEKTOR_MIN_VER}, {RUSTY_HEKTOR_MAX_VER}') + raise ValidationError( + f'The data you\'re trying to import has the wrong version {hektor_version}\n' + f'Requirements: {RUSTY_HEKTOR_MIN_VER}, {RUSTY_HEKTOR_MAX_VER}' + ) exam, _ = ExamType.objects.get_or_create(**exam_data['module']) @@ -131,7 +138,7 @@ def load_hektor_json(): _, created = SubmissionType.objects.update_or_create( name=submission_type['name'], defaults=submission_type) if not created: - log.warning(f"Updated submission type {submission_type}") + raise ValidationError(f"Updated submission type: {submission_type['name']}") for student in exam_data['students']: student_obj = user_factory.make_student(exam=exam, is_active=False, @@ -151,7 +158,7 @@ def load_reviewers(): store_pw=True) -def add_submission(student_obj, code, tests, type=None): +def add_submission(student_obj, code, tests, type=None, display_code=None): submission_type_obj = SubmissionType.objects.get(name=type) submission_obj, _ = Submission.objects.update_or_create(