From 744df9529e6c6a531449d5a7008e29ad67b53436 Mon Sep 17 00:00:00 2001 From: janmax <j.michal@stud.uni-goettingen.de> Date: Sat, 17 Feb 2018 15:34:26 +0100 Subject: [PATCH 1/2] Fixes a nasty race condition --- core/apps.py | 3 +++ .../0003_submissiondoneassignmentscount.py | 22 +++++++++++++++++++ core/models.py | 19 +++++++++++----- core/signals.py | 13 +++++++++++ .../test_subscription_assignment_service.py | 18 +++++---------- util/factories.py | 4 ++-- util/importer.py | 2 +- 7 files changed, 60 insertions(+), 21 deletions(-) create mode 100644 core/migrations/0003_submissiondoneassignmentscount.py create mode 100644 core/signals.py diff --git a/core/apps.py b/core/apps.py index 089650b7..17b18064 100644 --- a/core/apps.py +++ b/core/apps.py @@ -4,3 +4,6 @@ from django.apps import AppConfig class CoreConfig(AppConfig): name = 'core' verbose_name = 'where everything comes together' + + def ready(self): + import core.signals # noqa diff --git a/core/migrations/0003_submissiondoneassignmentscount.py b/core/migrations/0003_submissiondoneassignmentscount.py new file mode 100644 index 00000000..0d1f0b23 --- /dev/null +++ b/core/migrations/0003_submissiondoneassignmentscount.py @@ -0,0 +1,22 @@ +# Generated by Django 2.0.2 on 2018-02-17 12:01 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0002_auto_20180210_1727'), + ] + + operations = [ + migrations.CreateModel( + name='SubmissionDoneAssignmentsCount', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('done_assignments', models.PositiveIntegerField(default=0)), + ('submission', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='counter', to='core.Submission')), + ], + ), + ] diff --git a/core/models.py b/core/models.py index 7e680a27..6b823bdc 100644 --- a/core/models.py +++ b/core/models.py @@ -502,6 +502,7 @@ class SubmissionSubscription(models.Model): if self.query_type == self.RANDOM: return Submission.objects.all() + TutorSubmissionAssignment.objects.select_for_update() return Submission.objects.filter( **{self.type_query_mapper[self.query_type]: self.query_key}) @@ -521,11 +522,6 @@ class SubmissionSubscription(models.Model): Q(assignments__subscription__owner=self.owner) ).exclude( assignments__is_done=False - ).annotate( - done_assignments_count=Count( - Case(When(assignments__is_done=True, then=Value(1)), - output_field=IntegerField(),) - ) ) def _get_available_submissions_in_subscription_stage(self) -> QuerySet: @@ -546,7 +542,7 @@ class SubmissionSubscription(models.Model): done_assignments_count = self.assignment_count_on_stage[self.feedback_stage] # noqa stage_candiates = candidates.filter( - done_assignments_count=done_assignments_count, + counter__done_assignments=done_assignments_count, ) if stage_candiates.count() == 0: @@ -607,6 +603,14 @@ class DeletionOfDoneAssignmentsNotPermitted(Exception): pass +class SubmissionDoneAssignmentsCount(models.Model): + + submission = models.OneToOneField('submission', + related_name='counter', + on_delete=models.CASCADE) + done_assignments = models.PositiveIntegerField(default=0) + + class TutorSubmissionAssignment(models.Model): assignment_id = models.UUIDField(primary_key=True, @@ -623,6 +627,9 @@ class TutorSubmissionAssignment(models.Model): @transaction.atomic def set_done(self): + if not self.is_done: + self.submission.counter.done_assignments += 1 + self.submission.counter.save() self.is_done = True self.save() diff --git a/core/signals.py b/core/signals.py new file mode 100644 index 00000000..57173f12 --- /dev/null +++ b/core/signals.py @@ -0,0 +1,13 @@ +from core.models import Submission, SubmissionDoneAssignmentsCount + +from django.db.models.signals import post_save +from django.dispatch import receiver + + +@receiver(post_save, sender=Submission) +def create_counter_after_submission_create(sender, + instance, + created, + **kwargs): + if created: + SubmissionDoneAssignmentsCount.objects.create(submission=instance) diff --git a/core/tests/test_subscription_assignment_service.py b/core/tests/test_subscription_assignment_service.py index ea2533b8..b2c1c7e4 100644 --- a/core/tests/test_subscription_assignment_service.py +++ b/core/tests/test_subscription_assignment_service.py @@ -109,12 +109,6 @@ class TestApiEndpoints(APITestCase): 'full_score': 35, 'description': 'Very complicated', 'solution': 'Trivial!' - }, - { - 'name': '03. This one exists for the sole purpose to test', - 'full_score': 30, - 'description': 'Very complicated', - 'solution': 'Trivial!' } ], 'students': [ @@ -160,12 +154,12 @@ class TestApiEndpoints(APITestCase): ' on multi lines\n' ' asasxasx\n' ' lorem ipsum und so\n', - 'type': '03. This one exists for the sole purpose to test', - 'user': 'student01' + 'type': '01. Sort this or that', + 'user': 'student02' }, { 'text': 'function lorem ipsum etc\n', - 'type': '03. This one exists for the sole purpose to test', + 'type': '02. Merge this or that or maybe even this', 'user': 'student02' }, ]} @@ -242,17 +236,17 @@ class TestApiEndpoints(APITestCase): client = APIClient() client.force_authenticate(user=self.data['reviewers'][0]) - student01 = self.data['students'][0] + student = self.data['students'][0] response = client.post( '/api/subscription/', { 'query_type': 'student', - 'query_key': student01.student.student_id, + 'query_key': student.student.student_id, 'stage': 'feedback-creation' }) assignments = response.data['assignments'] - self.assertEqual(2, len(assignments)) + self.assertEqual(1, len(assignments)) def test_two_tutors_cant_have_assignments_for_same_submission(self): client = APIClient() diff --git a/util/factories.py b/util/factories.py index 89957819..db42ebcb 100644 --- a/util/factories.py +++ b/util/factories.py @@ -53,7 +53,7 @@ class GradyUserFactory: }[role] def _make_base_user(self, username, role, password=None, - store_pw=False, name='', **kwargs): + store_pw=False, fullname='', **kwargs): """ This is a specific wrapper for the django update_or_create method of objects. * If now username is passed, a generic one will be generated @@ -72,7 +72,7 @@ class GradyUserFactory: user, created = UserAccount.objects.update_or_create( username=username, - fullname=name, + fullname=fullname, role=role, defaults=kwargs) diff --git a/util/importer.py b/util/importer.py index 683dccc7..9f60042b 100644 --- a/util/importer.py +++ b/util/importer.py @@ -110,7 +110,7 @@ def add_tests(submission_obj, tests): for name, test_data in ((name, tests[name]) for name in TEST_ORDER): test_obj, created = Test.objects.update_or_create( - name=test_data['fullname'], + name=test_data['name'], submission=submission_obj, defaults={ 'label': test_data['label'], -- GitLab From 31c64f6931d145c1963c1bf560d5d710309c7c89 Mon Sep 17 00:00:00 2001 From: janmax <j.michal@stud.uni-goettingen.de> Date: Sat, 17 Feb 2018 20:13:22 +0100 Subject: [PATCH 2/2] Using signals to allow select on update --- .../0003_submissiondoneassignmentscount.py | 2 +- core/migrations/0004_auto_20180217_1723.py | 38 ++++++++++ core/migrations/0005_auto_20180217_1728.py | 19 +++++ core/migrations/0006_auto_20180217_1729.py | 19 +++++ core/models.py | 72 +++++++++--------- core/serializers/feedback.py | 7 -- core/signals.py | 74 +++++++++++++++++-- core/tests/test_export.py | 2 +- core/tests/test_feedback.py | 4 +- core/tests/test_tutor_api_endpoints.py | 2 +- core/views/common_views.py | 4 +- frontend/src/pages/SubscriptionWorkPage.vue | 2 +- 12 files changed, 189 insertions(+), 56 deletions(-) create mode 100644 core/migrations/0004_auto_20180217_1723.py create mode 100644 core/migrations/0005_auto_20180217_1728.py create mode 100644 core/migrations/0006_auto_20180217_1729.py diff --git a/core/migrations/0003_submissiondoneassignmentscount.py b/core/migrations/0003_submissiondoneassignmentscount.py index 0d1f0b23..61375c88 100644 --- a/core/migrations/0003_submissiondoneassignmentscount.py +++ b/core/migrations/0003_submissiondoneassignmentscount.py @@ -1,7 +1,7 @@ # Generated by Django 2.0.2 on 2018-02-17 12:01 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/core/migrations/0004_auto_20180217_1723.py b/core/migrations/0004_auto_20180217_1723.py new file mode 100644 index 00000000..b87bf368 --- /dev/null +++ b/core/migrations/0004_auto_20180217_1723.py @@ -0,0 +1,38 @@ +# Generated by Django 2.0.2 on 2018-02-17 17:23 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0003_submissiondoneassignmentscount'), + ] + + operations = [ + migrations.CreateModel( + name='MetaSubmission', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('done_assignments', models.PositiveIntegerField(default=0)), + ('has_active_assignment', models.BooleanField(default=False)), + ('has_feedback', models.BooleanField(default=False)), + ('has_final_feedback', models.BooleanField(default=False)), + ('feedback_authors', models.ManyToManyField(to=settings.AUTH_USER_MODEL)), + ('submission', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='meta', to='core.Submission')), + ], + ), + migrations.RemoveField( + model_name='submissiondoneassignmentscount', + name='submission', + ), + migrations.AlterUniqueTogether( + name='tutorsubmissionassignment', + unique_together={('submission', 'subscription')}, + ), + migrations.DeleteModel( + name='SubmissionDoneAssignmentsCount', + ), + ] diff --git a/core/migrations/0005_auto_20180217_1728.py b/core/migrations/0005_auto_20180217_1728.py new file mode 100644 index 00000000..85067337 --- /dev/null +++ b/core/migrations/0005_auto_20180217_1728.py @@ -0,0 +1,19 @@ +# Generated by Django 2.0.2 on 2018-02-17 17:28 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0004_auto_20180217_1723'), + ] + + operations = [ + migrations.AlterField( + model_name='metasubmission', + name='feedback_authors', + field=models.ManyToManyField(related_name='authors', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/core/migrations/0006_auto_20180217_1729.py b/core/migrations/0006_auto_20180217_1729.py new file mode 100644 index 00000000..3783adfb --- /dev/null +++ b/core/migrations/0006_auto_20180217_1729.py @@ -0,0 +1,19 @@ +# Generated by Django 2.0.2 on 2018-02-17 17:29 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('core', '0005_auto_20180217_1728'), + ] + + operations = [ + migrations.AlterField( + model_name='metasubmission', + name='feedback_authors', + field=models.ManyToManyField(to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/core/models.py b/core/models.py index 6b823bdc..22d63829 100644 --- a/core/models.py +++ b/core/models.py @@ -500,11 +500,11 @@ class SubmissionSubscription(models.Model): e.g. all submissions of one student or submission type. """ if self.query_type == self.RANDOM: - return Submission.objects.all() + return MetaSubmission.objects.all() - TutorSubmissionAssignment.objects.select_for_update() - return Submission.objects.filter( - **{self.type_query_mapper[self.query_type]: self.query_key}) + return MetaSubmission.objects.filter( + **{'submission__' + self.type_query_mapper[self.query_type]: + self.query_key}) def _get_submissions_that_do_not_have_final_feedback(self) -> QuerySet: """ There are a number of conditions to check for each submission @@ -516,12 +516,10 @@ class SubmissionSubscription(models.Model): Returns: QuerySet -- a list of all submissions ready for consumption """ - return self._get_submission_base_query().exclude( - Q(feedback__isnull=False) & Q(feedback__is_final=True) - ).exclude( - Q(assignments__subscription__owner=self.owner) - ).exclude( - assignments__is_done=False + return self._get_submission_base_query().select_for_update().exclude( + Q(has_final_feedback=True) | + Q(has_active_assignment=True) | + Q(feedback_authors=self.owner) ) def _get_available_submissions_in_subscription_stage(self) -> QuerySet: @@ -542,7 +540,7 @@ class SubmissionSubscription(models.Model): done_assignments_count = self.assignment_count_on_stage[self.feedback_stage] # noqa stage_candiates = candidates.filter( - counter__done_assignments=done_assignments_count, + done_assignments=done_assignments_count, ) if stage_candiates.count() == 0: @@ -552,9 +550,11 @@ class SubmissionSubscription(models.Model): return stage_candiates + @transaction.atomic def get_remaining_not_final(self) -> int: return self._get_submissions_that_do_not_have_final_feedback().count() + @transaction.atomic def get_available_in_stage(self) -> int: try: return self._get_available_submissions_in_subscription_stage().count() # noqa @@ -568,18 +568,19 @@ class SubmissionSubscription(models.Model): raise NotMoreThanTwoOpenAssignmentsAllowed( 'Not more than 2 active assignments allowed.') - log.info(f'{self.owner} is assignment to {task}.') - return TutorSubmissionAssignment.objects.get_or_create( + log.info(f'{self.owner} is assigned to {task} ({self.feedback_stage})') + return TutorSubmissionAssignment.objects.create( subscription=self, - submission=task)[0] + submission=task.submission) @transaction.atomic def reserve_all_assignments_for_a_student(self): assert self.query_type == self.STUDENT_QUERY - submissions = self._get_submissions_that_do_not_have_final_feedback() + meta_submissions = self._get_submissions_that_do_not_have_final_feedback() # noqa - for submission in submissions: + for meta in meta_submissions: + submission = meta.submission if hasattr(submission, 'assignments'): submission.assignments.filter(is_done=False).delete() TutorSubmissionAssignment.objects.create( @@ -603,12 +604,28 @@ class DeletionOfDoneAssignmentsNotPermitted(Exception): pass -class SubmissionDoneAssignmentsCount(models.Model): +class MetaSubmission(models.Model): submission = models.OneToOneField('submission', - related_name='counter', + related_name='meta', on_delete=models.CASCADE) done_assignments = models.PositiveIntegerField(default=0) + has_active_assignment = models.BooleanField(default=False) + + has_feedback = models.BooleanField(default=False) + has_final_feedback = models.BooleanField(default=False) + + feedback_authors = models.ManyToManyField(get_user_model()) + + def __str__(self): + return f''' Submission Meta of {self.submission} + + done_assignments = {self.done_assignments} + has_active_assignment = {self.has_active_assignment} + has_feedback = {self.has_feedback} + has_final_feedback = {self.has_final_feedback} + feedback_authors = {self.feedback_authors} + ''' class TutorSubmissionAssignment(models.Model): @@ -625,14 +642,6 @@ class TutorSubmissionAssignment(models.Model): is_done = models.BooleanField(default=False) created = models.DateTimeField(auto_now_add=True) - @transaction.atomic - def set_done(self): - if not self.is_done: - self.submission.counter.done_assignments += 1 - self.submission.counter.save() - self.is_done = True - self.save() - def __str__(self): return (f'{self.subscription.owner} assigned to {self.submission}' f' (done={self.is_done})') @@ -642,6 +651,9 @@ class TutorSubmissionAssignment(models.Model): raise DeletionOfDoneAssignmentsNotPermitted() super().delete(*args, **kwargs) + class Meta: + unique_together = ('submission', 'subscription') + class FeedbackComment(models.Model): """ This Class contains the Feedback for a specific line of a Submission""" @@ -664,14 +676,6 @@ class FeedbackComment(models.Model): null=True ) - def save(self, *args, **kwargs): - comments_on_the_same_line = FeedbackComment.objects.filter( - of_line=self.of_line, - of_feedback=self.of_feedback, - ) - comments_on_the_same_line.update(visible_to_student=False) - super().save(*args, **kwargs) - class Meta: verbose_name = "Feedback Comment" verbose_name_plural = "Feedback Comments" diff --git a/core/serializers/feedback.py b/core/serializers/feedback.py index 1b5c181c..b3a51819 100644 --- a/core/serializers/feedback.py +++ b/core/serializers/feedback.py @@ -83,11 +83,6 @@ class FeedbackCommentSerializer(serializers.ModelSerializer): class FeedbackSerializer(DynamicFieldsModelSerializer): feedback_lines = FeedbackCommentSerializer(many=True, required=False) - def _set_assignment_done(self, submission): - assignment = submission.assignments.get( - subscription__owner=self.context['request'].user) - assignment.set_done() - @transaction.atomic def create(self, validated_data) -> Feedback: submission = validated_data.pop('of_submission') @@ -103,7 +98,6 @@ class FeedbackSerializer(DynamicFieldsModelSerializer): **comment ) - self._set_assignment_done(submission) return Feedback.objects.get(of_submission=submission) @transaction.atomic @@ -115,7 +109,6 @@ class FeedbackSerializer(DynamicFieldsModelSerializer): of_line=comment.get('of_line'), defaults={'text': comment.get('text')}) - self._set_assignment_done(feedback.of_submission) return super().update(feedback, validated_data) def validate_of_submission(self, submission): diff --git a/core/signals.py b/core/signals.py index 57173f12..f74b8f2d 100644 --- a/core/signals.py +++ b/core/signals.py @@ -1,13 +1,73 @@ -from core.models import Submission, SubmissionDoneAssignmentsCount +import logging -from django.db.models.signals import post_save +from django.db.models.signals import post_save, pre_delete, pre_save from django.dispatch import receiver +from core import models +from core.models import (Feedback, FeedbackComment, MetaSubmission, Submission, + TutorSubmissionAssignment) + +log = logging.getLogger(__name__) + @receiver(post_save, sender=Submission) -def create_counter_after_submission_create(sender, - instance, - created, - **kwargs): +def create_meta_after_submission_create(sender, instance, created, **kwargs): + log.debug('SIGNAL -- create_meta_after_submission_create') if created: - SubmissionDoneAssignmentsCount.objects.create(submission=instance) + MetaSubmission.objects.create(submission=instance) + + +@receiver(post_save, sender=TutorSubmissionAssignment) +def update_active_after_assignment_save(sender, instance, created, **kwargs): + """ Assignments are created undone therefore save that no other + should use it. If it is already set to done it is not active. + """ + log.debug('SIGNAL -- update_active_after_assignment_save') + meta = instance.submission.meta + meta.has_active_assignment = created and not instance.is_done + meta.save() + + +@receiver(pre_delete, sender=TutorSubmissionAssignment) +def remove_active_assignment_on_delete(sender, instance, **kwargs): + log.debug('SIGNAL -- remove_active_assignment_on_delete') + if instance.is_done: + raise models.DeletionOfDoneAssignmentsNotPermitted() + meta = instance.submission.meta + meta.has_active_assignment = False + meta.save() + + +@receiver(post_save, sender=Feedback) +def update_after_feedback_save(sender, instance, created, **kwargs): + """ Do the following steps when feedback is saved: + + - set that feedback exists + - copy the final status of the feedback + - set all assignments of the submission done and remove active status + """ + log.debug('SIGNAL -- update_after_feedback_save') + meta = instance.of_submission.meta + meta.has_feedback = True + meta.has_final_feedback = instance.is_final + + undone_assignment = meta.submission.assignments.filter(is_done=False) + assert undone_assignment.count() <= 1 + if undone_assignment.count() > 0: + log.debug('SIGNAL -- Completed: %s' % undone_assignment.first()) + meta.feedback_authors.add(undone_assignment.first().subscription.owner) + meta.done_assignments += 1 + meta.has_active_assignment = False + undone_assignment.update(is_done=True) + + meta.save() + + +@receiver(pre_save, sender=FeedbackComment) +def set_comment_visibility_after_conflict(sender, instance, **kwargs): + log.debug('SIGNAL -- set_comment_visibility_after_conflict') + comments_on_the_same_line = FeedbackComment.objects.filter( + of_line=instance.of_line, + of_feedback=instance.of_feedback, + ) + comments_on_the_same_line.update(visible_to_student=False) diff --git a/core/tests/test_export.py b/core/tests/test_export.py index 182db206..9165bb47 100644 --- a/core/tests/test_export.py +++ b/core/tests/test_export.py @@ -1,5 +1,5 @@ +from django.test import Client, TestCase from rest_framework import status -from django.test import TestCase, Client from util.factories import make_test_data diff --git a/core/tests/test_feedback.py b/core/tests/test_feedback.py index 8b377de2..f904a785 100644 --- a/core/tests/test_feedback.py +++ b/core/tests/test_feedback.py @@ -361,7 +361,7 @@ class FeedbackPatchTestCase(APITestCase): self.assertEqual(status.HTTP_200_OK, response.status_code) def test_tutor_can_not_update_when_there_is_a_new_assignment(self): - # Step 1 - Create a new assignment + # Step 1 - Create a new assignment for Tutor 2 second_subs = models.SubmissionSubscription.objects.create( owner=self.tutor02, query_type='random', @@ -369,7 +369,7 @@ class FeedbackPatchTestCase(APITestCase): ) second_subs.get_or_create_work_assignment() - # Step 2 - Tutor 2 tries to patch + # Step 2 - Tutor 1 tries to patch data = { 'feedback_lines': { '2': {'text': 'Turns out this is rather bad.'}, diff --git a/core/tests/test_tutor_api_endpoints.py b/core/tests/test_tutor_api_endpoints.py index 0e9a5773..63a1cda6 100644 --- a/core/tests/test_tutor_api_endpoints.py +++ b/core/tests/test_tutor_api_endpoints.py @@ -10,7 +10,7 @@ from rest_framework.reverse import reverse from rest_framework.test import (APIClient, APIRequestFactory, APITestCase, force_authenticate) -from core.models import TutorSubmissionAssignment, SubmissionSubscription +from core.models import SubmissionSubscription, TutorSubmissionAssignment from core.views import TutorApiViewSet from util.factories import GradyUserFactory diff --git a/core/views/common_views.py b/core/views/common_views.py index b79dec5c..27e83892 100644 --- a/core/views/common_views.py +++ b/core/views/common_views.py @@ -13,8 +13,8 @@ from core.models import ExamType, StudentInfo, SubmissionType from core.permissions import IsReviewer, IsStudent from core.serializers import (ExamSerializer, StudentInfoSerializer, StudentInfoSerializerForListView, - SubmissionSerializer, SubmissionTypeSerializer, - TutorSerializer, SubmissionNoTypeSerializer) + SubmissionNoTypeSerializer, SubmissionSerializer, + SubmissionTypeSerializer, TutorSerializer) log = logging.getLogger(__name__) diff --git a/frontend/src/pages/SubscriptionWorkPage.vue b/frontend/src/pages/SubscriptionWorkPage.vue index 62b0f4d3..e3532b82 100644 --- a/frontend/src/pages/SubscriptionWorkPage.vue +++ b/frontend/src/pages/SubscriptionWorkPage.vue @@ -35,8 +35,8 @@ if (subscription['assignments'].length === 0) { store.dispatch('getAssignmentForSubscription', {subscription}).then(() => { next() - store.dispatch('getAssignmentForSubscription', {subscription}) }) + store.dispatch('getAssignmentForSubscription', {subscription}) } else { next() } -- GitLab