Merge branch '181-mailing' into 'master'

Resolve "Mailing"

Closes #181

See merge request !350

config/settings/base.py

@@ -243,7 +243,8 @@ X_FRAME_OPTIONS = "DENY"
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
 EMAIL_BACKEND = env(
-    "DJANGO_EMAIL_BACKEND", default="django.core.mail.backends.smtp.EmailBackend",
+    "DJANGO_EMAIL_BACKEND",
+    default="django.core.mail.backends.smtp.EmailBackend",
 )
 # https://docs.djangoproject.com/en/2.2/ref/settings/#email-timeout
 EMAIL_TIMEOUT = 5

discuss_data/core/djangoscripts/init_scheduler.py

+import arrow
+from django_q.models import Schedule
+
+
+"""
+    Definition of the task schedules to generate at startup
+    https://django-q.readthedocs.io/en/latest/schedules.html#reference
+"""
+SCHEDULES = [
+    # Schedule for the DAILY send_status_email task
+    # for user mailings. Called daily at 20:00
+    {
+        "name": "statusmails_daily",
+        "func": "discuss_data.ddusers.tasks.send_status_emails",
+        "schedule_type": Schedule.CRON,
+        "repeats": -1,
+        "cron": "0 20 * * *",
+    },
+    # Schedule for the WEEKLY send_status_email task
+    # or user mailings. Called every Saturday at 22:00
+    {
+        "name": "statusmails_weekly",
+        "func": "discuss_data.ddusers.tasks.send_status_emails",
+        "schedule_type": Schedule.CRON,
+        "repeats": -1,
+        "cron": "0 22 * * 6",
+    },
+]
+
+
+def run():
+    """
+        Initialize all schedules defined in SCHEDULES
+    """
+    for elem in SCHEDULES:
+        sched, created = Schedule.objects.get_or_create(name=elem.get("name"))
+        sched.func = elem.get("func")
+        sched.schedule_type = elem.get("schedule_type")
+        sched.minutes = elem.get("minutes")
+        sched.repeats = elem.get("repeats")
+        sched.cron = elem.get("cron")
+        sched.save()
+    print("Schedules defined ")

discuss_data/core/utils.py

 import logging
 
+from datetime import timedelta
+
 from datacite import DataCiteMDSClient, schema41
 from datacite.errors import DataCiteServerError
 
 from django.conf import settings
 from django.template.loader import render_to_string
+from django.utils import timezone
+from django.core.mail import send_mail
 
 logger = logging.getLogger(__name__)
 
@@ -89,3 +93,37 @@ def generate_discuss_data_doi(dataset):
             logger.error(e)
 
     return doi
+
+
+def weekly_td():
+    # week timedelta
+    startdate = timezone.now()
+    enddate = startdate - timedelta(days=7)
+    return startdate, enddate
+
+
+def daily_td():
+    # day timedelta
+    startdate = timezone.now()
+    enddate = startdate - timedelta(hours=24)
+    return startdate, enddate
+
+
+def send_update_email(subject, message, email_to):
+    email_from = "info@discuss-data.net"
+    subject_prefixed = "[Discuss Data] {}".format(subject,)
+
+    message_text = render_to_string(
+        "core/email.html", {"message": message, "subject": subject, "html": False}
+    )
+    # message_html = render_to_string(
+    #    "core/email.html", {"message": message, "subject": subject, "html": True, }
+    # )
+    send_mail(
+        subject_prefixed,
+        message_text,
+        email_from,
+        email_to,
+        fail_silently=False,
+        # html_message=message_html
+    )

discuss_data/dddatasets/admin.py

@@ -12,6 +12,7 @@ from discuss_data.dddatasets.models import (
     DisciplinesTags,
     LanguageTags,
     License,
+    DataSetAccessRequest,
 )
 
 
@@ -36,3 +37,4 @@ admin.site.register(LanguageTags)
 admin.site.register(CollectionMethodsTags)
 admin.site.register(AnalysisMethodsTags)
 admin.site.register(DisciplinesTags)
+admin.site.register(DataSetAccessRequest)

discuss_data/dddatasets/models.py

@@ -39,6 +39,7 @@ from discuss_data.ddcomments.models import Comment, Notification
 from discuss_data.ddusers.models import User
 from discuss_data.pages.models import LicensePage, ManualPage
 from discuss_data.utils.cropped_thumbnail import cropped_thumbnail
+from discuss_data.core.utils import send_update_email
 
 from wagtail.admin.edit_handlers import FieldPanel
 
@@ -97,6 +98,9 @@ class Category(models.Model):
         FieldPanel("curators"),
     ]
 
+    def get_curators_emails(self):
+        return self.curators.all().values_list("email", flat=True)
+
     def get_published_datasets_category(self):
         return DataSet.objects.filter(published=True, published_categories__id=self.id)
 
@@ -471,6 +475,16 @@ class DataSetAccessRequest(models.Model):
         )
         notification.save()
         self.notification = notification
+        # send email to dataset owner
+        subject = text
+        message = _(
+            'A user requested access to your dataset "{}" at {}.'.format(
+                self.dataset, self.dataset.get_absolute_url()
+            )
+        )
+        email_to = [self.dataset.owner.email]
+        send_update_email(subject, message, email_to)
+
         models.Model.save(self, *args, **kwargs)
 
     def __str__(self):
@@ -503,6 +517,16 @@ class DataSetPublicationRequest(models.Model):
         )
         notification.save()
         self.notification = notification
+        # send email to all curators
+        subject = "[Curation] {}".format(text,)
+        message = _(
+            "The dataset {} has been submitted for publication in the category {}:\n\n{}\n\nPlease conduct a technical review of the uploaded data and metadata prior to your decision about the publication.".format(
+                self.dataset, self.category, self.dataset.get_absolute_url_curation()
+            )
+        )
+        logger.debug(self.category.get_curators_emails())
+        email_to = self.category.get_curators_emails()
+        send_update_email(subject, message, email_to)
         models.Model.save(self, *args, **kwargs)
 
     def __str__(self):
@@ -1010,6 +1034,21 @@ class DataSet(models.Model):
             "dddatasets:detail", args=[str(self.uuid)]
         )
 
+    def get_absolute_url_prep(self):
+        return settings.DISCUSS_DATA_HOST + reverse(
+            "dddatasets:prep_edit", args=[str(self.uuid)]
+        )
+
+    def get_absolute_url_prep_versions(self):
+        return settings.DISCUSS_DATA_HOST + reverse(
+            "dddatasets:prep_edit_versions", args=[str(self.uuid)]
+        )
+
+    def get_absolute_url_curation(self):
+        return settings.DISCUSS_DATA_HOST + reverse(
+            "dddatasets:prep_curation", args=[str(self.uuid)]
+        )
+
     def get_fulltitle(self):
         if self.subtitle:
             fulltitle = "%s – %s" % (self.title, self.subtitle)
@@ -1328,8 +1367,9 @@ class DataSet(models.Model):
                     self.get_main_category()
                 )
             )
+            subject = text
             if message:
-                text = "{}\n\n{}".format(text, message)
+                text = "{}\n\n{}'s curators message:\n{}".format(text, user, message)
 
             notification = Notification(
                 owner=user,
@@ -1340,6 +1380,16 @@ class DataSet(models.Model):
                 notification_type=Notification.PUB_REQUEST,
             )
             notification.save()
+            # send email to dataset owner
+            mail_message = "Dataset {} at {}:\n{}".format(
+                self, self.get_absolute_url_prep(), text
+            )
+            email_to = [self.owner.email]
+            send_update_email(subject, mail_message, email_to)
+            # send emails to category curators
+            curators_subject = "[Curation] {}".format(subject,)
+            curators_to = self.get_main_category().get_curators_emails()
+            send_update_email(curators_subject, mail_message, curators_to)
             pubreq.delete()
         else:
             raise PermissionDenied
@@ -1365,8 +1415,9 @@ class DataSet(models.Model):
                     self.get_main_category()
                 )
             )
+            subject = text
             if message:
-                text = "{}\n\n{}".format(text, message)
+                text = "{}\n\n{}'s curators message:\n{}\n".format(text, user, message)
 
             notification = Notification(
                 owner=user,
@@ -1377,6 +1428,19 @@ class DataSet(models.Model):
                 notification_type=Notification.PUB_REQUEST,
             )
             notification.save()
+            # send email to dataset owner
+            mail_message = "Dataset {} at {}:\n{}".format(
+                self, self.get_absolute_url_prep(), text
+            )
+            mail_message += "\nYou can now publish the dataset at {}.\n".format(
+                self.get_absolute_url_prep_versions()
+            )
+            email_to = [self.owner.email]
+            send_update_email(subject, mail_message, email_to)
+            # send emails to category curators
+            curators_subject = "[Curation] {}".format(subject,)
+            curators_to = self.get_main_category().get_curators_emails()
+            send_update_email(curators_subject, mail_message, curators_to)
             pubreq.delete()
         else:
             raise PermissionDenied
@@ -1411,6 +1475,21 @@ class DataSet(models.Model):
             action.send(
                 user, verb="published the dataset", target=self,
             )
+            # send email to dataset owner
+            subject = "Dataset {} has been published successfully!".format(self,)
+            message = "{}\n\nIt is accessible via DOI https://doi.org/{}".format(
+                subject, self.doi
+            )
+            if self.dhdoi:
+                message += "\nDARIAH-Repository entry: https://doi.org/{}".format(
+                    self.doi
+                )
+            email_to = [self.owner.email]
+            send_update_email(subject, message, email_to)
+            # send emails to category curators
+            curators_subject = "[Curation] {}".format(subject,)
+            curators_to = self.get_main_category().get_curators_emails()
+            send_update_email(curators_subject, message, curators_to)
         else:
             raise PermissionDenied
         logger.debug("DS PUBLISH ended")

discuss_data/dddatasets/views/core.py

@@ -127,6 +127,9 @@ def files_download(request, uuid, df_uuid):
     ds = get_object_or_404(DataSet, uuid=uuid)
     datafile = DataFile.objects.get(uuid=df_uuid, dataset=ds)
     logger.info("datafile.file.path: %s", datafile.file.path, exc_info=1)
+    logger.info(
+        "datafile.get_download_file_name(): %s", datafile.get_download_file_name()
+    )
     # check permissions for the dataset if it is of filetype DATA or CONVERTED
     if datafile.data_file_type in (
         datafile.FILE_TYPE_DATA,

discuss_data/ddusers/models.py

@@ -11,11 +11,14 @@ from django.urls import reverse
 from django.utils import timezone
 from django.utils.translation import gettext as _
 from django_bleach.models import BleachField
+from django.template.loader import render_to_string
 from guardian.shortcuts import get_objects_for_user
 from PIL import Image
 from taggit.managers import TaggableManager
+from actstream.models import Action
 
 from discuss_data.core.models import AffiliationTags, EnglishTags, KeywordTags, Topic
+from discuss_data.core.utils import weekly_td, daily_td
 from discuss_data.ddcomments.models import Notification
 from discuss_data.utils.cropped_thumbnail import cropped_thumbnail
 
@@ -208,11 +211,18 @@ class User(AbstractUser):
     def __str__(self):
         return self.get_academic_name()
 
-    def get_notifications(self):
-        user_all_datasets = list(self.get_all_datasets().values_list("id", flat=True))
+    def get_notifications_curators(self, dataset_content_type):
         user_curated_datasets = list(
             self.get_curated_datasets().values_list("id", flat=True)
         )
+        notifications_curators = Notification.objects.filter(
+            content_type=dataset_content_type.id, object_id__in=user_curated_datasets
+        )
+        return notifications_curators
+
+    def get_notifications(self):
+        user_all_datasets = list(self.get_all_datasets().values_list("id", flat=True))
+
         user_admin_datasets = list(
             self.get_published_admin_datasets().values_list("id", flat=True)
         )
@@ -231,8 +241,8 @@ class User(AbstractUser):
             content_type=dataset_content_type.id, object_id__in=user_admin_datasets
         )
 
-        datasets_notifications_curators = Notification.objects.filter(
-            content_type=dataset_content_type.id, object_id__in=user_curated_datasets
+        datasets_notifications_curators = self.get_notifications_curators(
+            dataset_content_type
         )
 
         notifications = (
@@ -258,6 +268,83 @@ class User(AbstractUser):
     def get_notifications_recent_count(self):
         return self.get_notifications_recent().count()
 
+    def get_following_actions(self):
+        return user_stream(self)
+
+    def get_following_actions_recent_weekly(self):
+        startdate, enddate = weekly_td()
+        return (
+            self.get_following_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def get_following_actions_recent_daily(self):
+        startdate, enddate = daily_td()
+        return (
+            self.get_following_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def get_datasets_actions(self):
+        user_admin_datasets = list(
+            self.get_published_admin_datasets().values_list("id", flat=True)
+        )
+        actor_ctype = ContentType.objects.get(app_label="ddusers", model="user")
+        return Action.objects.datasets_actions(user_admin_datasets).exclude(
+            actor_content_type=actor_ctype, actor_object_id=self.id
+        )
+
+    def get_datasets_actions_recent_weekly(self):
+        startdate, enddate = weekly_td()
+        return (
+            self.get_datasets_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def get_datasets_actions_recent_daily(self):
+        startdate, enddate = daily_td()
+        return (
+            self.get_datasets_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def get_user_actions(self):
+        return Action.objects.any_everything(self)
+
+    def get_user_actions_recent_weekly(self):
+        startdate, enddate = weekly_td()
+        return (
+            self.get_user_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def get_user_actions_recent_daily(self):
+        startdate, enddate = daily_td()
+        return (
+            self.get_user_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def compose_status_email_daily(self):
+        # daily status email
+        feed_actions = self.get_user_actions_recent_daily()
+        return render_to_string(
+            "ddusers/status_email.html", {"user": self, "feed_actions": feed_actions}
+        )
+
+    def compose_status_email_weekly(self):
+        # weekly status email
+        feed_actions = self.get_user_actions_recent_weekly()
+        return render_to_string(
+            "ddusers/status_email.html", {"user": self, "feed_actions": feed_actions}
+        )
+
     def get_following(self):
         return following(self, User)
 

discuss_data/ddusers/tasks.py

+from django.utils.translation import gettext as _
+from django.utils import timezone
+
+from django_q.tasks import async_task
+
+from discuss_data.ddusers.models import User
+
+
+def send_status_emails_daily():
+    """
+        Task function for sending status emails to all users
+        Called by a django-q Schedule
+        Uses a django-q async_task
+        user.compose_status_email_daily() generates text
+    """
+    for user in User.objects.all():
+        email = user.email.split(";")[0]
+        subject = _("[Discuss Data] Daily Status")
+        text = user.compose_status_email_daily()
+        sender = "info@discuss-data.net"
+        to = [email]
+        async_task(
+            "django.core.mail.send_mail",
+            subject,
+            text,
+            sender,
+            to,
+            html_message=text,
+            fail_silently=False,
+        )

discuss_data/ddusers/views.py

@@ -181,15 +181,8 @@ def format_act_feed(request, feed, object_list=None):
 @dd_tou_accepted
 def datasets_act_feed(request):
     if request.user.is_authenticated:
-        user_admin_datasets = list(
-            request.user.get_published_admin_datasets().values_list("id", flat=True)
-        )
-        actor_ctype = ContentType.objects.get(app_label="ddusers", model="user")
-        datasets_actions = Action.objects.datasets_actions(user_admin_datasets).exclude(
-            actor_content_type=actor_ctype, actor_object_id=request.user.id
-        )
         feed_actions, paginator_range, paginator_last_page = format_act_feed(
-            request, datasets_actions
+            request, request.user.get_datasets_actions()
         )
         feed_type = "datasets-act-feed"
 
@@ -203,7 +196,6 @@ def datasets_act_feed(request):
                 "feed_type": feed_type,
             },
         )
-    raise Http404("Page not found.")
 
 
 @login_required
@@ -211,9 +203,8 @@ def datasets_act_feed(request):
 def following_act_feed(request):
     if request.user.is_authenticated:
         feed_actions, paginator_range, paginator_last_page = format_act_feed(
-            request, user_stream(request.user)
+            request, request.user.get_following_actions()
         )
-
         feed_type = "following-act-feed"
 
         return render(
@@ -226,7 +217,6 @@ def following_act_feed(request):
                 "feed_type": feed_type,
             },
         )
-    raise Http404("Page not found.")