Merge branch '181-mailing' into 'master'

Resolve "Mailing"

Closes #181

See merge request !350

config/settings/base.py

@@ -243,7 +243,8 @@ X_FRAME_OPTIONS = "DENY"
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
 EMAIL_BACKEND = env(
-    "DJANGO_EMAIL_BACKEND", default="django.core.mail.backends.smtp.EmailBackend",
+    "DJANGO_EMAIL_BACKEND",
+    default="django.core.mail.backends.smtp.EmailBackend",
 )
 # https://docs.djangoproject.com/en/2.2/ref/settings/#email-timeout
 EMAIL_TIMEOUT = 5

discuss_data/core/djangoscripts/init_scheduler.py

+import arrow
+from django_q.models import Schedule
+
+
+"""
+    Definition of the task schedules to generate at startup
+    https://django-q.readthedocs.io/en/latest/schedules.html#reference
+"""
+SCHEDULES = [
+    # Schedule for the DAILY send_status_email task
+    # for user mailings. Called daily at 20:00
+    {
+        "name": "statusmails_daily",
+        "func": "discuss_data.ddusers.tasks.send_status_emails",
+        "schedule_type": Schedule.CRON,
+        "repeats": -1,
+        "cron": "0 20 * * *",
+    },
+    # Schedule for the WEEKLY send_status_email task
+    # or user mailings. Called every Saturday at 22:00
+    {
+        "name": "statusmails_weekly",
+        "func": "discuss_data.ddusers.tasks.send_status_emails",
+        "schedule_type": Schedule.CRON,
+        "repeats": -1,
+        "cron": "0 22 * * 6",
+    },
+]
+
+
+def run():
+    """
+        Initialize all schedules defined in SCHEDULES
+    """
+    for elem in SCHEDULES:
+        sched, created = Schedule.objects.get_or_create(name=elem.get("name"))
+        sched.func = elem.get("func")
+        sched.schedule_type = elem.get("schedule_type")
+        sched.minutes = elem.get("minutes")
+        sched.repeats = elem.get("repeats")
+        sched.cron = elem.get("cron")
+        sched.save()
+    print("Schedules defined ")

discuss_data/core/utils.py

 import logging
 
+from datetime import timedelta
+
 from datacite import DataCiteMDSClient, schema41
 from datacite.errors import DataCiteServerError
 
 from django.conf import settings
 from django.template.loader import render_to_string
+from django.utils import timezone
+from django.core.mail import send_mail
 
 logger = logging.getLogger(__name__)
 
@@ -89,3 +93,37 @@ def generate_discuss_data_doi(dataset):
             logger.error(e)
 
     return doi
+
+
+def weekly_td():
+    # week timedelta
+    startdate = timezone.now()
+    enddate = startdate - timedelta(days=7)
+    return startdate, enddate
+
+
+def daily_td():
+    # day timedelta
+    startdate = timezone.now()
+    enddate = startdate - timedelta(hours=24)
+    return startdate, enddate
+
+
+def send_update_email(subject, message, email_to):
+    email_from = "info@discuss-data.net"
+    subject_prefixed = "[Discuss Data] {}".format(subject,)
+
+    message_text = render_to_string(
+        "core/email.html", {"message": message, "subject": subject, "html": False}
+    )
+    # message_html = render_to_string(
+    #    "core/email.html", {"message": message, "subject": subject, "html": True, }
+    # )
+    send_mail(
+        subject_prefixed,
+        message_text,
+        email_from,
+        email_to,
+        fail_silently=False,
+        # html_message=message_html
+    )

discuss_data/dddatasets/admin.py

@@ -12,6 +12,7 @@ from discuss_data.dddatasets.models import (
     DisciplinesTags,
     LanguageTags,
     License,
+    DataSetAccessRequest,
 )
 
 
@@ -36,3 +37,4 @@ admin.site.register(LanguageTags)
 admin.site.register(CollectionMethodsTags)
 admin.site.register(AnalysisMethodsTags)
 admin.site.register(DisciplinesTags)
+admin.site.register(DataSetAccessRequest)

discuss_data/dddatasets/models.py

@@ -39,6 +39,7 @@ from discuss_data.ddcomments.models import Comment, Notification
 from discuss_data.ddusers.models import User
 from discuss_data.pages.models import LicensePage, ManualPage
 from discuss_data.utils.cropped_thumbnail import cropped_thumbnail
+from discuss_data.core.utils import send_update_email
 
 from wagtail.admin.edit_handlers import FieldPanel
 
@@ -97,6 +98,9 @@ class Category(models.Model):
         FieldPanel("curators"),
     ]
 
+    def get_curators_emails(self):
+        return self.curators.all().values_list("email", flat=True)
+
     def get_published_datasets_category(self):
         return DataSet.objects.filter(published=True, published_categories__id=self.id)
 
@@ -471,6 +475,16 @@ class DataSetAccessRequest(models.Model):
         )
         notification.save()
         self.notification = notification
+        # send email to dataset owner
+        subject = text
+        message = _(
+            'A user requested access to your dataset "{}" at {}.'.format(
+                self.dataset, self.dataset.get_absolute_url()
+            )
+        )
+        email_to = [self.dataset.owner.email]
+        send_update_email(subject, message, email_to)
+
         models.Model.save(self, *args, **kwargs)
 
     def __str__(self):
@@ -503,6 +517,16 @@ class DataSetPublicationRequest(models.Model):
         )
         notification.save()
         self.notification = notification
+        # send email to all curators
+        subject = "[Curation] {}".format(text,)
+        message = _(
+            "The dataset {} has been submitted for publication in the category {}:\n\n{}\n\nPlease conduct a technical review of the uploaded data and metadata prior to your decision about the publication.".format(
+                self.dataset, self.category, self.dataset.get_absolute_url_curation()
+            )
+        )
+        logger.debug(self.category.get_curators_emails())
+        email_to = self.category.get_curators_emails()
+        send_update_email(subject, message, email_to)
         models.Model.save(self, *args, **kwargs)
 
     def __str__(self):
@@ -1010,6 +1034,21 @@ class DataSet(models.Model):
             "dddatasets:detail", args=[str(self.uuid)]
         )
 
+    def get_absolute_url_prep(self):
+        return settings.DISCUSS_DATA_HOST + reverse(
+            "dddatasets:prep_edit", args=[str(self.uuid)]
+        )
+
+    def get_absolute_url_prep_versions(self):
+        return settings.DISCUSS_DATA_HOST + reverse(
+            "dddatasets:prep_edit_versions", args=[str(self.uuid)]
+        )
+
+    def get_absolute_url_curation(self):
+        return settings.DISCUSS_DATA_HOST + reverse(
+            "dddatasets:prep_curation", args=[str(self.uuid)]
+        )
+
     def get_fulltitle(self):
         if self.subtitle:
             fulltitle = "%s – %s" % (self.title, self.subtitle)
@@ -1328,8 +1367,9 @@ class DataSet(models.Model):
                     self.get_main_category()
                 )
             )
+            subject = text
             if message:
-                text = "{}\n\n{}".format(text, message)
+                text = "{}\n\n{}'s curators message:\n{}".format(text, user, message)
 
             notification = Notification(
                 owner=user,
@@ -1340,6 +1380,16 @@ class DataSet(models.Model):
                 notification_type=Notification.PUB_REQUEST,
             )
             notification.save()
+            # send email to dataset owner
+            mail_message = "Dataset {} at {}:\n{}".format(
+                self, self.get_absolute_url_prep(), text
+            )
+            email_to = [self.owner.email]
+            send_update_email(subject, mail_message, email_to)
+            # send emails to category curators
+            curators_subject = "[Curation] {}".format(subject,)
+            curators_to = self.get_main_category().get_curators_emails()
+            send_update_email(curators_subject, mail_message, curators_to)
             pubreq.delete()
         else:
             raise PermissionDenied
@@ -1365,8 +1415,9 @@ class DataSet(models.Model):
                     self.get_main_category()
                 )
             )
+            subject = text
             if message:
-                text = "{}\n\n{}".format(text, message)
+                text = "{}\n\n{}'s curators message:\n{}\n".format(text, user, message)
 
             notification = Notification(
                 owner=user,
@@ -1377,6 +1428,19 @@ class DataSet(models.Model):
                 notification_type=Notification.PUB_REQUEST,
             )
             notification.save()
+            # send email to dataset owner
+            mail_message = "Dataset {} at {}:\n{}".format(
+                self, self.get_absolute_url_prep(), text
+            )
+            mail_message += "\nYou can now publish the dataset at {}.\n".format(
+                self.get_absolute_url_prep_versions()
+            )
+            email_to = [self.owner.email]
+            send_update_email(subject, mail_message, email_to)
+            # send emails to category curators
+            curators_subject = "[Curation] {}".format(subject,)
+            curators_to = self.get_main_category().get_curators_emails()
+            send_update_email(curators_subject, mail_message, curators_to)
             pubreq.delete()
         else:
             raise PermissionDenied
@@ -1411,6 +1475,21 @@ class DataSet(models.Model):
             action.send(
                 user, verb="published the dataset", target=self,
             )
+            # send email to dataset owner
+            subject = "Dataset {} has been published successfully!".format(self,)
+            message = "{}\n\nIt is accessible via DOI https://doi.org/{}".format(
+                subject, self.doi
+            )
+            if self.dhdoi:
+                message += "\nDARIAH-Repository entry: https://doi.org/{}".format(
+                    self.doi
+                )
+            email_to = [self.owner.email]
+            send_update_email(subject, message, email_to)
+            # send emails to category curators
+            curators_subject = "[Curation] {}".format(subject,)
+            curators_to = self.get_main_category().get_curators_emails()
+            send_update_email(curators_subject, message, curators_to)
         else:
             raise PermissionDenied
         logger.debug("DS PUBLISH ended")

discuss_data/dddatasets/views/core.py

@@ -127,6 +127,9 @@ def files_download(request, uuid, df_uuid):
     ds = get_object_or_404(DataSet, uuid=uuid)
     datafile = DataFile.objects.get(uuid=df_uuid, dataset=ds)
     logger.info("datafile.file.path: %s", datafile.file.path, exc_info=1)
+    logger.info(
+        "datafile.get_download_file_name(): %s", datafile.get_download_file_name()
+    )
     # check permissions for the dataset if it is of filetype DATA or CONVERTED
     if datafile.data_file_type in (
         datafile.FILE_TYPE_DATA,

discuss_data/ddusers/models.py

@@ -11,11 +11,14 @@ from django.urls import reverse
 from django.utils import timezone
 from django.utils.translation import gettext as _
 from django_bleach.models import BleachField
+from django.template.loader import render_to_string
 from guardian.shortcuts import get_objects_for_user
 from PIL import Image
 from taggit.managers import TaggableManager
+from actstream.models import Action
 
 from discuss_data.core.models import AffiliationTags, EnglishTags, KeywordTags, Topic
+from discuss_data.core.utils import weekly_td, daily_td
 from discuss_data.ddcomments.models import Notification
 from discuss_data.utils.cropped_thumbnail import cropped_thumbnail
 
@@ -208,11 +211,18 @@ class User(AbstractUser):
     def __str__(self):
         return self.get_academic_name()
 
-    def get_notifications(self):
-        user_all_datasets = list(self.get_all_datasets().values_list("id", flat=True))
+    def get_notifications_curators(self, dataset_content_type):
         user_curated_datasets = list(
             self.get_curated_datasets().values_list("id", flat=True)
         )
+        notifications_curators = Notification.objects.filter(
+            content_type=dataset_content_type.id, object_id__in=user_curated_datasets
+        )
+        return notifications_curators
+
+    def get_notifications(self):
+        user_all_datasets = list(self.get_all_datasets().values_list("id", flat=True))
+
         user_admin_datasets = list(
             self.get_published_admin_datasets().values_list("id", flat=True)
         )
@@ -231,8 +241,8 @@ class User(AbstractUser):
             content_type=dataset_content_type.id, object_id__in=user_admin_datasets
         )
 
-        datasets_notifications_curators = Notification.objects.filter(
-            content_type=dataset_content_type.id, object_id__in=user_curated_datasets
+        datasets_notifications_curators = self.get_notifications_curators(
+            dataset_content_type
         )
 
         notifications = (
@@ -258,6 +268,83 @@ class User(AbstractUser):
     def get_notifications_recent_count(self):
         return self.get_notifications_recent().count()
 
+    def get_following_actions(self):
+        return user_stream(self)
+
+    def get_following_actions_recent_weekly(self):
+        startdate, enddate = weekly_td()
+        return (
+            self.get_following_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def get_following_actions_recent_daily(self):
+        startdate, enddate = daily_td()
+        return (
+            self.get_following_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def get_datasets_actions(self):
+        user_admin_datasets = list(
+            self.get_published_admin_datasets().values_list("id", flat=True)
+        )
+        actor_ctype = ContentType.objects.get(app_label="ddusers", model="user")
+        return Action.objects.datasets_actions(user_admin_datasets).exclude(
+            actor_content_type=actor_ctype, actor_object_id=self.id
+        )
+
+    def get_datasets_actions_recent_weekly(self):
+        startdate, enddate = weekly_td()
+        return (
+            self.get_datasets_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def get_datasets_actions_recent_daily(self):
+        startdate, enddate = daily_td()
+        return (
+            self.get_datasets_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def get_user_actions(self):
+        return Action.objects.any_everything(self)
+
+    def get_user_actions_recent_weekly(self):
+        startdate, enddate = weekly_td()
+        return (
+            self.get_user_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def get_user_actions_recent_daily(self):
+        startdate, enddate = daily_td()
+        return (
+            self.get_user_actions()
+            .filter(timestamp__lte=startdate)
+            .filter(timestamp__gte=enddate)
+        )
+
+    def compose_status_email_daily(self):
+        # daily status email
+        feed_actions = self.get_user_actions_recent_daily()
+        return render_to_string(
+            "ddusers/status_email.html", {"user": self, "feed_actions": feed_actions}
+        )
+
+    def compose_status_email_weekly(self):
+        # weekly status email
+        feed_actions = self.get_user_actions_recent_weekly()
+        return render_to_string(
+            "ddusers/status_email.html", {"user": self, "feed_actions": feed_actions}
+        )
+
     def get_following(self):
         return following(self, User)
 

discuss_data/ddusers/tasks.py

+from django.utils.translation import gettext as _
+from django.utils import timezone
+
+from django_q.tasks import async_task
+
+from discuss_data.ddusers.models import User
+
+
+def send_status_emails_daily():
+    """
+        Task function for sending status emails to all users
+        Called by a django-q Schedule
+        Uses a django-q async_task
+        user.compose_status_email_daily() generates text
+    """
+    for user in User.objects.all():
+        email = user.email.split(";")[0]
+        subject = _("[Discuss Data] Daily Status")
+        text = user.compose_status_email_daily()
+        sender = "info@discuss-data.net"
+        to = [email]
+        async_task(
+            "django.core.mail.send_mail",
+            subject,
+            text,
+            sender,
+            to,
+            html_message=text,
+            fail_silently=False,
+        )

discuss_data/ddusers/views.py

@@ -181,15 +181,8 @@ def format_act_feed(request, feed, object_list=None):
 @dd_tou_accepted
 def datasets_act_feed(request):
     if request.user.is_authenticated:
-        user_admin_datasets = list(
-            request.user.get_published_admin_datasets().values_list("id", flat=True)
-        )
-        actor_ctype = ContentType.objects.get(app_label="ddusers", model="user")
-        datasets_actions = Action.objects.datasets_actions(user_admin_datasets).exclude(
-            actor_content_type=actor_ctype, actor_object_id=request.user.id
-        )
         feed_actions, paginator_range, paginator_last_page = format_act_feed(
-            request, datasets_actions
+            request, request.user.get_datasets_actions()
         )
         feed_type = "datasets-act-feed"
 
@@ -203,7 +196,6 @@ def datasets_act_feed(request):
                 "feed_type": feed_type,
             },
         )
-    raise Http404("Page not found.")
 
 
 @login_required
@@ -211,9 +203,8 @@ def datasets_act_feed(request):
 def following_act_feed(request):
     if request.user.is_authenticated:
         feed_actions, paginator_range, paginator_last_page = format_act_feed(
-            request, user_stream(request.user)
+            request, request.user.get_following_actions()
         )
-
         feed_type = "following-act-feed"
 
         return render(
@@ -226,7 +217,6 @@ def following_act_feed(request):
                 "feed_type": feed_type,
             },
         )
-    raise Http404("Page not found.")
 
 
 @login_required
@@ -361,10 +351,8 @@ def user_curation(request):
 @login_required
 @dd_tou_accepted
 def user_act_feed(request):
-    actions = Action.objects.any_everything(request.user)
-
     feed_actions, paginator_range, paginator_last_page = format_act_feed(
-        request, actions
+        request, request.user.get_user_actions()
     )
 
     feed_type = "user-act-feed"

discuss_data/static/sass/project.scss

@@ -349,6 +349,7 @@ a:hover
 */
 .btn-pill {
   border-radius: 50em;
+  margin-bottom: .2rem !important;
 }
 
 .nav-datasets {

discuss_data/templates/base_email.html

+{% load static i18n compress%}<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="x-ua-compatible" content="ie=edge">
+    <title>{% block title %}Discuss Data{% endblock title %}</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="">
+    <meta name="author" content="">
+
+    {% block metatags %}
+    {% endblock %}
+
+    <link rel="icon" type="image/svg+xml" href="{% static 'images/logo-sm-primary.svg' %}" sizes="any">
+
+    {% block css %}
+
+
+    <!-- Your stuff: Third-party CSS libraries go here -->
+    {% compress css %}
+    <!-- This file stores project-specific CSS -->
+
+    <link href="{% static 'css/project.css' %}" rel="stylesheet">
+
+
+    {% endcompress %}
+    {% endblock %}
+    <!-- jsDelivr :: Sortable :: Latest (https://www.jsdelivr.com/package/npm/sortablejs) -->
+    <script src="https://cdn.jsdelivr.net/npm/sortablejs@1.12.0/dist/sortable.umd.js" integrity="sha256-5fXAU9N22Tec41bUvuStMzIywttdqVlleaOm24H5rGw=" crossorigin="anonymous"></script>
+    {% block javascript %}
+    <!-- Vendor dependencies bundled as one file-->
+    {% compress js %}
+      <script src="{% static 'js/vendors.js' %}"></script>
+      {% endcompress %}
+
+      <!-- place project specific Javascript in this file -->
+      {% compress js %}
+      <script src="{% static 'js/project.js' %}"></script>
+      {% endcompress %}
+    {% endblock javascript %}
+  </head>
+
+  <body>
+    <header>
+    {# main navigation #}
+    <nav class="navbar navbar-light fixed-top flex-md-nowrap p-0 border-bottom navbar-expand-sm" id="navbar-top">
+      <a class="navbar-brand px-3" href="{% url 'core.landing_page' %}"><img class="navbar-logo" src="{% static 'images/logo-primary.svg' %}" /></a>
+
+      <button class="navbar-toggler navbar-toggler-right" type="button" data-toggle="collapse" data-target="#navbarSupportedContent" aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
+          <span class="navbar-toggler-icon"></span>
+      </button>
+    </nav>
+    </header>
+
+    {# main container #}
+    <div class="container">
+      <div id="content">
+      {% block content %}
+      {% endblock content %}
+      </div>
+    </div>
+
+    {# end main container #}
+
+    {# Footer #}
+      {% include "footer.html" %}
+    {# end footer #}
+
+
+  </body>
+</html>

discuss_data/templates/core/email.html

+{% load static i18n activity_tags core_tags %}
+
+{% if html %}<h1>{% trans "Update from Discuss Data" %}</h1>{% else %}{% trans "Update from Discuss Data" %}{% endif %}
+
+{% if html %}
+<p>{{ message|urlize }}</p>
+<p><a href="https://discuss-data.net/users/notifications">{% trans "View all your notifications" %}</a></p>
+{% else %}
+{{ message|safe }}
+
+{% trans "You can view all your notifications at https://discuss-data.net/users/notifications" %}.
+{% endif %}
+
+--
+
+{% if html %}<p>{% endif %}{% trans "Auto-generated update email from Discuss Data: Open platform for archiving, sharing and discussing research data with a focus on the post-Soviet region" %}{% if html %}</p>{% endif %}
+{% if html %}<a href="https://discuss-data.net"></a>{% else %}https://discuss-data.net{% endif %}
+

discuss_data/templates/ddusers/status_email.html

+{% extends "base_email.html" %}
+{% load static i18n activity_tags core_tags %}
+
+{% block title %}Your Discuss Data Status{% endblock title %}
+
+{% block content %}
+
+<h1>{% trans "Your Discuss Data Status" %}</h1>
+Dear {{ user.get_academic_name }},<br>
+