Commit ebbe9890 authored by felix.herrmann's avatar felix.herrmann
Browse files

Merge branch '181-mailing' into 'cherry-pick-de0d44c7'

# Conflicts:
#   config/settings/base.py
parents 624a52d2 463d592f
# PostgreSQL
# ------------------------------------------------------------------------------
POSTGRES_HOST=postgres
POSTGRES_PORT=5432
POSTGRES_DB=discuss_data
POSTGRES_USER=debug
POSTGRES_PASSWORD=debug
......@@ -54,14 +54,12 @@ stages:
.default_rules_template: &default_rules
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
when: always
.default_not_scheduled_rules_template: &default_not_scheduled_rules
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule"'
when: never
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
when: always
.default_merge_request_rules_template: &default_merge_request_rules
rules:
......@@ -274,11 +272,13 @@ deploy_latest_to_staging:
- $DOCKER_CMD pull $DISCUSS_DATA_IMAGE
- $DOCKER_CMD tag $DISCUSS_DATA_IMAGE $CI_REGISTRY_IMAGE:$CI_ENVIRONMENT_NAME
- $DOCKER_CMD push $CI_REGISTRY_IMAGE:$CI_ENVIRONMENT_NAME
- $DOCKER_COMPOSE_CMD up -d
- $DOCKER_COMPOSE_CMD up -d --force-recreate --no-deps django cms qcluster
- $DOCKER_COMPOSE_CMD run --rm django ./manage.py migrate
- $DOCKER_COMPOSE_CMD run --rm django ./manage.py search_index --rebuild -f
- $DOCKER_COMPOSE_CMD run --rm django ./manage.py wagtail_update_index
<<: *default_not_scheduled_rules
rules:
- if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
when: manual
pages:
stage: deploy
......@@ -292,7 +292,7 @@ pages:
paths:
- public
expire_in: 5 days
<<: *default_rules
<<: *default_not_scheduled_rules
sentry_deploy_staging:
......@@ -344,7 +344,7 @@ deploy_release_to_production:
- $DOCKER_CMD pull $RELEASE_IMAGE
- $DOCKER_CMD tag $RELEASE_IMAGE $CI_REGISTRY_IMAGE:$CI_ENVIRONMENT_NAME
- $DOCKER_CMD push $CI_REGISTRY_IMAGE:$CI_ENVIRONMENT_NAME
- $DOCKER_COMPOSE_CMD up -d
- $DOCKER_COMPOSE_CMD up -d --force-recreate --no-deps django cms qcluster
- $DOCKER_COMPOSE_CMD run --rm django ./manage.py migrate
- $DOCKER_COMPOSE_CMD run --rm django ./manage.py search_index --rebuild -f
- $DOCKER_COMPOSE_CMD run --rm django ./manage.py wagtail_update_index
......
......@@ -6,4 +6,4 @@ set -o nounset
python /app/manage.py collectstatic --noinput
/usr/local/bin/gunicorn config.wsgi --bind 0.0.0.0:5000 --chdir=/app
/usr/local/bin/gunicorn config.wsgi --timeout 3600 --bind 0.0.0.0:5000 --chdir=/app
......@@ -81,6 +81,7 @@ THIRD_PARTY_APPS = [
"taggit",
"django_elasticsearch_dsl",
"mptt",
"django_sendfile",
]
LOCAL_APPS = [
......@@ -251,9 +252,10 @@ EMAIL_TIMEOUT = 5
# ADMIN
# ------------------------------------------------------------------------------
# Django Admin URL
ADMIN_URL = "admin/"
ADMIN_URL = env("DJANGO_ADMIN_URL", default="admin/")
# Wagtail Admin URL
CMS_URL = "cms/"
CMS_URL = env("DJANGO_CMS_URL", default="cms/")
# https://docs.djangoproject.com/en/dev/ref/settings/#admins
ADMINS = env("DJANGO_ADMINS", default=[])
# https://docs.djangoproject.com/en/dev/ref/settings/#managers
......@@ -273,14 +275,8 @@ LOGGING = {
"%(process)d %(thread)d %(message)s"
}
},
"handlers": {
"console": {
"level": "DEBUG",
"class": "logging.StreamHandler",
"formatter": "verbose",
}
},
"root": {"level": "INFO", "handlers": ["console"]},
"handlers": {"console": {"class": "logging.StreamHandler", "formatter": "verbose"}},
"root": {"level": "DEBUG", "handlers": ["console"]},
}
# django-compressor
......@@ -327,6 +323,12 @@ WAGTAIL_LANDING_PAGE = env("WAGTAIL_LANDING_PAGE", default="discuss-data-landing
WAGTAIL_CONTACTS_PAGE = env("WAGTAIL_CONTACTS_PAGE", default="contacts")
WAGTAIL_SECURITY_PAGE = env("WAGTAIL_SECURITY_PAGE", default="security")
# If WAGTAILDOCS_SERVE_METHOD is unspecified or set to None, the default method is
# 'redirect' when a remote storage backend is in use (i.e. one that exposes a URL but
# not a local filesystem path), and 'serve_view' otherwise.
# see https://docs.wagtail.io/en/stable/reference/settings.html#documents
WAGTAILDOCS_SERVE_METHOD = "redirect"
# Discuss Data
DISCUSS_DATA_HOST = env("DISCUSS_DATA_HOST", default="https://discuss-data.net")
......@@ -403,3 +405,53 @@ DC_USER = env("DATACITE_USERNAME", default="")
DC_PWD = env("DATACITE_PASSWORD", default="")
DC_URL = env("DATACITE_URL", default="https://mds.test.datacite.org")
DC_PREFIX = env("DATACITE_PREFIX", default="10.5072")
# Sendfile
SENDFILE_ROOT = env("SENDFILE_ROOT", default="/app/data/")
SENDFILE_URL = env("SENDFILE_URL", default="/datasets/")
# CACHES
# ------------------------------------------------------------------------------
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": env("REDIS_URL"),
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
# Mimicing memcache behavior.
# http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
"IGNORE_EXCEPTIONS": True,
},
}
}
# https://docs.djangoproject.com/en/2.2/topics/http/sessions/#using-cached-sessions
SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
# django-Q
# ------------------------------------------------------------------------------
# https://django-q.readthedocs.io/en/latest/configure.html
Q_CLUSTER = {
"name": "discuss_data",
"workers": 8,
"recycle": 500,
"timeout": 3600, # =1h
"compress": True,
"save_limit": 250,
"queue_limit": 500,
"cpu_affinity": 1,
"label": "Django Q",
"django_redis": "default",
}
# Discuss Data: Matrix for Dataset licenses and access model validation
DD_LICENSE_MATRIX = {
"OA": (
"license-odc-by-v1-0",
"license-odbl",
"license-no-sharing",
"license-individual",
),
"RA": ("license-no-sharing", "license-individual"),
"MO": ("nolicense"),
}
......@@ -13,17 +13,8 @@ SECRET_KEY = env(
default="vug6D5vrVBwICIW8iqLno3I4lKAbrWmuruYpgTaHNc7k12WzS61BNswdg4w4lwI2",
)
# https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
ALLOWED_HOSTS = ["localhost", "0.0.0.0", "127.0.0.1"]
ALLOWED_HOSTS = ["django", "localhost", "0.0.0.0", "127.0.0.1"]
# CACHES
# ------------------------------------------------------------------------------
# https://docs.djangoproject.com/en/dev/ref/settings/#caches
CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.locmem.LocMemCache",
"LOCATION": "",
}
}
# EMAIL
# ------------------------------------------------------------------------------
......@@ -61,18 +52,4 @@ INTERNAL_IPS += [ip[:-1] + "1" for ip in ips]
# https://django-extensions.readthedocs.io/en/latest/installation_instructions.html#configuration
INSTALLED_APPS += ["django_extensions"] # noqa F405
# django-Q
# ------------------------------------------------------------------------------
# https://django-q.readthedocs.io/en/latest/configure.html
Q_CLUSTER = {
"name": "discuss_data",
"workers": 8,
"recycle": 500,
"timeout": 60,
"compress": True,
"save_limit": 250,
"queue_limit": 500,
"cpu_affinity": 1,
"label": "Django Q",
"redis": {"host": "redis", "port": 6379, "db": 0},
}
SENDFILE_BACKEND = "django_sendfile.backends.development"
......@@ -18,23 +18,6 @@ DATABASES["default"] = env.db("DATABASE_URL") # noqa F405
DATABASES["default"]["ATOMIC_REQUESTS"] = True # noqa F405
DATABASES["default"]["CONN_MAX_AGE"] = env.int("CONN_MAX_AGE", default=60) # noqa F405
# CACHES
# ------------------------------------------------------------------------------
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": env("REDIS_URL"),
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
# Mimicing memcache behavior.
# http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
"IGNORE_EXCEPTIONS": True,
},
}
}
# https://docs.djangoproject.com/en/2.2/topics/http/sessions/#using-cached-sessions
SESSION_ENGINE = "django.contrib.sessions.backends.cached_db"
# SECURITY
# ------------------------------------------------------------------------------
# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header
......@@ -115,28 +98,6 @@ SERVER_EMAIL = env("DJANGO_SERVER_EMAIL", default=DEFAULT_FROM_EMAIL)
# https://docs.djangoproject.com/en/dev/ref/settings/#email-subject-prefix
EMAIL_SUBJECT_PREFIX = env("DJANGO_EMAIL_SUBJECT_PREFIX", default="[Discuss Data] ")
# ADMIN
# ------------------------------------------------------------------------------
# Django Admin URL regex.
ADMIN_URL = env("DJANGO_ADMIN_URL", default="admin/")
CMS_URL = env("DJANGO_CMS_URL", default="cms/")
# TO BE REMOVED BEGIN
# Anymail (Mailgun)
# ------------------------------------------------------------------------------
# https://anymail.readthedocs.io/en/stable/installation/#installing-anymail
# INSTALLED_APPS += ["anymail"] # noqa F405
# EMAIL_BACKEND = "anymail.backends.mailgun.EmailBackend"
# https://anymail.readthedocs.io/en/stable/installation/#anymail-settings-reference
# ANYMAIL = {
# "MAILGUN_API_KEY": env("MAILGUN_API_KEY"),
# "MAILGUN_SENDER_DOMAIN": env("MAILGUN_DOMAIN"),
# "MAILGUN_API_URL": env("MAILGUN_API_URL", default="https://api.mailgun.net/v3"),
# }
# TO BE REMOVED END
# django-compressor
# ------------------------------------------------------------------------------
# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_ENABLED
......@@ -197,9 +158,7 @@ sentry_sdk.init(
dsn="https://824e439c530d423e989185abee8c5cc2@dev2.discuss-data.net/2",
integrations=[DjangoIntegration(), RedisIntegration()],
environment=env("ENVIRONMENT", default="local"),
# If you wish to associate users to errors (assuming you are using
# django.contrib.auth) you may enable sending PII data.
send_default_pii=True,
traces_sample_rate=0.1,
)
# Sentry django-Q reporter
......@@ -227,3 +186,5 @@ Q_CLUSTER = {
# }
# }
}
SENDFILE_BACKEND = "django_sendfile.backends.nginx"
......@@ -57,3 +57,6 @@ EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"
# put fresh DARIAH_STORAGE_TOKEN to .env for integration tests
DARIAH_STORAGE_TOKEN = env("DARIAH_STORAGE_TOKEN", default="SET-FOR-INTEGRATION-TEST")
# sendfile
SENDFILE_BACKEND = "django_sendfile.backends.development"
# from django.contrib import admin
# Register your models here.
from django.contrib import admin
from discuss_data.core.models import KeywordTagged
admin.site.register(KeywordTagged)
......@@ -52,6 +52,26 @@ SEARCH_FIELDS_DATASET = [
# }
# Commented out, as filters are not applied using this code.
# Switching back to older version below.
# def add_tag_filter(search, filter_name, filter_list, filter_field):
# """
# Add specific filters dynamically to search object using the alternative elasticsearch-dsl syntax
# see: https://elasticsearch-dsl.readthedocs.io/en/latest/search_dsl.html#dotted-fields
# """
# if filter_list:
# query_list = list()
# # construct list of Q match clauses for filtering elements
# for elem in filter_list:
# if elem != "":
# query_list.append(Q("match", **{"{}.name".format(filter_name): elem}))
# # add the list to the search object
# return search.filter("terms", **{filter_name + "." + filter_field: query_list})
# else:
# # return original search object if filter_list is None
# return search
def add_tag_filter(search, filter_name, filter_list):
"""
Add specific filters dynamically using the alternative elasticsearch-dsl syntax
......@@ -102,6 +122,7 @@ def index_search(index, term, **kwargs):
document = DataSetDocument
search_fields = SEARCH_FIELDS_DATASET
# create an elastic simple query clause
q = Q(
"simple_query_string",
query=wc_term,
......@@ -109,21 +130,28 @@ def index_search(index, term, **kwargs):
default_operator="AND",
)
# "filter" does not work with categories as they come from 2 different fields.
# Using "query" clause instead, but only AND concatenation with q produces reasonable results
if categories:
for elem in categories:
if elem != "":
q_cat = Q("match", published_main_category__name=elem) | Q(
"match", published_categories__name=elem
)
# using & instead of | to add to query
q = q & q_cat
# create elastic bool query and include q query
filter_query = Q("bool", must=q)
# create search object from document
search = document.search().query(q)
search = document.search().query(filter_query)
# add filters for both dataset_index and user_index
# add filters for both dataset_index and user_index to search
search = add_tag_filter(search, "countries", countries)
# add filters for dataset_index
if index == "dataset_index":
if categories:
for category in categories:
if category != "":
q = Q("match", published_main_category__name=category) | Q(
"match", published_categories__name=category
)
search = search.query(q)
search = add_tag_filter(search, "keywords", keywords)
search = add_tag_filter(search, "languages_of_data", languages)
search = add_tag_filter(search, "disciplines", disciplines)
......@@ -138,7 +166,7 @@ def index_search(index, term, **kwargs):
if index == "user_index":
search = add_tag_filter(search, "interests", keywords)
# give back all results, not just 10 (elastic default)
# return all results, not just 10 (elastic default)
total = search.count()
search = search[0:total]
return search.to_queryset()
......
......@@ -3,6 +3,7 @@ import logging
from django.db import models
from django_bleach.models import BleachField
from taggit.models import GenericTaggedItemBase, TagBase
from wagtail.admin.edit_handlers import FieldPanel
logger = logging.getLogger(__name__)
......@@ -53,6 +54,10 @@ class KeywordTagged(TagBase):
verbose_name = "Keyword Tag"
verbose_name_plural = "Keyword Tags"
panels = [
FieldPanel("name"),
]
class KeywordTags(GenericTaggedItemBase):
tag = models.ForeignKey(
......@@ -70,6 +75,10 @@ class LanguageTagged(TagBase):
verbose_name = "Language Tag"
verbose_name_plural = "Language Tags"
panels = [
FieldPanel("name"),
]
class LanguageTags(GenericTaggedItemBase):
tag = models.ForeignKey(
......
......@@ -8,6 +8,7 @@ from discuss_data.ddcomments.forms import CommentForm, NotificationForm
from discuss_data.ddcomments.models import Notification
from discuss_data.ddusers.models import User
from discuss_data.pages.models import ManualPage, TocTree
from discuss_data.dddatasets.models import License
register = template.Library()
DEFAULT_USER_IMAGE = "/static/images/user_default.png"
......@@ -259,3 +260,13 @@ def get_manpage(page_slug):
except ManualPage.DoesNotExist:
man_page = None
return man_page
@register.filter
def get_license(slug):
""" get a license from a slug
"""
try:
return License.objects.get(license_slug=slug)
except License.DoesNotExist:
return None
......@@ -8,6 +8,7 @@ from datacite.errors import DataCiteServerError
from django.conf import settings
from django.template.loader import render_to_string
from django.utils import timezone
from django.core.mail import send_mail
logger = logging.getLogger(__name__)
......@@ -95,12 +96,34 @@ def generate_discuss_data_doi(dataset):
def weekly_td():
# week timedelta
startdate = timezone.now()
enddate = startdate - timedelta(days=7)
return startdate, enddate
def daily_td():
# day timedelta
startdate = timezone.now()
enddate = startdate - timedelta(hours=24)
return startdate, enddate
def send_update_email(subject, message, email_to):
email_from = "info@discuss-data.net"
subject_prefixed = "[Discuss Data] {}".format(subject,)
message_text = render_to_string(
"core/email.html", {"message": message, "subject": subject, "html": False}
)
# message_html = render_to_string(
# "core/email.html", {"message": message, "subject": subject, "html": True, }
# )
send_mail(
subject_prefixed,
message_text,
email_from,
email_to,
fail_silently=False,
# html_message=message_html
)
......@@ -117,13 +117,13 @@ def core_search_view(request, search_index, objects_on_page):
methods_of_data_collection = request.GET.getlist("methods_of_data_collection")
if len(methods_of_data_collection) == 1 and methods_of_data_collection[0] == "":
methods_of_data_collection = None
methods_of_data_collection_all = AnalysisMethodsTagged.objects.all()
methods_of_data_collection_all = CollectionMethodsTagged.objects.all()
# extract methods_of_data_analysis slugs from GET
methods_of_data_analysis = request.GET.getlist("methods_of_data_analysis")
if len(methods_of_data_analysis) == 1 and methods_of_data_analysis[0] == "":
methods_of_data_analysis = None
methods_of_data_analysis_all = CollectionMethodsTagged.objects.all()
methods_of_data_analysis_all = AnalysisMethodsTagged.objects.all()
if search_index == "dataset_index":
template = "dddatasets/search_results.html"
......
# Generated by Django 2.2.17 on 2020-12-09 13:37
from django.db import migrations
import django_bleach.models
class Migration(migrations.Migration):
dependencies = [
('ddcomments', '0025_auto_20200614_0653'),
]
operations = [
migrations.AlterField(
model_name='comment',
name='text',
field=django_bleach.models.BleachField(max_length=12000),
),
]
......@@ -32,7 +32,9 @@ class Notification(MPTTModel):
uuid = models.UUIDField(default=uuid.uuid4, editable=False)
notification_type = models.CharField(
max_length=4, choices=NOTIFICATION_TYPE_CHOICES, default=PUBLIC,
max_length=4,
choices=NOTIFICATION_TYPE_CHOICES,
default=PUBLIC,
)
text = BleachField(max_length=12000)
date_added = models.DateTimeField(auto_now_add=True)
......@@ -96,9 +98,11 @@ class Comment(MPTTModel):
uuid = models.UUIDField(default=uuid.uuid4, editable=False)
doi = models.CharField(max_length=200, blank=True)
comment_type = models.CharField(
max_length=4, choices=COMMENT_TYPE_CHOICES, default=PUBLIC,
max_length=4,
choices=COMMENT_TYPE_CHOICES,
default=PUBLIC,
)
text = BleachField(max_length=1200)
text = BleachField(max_length=12000)
date_added = models.DateTimeField(auto_now_add=True)
date_edited = models.DateTimeField(auto_now=True)
owner = models.ForeignKey(
......
......@@ -12,12 +12,24 @@ from discuss_data.dddatasets.models import (
DisciplinesTags,
LanguageTags,
License,
DataSetAccessRequest,
)
class LicenseAdmin(admin.ModelAdmin):
list_display = (
"license_type",
"standard_license",
"license_name",
"license_slug",
"get_datasets",
)
admin.site.register(DataFile)
admin.site.register(DataSet)
admin.site.register(DataSetManagementObject)
admin.site.register(License)
admin.site.register(License, LicenseAdmin)
admin.site.register(DataType)
admin.site.register(Category)
admin.site.register(DataList)
......@@ -25,3 +37,4 @@ admin.site.register(LanguageTags)
admin.site.register(CollectionMethodsTags)
admin.site.register(AnalysisMethodsTags)
admin.site.register(DisciplinesTags)
admin.site.register(DataSetAccessRequest)
......@@ -55,7 +55,7 @@ class DataSetIndividualLicenseForm(ModelForm):
class Meta:
model = License
fields = [
"individual_license",
"license_text",
]
help_texts = get_help_texts("edit-license-access")
manpage = get_man_page("edit-license-access")
......@@ -69,7 +69,7 @@ class DataSetIndividualLicenseForm(ModelForm):
HTML(
"{% include 'dddatasets/_manual_page.html' with manpage=Meta.manpage key='D-Customized-Data-Usage-Licence-Agreement' %}"
),
Div(Div("individual_license", css_class="col-md-12",), css_class="row",),
Div(Div("license_text", css_class="col-md-12",), css_class="row",),
)
......@@ -201,7 +201,7 @@ class DataSetForm(ModelForm):
"institutional_affiliation",
"funding",
"related_dataset_text",
"related_dataset",
# "related_dataset", TODO: commenting out as the field lists also unpublished datasets. We need a custom widget/control for this.
"related_projects",
)
help_texts = get_help_texts("edit-metadata")
......@@ -245,16 +245,17 @@ class DataSetForm(ModelForm):
css_class="row",
),
Div(
Div("institutional_affiliation", css_class="col-md-4",),
Div("funding", css_class="col-md-4", wrapper_class="controls"),
Div("related_projects", css_class="col-md-4",),
css_class="row",
),
Div(
Div("related_dataset_text", css_class="col-md-6",),
Div("related_dataset", css_class="col-md-6",),
Div("institutional_affiliation", css_class="col-md-3",),
Div("funding", css_class="col-md-3", wrapper_class="controls"),
Div("related_projects", css_class="col-md-3",),
Div("related_dataset_text", css_class="col-md-3",),
css_class="row",
),
# Div(
# Div("related_dataset_text", css_class="col-md-6",),
# Div("related_dataset", css_class="col-md-6",),
# css_class="row",
# ),
HTML(REQ_FIELD),
HTML(EDIT_DATASET_BTN),
# FormActions(
......
# Generated by Django 2.2.16 on 2020-12-04 14:51
import discuss_data.dddatasets.models
import django.core.files.storage
from django.db import migrations, models