Commit 38905ce6 authored by felix.herrmann's avatar felix.herrmann
Browse files

Merge branch 'develop' into 'master'

Update master

See merge request !46
parents abd5a731 2f253662
image: docker:latest
image: docker:19.03.0
services:
- docker:dind
- docker:19.03.0-dind
include:
- template: Dependency-Scanning.gitlab-ci.yml
- template: Code-Quality.gitlab-ci.yml
- template: Container-Scanning.gitlab-ci.yml
- template: SAST.gitlab-ci.yml
- template: License-Management.gitlab-ci.yml
- template: License-Scanning.gitlab-ci.yml
variables:
CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA
CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest
DS_PIP_DEPENDENCY_PATH: requirements/production.txt
CONTAINER_DEPLOY_IMAGE: $CI_REGISTRY_IMAGE/$CI_ENVIRONMENT_SLUG:latest
DARIAH_STORAGE_TOKEN: $DH_TOKEN
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
DS_PIP_DEPENDENCY_PATH: requirements/production.txt
stages:
- build
- test
- deploy
- release
- deploy
.docker-setup: &docker-setup
- docker info
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
.only-default: &only-default
only:
- branches
- merge_requests
- tags
build_production:
stage: build
before_script:
- *docker-setup
script:
- docker version
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker build -t $CONTAINER_TEST_IMAGE -f compose/production/django/Dockerfile .
- docker push $CONTAINER_TEST_IMAGE
only:
- master
build_develop:
<<: *only-default
stage: build
before_script:
- *docker-setup
script:
- docker version
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker build -t $CONTAINER_TEST_IMAGE -f compose/local/django/Dockerfile .
- docker push $CONTAINER_TEST_IMAGE
except:
- master
tests:
<<: *only-default
stage: test
image: tiangolo/docker-with-compose
image: docker/compose
before_script:
- *docker-setup
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- echo "Composing CI setup with $CONTAINER_TEST_IMAGE"
- docker-compose -f ci.yml build
- docker-compose -f ci.yml run django /bin/sh -c "./run_pytest"
- docker-compose -f ci.yml run --rm django coverage html
- docker-compose -f ci.yml run --rm django /bin/sh -c "cd docs && apk add make && make html"
- docker-compose -f ci.yml run django coverage report
- docker-compose -f ci.yml run --rm django /bin/sh -c "./run_pytest"
coverage: "/TOTAL.+ ([0-9]{1,3}%)/"
artifacts:
paths:
......@@ -63,18 +77,6 @@ tests:
except:
- master
merge_tests:
stage: test
image: tiangolo/docker-with-compose
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- echo "Composing CI setup with $CONTAINER_TEST_IMAGE"
- docker-compose -f ci.yml build
- docker-compose -f ci.yml run django /bin/sh -c "./run_pytest"
allow_failure: true
only:
- merge_requests
code_quality:
stage: test
artifacts:
......@@ -89,29 +91,22 @@ dependency_scanning:
only:
- master
- develop
- merge_requests
container_scanning:
stage: test
variables:
GIT_STRATEGY: fetch
DOCKER_USER: gitlab-ci-token
DOCKER_PASSWORD: $CI_BUILD_TOKEN
artifacts:
paths: [gl-container-scanning-report.json]
only:
- master
- develop
- merge_requests
license_management:
license_scanning:
stage: test
variables:
LM_PYTHON_VERSION: 3
only:
- master
- develop
- merge_requests
create_release:
image: node:8
......@@ -123,18 +118,38 @@ create_release:
- master
release_image:
deploy_staging:
stage: deploy
before_script:
- *docker-setup
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker pull $CONTAINER_TEST_IMAGE
- docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
- docker push $CONTAINER_RELEASE_IMAGE
environment:
name: staging
url: https://dev.discuss-data.net/
only:
- master
when: manual
pages:
deploy_production:
stage: deploy
before_script:
- *docker-setup
script:
- docker pull $CONTAINER_TEST_IMAGE
- docker tag $CONTAINER_TEST_IMAGE $CONTAINER_DEPLOY_IMAGE
- docker push $CONTAINER_DEPLOY_IMAGE
environment:
name: production
url: https://discuss-data.net/
only:
- master
when: manual
deploy_pages:
stage: deploy
script:
- mkdir -p public/coverage
......@@ -147,3 +162,4 @@ pages:
expire_in: 30 days
only:
- develop
- master
......@@ -62,10 +62,10 @@ Test coverage
To run the tests, check your test coverage, and generate an HTML coverage report::
$ docker-compose -f local.yml run django coverage run -m pytest
$ docker-compose -f local.yml run django ./run_pytest
$ docker-compose -f local.yml run django coverage report
Django templace check
Django template check
^^^^^^^^^^^^^^^^^^^^^
::
......@@ -90,6 +90,16 @@ With MailHog running, to view messages that are sent by your application, open y
.. _mailhog: https://github.com/mailhog/MailHog
Search
^^^^^^
To initialize or rebuild the elasticsearch index run:
$ docker-compose -f local.yml run --rm django python manage.py search_index --rebuild
Print the whole index in development by accessing:
``http://localhost:9200/users/_search?pretty``
Deployment
......
......@@ -3,6 +3,7 @@ version: '3'
volumes:
local_postgres_data: {}
local_postgres_data_backups: {}
local_elasticsearch_data: {}
services:
django:
......@@ -10,6 +11,7 @@ services:
depends_on:
- postgres
- mailhog
- elasticsearch
volumes:
- .:/app
env_file:
......@@ -30,6 +32,16 @@ services:
env_file:
- ./.envs/.local/.postgres
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2
container_name: elasticsearch
ports:
- "9200:9200"
environment:
- discovery.type=single-node
volumes:
- local_elasticsearch_data:/usr/share/elasticsearch/data
mailhog:
image: mailhog/mailhog:v1.0.0
ports:
......
# TODO: change to python:3.7-slim (i.e. buster-slim)
FROM python:3.7-alpine
ENV PYTHONUNBUFFERED 1
......@@ -21,11 +22,18 @@ RUN apk update \
# weasy dependencies:
&& apk add cairo-dev pango-dev gdk-pixbuf-dev \
# pip requires git
&& apk add git
&& apk add git \
# waiting for elasticsearch needs curl
&& apk add curl
# Requirements are installed here to ensure they will be cached.
COPY ./requirements /requirements
RUN pip install -r /requirements/local.txt
RUN pip install --upgrade pip && pip install -r /requirements/local.txt
# Shibboleth --> moved from requirements/local.txt
# ------------------------------------------------------------------------------
RUN pip install git+https://gitlab.gwdg.de/discuss-data/django-shibboleth-remoteuser.git@5287a46e00f8f6a0d835212e6328acb8f32c5d80
COPY ./compose/production/django/entrypoint /entrypoint
RUN sed -i 's/\r$//g' /entrypoint
......
......@@ -6,4 +6,5 @@ set -o nounset
python manage.py makemigrations
python manage.py migrate
python manage.py search_index --rebuild -f
python manage.py runserver_plus 0.0.0.0:8000
......@@ -26,7 +26,9 @@ RUN apk update \
# weasy dependencies:
&& apk add cairo-dev pango-dev gdk-pixbuf-dev \
# pip requires git
&& apk add git
&& apk add git \
# waiting for elasticsearch needs curl
&& apk add curl
RUN addgroup -S django \
&& adduser -S -G django django
......@@ -36,19 +38,15 @@ COPY ./requirements /requirements
RUN pip install --no-cache-dir -r /requirements/production.txt \
&& rm -rf /requirements
COPY ./compose/production/django/entrypoint /entrypoint
COPY --chown=django:django ./compose/production/django/entrypoint /entrypoint
RUN sed -i 's/\r$//g' /entrypoint
RUN chmod +x /entrypoint
RUN chown django /entrypoint
COPY ./compose/production/django/start /start
COPY --chown=django:django ./compose/production/django/start /start
RUN sed -i 's/\r$//g' /start
RUN chmod +x /start
RUN chown django /start
COPY --from=client-builder /app /app
RUN chown -R django /app
COPY --chown=django:django --from=client-builder /app /app
USER django
......
......@@ -39,4 +39,12 @@ until postgres_ready; do
done
>&2 echo 'PostgreSQL is available'
until echo $(curl --silent http://elasticsearch:9200/_cat/health?h=st) | grep "green"
do
>&2 echo 'Waiting for ElasticSearch to become available...'
sleep 1
done
>&2 echo 'ElasticSearch is available'
exec "$@"
......@@ -66,22 +66,22 @@ DJANGO_APPS = [
]
THIRD_PARTY_APPS = [
"actstream",
"allauth",
"allauth.account",
"allauth.socialaccount",
"crispy_forms",
"django_activeurl",
"guardian",
"intercoolerjs",
"rest_framework",
"reversion",
"shibboleth",
"taggit",
"django_elasticsearch_dsl",
]
LOCAL_APPS = [
# "discuss_data.users.apps.UsersConfig",
# Your stuff: custom apps go here
"discuss_data.ddusers", # discuss_data.ddusers.apps.DdusersConfig does not work
# "discuss_data.ddusers.apps.DdusersConfig",# does not work
"discuss_data.ddusers",
"discuss_data.dddatasets",
"discuss_data.ddcomments",
"discuss_data.core",
......@@ -103,7 +103,6 @@ MIGRATION_MODULES = {"sites": "discuss_data.contrib.sites.migrations"}
# https://docs.djangoproject.com/en/dev/ref/settings/#authentication-backends
AUTHENTICATION_BACKENDS = [
"django.contrib.auth.backends.ModelBackend",
"allauth.account.auth_backends.AuthenticationBackend",
# custom authentication backends
"shibboleth.backends.ShibbolethRemoteUserBackend",
"guardian.backends.ObjectPermissionBackend",
......@@ -113,7 +112,8 @@ AUTH_USER_MODEL = "ddusers.User"
# https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url
LOGIN_REDIRECT_URL = "ddusers.dashboard_page"
# https://docs.djangoproject.com/en/dev/ref/settings/#login-url
LOGIN_URL = "login"
LOGIN_URL = "/shib/login/"
LOGOUT_URL = "/shib/logout/"
# PASSWORDS
# ------------------------------------------------------------------------------
......@@ -273,21 +273,6 @@ LOGGING = {
"root": {"level": "INFO", "handlers": ["console"]},
}
# django-allauth
# ------------------------------------------------------------------------------
ACCOUNT_ALLOW_REGISTRATION = env.bool("DJANGO_ACCOUNT_ALLOW_REGISTRATION", True)
# https://django-allauth.readthedocs.io/en/latest/configuration.html
ACCOUNT_AUTHENTICATION_METHOD = "username"
# https://django-allauth.readthedocs.io/en/latest/configuration.html
ACCOUNT_EMAIL_REQUIRED = True
# https://django-allauth.readthedocs.io/en/latest/configuration.html
ACCOUNT_EMAIL_VERIFICATION = "mandatory"
# https://django-allauth.readthedocs.io/en/latest/configuration.html
ACCOUNT_ADAPTER = "discuss_data.users.adapters.AccountAdapter"
# https://django-allauth.readthedocs.io/en/latest/configuration.html
SOCIALACCOUNT_ADAPTER = "discuss_data.users.adapters.SocialAccountAdapter"
# django-compressor
# ------------------------------------------------------------------------------
# https://django-compressor.readthedocs.io/en/latest/quickstart/#installation
......@@ -326,9 +311,10 @@ WAGTAIL_SITE_NAME = "Discuss Data"
# shibboleth
SHIBBOLETH_ATTRIBUTE_MAP = {
"REMOTE_USER": (False, "username"),
"displayName": (False, "first_name"),
"mail": (False, "email"),
"REMOTE_USER": (True, "username"),
"GIVENNAME": (False, "first_name"),
"SN": (False, "last_name"),
"MAIL": (False, "email"),
}
SHIBBOLETH_LOGIN_URL = "/Shibboleth.sso/Login"
......@@ -336,3 +322,16 @@ SHIBBOLETH_LOGOUT_URL = "/Shibboleth.sso/Logout?return=%s"
SHIBBOLETH_LOGIN_REDIRECT_URL = "ddusers.dashboard_page"
SHIBBOLETH_LOGOUT_REDIRECT_URL = "ddusers.dashboard_page"
DARIAH_STORAGE_LOCATION = "https://cdstar.de.dariah.eu/test/dariah/"
DARIAH_PUBLISH_URL = "https://trep.de.dariah.eu/1.0/dhpublish/"
DARIAH_PDP_URL = "https://pdpdev.de.dariah.eu/"
DARIAH_PDP_CLIENT_ID = "discussdata"
DARIAH_PDP_REDIRECT_URI = "https://dev2.discuss-data.net/portal/dhrep/token"
ELASTICSEARCH_DSL = {
"default": {"hosts": "elasticsearch:9200"},
}
RUNSCRIPT_SCRIPT_DIR = "djangoscripts"
......@@ -49,8 +49,6 @@ EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"
# Your stuff...
# ------------------------------------------------------------------------------
DARIAH_STORAGE_LOCATION = "https://cdstar.de.dariah.eu/test/dariah/"
DARIAH_PUBLISH_URL = "https://trep.de.dariah.eu/1.0/dhpublish/"
# put fresh DARIAH_STORAGE_TOKEN to .env for integration tests
DARIAH_STORAGE_TOKEN = env("DARIAH_STORAGE_TOKEN", default="SET-FOR-INTEGRATION-TEST")
......@@ -4,7 +4,6 @@ from django.conf.urls.static import static
from django.contrib import admin
from django.views.generic import TemplateView
from django.views import defaults as default_views
from django.contrib.auth import views as auth_views
from discuss_data.ddusers.views import dashboard_page
import discuss_data.dddatasets
......@@ -26,22 +25,17 @@ urlpatterns = [
path(
"users/", include(("discuss_data.ddusers.urls", "ddusers"), namespace="ddusers")
),
# next line to be removed
# path("accounts/", include("allauth.urls")),
# Your stuff: custom urls includes go here
path("core/", include(("discuss_data.core.urls", "core"), namespace="core")),
# path(
# "publication/",
# include(
# ("discuss_data.ddpublications.urls", "ddpublications"),
# namespace="ddpublications",
# ),
# ),
path("dataset/", include("discuss_data.dddatasets.urls", namespace="dddatasets")),
path("dhrep/", include("discuss_data.dhrep.urls", namespace="dhrep")),
path(
"login/",
auth_views.LoginView.as_view(template_name="auth/login.html"),
name="login",
),
path(
"logout/",
auth_views.LogoutView.as_view(template_name="auth/logout.html"),
name="logout",
),
path("shib/", include("shibboleth.urls", namespace="shibboleth")),
path("", dashboard_page, name="ddusers.dashboard_page"),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
......
from django.contrib import admin
# Register your models here.
from discuss_data.ddcomments.models import Comment
admin.site.register(Comment)
from django.contrib import admin
# Register your models here.
from discuss_data.dddatasets.models import DataSet
admin.site.register(DataSet)
......@@ -17,7 +17,11 @@ from crispy_forms.layout import (
)
from crispy_forms.bootstrap import FormActions, TabHolder, Tab
from discuss_data.dddatasets.models import DataSet, DataFile
from discuss_data.dddatasets.models import (
DataSet,
DataFile,
DataSetPublication,
)
UPDATED_MSG = """
{% if updated %}<div id='updated-msg' class='alert alert-success' ic-trigger-on='scrolled-into-view' ic-action='delay:2500;fadeOut;remove' ic-target='#updated-msg' role='alert'>
......@@ -31,6 +35,20 @@ ADD_EDIT_HEADING = (
REQ_FIELD = "<p class='help-block'>[*] required field</p><hr>"
class DataSetPublicationForm(ModelForm):
class Meta:
model = DataSetPublication
fields = [
"pub_type",
]
helper = FormHelper()
helper.form_tag = False
helper.layout = Layout(
Div(Div("pub_type", css_class="col-md-12",), css_class="row",),
)
class DataSetForm(ModelForm):
class Meta:
model = DataSet
......@@ -183,18 +201,18 @@ class DataFileUploadForm(ModelForm):
helper = FormHelper()
helper.form_tag = False
helper.use_custom_control = True
helper.layout = Layout(
HTML(ADD_EDIT_HEADING),
Div(
Div("file", css_class="col-md-6",),
Div("data_file_type", css_class="col-md-6",),
Div("file", css_class="col-md-9",),
Div("data_file_type", css_class="col-md-3",),
css_class="row",
),
HTML(REQ_FIELD),
# HTML(mark_safe(progress_id)),
# HTML(mark_safe(progress_bar_tag + upload_progress_url)),
FormActions(
Submit("save", "Save {{ target }}"),
Submit("save", "Save {{ target }}", css_class="btn btn-primary",),
# Button('cancel', 'Cancel', data_toggle="collapse", data_target="#collapse-{{ object_type }}-{{ object.id }}"),
),
Div(HTML(UPDATED_MSG),),
......
# Generated by Django 2.2.9 on 2020-02-27 09:44
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('dddatasets', '0002_auto_20191028_1801'),
]
operations = [
migrations.AlterModelOptions(
name='datasetmanagementobject',
options={'permissions': (('view_dsmo', 'View Data Set'), ('edit_dsmo', 'Edit Data Set'), ('admin_dsmo', 'Admin Data Set'))},
),
]
# Generated by Django 2.2.10 on 2020-03-02 09:29
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('dddatasets', '0003_auto_20200227_0944'),
]
operations = [
migrations.AlterModelOptions(
name='datasetmanagementobject',
options={'permissions': (('view_dsmo', 'View Data Set Management Object'), ('edit_dsmo', 'Edit Data Set Management Object'), ('admin_dsmo', 'Admin Data Set Management Object'))},
),
]
......@@ -245,7 +245,7 @@ class DataSetPublication(models.Model):
)
def __str__(self):
return format_html("%s (%s)", self.publication, self.get_pub_type_display())
return format_html("{} ({})", self.publication, self.get_pub_type_display())
class DataSetExternalLink(models.Model):
......@@ -672,45 +672,42 @@ class DataSetManagementObject(models.Model):
# max_version_published = models.DecimalField(max_digits=5, decimal_places=1,default=1.0)
class Meta:
"""
Definition of individual dsmo permissions.
DSMO permissions override DataSet permissions which are
deprecated and will be removed in the near future
"""
permissions = (
("view_dataset", "View Data Set"),
("edit_dataset", "Edit Data Set"),
("admin_dataset", "Edit Data Set"),
("view_dsmo", "View Data Set Management Object"),
("edit_dsmo", "Edit Data Set Management Object"),
("admin_dsmo", "Admin Data Set Management Object"),
)
groups_dict = {
"view": ("view_dataset",),
"edit": ("view_dataset", "edit_dataset"),
"admin": ("view_dataset", "edit_dataset", "admin_dataset"),
# group individual permissions together
perms_groups = {
"view": ("view_dsmo",),
"edit": ("view_dsmo", "edit_dsmo"),
"admin": ("view_dsmo", "edit_dsmo", "admin_dsmo"),
}
def create_groups(self, groups_dict):
def clear_user_permissions(self, user):
for perm in get_perms(user, self):
remove_perm(perm, user, self)
dsmoid = str(self.id)
# print('creating groups for dsmo %s' % (dsmoid,))
for gkey in groups_dict.keys():
group_name = "%s_%s" % (dsmoid, gkey)
# print('create %s' % (group_name,) )
group = Group.objects.create(name=group_name)
# print('add %s to dataset management object' % (group_name,) )
self.groups.add(group)
# assign permissions to group
for perm in groups_dict[gkey]:
# print('assign %s to %s' % (perm, group,) )
assign_perm(perm, group, self)