Commit 550a378d authored by hynek's avatar hynek 🤤
Browse files

Merge branch 'release/0.5.2' into 'master'

Release/0.5.2

See merge request !22
parents 8ebf8a53 250cd230
......@@ -13,7 +13,7 @@ indent_style = space
indent_size = 4
[*.py]
line_length = 120
line_length = 88
known_first_party = discuss_data
multi_line_output = 3
default_section = THIRDPARTY
......
[flake8]
ignore = E203, E266, E501, W503, F403, F401
max-line-length = 88
max-complexity = 18
select = B,C,E,F,W,T4,B9
exclude = .tox,.git,*/migrations/*,*/static/CACHE/*,docs,node_modules
......@@ -152,10 +152,10 @@ typings/
### VisualStudioCode template
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
#!.vscode/settings.json
#!.vscode/tasks.json
#!.vscode/launch.json
#!.vscode/extensions.json
......
......@@ -7,14 +7,13 @@ include:
- template: Code-Quality.gitlab-ci.yml
- template: Container-Scanning.gitlab-ci.yml
- template: SAST.gitlab-ci.yml
- template: License-Management.gitlab-ci.yml
variables:
DOCKER_HOST: tcp://docker:2375
DOCKER_DRIVER: overlay2
CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA
CONTAINER_RELEASE_IMAGE: $CI_REGISTRY_IMAGE:latest
DS_PIP_DEPENDENCY_PATH: requirements/production.txt
DARIAH_STORAGE_TOKEN: $DH_TOKEN
stages:
- build
......@@ -23,33 +22,73 @@ stages:
- release
build_production:
stage: build
script:
- docker version
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker build -t $CONTAINER_TEST_IMAGE -f compose/production/django/Dockerfile .
- docker push $CONTAINER_TEST_IMAGE
only:
- master
build_develop:
stage: build
script:
- docker version
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker build -t $CONTAINER_TEST_IMAGE -f compose/local/django/Dockerfile .
- docker push $CONTAINER_TEST_IMAGE
tests:
stage: test
image: tiangolo/docker-with-compose
script:
- docker-compose -f local.yml build
# - docker-compose -f local.yml run --rm django pydocstyle
- docker-compose -f local.yml run --rm django flake8
- docker-compose -f local.yml run django coverage run -m pytest
- docker-compose -f local.yml run --rm django coverage html
- docker-compose -f local.yml run --rm django /bin/sh -c "cd docs && apk add make && make html"
- docker-compose -f local.yml run django coverage report
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- echo "Composing CI setup with $CONTAINER_TEST_IMAGE"
- docker-compose -f ci.yml build
- docker-compose -f ci.yml run django /bin/sh -c "./run_pytest"
- docker-compose -f ci.yml run --rm django coverage html
- docker-compose -f ci.yml run --rm django /bin/sh -c "cd docs && apk add make && make html"
- docker-compose -f ci.yml run django coverage report
coverage: "/TOTAL.+ ([0-9]{1,3}%)/"
artifacts:
paths:
- htmlcov
- docs/_build
expire_in: 5 days
allow_failure: true
except:
- master
dependency_scanning:
merge_tests:
stage: test
artifacts:
paths: [gl-dependency-scanning-report.json]
image: tiangolo/docker-with-compose
script:
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- echo "Composing CI setup with $CONTAINER_TEST_IMAGE"
- docker-compose -f ci.yml build
- docker-compose -f ci.yml run django /bin/sh -c "./run_pytest"
allow_failure: true
only:
- merge_requests
code_quality:
stage: test
artifacts:
paths: [gl-code-quality-report.json]
except:
- master
dependency_scanning:
stage: test
artifacts:
paths: [gl-dependency-scanning-report.json]
only:
- master
- develop
- merge_requests
container_scanning:
stage: test
......@@ -59,26 +98,30 @@ container_scanning:
DOCKER_PASSWORD: $CI_BUILD_TOKEN
artifacts:
paths: [gl-container-scanning-report.json]
only:
- master
- develop
- merge_requests
build:
stage: build
script:
- docker version
- docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY
- docker build -t $CONTAINER_TEST_IMAGE -f compose/production/django/Dockerfile .
- docker push $CONTAINER_TEST_IMAGE
license_management:
stage: test
variables:
LM_PYTHON_VERSION: 3
only:
- master
- develop
- merge_requests
create_release:
image: node:8
stage: release
script:
script:
- npm install
- npx semantic-release
only:
- master
- master
release_image:
stage: deploy
script:
......@@ -87,7 +130,8 @@ release_image:
- docker tag $CONTAINER_TEST_IMAGE $CONTAINER_RELEASE_IMAGE
- docker push $CONTAINER_RELEASE_IMAGE
only:
- tags
- master
when: manual
pages:
stage: deploy
......@@ -101,4 +145,4 @@ pages:
- public
expire_in: 30 days
only:
- tags
- develop
......@@ -3,13 +3,19 @@ default_stages: [commit]
fail_fast: true
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: master
hooks:
- id: trailing-whitespace
files: (^|/)a/.+\.(py|html|sh|css|js)$
# files: (^|/)a/.+\.(py|html|sh|css|js)$
- repo: local
- repo: https://github.com/psf/black
rev: stable
hooks:
- id: black
#language_version: python3.6
- repo: local
hooks:
- id: flake8
name: flake8
......@@ -17,3 +23,8 @@ repos:
language: python
types: [python]
- repo: https://github.com/adrienverge/yamllint.git
rev: master
hooks:
- id: yamllint
args: ['-d {extends: relaxed, rules: {line-length: disable}}', '-s']
......@@ -2,7 +2,7 @@
load-plugins=pylint_django
[FORMAT]
max-line-length=120
max-line-length=88
[MESSAGES CONTROL]
disable=missing-docstring,invalid-name
......
============================
Contributing to Discuss Data
============================
As an open source project, Discuss Data welcomes contributions of many forms.
Examples of contributions include:
* Code patches
* Documentation improvements
* Bug reports and patch reviews
Felix Herrmann, Stefan Hynek, Ubbo Veentjer
Felix Herrmann
Stefan Hynek
Ubbo Veentjer
Discuss Data
============
Open Platform for the Interactive Discussion of Research Data Quality (on the example of area studies on the post-Soviet region)
Open Platform for the Interactive Discussion of Research Data Quality
(on the example of area studies on the post-Soviet region)
.. image:: https://img.shields.io/badge/built%20with-Cookiecutter%20Django-ff69b4.svg
:target: https://github.com/pydanny/cookiecutter-django/
......@@ -15,32 +16,32 @@ Open Platform for the Interactive Discussion of Research Data Quality (on the ex
.. image:: https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg
:target: https://github.com/semantic-release/semantic-release
:alt: semantic-release
.. image:: https://img.shields.io/badge/License-GPLv3-blue.svg
:target: https://www.gnu.org/licenses/gpl-3.0
:alt: License: GPL v3
Documentation
-------------
:License: GPLv3
Documentation_
`Coverage Report`_
.. _Documentation: https://discuss-data.pages.gwdg.de/discuss-data/docs/
.. _`Coverage Report`: https://discuss-data.pages.gwdg.de/discuss-data/coverage/
Settings
--------
Moved to settings_.
Settings are read from environment configuration files located in ``.envs/``.
Bear in mind that no settings for the production environment are provided.
You could easily create your own environment settings from ``.envs/.local``
and from the documentation on `Configuring the Environment`_.
.. _settings: http://cookiecutter-django.readthedocs.io/en/latest/settings.html
.. _`Configuring the Environment`: https://cookiecutter-django.readthedocs.io/en/latest/developing-locally-docker.html#configuring-the-environment
Basic Commands
--------------
Setting Up Your Users
^^^^^^^^^^^^^^^^^^^^^
* To create a **normal user account**, just go to Sign Up and fill out the form. Once you submit it, you'll see a "Verify Your E-mail Address" page. Go to your console to see a simulated email verification message. Copy the link into your browser. Now the user's email should be verified and ready to go.
* To create an **superuser account**, use this command::
$ python manage.py createsuperuser
For convenience, you can keep your normal user logged in on Chrome and your superuser logged in on Firefox (or similar), so that you can see how the site behaves for both kinds of users.
Type checks
^^^^^^^^^^^
......@@ -50,39 +51,40 @@ Running type checks with mypy:
$ mypy discuss_data
Django unit tests
^^^^^^^^^^^^^^^^^
::
$ docker-compose -f local.yml run django python manage.py test
Test coverage
^^^^^^^^^^^^^
To run the tests, check your test coverage, and generate an HTML coverage report::
$ coverage run -m pytest
$ coverage html
$ open htmlcov/index.html
Running tests with py.test
~~~~~~~~~~~~~~~~~~~~~~~~~~
$ docker-compose -f local.yml run django coverage run -m pytest
$ docker-compose -f local.yml run django coverage report
Django templace check
^^^^^^^^^^^^^^^^^^^^^
::
$ pytest
$ docker-compose -f local.yml run django python manage.py templatecheck
Live reloading and Sass CSS compilation
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Moved to `Live reloading and SASS compilation`_.
.. _`Live reloading and SASS compilation`: http://cookiecutter-django.readthedocs.io/en/latest/live-reloading-and-sass-compilation.html
I18N
^^^^
For further information see `Localization`_.
.. _`Localization`: https://docs.djangoproject.com/en/3.0/topics/i18n/translation/#localization-how-to-create-language-files
Email Server
^^^^^^^^^^^^
In development, it is often nice to be able to see emails that are being sent from your application. For that reason local SMTP server `MailHog`_ with a web interface is available as docker container.
Container mailhog will start automatically when you will run all docker containers.
Please check `cookiecutter-django Docker documentation`_ for more details how to start all containers.
The local SMTP server `MailHog`_ with a web interface is available as docker
container. It will start automatically when composing the "local"-setup.
With MailHog running, to view messages that are sent by your application, open your browser and go to ``http://127.0.0.1:8025``
......@@ -93,30 +95,84 @@ With MailHog running, to view messages that are sent by your application, open y
Deployment
----------
The following details how to deploy this application.
The following details how to deploy this application to the different
environments.
Development (w/ Docker)
^^^^^^^^^^^^^^^^^^^^^^^
This project is configured to use Docker for development.
Build images:
::
$ docker-compose -f local.yml build
Run container:
::
$ docker-compose -f local.yml up
Make migrations:
::
$ docker-compose -f local.yml run --rm django python manage.py makemigrations
Run migrations:
::
$ docker-compose -f local.yml run --rm django python manage.py migrate
Docker
^^^^^^
See detailed `cookiecutter-django Docker documentation`_.
Consult `cookiecutter-django Docker documentation`_ for further details.
.. _`cookiecutter-django Docker documentation`: http://cookiecutter-django.readthedocs.io/en/latest/deployment-with-docker.html
Development (w/o Docker)
^^^^^^^^^^^^^^^^^^^^^^^^
This is not recommended. Since this project is configured to use Docker you
have to override the Docker-setting with an environment variable:
::
$ export USE_DOCKER=false
Also, set your local Database url:
::
$ export DATABASE_URL=postgres://<postgres_user>:<password>@127.0.0.1:5432/<dbname>
Custom Bootstrap Compilation
^^^^^^
After having installed the node dependencies with ``npm install`` the project
can be run with ``npm run dev`` with Live reloading and Sass CSS compilation.
The generated CSS is set up with automatic Bootstrap recompilation with variables of your choice.
Bootstrap v4 is installed using npm and customised by tweaking your variables in ``static/sass/custom_bootstrap_vars``.
Production
^^^^^^^^^^
You can find a list of available variables `in the bootstrap source`_, or get explanations on them in the `Bootstrap docs`_.
To deploy to the production environment see the `Discuss Data Docker`_ project.
.. _`Discuss Data Docker`: https://gitlab.gwdg.de/discuss-data/discuss-data-docker
Bootstrap's javascript as well as its dependencies is concatenated into a single file: ``static/js/vendors.js``.
Backup
------
run backup:
::
$ docker-compose -f production.yml exec postgres backup
show backups:
::
$ docker-compose -f production.yml exec postgres backups
check backup mount:
::
$ docker volume inspect discussdata_production_postgres_data_backups
current backup mount:
::
.. _in the bootstrap source: https://github.com/twbs/bootstrap/blob/v4-dev/scss/_variables.scss
.. _Bootstrap docs: https://getbootstrap.com/docs/4.1/getting-started/theming/
$ /var/lib/docker/volumes/discussdata_production_postgres_data_backups/_data
version: '3'
volumes:
local_postgres_data: {}
local_postgres_data_backups: {}
services:
django:
image: $CONTAINER_TEST_IMAGE
depends_on:
- postgres
- mailhog
volumes:
- .:/app
env_file:
- ./.envs/.local/.django
- ./.envs/.local/.postgres
ports:
- "8000:8000"
command: /start
postgres:
build:
context: .
dockerfile: ./compose/production/postgres/Dockerfile
image: discuss_data_production_postgres
volumes:
- local_postgres_data:/var/lib/postgresql/data
- local_postgres_data_backups:/backups
env_file:
- ./.envs/.local/.postgres
mailhog:
image: mailhog/mailhog:v1.0.0
ports:
- "8025:8025"
node:
build:
context: .
dockerfile: ./compose/local/node/Dockerfile
image: discuss_data_local_node
depends_on:
- django
volumes:
- .:/app
# http://jdlm.info/articles/2016/03/06/lessons-building-node-app-docker.html
- /app/node_modules
command: npm run dev
ports:
- "3000:3000"
# Expose browsersync UI: https://www.browsersync.io/docs/options/#option-ui
- "3001:3001"
......@@ -13,7 +13,15 @@ RUN apk update \
# Translations dependencies
&& apk add gettext \
# https://docs.djangoproject.com/en/dev/ref/django-admin/#dbshell
&& apk add postgresql-client
&& apk add postgresql-client \
# custom installs: if they would become more than this, a file import must be provided
# unknown dependencies
&& apk add libxml2-dev \
&& apk add libxslt-dev \
# weasy dependencies:
&& apk add cairo-dev pango-dev gdk-pixbuf-dev \
# pip requires git
&& apk add git
# Requirements are installed here to ensure they will be cached.
COPY ./requirements /requirements
......
......@@ -4,6 +4,6 @@ set -o errexit
set -o pipefail
set -o nounset
python manage.py makemigrations
python manage.py migrate
python manage.py runserver_plus 0.0.0.0:8000
......@@ -18,7 +18,15 @@ RUN apk update \
# Pillow dependencies
&& apk add jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \
# CFFI dependencies
&& apk add libffi-dev py-cffi
&& apk add libffi-dev py-cffi \
# custom installs: if they would become more than this, a file import must be provided
# unknown dependencies
&& apk add libxml2-dev \
&& apk add libxslt-dev \
# weasy dependencies:
&& apk add cairo-dev pango-dev gdk-pixbuf-dev \
# pip requires git
&& apk add git
RUN addgroup -S django \
&& adduser -S -G django django
......
......@@ -62,18 +62,33 @@ DJANGO_APPS = [
"django.contrib.staticfiles",
# "django.contrib.humanize", # Handy template tags
"django.contrib.admin",
"django.contrib.admindocs", # check pip dependency
]
THIRD_PARTY_APPS = [
"crispy_forms",
"actstream",
"allauth",
"allauth.account",
"allauth.socialaccount",
"crispy_forms",
"django_activeurl",
"guardian",
"intercoolerjs",
"rest_framework",
"reversion",
"taggit",
]
LOCAL_APPS = [
"discuss_data.users.apps.UsersConfig",
# "discuss_data.users.apps.UsersConfig",
# Your stuff: custom apps go here
"discuss_data.ddusers", # discuss_data.ddusers.apps.DdusersConfig does not work
"discuss_data.dddatasets",
"discuss_data.ddcomments",
"discuss_data.core",
"discuss_data.ddpublications",
"discuss_data.dhrep",
"discuss_data.pages",
"discuss_data.utils",
]
# https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps
INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS
......@@ -89,13 +104,16 @@ MIGRATION_MODULES = {"sites": "discuss_data.contrib.sites.migrations"}
AUTHENTICATION_BACKENDS = [
"django.contrib.auth.backends.ModelBackend",
"allauth.account.auth_backends.AuthenticationBackend",
# custom authentication backends
"shibboleth.backends.ShibbolethRemoteUserBackend",
"guardian.backends.ObjectPermissionBackend",
]
# https://docs.djangoproject.com/en/dev/ref/settings/#auth-user-model
AUTH_USER_MODEL = "users.User"
AUTH_USER_MODEL = "ddusers.User"
# https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url
LOGIN_REDIRECT_URL = "users:redirect"
LOGIN_REDIRECT_URL = "ddusers.dashboard_page"
# https://docs.djangoproject.com/en/dev/ref/settings/#login-url
LOGIN_URL = "account_login"
LOGIN_URL = "login"
# PASSWORDS
# ------------------------------------------------------------------------------
......@@ -128,8 +146,14 @@ MIDDLEWARE = [
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"shibboleth.middleware.ShibbolethRemoteUserMiddleware",
"django.contrib.admindocs.middleware.XViewMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
# custom middlewares
"intercooler_helpers.middleware.HttpMethodOverride",
"intercooler_helpers.middleware.IntercoolerData",
"intercooler_helpers.middleware.IntercoolerRedirector",
]
# STATIC
......@@ -180,6 +204,10 @@ TEMPLATES = [
"django.template.context_processors.tz",
"django.contrib.messages.context_processors.messages",
"discuss_data.utils.context_processors.settings_context",
"shibboleth.context_processors.login_link",
"shibboleth.context_processors.logout_link",
"wagtail.contrib.settings.context_processors.settings",
"wagtailmenus.context_processors.wagtailmenus",
],
},
}
......@@ -207,7 +235,7 @@ X_FRAME_OPTIONS = "DENY"
# ------------------------------------------------------------------------------
# https://docs.djangoproject.com/en/dev/ref/settings/#email-backend
EMAIL_BACKEND = env(
"DJANGO_EMAIL_BACKEND", default="django.core.mail.backends.smtp.EmailBackend"
"DJANGO_EMAIL_BACKEND", default="django.core.mail.backends.smtp.EmailBackend",
)
# https://docs.djangoproject.com/en/2.2/ref/settings/#email-timeout
EMAIL_TIMEOUT = 5
......@@ -217,7 +245,7 @@ EMAIL_TIMEOUT = 5
# Django Admin URL.
ADMIN_URL = "admin/"
# https://docs.djangoproject.com/en/dev/ref/settings/#admins
ADMINS = [("""Felix Herrmann, Stefan Hynek, Ubbo Veentjer""", "info@discuss-data.net")]
ADMINS = [("""Felix Herrmann, Stefan Hynek, Ubbo Veentjer""", "info@discuss-data.net",)]