From 341b9b105c168fedff72d3ba28f1d9ae78f9024f Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Sat, 22 Feb 2020 00:08:18 +0100 Subject: [PATCH 01/10] local docker setup working --- .envs/.django | 3 + .envs/.postgres | 8 + README.md | 31 +++- compose/django/Dockerfile | 38 +++++ compose/django/Dockerfile.0.11.0 | 43 +++++ compose/django/entrypoint | 41 +++++ compose/django/start | 15 ++ compose/postgres/Dockerfile | 7 + config/local.py | 263 +++++++++++++++++++++++++++++++ docker-compose.yaml | 36 +++++ 10 files changed, 484 insertions(+), 1 deletion(-) create mode 100644 .envs/.django create mode 100644 .envs/.postgres create mode 100644 compose/django/Dockerfile create mode 100644 compose/django/Dockerfile.0.11.0 create mode 100644 compose/django/entrypoint create mode 100644 compose/django/start create mode 100644 compose/postgres/Dockerfile create mode 100644 config/local.py create mode 100644 docker-compose.yaml diff --git a/.envs/.django b/.envs/.django new file mode 100644 index 0000000..cb22ce2 --- /dev/null +++ b/.envs/.django @@ -0,0 +1,3 @@ +# General +# ------------------------------------------------------------------------------ +DJANGO_SECRET_KEY=LP3CtSG5ZjMaVwgWa1ahKIT5XrPsbnGPAM1oE6OeXD3ZwSWdmDp6DSv3tQob diff --git a/.envs/.postgres b/.envs/.postgres new file mode 100644 index 0000000..eaf9052 --- /dev/null +++ b/.envs/.postgres @@ -0,0 +1,8 @@ +# PostgreSQL +# ------------------------------------------------------------------------------ +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +POSTGRES_DB=rdmo +POSTGRES_USER=rdmo +POSTGRES_PASSWORD=debug + diff --git a/README.md b/README.md index dc0a717..2bf200f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,36 @@ eRA template modifications for the rdmo app - https://github.com/rdmorganiser/rdmo-app -# Usage + +# Docker + +list running containers + + docker ps + +enter running container + + docker exec -it plan-git_django_1 bash + +## compose + +build images: + + docker-compose build + +run container: + + docker-compose build + +create super-user (on first run) + + docker exec -it plan-git_django_1 python3 manage.py createsuperuser + + + + + +# Usage (without docker - old) Clone the repo next to the rdmo-app dir and add symbolic links diff --git a/compose/django/Dockerfile b/compose/django/Dockerfile new file mode 100644 index 0000000..4e2c5cd --- /dev/null +++ b/compose/django/Dockerfile @@ -0,0 +1,38 @@ +# FROM debian:buster +FROM debian:buster-slim + +RUN apt update -y && apt upgrade -y && apt install -y \ + python3 \ + python3-dev \ + python3-pip \ + git + + +# postgres +RUN apt install -y \ + libpq-dev \ + postgresql-client \ + python3-psycopg2 + + +RUN git clone https://github.com/rdmorganiser/rdmo-app.git /app + +WORKDIR /app + +RUN pip3 install rdmo + +#COPY ./theme /app/theme +COPY ./locale /app/locale + +COPY ./config/local.py /app/config/settings/local.py + +COPY ./compose/django/entrypoint /entrypoint +RUN chmod +x /entrypoint + +COPY ./compose/django/start /start +RUN chmod +x /start + +ENV DJANGO_SECRET_KEY=NOT_SO_SECRET_TEMP_KEY +RUN python3 manage.py download_vendor_files + +ENTRYPOINT ["/entrypoint"] diff --git a/compose/django/Dockerfile.0.11.0 b/compose/django/Dockerfile.0.11.0 new file mode 100644 index 0000000..a623a55 --- /dev/null +++ b/compose/django/Dockerfile.0.11.0 @@ -0,0 +1,43 @@ +FROM debian:buster +# FROM debian:buster-slim ? + +RUN apt update -y && apt upgrade -y && apt install -y \ + python3 \ + python3-dev \ + python3-pip + +# build-deps +RUN apt install -y \ + git + +# postgres +#RUN apt install -y \ +# libpq-dev \ +# postgresql \ +# postgresql-client \ +# python-psycopg2 + + +RUN git clone https://github.com/rdmorganiser/rdmo-app.git /app + +WORKDIR /app + +# create docker image for 0.11.0 first +RUN git checkout 89749332435a00d096afc2a16704c09f6f3525c7 +#RUN pip3 install --upgrade pip setuptools +RUN pip3 install 'rdmo==0.11.0' + +COPY ./theme /app/theme +COPY ./locale /app/locale + +COPY ./config/local.py /app/config/settings/local.py + +RUN ls + +RUN python3 manage.py migrate # initializes the database +RUN python3 manage.py setup_groups # creates groups with different permissions +RUN python3 manage.py createsuperuser # creates the admin user + +RUN python3 manage.py download_vendor_files + + diff --git a/compose/django/entrypoint b/compose/django/entrypoint new file mode 100644 index 0000000..ba5831b --- /dev/null +++ b/compose/django/entrypoint @@ -0,0 +1,41 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +if [ -z "${POSTGRES_USER}" ]; then + base_postgres_image_default_user='postgres' + export POSTGRES_USER="${base_postgres_image_default_user}" +fi + +postgres_ready() { +python3 << END +import sys + +import psycopg2 + +try: + psycopg2.connect( + dbname="${POSTGRES_DB}", + user="${POSTGRES_USER}", + password="${POSTGRES_PASSWORD}", + host="${POSTGRES_HOST}", + port="${POSTGRES_PORT}", + ) +except psycopg2.OperationalError: + sys.exit(-1) +sys.exit(0) + +END +} + +until postgres_ready; do + >&2 echo 'Waiting for PostgreSQL to become available...' + sleep 1 +done +>&2 echo 'PostgreSQL is available' + + +exec "$@" + diff --git a/compose/django/start b/compose/django/start new file mode 100644 index 0000000..d7c985b --- /dev/null +++ b/compose/django/start @@ -0,0 +1,15 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +# todo: +# https://rdmo.readthedocs.io/en/latest/installation/setup.html +# - do this commands need to run every time? +# - should setup_groups only run once per db setup? +# - would this also handle rdmo upgrades on the db? -> https://rdmo.readthedocs.io/en/latest/upgrade/index.html +python3 manage.py migrate # initializes the database +python3 manage.py setup_groups # creates groups with different permissions + +python3 manage.py runserver 0.0.0.0:8000 diff --git a/compose/postgres/Dockerfile b/compose/postgres/Dockerfile new file mode 100644 index 0000000..267e0c8 --- /dev/null +++ b/compose/postgres/Dockerfile @@ -0,0 +1,7 @@ +FROM postgres:11.3 + +#COPY ./compose/production/postgres/maintenance /usr/local/bin/maintenance +#RUN chmod +x /usr/local/bin/maintenance/* +#RUN mv /usr/local/bin/maintenance/* /usr/local/bin \ +# && rmdir /usr/local/bin/maintenance + diff --git a/config/local.py b/config/local.py new file mode 100644 index 0000000..41525af --- /dev/null +++ b/config/local.py @@ -0,0 +1,263 @@ +import os +from . import BASE_DIR + +''' +Debug mode, don't use this in production +''' + +DEBUG = True + + +''' +A secret key for a particular Django installation. This is used to provide +cryptographic signing, and should be set to a unique, unpredictable value. +''' + +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY') + +''' +The list of URLs und which this application available +''' + +ALLOWED_HOSTS = ['localhost', 'ip6-localhost', '127.0.0.1', '[::1]'] + +''' +The root url of your application, only needed when its not '/' +''' + +# BASE_URL = '/path' + +''' +Language code and time zone +''' +LANGUAGE_CODE = 'de-de' +TIME_ZONE = 'Europe/Berlin' + +''' +The database connection to be used, see also: +http://rdmo.readthedocs.io/en/latest/configuration/databases.html +''' + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': os.getenv('POSTGRES_DB'), + 'USER': os.getenv('POSTGRES_USER'), + 'PASSWORD': os.getenv('POSTGRES_PASSWORD'), + 'HOST': os.getenv('POSTGRES_HOST'), + 'PORT': os.getenv('POSTGRES_PORT'), + } +} + +''' +E-Mail configuration, see also: +http://rdmo.readthedocs.io/en/latest/configuration/email.html +''' + +# EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +# EMAIL_HOST = 'localhost' +# EMAIL_PORT = '25' +# EMAIL_HOST_USER = '' +# EMAIL_HOST_PASSWORD = '' +# EMAIL_USE_TLS = False +# EMAIL_USE_SSL = False +# DEFAULT_FROM_EMAIL = '' + +''' +Allauth configuration, see also: +http://rdmo.readthedocs.io/en/latest/configuration/authentication/allauth.html +''' + +# from rdmo.core.settings import INSTALLED_APPS, AUTHENTICATION_BACKENDS +# +# ACCOUNT = True +# ACCOUNT_SIGNUP = True +# SOCIALACCOUNT = False +# +# INSTALLED_APPS += [ +# 'allauth', +# 'allauth.account', +# 'allauth.socialaccount', +# 'allauth.socialaccount.providers.facebook', +# 'allauth.socialaccount.providers.github', +# 'allauth.socialaccount.providers.google', +# 'allauth.socialaccount.providers.orcid', +# 'allauth.socialaccount.providers.twitter', +# ] +# +# AUTHENTICATION_BACKENDS.append('allauth.account.auth_backends.AuthenticationBackend') + +''' +LDAP, see also: +http://rdmo.readthedocs.io/en/latest/configuration/authentication/ldap.html +''' + +# import ldap +# from django_auth_ldap.config import LDAPSearch +# from rdmo.core.settings import AUTHENTICATION_BACKENDS +# +# PROFILE_UPDATE = False +# +# AUTH_LDAP_SERVER_URI = "ldap://ldap.example.com" +# AUTH_LDAP_BIND_DN = "cn=admin,dc=ldap,dc=example,dc=com" +# AUTH_LDAP_BIND_PASSWORD = "admin" +# AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=ldap,dc=example,dc=com", ldap.SCOPE_SUBTREE, "(uid=%(user)s)") +# +# AUTH_LDAP_USER_ATTR_MAP = { +# "first_name": "givenName", +# "last_name": "sn", +# 'email': 'mail' +# } +# +# AUTHENTICATION_BACKENDS.insert( +# AUTHENTICATION_BACKENDS.index('django.contrib.auth.backends.ModelBackend'), +# 'django_auth_ldap.backend.LDAPBackend' +# ) + +''' +Shibboleth, see also: +http://rdmo.readthedocs.io/en/latest/configuration/authentication/shibboleth.html +''' + +# from rdmo.core.settings import INSTALLED_APPS, AUTHENTICATION_BACKENDS, MIDDLEWARE_CLASSES +# +# SHIBBOLETH = True +# PROFILE_UPDATE = False +# +# INSTALLED_APPS += ['shibboleth'] +# +# SHIBBOLETH_ATTRIBUTE_MAP = { +# 'uid': (True, 'username'), +# 'givenName': (True, 'first_name'), +# 'sn': (True, 'last_name'), +# 'mail': (True, 'email'), +# } +# +# AUTHENTICATION_BACKENDS.append('shibboleth.backends.ShibbolethRemoteUserBackend') +# +# MIDDLEWARE_CLASSES.insert( +# MIDDLEWARE_CLASSES.index('django.contrib.auth.middleware.AuthenticationMiddleware') + 1, +# 'shibboleth.middleware.ShibbolethRemoteUserMiddleware' +# ) +# +# LOGIN_URL = '/Shibboleth.sso/Login?target=/projects' +# LOGOUT_URL = '/Shibboleth.sso/Logout' + +''' +Theme, see also: +http://rdmo.readthedocs.io/en/latest/configuration/themes.html +''' + +# THEME_DIR = os.path.join(BASE_DIR, 'theme') + +''' +Export Formats +''' + +# from django.utils.translation import ugettext_lazy as _ +# EXPORT_FORMATS = ( +# ('pdf', _('PDF')), +# ('rtf', _('Rich Text Format')), +# ('odt', _('Open Office')), +# ('docx', _('Microsoft Office')), +# ('html', _('HTML')), +# ('markdown', _('Markdown')), +# ('mediawiki', _('mediawiki')), +# ('tex', _('LaTeX')) +# ) + +''' +Cache, see also: +http://rdmo.readthedocs.io/en/latest/configuration/cache.html +''' + +# CACHES = { +# 'default': { +# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', +# 'LOCATION': '127.0.0.1:11211', +# 'KEY_PREFIX': 'rdmo_default' +# }, +# 'api': { +# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', +# 'LOCATION': '127.0.0.1:11211', +# 'KEY_PREFIX': 'rdmo_api' +# }, +# } + +''' +Logging configuration +''' + +# import os +# from . import BASE_DIR + +# LOGGING_DIR = os.path.join(BASE_DIR, 'log') +# LOGGING = { +# 'version': 1, +# 'disable_existing_loggers': True, +# 'filters': { +# 'require_debug_false': { +# '()': 'django.utils.log.RequireDebugFalse' +# }, +# 'require_debug_true': { +# '()': 'django.utils.log.RequireDebugTrue' +# } +# }, +# 'formatters': { +# 'default': { +# 'format': '[%(asctime)s] %(levelname)s: %(message)s' +# }, +# 'name': { +# 'format': '[%(asctime)s] %(levelname)s %(name)s: %(message)s' +# }, +# 'console': { +# 'format': '[%(asctime)s] %(message)s' +# } +# }, +# 'handlers': { +# 'mail_admins': { +# 'level': 'ERROR', +# 'filters': ['require_debug_false'], +# 'class': 'django.utils.log.AdminEmailHandler' +# }, +# 'error_log': { +# 'level': 'ERROR', +# 'class':'logging.FileHandler', +# 'filename': os.path.join(LOGGING_DIR, 'error.log'), +# 'formatter': 'default' +# }, +# 'rdmo_log': { +# 'level': 'DEBUG', +# 'class':'logging.FileHandler', +# 'filename': os.path.join(LOGGING_DIR, 'rdmo.log'), +# 'formatter': 'name' +# }, +# 'console': { +# 'level': 'DEBUG', +# 'filters': ['require_debug_true'], +# 'class': 'logging.StreamHandler', +# 'formatter': 'console' +# } +# }, +# 'loggers': { +# 'django': { +# 'handlers': ['console'], +# 'level': 'INFO', +# }, +# 'django.request': { +# 'handlers': ['mail_admins', 'error_log'], +# 'level': 'ERROR', +# 'propagate': True +# }, +# 'rdmo': { +# 'handlers': ['rdmo_log'], +# 'level': 'DEBUG', +# 'propagate': False +# } +# } +# } + +VENDOR_CDN = False + +THEME_DIR = os.path.join(BASE_DIR, 'theme') + diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..bb583dd --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,36 @@ +--- +version: '3' + +volumes: + local_postgres_data: {} + local_postgres_data_backups: {} + +services: + django: + build: + context: . + dockerfile: ./compose/django/Dockerfile + image: discuss_data_local_django +# image: docker.gitlab.gwdg.de/discuss-data/discuss-data:latest + depends_on: + - postgres + volumes: + - ./theme:/app/theme + env_file: + - ./.envs/.django + - ./.envs/.postgres + ports: + - "8000:8000" + command: /start + + postgres: + build: + context: . + dockerfile: ./compose/postgres/Dockerfile + image: discuss_data_production_postgres + volumes: + - local_postgres_data:/var/lib/postgresql/data + - local_postgres_data_backups:/backups + env_file: + - ./.envs/.postgres + -- GitLab From fcf43aa09cce7c44baef9134e14e06834720367d Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Sat, 22 Feb 2020 00:48:23 +0100 Subject: [PATCH 02/10] use gitlab-ci to publish docker images --- .gitlab-ci.yml | 24 +++ compose/django/Dockerfile | 1 + compose/django/Dockerfile.deploy | 38 +++++ config/deploy.py | 263 +++++++++++++++++++++++++++++++ 4 files changed, 326 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 compose/django/Dockerfile.deploy create mode 100644 config/deploy.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..827615b --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,24 @@ +image: docker:19.03.0 + +services: + - docker:19.03.0-dind + +variables: + CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA + +before_script: + - docker info + - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN $CI_REGISTRY + +stages: + - build + +build_develop: + stage: build + script: + - docker build -t $CONTAINER_TEST_IMAGE -f compose/django/Dockerfile . + - docker push $CONTAINER_TEST_IMAGE + except: + - master + + diff --git a/compose/django/Dockerfile b/compose/django/Dockerfile index 4e2c5cd..d08fa80 100644 --- a/compose/django/Dockerfile +++ b/compose/django/Dockerfile @@ -36,3 +36,4 @@ ENV DJANGO_SECRET_KEY=NOT_SO_SECRET_TEMP_KEY RUN python3 manage.py download_vendor_files ENTRYPOINT ["/entrypoint"] + diff --git a/compose/django/Dockerfile.deploy b/compose/django/Dockerfile.deploy new file mode 100644 index 0000000..802b69c --- /dev/null +++ b/compose/django/Dockerfile.deploy @@ -0,0 +1,38 @@ +FROM debian:buster-slim + +RUN apt update -y && apt upgrade -y && apt install -y \ + python3 \ + python3-dev \ + python3-pip \ + git + + +# postgres +RUN apt install -y \ + libpq-dev \ + postgresql-client \ + python3-psycopg2 + + +RUN git clone https://github.com/rdmorganiser/rdmo-app.git /app + +WORKDIR /app + +RUN pip3 install rdmo + +COPY ./theme /app/theme +COPY ./locale /app/locale + +COPY ./config/deploy.py /app/config/settings/local.py + +COPY ./compose/django/entrypoint /entrypoint +RUN chmod +x /entrypoint + +COPY ./compose/django/start /start +RUN chmod +x /start + +ENV DJANGO_SECRET_KEY=NOT_SO_SECRET_TEMP_KEY +RUN python3 manage.py download_vendor_files + +ENTRYPOINT ["/entrypoint"] + diff --git a/config/deploy.py b/config/deploy.py new file mode 100644 index 0000000..ba54eba --- /dev/null +++ b/config/deploy.py @@ -0,0 +1,263 @@ +import os +from . import BASE_DIR + +''' +Debug mode, don't use this in production +''' + +DEBUG = False + + +''' +A secret key for a particular Django installation. This is used to provide +cryptographic signing, and should be set to a unique, unpredictable value. +''' + +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY') + +''' +The list of URLs und which this application available +''' + +ALLOWED_HOSTS = ['localhost', 'ip6-localhost', '127.0.0.1', '[::1]'] + +''' +The root url of your application, only needed when its not '/' +''' + +# BASE_URL = '/path' + +''' +Language code and time zone +''' +LANGUAGE_CODE = 'de-de' +TIME_ZONE = 'Europe/Berlin' + +''' +The database connection to be used, see also: +http://rdmo.readthedocs.io/en/latest/configuration/databases.html +''' + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': os.getenv('POSTGRES_DB'), + 'USER': os.getenv('POSTGRES_USER'), + 'PASSWORD': os.getenv('POSTGRES_PASSWORD'), + 'HOST': os.getenv('POSTGRES_HOST'), + 'PORT': os.getenv('POSTGRES_PORT'), + } +} + +''' +E-Mail configuration, see also: +http://rdmo.readthedocs.io/en/latest/configuration/email.html +''' + +# EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend' +# EMAIL_HOST = 'localhost' +# EMAIL_PORT = '25' +# EMAIL_HOST_USER = '' +# EMAIL_HOST_PASSWORD = '' +# EMAIL_USE_TLS = False +# EMAIL_USE_SSL = False +# DEFAULT_FROM_EMAIL = '' + +''' +Allauth configuration, see also: +http://rdmo.readthedocs.io/en/latest/configuration/authentication/allauth.html +''' + +# from rdmo.core.settings import INSTALLED_APPS, AUTHENTICATION_BACKENDS +# +# ACCOUNT = True +# ACCOUNT_SIGNUP = True +# SOCIALACCOUNT = False +# +# INSTALLED_APPS += [ +# 'allauth', +# 'allauth.account', +# 'allauth.socialaccount', +# 'allauth.socialaccount.providers.facebook', +# 'allauth.socialaccount.providers.github', +# 'allauth.socialaccount.providers.google', +# 'allauth.socialaccount.providers.orcid', +# 'allauth.socialaccount.providers.twitter', +# ] +# +# AUTHENTICATION_BACKENDS.append('allauth.account.auth_backends.AuthenticationBackend') + +''' +LDAP, see also: +http://rdmo.readthedocs.io/en/latest/configuration/authentication/ldap.html +''' + +# import ldap +# from django_auth_ldap.config import LDAPSearch +# from rdmo.core.settings import AUTHENTICATION_BACKENDS +# +# PROFILE_UPDATE = False +# +# AUTH_LDAP_SERVER_URI = "ldap://ldap.example.com" +# AUTH_LDAP_BIND_DN = "cn=admin,dc=ldap,dc=example,dc=com" +# AUTH_LDAP_BIND_PASSWORD = "admin" +# AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=ldap,dc=example,dc=com", ldap.SCOPE_SUBTREE, "(uid=%(user)s)") +# +# AUTH_LDAP_USER_ATTR_MAP = { +# "first_name": "givenName", +# "last_name": "sn", +# 'email': 'mail' +# } +# +# AUTHENTICATION_BACKENDS.insert( +# AUTHENTICATION_BACKENDS.index('django.contrib.auth.backends.ModelBackend'), +# 'django_auth_ldap.backend.LDAPBackend' +# ) + +''' +Shibboleth, see also: +http://rdmo.readthedocs.io/en/latest/configuration/authentication/shibboleth.html +''' + +# from rdmo.core.settings import INSTALLED_APPS, AUTHENTICATION_BACKENDS, MIDDLEWARE_CLASSES +# +# SHIBBOLETH = True +# PROFILE_UPDATE = False +# +# INSTALLED_APPS += ['shibboleth'] +# +# SHIBBOLETH_ATTRIBUTE_MAP = { +# 'uid': (True, 'username'), +# 'givenName': (True, 'first_name'), +# 'sn': (True, 'last_name'), +# 'mail': (True, 'email'), +# } +# +# AUTHENTICATION_BACKENDS.append('shibboleth.backends.ShibbolethRemoteUserBackend') +# +# MIDDLEWARE_CLASSES.insert( +# MIDDLEWARE_CLASSES.index('django.contrib.auth.middleware.AuthenticationMiddleware') + 1, +# 'shibboleth.middleware.ShibbolethRemoteUserMiddleware' +# ) +# +# LOGIN_URL = '/Shibboleth.sso/Login?target=/projects' +# LOGOUT_URL = '/Shibboleth.sso/Logout' + +''' +Theme, see also: +http://rdmo.readthedocs.io/en/latest/configuration/themes.html +''' + +# THEME_DIR = os.path.join(BASE_DIR, 'theme') + +''' +Export Formats +''' + +# from django.utils.translation import ugettext_lazy as _ +# EXPORT_FORMATS = ( +# ('pdf', _('PDF')), +# ('rtf', _('Rich Text Format')), +# ('odt', _('Open Office')), +# ('docx', _('Microsoft Office')), +# ('html', _('HTML')), +# ('markdown', _('Markdown')), +# ('mediawiki', _('mediawiki')), +# ('tex', _('LaTeX')) +# ) + +''' +Cache, see also: +http://rdmo.readthedocs.io/en/latest/configuration/cache.html +''' + +# CACHES = { +# 'default': { +# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', +# 'LOCATION': '127.0.0.1:11211', +# 'KEY_PREFIX': 'rdmo_default' +# }, +# 'api': { +# 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', +# 'LOCATION': '127.0.0.1:11211', +# 'KEY_PREFIX': 'rdmo_api' +# }, +# } + +''' +Logging configuration +''' + +# import os +# from . import BASE_DIR + +# LOGGING_DIR = os.path.join(BASE_DIR, 'log') +# LOGGING = { +# 'version': 1, +# 'disable_existing_loggers': True, +# 'filters': { +# 'require_debug_false': { +# '()': 'django.utils.log.RequireDebugFalse' +# }, +# 'require_debug_true': { +# '()': 'django.utils.log.RequireDebugTrue' +# } +# }, +# 'formatters': { +# 'default': { +# 'format': '[%(asctime)s] %(levelname)s: %(message)s' +# }, +# 'name': { +# 'format': '[%(asctime)s] %(levelname)s %(name)s: %(message)s' +# }, +# 'console': { +# 'format': '[%(asctime)s] %(message)s' +# } +# }, +# 'handlers': { +# 'mail_admins': { +# 'level': 'ERROR', +# 'filters': ['require_debug_false'], +# 'class': 'django.utils.log.AdminEmailHandler' +# }, +# 'error_log': { +# 'level': 'ERROR', +# 'class':'logging.FileHandler', +# 'filename': os.path.join(LOGGING_DIR, 'error.log'), +# 'formatter': 'default' +# }, +# 'rdmo_log': { +# 'level': 'DEBUG', +# 'class':'logging.FileHandler', +# 'filename': os.path.join(LOGGING_DIR, 'rdmo.log'), +# 'formatter': 'name' +# }, +# 'console': { +# 'level': 'DEBUG', +# 'filters': ['require_debug_true'], +# 'class': 'logging.StreamHandler', +# 'formatter': 'console' +# } +# }, +# 'loggers': { +# 'django': { +# 'handlers': ['console'], +# 'level': 'INFO', +# }, +# 'django.request': { +# 'handlers': ['mail_admins', 'error_log'], +# 'level': 'ERROR', +# 'propagate': True +# }, +# 'rdmo': { +# 'handlers': ['rdmo_log'], +# 'level': 'DEBUG', +# 'propagate': False +# } +# } +# } + +VENDOR_CDN = False + +THEME_DIR = os.path.join(BASE_DIR, 'theme') + -- GitLab From aa46537f20c86f3617ac2ea90ffdc027655fba14 Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Mon, 24 Feb 2020 14:01:14 +0100 Subject: [PATCH 03/10] minor changes --- README.md | 5 ++++- compose/django/start | 2 +- docker-compose.yaml | 5 ++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 2bf200f..2838419 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ enter running container docker exec -it plan-git_django_1 bash +logs + + docker logs rdmo_django_1 + ## compose build images: @@ -30,7 +34,6 @@ create super-user (on first run) - # Usage (without docker - old) Clone the repo next to the rdmo-app dir and add symbolic links diff --git a/compose/django/start b/compose/django/start index d7c985b..10d595b 100644 --- a/compose/django/start +++ b/compose/django/start @@ -12,4 +12,4 @@ set -o nounset python3 manage.py migrate # initializes the database python3 manage.py setup_groups # creates groups with different permissions -python3 manage.py runserver 0.0.0.0:8000 +python3 manage.py runserver 0.0.0.0:8001 diff --git a/docker-compose.yaml b/docker-compose.yaml index bb583dd..51fe6fd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,8 +10,7 @@ services: build: context: . dockerfile: ./compose/django/Dockerfile - image: discuss_data_local_django -# image: docker.gitlab.gwdg.de/discuss-data/discuss-data:latest + image: rdmo_local_django depends_on: - postgres volumes: @@ -27,7 +26,7 @@ services: build: context: . dockerfile: ./compose/postgres/Dockerfile - image: discuss_data_production_postgres + image: rdmo_production_postgres volumes: - local_postgres_data:/var/lib/postgresql/data - local_postgres_data_backups:/backups -- GitLab From 5557ecd717f0f155d269435f012f67fe63ce690e Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Mon, 24 Feb 2020 14:03:31 +0100 Subject: [PATCH 04/10] typo --- compose/django/start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/django/start b/compose/django/start index 10d595b..d7c985b 100644 --- a/compose/django/start +++ b/compose/django/start @@ -12,4 +12,4 @@ set -o nounset python3 manage.py migrate # initializes the database python3 manage.py setup_groups # creates groups with different permissions -python3 manage.py runserver 0.0.0.0:8001 +python3 manage.py runserver 0.0.0.0:8000 -- GitLab From c2e79529f9e888168129949e46ad25e6caa3b452 Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Mon, 24 Feb 2020 14:05:40 +0100 Subject: [PATCH 05/10] do we need dind? --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 827615b..d567eea 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ image: docker:19.03.0 -services: - - docker:19.03.0-dind +#services: +# - docker:19.03.0-dind variables: CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA -- GitLab From 49b2bd24893326c1d8556cbd2726a8d5ce3c50fd Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Mon, 24 Feb 2020 14:08:38 +0100 Subject: [PATCH 06/10] sure we need dind! --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d567eea..827615b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ image: docker:19.03.0 -#services: -# - docker:19.03.0-dind +services: + - docker:19.03.0-dind variables: CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA -- GitLab From 5493d62145af7031be44624e046d8100a811c8fe Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Mon, 24 Feb 2020 23:56:49 +0100 Subject: [PATCH 07/10] dockerfile for production --- .gitlab-ci.yml | 2 +- compose/django/Dockerfile.deploy | 38 -------------------- compose/django/Dockerfile.production | 52 ++++++++++++++++++++++++++++ config/{deploy.py => production.py} | 4 +-- 4 files changed, 55 insertions(+), 41 deletions(-) delete mode 100644 compose/django/Dockerfile.deploy create mode 100644 compose/django/Dockerfile.production rename config/{deploy.py => production.py} (99%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 827615b..c916b66 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ stages: build_develop: stage: build script: - - docker build -t $CONTAINER_TEST_IMAGE -f compose/django/Dockerfile . + - docker build -t $CONTAINER_TEST_IMAGE -f compose/django/Dockerfile.production . - docker push $CONTAINER_TEST_IMAGE except: - master diff --git a/compose/django/Dockerfile.deploy b/compose/django/Dockerfile.deploy deleted file mode 100644 index 802b69c..0000000 --- a/compose/django/Dockerfile.deploy +++ /dev/null @@ -1,38 +0,0 @@ -FROM debian:buster-slim - -RUN apt update -y && apt upgrade -y && apt install -y \ - python3 \ - python3-dev \ - python3-pip \ - git - - -# postgres -RUN apt install -y \ - libpq-dev \ - postgresql-client \ - python3-psycopg2 - - -RUN git clone https://github.com/rdmorganiser/rdmo-app.git /app - -WORKDIR /app - -RUN pip3 install rdmo - -COPY ./theme /app/theme -COPY ./locale /app/locale - -COPY ./config/deploy.py /app/config/settings/local.py - -COPY ./compose/django/entrypoint /entrypoint -RUN chmod +x /entrypoint - -COPY ./compose/django/start /start -RUN chmod +x /start - -ENV DJANGO_SECRET_KEY=NOT_SO_SECRET_TEMP_KEY -RUN python3 manage.py download_vendor_files - -ENTRYPOINT ["/entrypoint"] - diff --git a/compose/django/Dockerfile.production b/compose/django/Dockerfile.production new file mode 100644 index 0000000..7a1250e --- /dev/null +++ b/compose/django/Dockerfile.production @@ -0,0 +1,52 @@ +FROM debian:buster-slim + +ENV PYTHONUNBUFFERED 1 + +# install required +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 \ + python3-pkg-resources \ + python3-psycopg2 \ + && rm -rf /var/lib/apt/lists/* + +# download rdmo-app rom github +RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates \ + && mkdir /app \ + && curl -SL https://github.com/rdmorganiser/rdmo-app/archive/master.tar.gz \ + | tar --strip-components=1 -xzC /app \ + && apt-get remove -y curl ca-certificates\ + && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# pip3 install rdmo +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3-pip python3-dev python3-setuptools gcc python3-wheel \ + && pip3 install rdmo \ + && apt-get remove -y python3-pip python3-dev python3-setuptools gcc python3-wheel \ + && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* + +#COPY ./theme /app/theme +COPY ./locale /app/locale + +COPY ./config/production.py /app/config/settings/local.py + +# download vendor files +ENV DJANGO_SECRET_KEY=NOT_SO_SECRET_TEMP_KEY +RUN python3 manage.py download_vendor_files + +RUN addgroup --system django \ + && adduser --system --ingroup django django \ + && chown -R django /app + +COPY ./compose/django/entrypoint /entrypoint +COPY ./compose/django/start /start + +RUN chmod +x /entrypoint /start \ + && chown django /entrypoint /start + +USER django +ENTRYPOINT ["/entrypoint"] + diff --git a/config/deploy.py b/config/production.py similarity index 99% rename from config/deploy.py rename to config/production.py index ba54eba..9c13143 100644 --- a/config/deploy.py +++ b/config/production.py @@ -5,8 +5,8 @@ from . import BASE_DIR Debug mode, don't use this in production ''' -DEBUG = False - +#DEBUG = False +DEBUG = True # todo: serve assets on docker ''' A secret key for a particular Django installation. This is used to provide -- GitLab From 1d072d523b248d65f929c37f18440cc1db8a2440 Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Tue, 25 Feb 2020 00:58:33 +0100 Subject: [PATCH 08/10] use gunicorn for production --- compose/django/Dockerfile.production | 9 +++---- compose/django/start.production | 8 +++++++ config/production.py | 5 ++-- docker-compose.prod.yaml | 35 ++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 7 deletions(-) create mode 100644 compose/django/start.production create mode 100644 docker-compose.prod.yaml diff --git a/compose/django/Dockerfile.production b/compose/django/Dockerfile.production index 7a1250e..91c38a4 100644 --- a/compose/django/Dockerfile.production +++ b/compose/django/Dockerfile.production @@ -20,10 +20,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certifi WORKDIR /app -# pip3 install rdmo +# install rdmo and gunicorn RUN apt-get update && apt-get install -y --no-install-recommends \ python3-pip python3-dev python3-setuptools gcc python3-wheel \ && pip3 install rdmo \ + && pip3 install gunicorn \ && apt-get remove -y python3-pip python3-dev python3-setuptools gcc python3-wheel \ && apt-get -y autoremove \ && rm -rf /var/lib/apt/lists/* @@ -37,12 +38,12 @@ COPY ./config/production.py /app/config/settings/local.py ENV DJANGO_SECRET_KEY=NOT_SO_SECRET_TEMP_KEY RUN python3 manage.py download_vendor_files -RUN addgroup --system django \ - && adduser --system --ingroup django django \ +RUN addgroup --system --gid 200 django \ + && adduser --system --uid 200 --ingroup django django \ && chown -R django /app COPY ./compose/django/entrypoint /entrypoint -COPY ./compose/django/start /start +COPY ./compose/django/start.production /start RUN chmod +x /entrypoint /start \ && chown django /entrypoint /start diff --git a/compose/django/start.production b/compose/django/start.production new file mode 100644 index 0000000..554f4e6 --- /dev/null +++ b/compose/django/start.production @@ -0,0 +1,8 @@ +#!/bin/bash + +set -o errexit +set -o pipefail +set -o nounset + +python3 /app/manage.py collectstatic --noinput +/usr/local/bin/gunicorn config.wsgi --bind 0.0.0.0:5000 --chdir=/app diff --git a/config/production.py b/config/production.py index 9c13143..df0eaa1 100644 --- a/config/production.py +++ b/config/production.py @@ -5,8 +5,7 @@ from . import BASE_DIR Debug mode, don't use this in production ''' -#DEBUG = False -DEBUG = True # todo: serve assets on docker +DEBUG = False ''' A secret key for a particular Django installation. This is used to provide @@ -259,5 +258,5 @@ Logging configuration VENDOR_CDN = False -THEME_DIR = os.path.join(BASE_DIR, 'theme') +#THEME_DIR = os.path.join(BASE_DIR, 'theme') diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml new file mode 100644 index 0000000..3945bfa --- /dev/null +++ b/docker-compose.prod.yaml @@ -0,0 +1,35 @@ +--- +version: '3' + +volumes: + local_postgres_data: {} + local_postgres_data_backups: {} + +services: + django: + build: + context: . + dockerfile: ./compose/django/Dockerfile.production + image: rdmo_local_django + depends_on: + - postgres + volumes: + - /var/www/html/static:/app/static_root + env_file: + - ./.envs/.django + - ./.envs/.postgres + ports: + - "5000:5000" + command: /start + + postgres: + build: + context: . + dockerfile: ./compose/postgres/Dockerfile + image: rdmo_production_postgres + volumes: + - local_postgres_data:/var/lib/postgresql/data + - local_postgres_data_backups:/backups + env_file: + - ./.envs/.postgres + -- GitLab From 5602fabb21e30498676cf842acc78aca30c74d71 Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Tue, 25 Feb 2020 01:22:17 +0100 Subject: [PATCH 09/10] devkey should not look to secret ;-) --- .envs/.django | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.envs/.django b/.envs/.django index cb22ce2..6bc9faa 100644 --- a/.envs/.django +++ b/.envs/.django @@ -1,3 +1,3 @@ # General # ------------------------------------------------------------------------------ -DJANGO_SECRET_KEY=LP3CtSG5ZjMaVwgWa1ahKIT5XrPsbnGPAM1oE6OeXD3ZwSWdmDp6DSv3tQob +DJANGO_SECRET_KEY=NOT_SO_SECRET_DEVKEY -- GitLab From 9b56bac9c1470dae27533a21101f740776ca716d Mon Sep 17 00:00:00 2001 From: Ubbo Veentjer <veentjer@sub.uni-goettingen.de> Date: Tue, 25 Feb 2020 01:32:50 +0100 Subject: [PATCH 10/10] tag docker images from dev branch --- .gitlab-ci.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c916b66..088c4d8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ services: variables: CONTAINER_TEST_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA + CONTAINER_DEPLOY_IMAGE: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:latest before_script: - docker info @@ -12,6 +13,7 @@ before_script: stages: - build + - deploy build_develop: stage: build @@ -21,4 +23,12 @@ build_develop: except: - master +deploy_develop: + stage: deploy + script: + - docker pull $CONTAINER_TEST_IMAGE + - docker tag $CONTAINER_TEST_IMAGE $CONTAINER_DEPLOY_IMAGE + - docker push $CONTAINER_DEPLOY_IMAGE + only: + - develop -- GitLab