Configured and tested new logging setup for ELK

.gitignore

@@ -39,3 +39,4 @@ anon-export/
 node_modules
 secret
 dump.sql
+.pytest_cache/

Makefile

@@ -35,4 +35,4 @@ coverage:
 	coverage html
 
 db:
-	docker run --rm --name $(DB_NAME) -p 5432:5432 postgres:9.5
+	docker run -d --name $(DB_NAME) -p 5432:5432 postgres:9.5

docs/deployment.md

+# Grady deployment instructions
+
+For every commit on master a Docker image is build which can be deployed
+anywhere. The current deployment configurations is explained in another repo.
+
+## Environment variables
+
+- `GRADY_LOG_LEVEL` Sets the log level for our custom logging configuration
+  (default: DEBUG).
+- `GRADY_LOG_FORMAT` Can be set to `json` in order to make logs readable
+  by Logstash (default: `default-format`).

grady/settings/default.py

@@ -87,11 +87,11 @@ WSGI_APPLICATION = 'grady.wsgi.application'
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
-        'NAME': 'postgres',
-        'USER': 'postgres',
-        'PASSWORD': 'postgres',
-        'HOST': 'localhost',
-        'PORT': '5432',
+        'NAME': os.environ.get('DB_NAME', 'postgres'),
+        'USER': os.environ.get('DB_USER', 'postgres'),
+        'PASSWORD': os.environ.get('DB_PASSWORD', 'postgres'),
+        'HOST': os.environ.get('DB_HOST', 'localhost'),
+        'PORT': os.environ.get('DB_PORT', '5432'),
     },
 }
 
@@ -146,47 +146,45 @@ JWT_AUTH = {
     'JWT_ALLOW_REFRESH': True,
 }
 
+
+LOG_LEVEL = os.environ.get('GRADY_LOG_LEVEL', 'DEBUG')
+LOG_FORMAT = os.environ.get('GRADY_LOG_FORMAT', 'default-format')
 LOGGING = {
     "version": 1,
-    "disable_existing_loggers": False,
+    "disable_existing_loggers": not DEBUG,
     "formatters": {
-        'core': {
-            'datefmt': '%Y-%m-%d %H:%M:%S',
+        'default-format': {
+            # 'datefmt': the default is ISO8601 which is what we want
             'format': '[%(asctime)s] [%(levelname)s] %(name)-20s %(message)s',
         },
-    },
-    'filters': {
-        'require_debug_true': {
-            '()': 'django.utils.log.RequireDebugTrue',
+        'json': {
+            'class': 'pythonjsonlogger.jsonlogger.JsonFormatter',
+            'format': '(asctime) (levelname) (name) (funcName) (message)'
         },
     },
     'handlers': {
         'console': {
-            'level': 'DEBUG',
-            'filters': ['require_debug_true'],
+            'level': LOG_LEVEL,
             'class': 'logging.StreamHandler',
-            'formatter': 'core'
+            'formatter': LOG_FORMAT
         },
-        'django': {
-            'level': 'INFO',
-            'class': 'logging.StreamHandler',
-            'formatter': 'core'
-        },
-        'mail_admins': {  # TODO: configuration
-            'level': 'ERROR',
-            'class': 'django.utils.log.AdminEmailHandler',
-        }
     },
     'loggers': {
         'django': {
-            'handlers': [],
+            'handlers': ['console'],
         },
         'django.request': {
-            'handlers': ['django'],
+            'handlers': ['console'],
+        },
+        'gunicorn.error': {
+            'handlers': ['console'],
+        },
+        'gunicorn.access': {
+            'handlers': ['console'],
         },
         'core': {
-            'handlers': ['console', 'mail_admins'],
-            'level': 'DEBUG',
+            'handlers': ['console'],
+            'level': LOG_LEVEL,
         }
     }
 }

requirements.txt

@@ -5,9 +5,10 @@ djangorestframework-jwt~=1.11.0
 djangorestframework~=3.8
 Django~=2.1
 drf-dynamic-fields~=0.3.0
-gevent~=1.2.2
+gevent~=1.3.2
 gunicorn~=19.7.0
 psycopg2-binary~=2.7.4
+python-json-logger~=0.1.9
 tqdm~=4.19.5
 whitenoise~=3.3.1
 xlrd~=1.0.0