Added metrics logging to docker images. Allows to be run as cron

or in a while loop.

.dockerignore

@@ -38,6 +38,13 @@ node_modules
 !docker/common.sh
 !packages/configs/snip/defaults/*.yaml
 
+# Metrics scripts
+!docker/metrics/*
+# Logrotate
+!docker/logrotate/*
+# Entrypoints
+!docker/entrypoints/*
+
 # Allow test db
 !packages/database/test_setup
 

apps/nginx/nginx.conf

@@ -20,13 +20,16 @@ http {
     proxy_ssl_server_name on;
 
     # Get the port that is show in the url of the request
-    # needed for propper rewrite of the url using x-forwarded-port
+    # needed for proper rewrite of the url using x-forwarded-port
     # see proxy_header.conf for usage
     map $http_host $port {
         default 443;
         "~^[^\:]+:(?<p>\d+)$" $p;
     }
 
+    map "$time_iso8601 # $msec" $time_iso8601_ms {
+        "~(^[^+]+)(\+[0-9:]+) # \d+\.(\d+)$" $1.$3$2; 
+    }
     server {
         listen 80;
         return 301 https://$host$request_uri;
@@ -132,16 +135,21 @@ http {
         gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
     }
 
-    # Logging everything but 1xx and 2xx and 3xx responses otherwise logs get very big very quickly
-    map $status $loggable {
-        ~^[123]  0;
-        default 1;
-    }
-
-    log_format form '$remote_addr - $remote_user [$time_local] '
-                    '"$request" $status $body_bytes_sent '
-                    '"$http_referer" "$http_user_agent" "$http_cookie"';
-
-
-    access_log /var/log/nginx/access.log form if=$loggable;
+    # Just log all :)
+    log_format json_combined escape=json
+    '{'
+        '"time_local":"$time_iso8601_ms",'
+        '"remote_addr":"$remote_addr",'
+        '"request":"$request",'
+        '"status": $status,'
+        '"body_bytes_sent":$body_bytes_sent,'
+        '"request_time":$request_time,'
+        '"http_referrer":"$http_referer",'
+        '"http_user_agent":"$http_user_agent",'
+        '"upstream_response_time":$upstream_response_time'
+    '}';
+
+
+    access_log /logs/nginx/access.log json_combined;
+    error_log /logs/nginx/error.log;
 }

docker/Dockerfile

 FROM node:22-slim AS base
 
-ENV PNPM_HOME="/pnpm"
-ENV PATH="$PNPM_HOME:$PATH"
-RUN corepack enable
 
-WORKDIR /app
+# Create config and logs folder
+# all apps should read and write to /config and /logs
+RUN mkdir -p /config
+RUN mkdir -p /logs
 
-# Install yq for yaml parsing
-RUN apt-get update && apt-get install -y curl pwgen
+# Dependencies used in all container targets (config parsing, metrics, etc.)
+RUN apt-get update && apt-get install -y curl pwgen logrotate util-linux 
 RUN curl -LO https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64
 RUN mv yq_linux_amd64 /usr/bin/yq
 RUN chmod +x /usr/bin/yq
 
+# Install corepack and pnpm
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable && corepack prepare pnpm@9.x.x --activate
+
+# Allow to collect metrics (is setup in the entrypoint)
+# also rotate logs to not fill up the disk too 
+COPY ./docker/metrics /entry/metrics
+COPY ./docker/logrotate/service.sh /entry/logrotate/service.sh
+COPY ./docker/logrotate/metrics.conf /etc/logrotate.d/metrics
+RUN chmod 0644 /etc/logrotate.d/metrics
+# Copy shared entrypoint (runs the metrics and logrotate service)
+COPY ./docker/entrypoints/common.sh /entry/common.sh
+COPY ./docker/entrypoints/entrypoint.sh /entry/entrypoint.sh
+# Configuration templates (needed if the app is started without a config)
+COPY ./packages/configs/snip/defaults /config_templates
+
+WORKDIR /app
+
 # Set timezone
 # todo: allow to set timezone via env variable
 ENV TZ=Europe/Berlin
@@ -36,17 +55,13 @@ FROM build_deps AS build
 COPY pnpm-workspace.yaml pnpm-lock.yaml package.json ./
 COPY packages ./packages
 
-# Build the packages
+# Build all the packages
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile 
 RUN pnpm --filter "./packages/**" -r build
 RUN pnpm --filter "./packages/**" -r build:cleanup
 
-
-# ##############################
-# Image service
-# ##############################
+# --------------------------- Image render service --------------------------- #
 FROM build AS build_images
-
 COPY ./apps/images ./apps/images
 
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile
@@ -57,17 +72,17 @@ RUN pnpm ---filter "./apps/images" bundle
 RUN pnpm --filter=images --prod deploy /prod/images
 
 FROM base AS images
-
-RUN apt-get update && apt-get install -y -q --no-install-recommends  \
-    libfontconfig1 
+# Copy the bundled app to the final image
 COPY --from=build_images /prod/images /app
+RUN apt-get update && apt-get install -y -q --no-install-recommends libfontconfig1 
 COPY ./assets/fonts /assets/fonts
+
+ENV LOG_DIR="/logs/images"
+ENTRYPOINT [ "/entry/entrypoint.sh"]
 CMD ["npm", "run","--silent", "prod"]
 
 
-# ##############################
-# Email service
-# ##############################
+# ------------------------------- Email service ------------------------------ #
 FROM build AS build_email 
 
 COPY ./apps/email ./apps/email
@@ -81,14 +96,14 @@ RUN pnpm --filter "./apps/email" bundle
 RUN pnpm --filter=email --prod deploy /prod/email
 
 FROM base AS email
-
 COPY --from=build_email /prod/email /app
+
+ENV LOG_DIR="/logs/email"
+ENTRYPOINT [ "/entry/entrypoint.sh"]
 CMD ["npm", "run","--silent", "prod"]
 
 
-# ##############################
-# Websocket service
-# ##############################
+# ----------------------------- Websocket service ---------------------------- #
 FROM build AS build_socket
 
 COPY ./apps/socket ./apps/socket
@@ -103,13 +118,14 @@ RUN pnpm --filter=socket --prod deploy /prod/socket
 
 FROM base AS socket
 COPY --from=build_socket /prod/socket /app
+
+ENV LOG_DIR="/logs/socket"
+ENTRYPOINT [ "/entry/entrypoint.sh"]
 CMD ["npm", "run","--silent", "prod"]
 
-# ##############################
-# Next (frontend) service
-# ##############################
+# ----------------------------- Fullstack service ---------------------------- #
 # App build with nextjs needs types from the socket app
-# that's why we build the socket app first
+# that's why we build the socket app first and derive from it
 FROM build_socket AS build_next
 
 COPY ./apps/fullstack ./apps/fullstack
@@ -127,7 +143,6 @@ RUN pnpm --filter database_migrate build
 RUN pnpm --filter database_migrate bundle
 RUN pnpm --filter database_migrate --prod deploy /prod/database_migrate
 
-
 FROM base AS next
 
 RUN apt-get update && apt-get install -y apt-transport-https
@@ -145,21 +160,17 @@ COPY --from=build_next /prod/database_migrate /database/migration_script
 COPY ./database/migration /database/migration
 COPY ./docs /docs
 
-# Configuration
-COPY ./packages/configs/snip/defaults /config_templates
-
 # Entrypoint
-COPY ./docker/entrypoint_next.sh /entry/entrypoint_next.sh
-COPY ./docker/common.sh /entry/common.sh
-RUN chmod +x /entry/entrypoint_next.sh
-ENTRYPOINT ["/entry/entrypoint_next.sh"]
+COPY ./docker/entrypoints/entrypoint.next.sh /entry/entrypoint.next.sh
+
+ENV LOG_DIR="/logs/next"
+ENTRYPOINT ["/entry/entrypoint.sh", "/entry/entrypoint.next.sh"]
 CMD ["pnpm", "--filter", "fullstack", "prod"]
 
 
 # ---------------------------------------------------------------------------- #
 #                                DEV environment                               #
 # ---------------------------------------------------------------------------- #
-
 FROM build_deps AS dev_deps
 
 # Install all pnpm dependencies so we can use them in the dev container
@@ -169,6 +180,7 @@ COPY ./packages ./packages
 COPY ./apps/**/package.json ./apps/**/
 
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install
+
 # We also expect the apps to be live mounted from this point
 
 FROM dev_deps AS dev_next
@@ -188,21 +200,29 @@ COPY ./apps/fullstack ./apps/fullstack
 RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install
 
 # Entrypoint
-COPY ./docker/entrypoint_next.sh /entry/entrypoint_next.sh
-COPY ./docker/common.sh /entry/common.sh
-RUN chmod +x /entry/entrypoint_next.sh
-ENTRYPOINT ["/entry/entrypoint_next.sh"]
-
+USER root
+COPY ./docker/entrypoints/entrypoint.next.sh /entry/entrypoint.next.sh
+ENV LOG_DIR="/logs/next"
+ENTRYPOINT ["/entry/entrypoint.sh", "/entry/entrypoint.next.sh"]
 CMD ["pnpm", "--filter", "fullstack", "dev"]
 
 FROM dev_deps AS dev_images
 COPY ./apps/images ./apps/images
+
+ENV LOG_DIR="/logs/images"
+ENTRYPOINT [ "/entry/entrypoint.sh"]
 CMD ["pnpm", "--filter", "images", "dev"]
 
 FROM dev_deps AS dev_email
 COPY ./apps/email ./apps/email
+
+ENV LOG_DIR="/logs/email"
+ENTRYPOINT [ "/entry/entrypoint.sh"]
 CMD ["pnpm", "--filter", "email", "dev"]
 
 FROM dev_deps AS dev_socket
+
 COPY ./apps/socket ./apps/socket
+ENV LOG_DIR="/logs/socket"
+ENTRYPOINT [ "/entry/entrypoint.sh"]
 CMD ["pnpm", "--filter", "socket", "dev"]

docker/Dockerfile.mariadb

@@ -9,20 +9,14 @@ RUN chmod +x /usr/bin/yq
 # Set the working directory
 WORKDIR /usr/local/bin
 
-# Copy the custom entrypoint script into the container
-COPY ./docker/common.sh /usr/local/bin/common.sh
-COPY ./docker/entrypoint_mariadb.sh /usr/local/bin/entrypoint_mariadb.sh
-
 # COPY default yaml configuration
 COPY ./packages/configs/snip/defaults/database.yaml /config_templates/database.yaml
 
-# Ensure the script is executable
-RUN chmod +x /usr/local/bin/entrypoint_mariadb.sh
 
 ENV TZ=Europe/Berlin
 
 # Use the custom entrypoint script as the entrypoint
+COPY ./docker/entrypoints/common.sh /usr/local/bin/common.sh
+COPY ./docker/entrypoints/entrypoint.mariadb.sh /usr/local/bin/entrypoint_mariadb.sh
 ENTRYPOINT ["entrypoint_mariadb.sh"]
-
-# Default command (optional)
 CMD ["mariadbd"]
\ No newline at end of file

docker/Dockerfile.nginx

 FROM nginx:latest
 
-# Install openssl
-RUN apt-get update && apt-get install -y openssl
-
-# Create directories for SSL certificates
-RUN mkdir -p /ssl
+# Install dependencies
+RUN apt-get update && apt-get install -y openssl logrotate
 
 # Copy Nginx configuration files
 COPY ./apps/nginx /etc/nginx
 
+# Copy logrotate configuration files
+COPY ./docker/logrotate/service.sh /entry/logrotate/service.sh
+COPY ./docker/logrotate/nginx.conf /etc/logrotate.d/nginx
+RUN chmod 0644 /etc/logrotate.d/nginx
+
+# Remove default Nginx link to stdout and stderr
+RUN rm /var/log/nginx/*
+RUN mkdir -p /logs/nginx
 
 # Generate self-signed SSL certificates
+RUN mkdir -p /ssl
 RUN openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
     -keyout /ssl/certificate.key \
     -out /ssl/certificate.crt \
     -config /etc/nginx/ssl/openssl.cnf \
     -subj "/C=DE/ST=Lower Saxony/L=Goettingen/O=Snip Lab/"
 
+# Entrypoint
+ENV DISABLE_METRICS=true
+COPY ./docker/entrypoints/entrypoint.sh /entry/entrypoint.sh
+COPY ./docker/entrypoints/entrypoint.nginx.sh /entry/entrypoint.nginx.sh
+COPY ./docker/entrypoints/common.sh /entry/common.sh
+ENTRYPOINT ["/entry/entrypoint.sh", "/entry/entrypoint.nginx.sh", "/docker-entrypoint.sh"]
+# Have to reset CMD since it gets cleared when we set ENTRYPOINT
+CMD ["nginx", "-g", "daemon off;"]
\ No newline at end of file

docker/common.sh → docker/entrypoints/common.sh

@@ -59,4 +59,3 @@ wait_database_ready() {
         sleep 1
     done
 }
-

docker/entrypoint_next.sh → docker/entrypoints/entrypoint.next.sh

@@ -3,13 +3,6 @@
 # Source common functions
 source /entry/common.sh
 
-# Check if /config is mounted or available
-if [ ! -d /config ]; then
-    log "No /config directory found."
-    mkdir /config
-    log "Created /config directory."
-fi
-
 # Check if database and secrets config exists
 create_default_config "/config_templates/database.yaml" "/config/database.yaml"
 create_default_config "/config_templates/secrets.yaml" "/config/secrets.yaml"
@@ -52,6 +45,6 @@ else
     fi
 fi
 
-
-log "Running $@"
-exec "$@"
\ No newline at end of file
+# Run the command
+log "Running $*"
+exec "$@"

docker/entrypoints/entrypoint.nginx.sh

+# Make sure the logs dir exists
+mkdir -p /logs/nginx
+
+exec "$@"
\ No newline at end of file

docker/entrypoints/entrypoint.sh

+#!/bin/bash
+source /entry/common.sh
+
+# Setup metrics logging and log rotation
+if [ -z "$DISABLE_METRICS" ]; then
+    /entry/metrics/service.sh &
+fi
+/entry/logrotate/service.sh &
+
+# Run the secondary entrypoint (see Dockerfile)
+exec "$@"

docker/logrotate/metrics.conf

+/logs/*/metrics.log {
+    daily
+    missingok
+    rotate 31
+    notifempty
+    dateext
+    dateformat -%Y-%m-%d
+    compress
+}
\ No newline at end of file

docker/logrotate/nginx.conf

+/logs/nginx/access.log {
+	daily
+	missingok
+	rotate 31
+	notifempty
+	dateext
+	dateformat -%Y-%m-%d
+	compress
+	sharedscripts
+	postrotate
+		[ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`
+	endscript
+}
\ No newline at end of file

docker/logrotate/service.sh

+#!/bin/bash
+
+## Description:
+# Simple script that executes the logrotate every day at 1:00 AM
+
+# Function to print an error and exit
+function error_exit {
+    echo -e "\033[0;31m[Logrotate service] $1\033[0m"
+    exit 1
+}
+
+function log {
+    echo -e "[Logrotate service] $1"
+}
+
+# Change dir to the script path
+cd "$(dirname "$0")"
+
+# Test if cron is installed
+if ! command -v cron &> /dev/null; then
+    # While loop (infinite) to run the script every day at 1:00 AM
+    log "Running logrotate service..."
+    while true; do
+        # Get the current time in HH:MM format
+        current_time=$(date +%H:%M)
+
+        # Check if the current time is 01:00
+        if [[ "$current_time" == "01:00" ]]; then
+            # Run your command
+            logrotate /etc/logrotate.conf
+
+            # Sleep for 60 seconds to avoid running the command multiple times within the same minute
+            sleep 120
+        fi
+
+        # Sleep for 30 seconds to minimize CPU usage
+        sleep 30
+    done
+else
+    # Install as cron
+    log "Installing logrotate service as cron job..."
+    cron_job="0 1 * * * logrotate /etc/logrotate.conf"
+    # Write the cron job to the crontab
+    (crontab -l 2>/dev/null; echo "$cron_job") | crontab -
+
+    # Make sure cron is running
+    cron > /dev/null 2>&1
+fi
+
+

docker/metrics/collect.sh

+# Function to get metrics
+get_metrics() {
+    if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+        # cgroup v2
+        time=$(date +%s%N)
+
+        # In microseconds (*1000 to convert to nanoseconds)
+        cpu_usage=$(awk '/usage_usec/ {print $2 * 1000}' /sys/fs/cgroup/cpu.stat)
+        memory_usage=$(cat /sys/fs/cgroup/memory.current)
+        memory_limit=$(cat /sys/fs/cgroup/memory.max)
+    else
+        # cgroup v1
+        time=$(date +%s%N)
+
+        # In nanoseconds
+        cpu_usage=$(cat /sys/fs/cgroup/cpu/cpuacct.usage)
+        memory_usage=$(cat /sys/fs/cgroup/memory/memory.usage_in_bytes)
+        memory_limit=$(cat /sys/fs/cgroup/memory/memory.limit_in_bytes)
+    fi
+}
+
+# Get initial metrics
+get_metrics
+time_start=$time
+cpu_start=$cpu_usage
+
+# Wait for 1 second
+sleep 10
+
+# Get metrics again
+get_metrics
+time_end=$time
+cpu_end=$cpu_usage
+
+# Compute percentage CPU usage
+cpu_usage_percent=$(awk -v cpu_start="$cpu_start" -v cpu_end="$cpu_end" -v time_start="$time_start" -v time_end="$time_end" 'BEGIN { printf "%.4f", (cpu_end - cpu_start) / (time_end - time_start) * 100}')
+# Compute memory usage in MB
+memory_usage_mb=$(awk -v memory_usage="$memory_usage" 'BEGIN { printf "%.4f", memory_usage / 1024 / 1024}')
+# Get memory limit in MB
+if [ "$memory_limit" = "max" ] || [ "$memory_limit" = "max\n" ]; then
+    memory_limit=$(cat /proc/meminfo | grep MemTotal | awk '{print $2 * 1024}')
+    memory_limit_mb=$(awk -v memory_limit="$memory_limit" 'BEGIN { printf "%.4f", memory_limit / 1024 / 1024}')
+else
+    memory_limit_mb=$(awk -v memory_limit="$memory_limit" 'BEGIN { printf "%.4f", memory_limit / 1024 / 1024}')
+fi
+
+############################
+
+# Convert time to UTC ISO 8601 format
+time_start_iso=$(date -u -d "@$(($time_start / 1000000000))" +"%Y-%m-%dT%H:%M:%SZ")
+time_end_iso=$(date -u -d "@$(($time_end / 1000000000))" +"%Y-%m-%dT%H:%M:%SZ")
+
+# Echo measurement as json
+echo "{\"cpu_usage_percent\": $cpu_usage_percent, \"time_start\": \"$time_start_iso\", \"time_end\": \"$time_end_iso\", \"memory_usage_gb\": $memory_usage_mb, \"memory_limit_mb\": $memory_limit_mb}"

docker/metrics/service.sh

+#!/bin/bash
+
+## Description:
+# Simple script that executes the collect script in a loop every 5 minutes
+
+# Function to print an error and exit
+function error_exit {
+    echo -e "\033[0;31m[Metrics service] $1\033[0m"
+    exit 1
+}
+
+function log {
+    echo -e "[Metrics service] $1"
+}
+
+# Check if log folder exits
+if [[ ! -d "${LOG_DIR}" ]]; then
+    mkdir -p "${LOG_DIR}" || error_exit "Failed to create log folder."
+fi
+
+if [[ ! -f "${LOG_DIR}/metrics.log" ]]; then
+    touch "${LOG_DIR}/metrics.log" || error_exit "Failed to create log file."
+fi
+# Change dir to the script path
+cd "$(dirname "$0")"
+
+# While loop (infinite) to run the script every 5 minutes
+log "Running metrics service..."
+while true; do
+    start_time=$(date +%s)
+
+    ./collect.sh >> "${LOG_DIR}/metrics.log" 2>&1
+    
+    if [[ $? -ne 0 ]]; then
+        error_exit "Metrics service failed to run!"
+    fi
+
+    end_time=$(date +%s)
+    elapsed_time=$((end_time - start_time))
+    sleep_time=$((300 - elapsed_time))
+    
+    if [[ $sleep_time -gt 0 ]]; then
+        sleep $sleep_time
+    fi
+
+    if [[ $sleep_time -lt 0 ]]; then
+        error_exit "Metrics service took longer than 5 minutes to run! Stopping..."
+    fi
+done
+
+
+# Test if cron is installed
+if ! command -v cron &> /dev/null; then
+    # While loop (infinite) to run the script every day at 1:00 AM
+    log "Running metrics service..."
+    while true; do
+        start_time=$(date +%s)
+
+        ./collect.sh >> "${LOG_DIR}/metrics.log" 2>&1
+        
+        if [[ $? -ne 0 ]]; then
+            error_exit "Metrics service failed to run!"
+        fi
+
+        end_time=$(date +%s)
+        elapsed_time=$((end_time - start_time))
+        sleep_time=$((300 - elapsed_time))
+        
+        if [[ $sleep_time -gt 0 ]]; then
+            sleep $sleep_time
+        fi
+
+        if [[ $sleep_time -lt 0 ]]; then
+            error_exit "Metrics service took longer than 5 minutes to run! Stopping..."
+        fi
+    done
+else
+    # Install as cron
+    log "Installing metrics service as cron job..."
+    cron_job="*/5 * * * * /entry/metrics/collect.sh"
+    # Write the cron job to the crontab
+    (crontab -l 2>/dev/null; echo "$cron_job") | crontab -
+
+    # Make sure cron is running
+    cron > /dev/null 2>&1
+fi
\ No newline at end of file

docs/deployment/maintenance.mdx

@@ -48,3 +48,25 @@ This script uses the [mariabackup](https://mariadb.com/kb/en/mariabackup-overvie
 For an example you can see the `.gitlab/backup.yml` file in the root directory of the project. This file is used to create a backup of our main instance.
 
 Alternatively you may backup the full docker volume but this will increase the size of the backup significantly, and we do not recommend it. 
+
+## Logging and metrics
+
+By default each of our containers records its own usage metrics. You may find them within the `/logs` folder inside the containers. The files are in jsonl format, i.e. contain 
+
+Additionally, the nginx container also writes the access log to te logs folder. 
+
+We rotate all log files every day and keep the files for 31 days. At the moment this is not configurable. If you have a need to change this
+let us know.
+
+You may mount the `/logs` folder to a persistent storage to keep the logs or further process them.
+
+```yaml
+# e.g.
+services:
+    nginx:
+        volumes:
+        - ./logs:/logs
+    ...
+```
+
+We choose this approach as it is quite lightweight and doesn't require another container running for the only purpose of logging and monitoring.
\ No newline at end of file