↓Skip to main content

Loki & Grafana - Docker Compose Stack: Promtail Container for Container-, /var/log- and Systemd-Journal Logs; Promtail Bare-Metal for Systemd-Journal Logs; Traefik Reverse Proxy

2280 words·
Loki Grafana Promtail Docker-Compose
Table of Contents
Loki-Stack - This article is part of a series.
Part 1: This Article
Part 2: Loki, Grafana & Traefik - Docker Compose Stack: Deployment with GitLab CI Pipeline, Git Commands
Part 3: Loki, Grafana & Promtail Stack - Deployment with Helm Chart in a K3s Kubernetes Cluster, NodePort Service and Traefik Ingress with TLS Secret; Promtail Helm Deployment for External Loki Data Source
Part 4: Loki & Grafana - Kubernetes K8s Deployment with Several Loki Instances for Seperated Data Sources; Nginx Ingress with TLS Secret; Promtail Helm and Docker Container Deployment
Part 5: Loki: Automated Promtail Container Deployment with Ansible

Overview
#

Tutorial Infrastructure
#

In this tutorial I’m using five Debian 12 servers for the following deployment:

192.168.30.20 debian-01 # Loki & Grafana Docker Stack, Traefik Reverse Proxy Container
192.168.30.21 debian-02 # Promtail Container for Containers
192.168.30.22 ubuntu-01 # Promtail Container for /var/log
192.168.30.21 debian-03 # Promtail Container for Systemd-Journal
192.168.30.21 debian-04 # Promtail Bare-Metal for Systemd-Journal

Since the different Promtail configuration for containers, /var/log and systemd journal produce different labels, I have deployed 3 Loki instances in the Docker Compose stacks, so that I can use different data sources for each Promtail configuration type.

Loki-Stack Overview
#

  • Loki Is responsible for storing and querying log data. It receives logs from various agents like Promtail and offers a querying interface that can be utilized by tools like Grafana for visualizing and analyzing the logs.

  • Promtail is an agent which ships the contents of local logs to a Loki instance.

  • Grafana Used to visualize the logs collected by Loki.

Loki, Grafana & Traefik Reverse Proxy
#

Prerequisites
#

Docker Network
# 

# Create network used for Traefik to communicate with other Docker containers
sudo docker network create traefik

Docker System User
# 

# Create a system user for Docker
sudo adduser --system --ingroup docker --shell /sbin/nologin docker-system

Loki & Grafana Docker Compose
#

Folder Structure
# 

# Create folder struture
sudo mkdir -p /opt/loki && cd /opt/loki

Docker Compose File
#

No Authentication 
# Create Docker Compose file
sudo vi docker-compose.yml

services:
  grafana:
    image: grafana/grafana:latest
    ports:
      - 3000:3000
    volumes:
      - ./grafana-datasource.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
      - GF_AUTH_DISABLE_LOGIN_FORM=true
    restart: unless-stopped
    labels:
      - "traefik.enable=true"
      - "traefik.docker.network=traefik"
      - "traefik.http.services.loki.loadbalancer.server.port=3000"
      # HTTPS Router
      - "traefik.http.routers.loki.entrypoints=websecure"
      - "traefik.http.routers.loki.tls=true"
      - "traefik.http.routers.loki.rule=Host(`loki.jklug.work`)"
      # HTTP Router
      - "traefik.http.routers.loki-http.rule=Host(`loki.jklug.work`)"
      - "traefik.http.routers.loki-http.entrypoints=web"
      - "traefik.http.routers.loki-http.middlewares=redirect-to-https"
      # Middleware for HTTP to HTTPS redirection
      - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"
    networks:
      - loki
      - traefik

  loki:
    image: grafana/loki:latest
    ports:
      - 3101:3100
    command: -config.file=/etc/loki/local-config.yaml
    restart: unless-stopped
    labels:
      - "traefik.enable=false"
    networks:
      - loki

  loki2:
    image: grafana/loki:latest
    ports:
      - 3102:3100
    command: -config.file=/etc/loki/local-config.yaml
    restart: unless-stopped
    labels:
      - "traefik.enable=false"
    networks:
      - loki

  loki3:
    image: grafana/loki:latest
    ports:
      - 3103:3100
    command: -config.file=/etc/loki/local-config.yaml
    restart: unless-stopped
    labels:
      - "traefik.enable=false"
    networks:
      - loki

networks:
  loki:
    name: loki
  traefik:
    external: true
Basic Authentication 
# Create file for environment varibales
sudo vi .env

# Define username and password
GRAFANA_USERNAME=admin
GRAFANA_PW=mypassword

# Create Docker Compose file
sudo vi docker-compose.yml

services:
  grafana:
    image: grafana/grafana:latest
    ports:
      - 3000:3000
    volumes:
      - ./grafana-datasource.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=false
      - GF_AUTH_DISABLE_LOGIN_FORM=false
      - GF_SECURITY_ADMIN_USER=${GRAFANA_USERNAME}
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PW}
    restart: unless-stopped
    labels:
      - "traefik.enable=true"
      - "traefik.docker.network=traefik"
      - "traefik.http.services.loki.loadbalancer.server.port=3000"
      # HTTPS Router
      - "traefik.http.routers.loki.entrypoints=websecure"
      - "traefik.http.routers.loki.tls=true"
      - "traefik.http.routers.loki.rule=Host(`loki.jklug.work`)"
      # HTTP Router
      - "traefik.http.routers.loki-http.rule=Host(`loki.jklug.work`)"
      - "traefik.http.routers.loki-http.entrypoints=web"
      - "traefik.http.routers.loki-http.middlewares=redirect-to-https"
      # Middleware for HTTP to HTTPS redirection
      - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"
    networks:
      - loki
      - traefik

  loki:
    image: grafana/loki:latest
    ports:
      - 3101:3100
    command: -config.file=/etc/loki/local-config.yaml
    restart: unless-stopped
    labels:
      - "traefik.enable=false"
    networks:
      - loki

  loki2:
    image: grafana/loki:latest
    ports:
      - 3102:3100
    command: -config.file=/etc/loki/local-config.yaml
    restart: unless-stopped
    labels:
      - "traefik.enable=false"
    networks:
      - loki

  loki3:
    image: grafana/loki:latest
    ports:
      - 3103:3100
    command: -config.file=/etc/loki/local-config.yaml
    restart: unless-stopped
    labels:
      - "traefik.enable=false"
    networks:
      - loki

networks:
  loki:
    name: loki
  traefik:
    external: true
Persistent Data 
# Create folder structure for the volume mapping
sudo mkdir -p /opt/loki/{loki_data,loki_data2,loki_data3}

# Change owner
sudo chown 10001:10001 /opt/loki/{loki_data,loki_data2,loki_data3}

# Create Docker Compose file
sudo vi docker-compose.yml

services:
  grafana:
    image: grafana/grafana:latest
    ports:
      - 3000:3000
    volumes:
      - ./grafana-datasource.yaml:/etc/grafana/provisioning/datasources/datasources.yaml
    environment:
      - GF_AUTH_ANONYMOUS_ENABLED=true
      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
      - GF_AUTH_DISABLE_LOGIN_FORM=true
    restart: unless-stopped
    labels:
      - "traefik.enable=true"
      - "traefik.docker.network=traefik"
      - "traefik.http.services.loki.loadbalancer.server.port=3000"
      # HTTPS Router
      - "traefik.http.routers.loki.entrypoints=websecure"
      - "traefik.http.routers.loki.tls=true"
      - "traefik.http.routers.loki.rule=Host(`loki.jklug.work`)"
      # HTTP Router
      - "traefik.http.routers.loki-http.rule=Host(`loki.jklug.work`)"
      - "traefik.http.routers.loki-http.entrypoints=web"
      - "traefik.http.routers.loki-http.middlewares=redirect-to-https"
      # Middleware for HTTP to HTTPS redirection
      - "traefik.http.middlewares.redirect-to-https.redirectscheme.scheme=https"
    networks:
      - loki
      - traefik

  loki:
    image: grafana/loki:latest
    ports:
      - 3101:3100
    command: -config.file=/etc/loki/local-config.yaml
    volumes:
      - ./loki_data:/loki
    restart: unless-stopped
    labels:
      - "traefik.enable=false"
    networks:
      - loki

  loki2:
    image: grafana/loki:latest
    ports:
      - 3102:3100
    command: -config.file=/etc/loki/local-config.yaml
    volumes:
      - ./loki_data2:/loki
    restart: unless-stopped
    labels:
      - "traefik.enable=false"
    networks:
      - loki

  loki3:
    image: grafana/loki:latest
    ports:
      - 3103:3100
    command: -config.file=/etc/loki/local-config.yaml
    volumes:
      - ./loki_data3:/loki
    restart: unless-stopped
    labels:
      - "traefik.enable=false"
    networks:
      - loki

networks:
  loki:
    name: loki
  traefik:
    external: true

Grafana Data Source
#

Configurate Grafana to use Loki as data source:

# Create Grafana configuration file
sudo vi grafana-datasource.yaml

apiVersion: 1

datasources:
  - name: Loki-Container
    type: loki
    access: proxy
    url: http://loki:3100
    version: 1
    editable: false
    isDefault: true

  - name: Loki-VarLog
    type: loki
    access: proxy
    url: http://loki2:3100
    version: 1
    editable: false
    isDefault: false

  - name: Loki-Systemd-Journal
    type: loki
    access: proxy
    url: http://loki3:3100
    version: 1
    editable: false
    isDefault: false

Note: Within the Docker network the Loki endpoints are referenced by it’s DNS names loki, loki2 & loki3.


Change Ownership
# 

# Change ownership
sudo chown docker-system:docker /opt/loki &&
sudo chown docker-system:docker /opt/loki/{docker-compose.yml,grafana-datasource.yaml}

Create & Start Docker Stack
# 

# Create & start containers
sudo -u docker-system docker compose up -d

Note: The external docker network for traefik must already be existing.


Traefik Reverse Proxy Container
#

Folder Structure
# 

# Create folder structure
sudo mkdir -p /opt/traefik/certs && cd /opt/traefik

Certificates
# 

# Place the certificates into the "/opt/traefik/certs" directory
sudo cp fullchain.pem privkey.pem /opt/traefik/certs/

Traefik Dynamic Configuration
# 

# Create file for dynamic configuration
sudo vi tls-configuration.yaml

tls:
  certificates:
    - certFile: /etc/certs/fullchain.pem
      keyFile: /etc/certs/privkey.pem

Docker Compose File
# 

# Create Docker Compose file
sudo vi docker-compose.yml

version: '3'
services:
  reverse-proxy:
    image: traefik:v2.10
    container_name: Traefik-Reverse-Proxy
    restart: unless-stopped
    command:
      - --api.insecure=true
      - --providers.docker
      - --providers.file.directory=/etc/traefik/dynamic
      - --entryPoints.web.address=:80
      - --entryPoints.websecure.address=:443
    ports:
      - "80:80"
      - "443:443"
      - "8080:8080"
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./tls-configuration.yaml:/etc/traefik/dynamic/tls-configuration.yaml
      - ./certs/:/etc/certs/
    networks:
      - traefik

networks:
  traefik:
    external: true

Change Ownership
# 

# Change ownership
sudo chown -R docker-system:docker /opt/traefik

Create & Start Docker Stack
# 

# Create & start containers
sudo -u docker-system docker compose up -d

Note: The external docker network for traefik must already be existing.

Grafana & Loki
#

Loki Status & Troubleshooting
# 

# Check the readiness status of Loki (Health check): Wait till the Loki instances are ready!
curl http://localhost:3101/ready
curl http://localhost:3102/ready
curl http://localhost:3103/ready

# Shell output:
ready

# Metrics endpoint of the Loki service: This endpoint delivers data about the Loki service itself and not the log data that Loki collects
http://192.168.30.20:3101/metrics
http://192.168.30.20:3102/metrics
http://192.168.30.20:3103/metrics

Grafana Webinterface
# 

# Access the Grafana webinterface
https://loki.jklug.work/

Label Filters
# 

# Label filter: Select all containers
container =~ .+
  • .+ Regular expression that matches any string of at least one character

Line Contains
#

  • “+ Operations” > “Line filters” > “Line containers”: “error”

Log Context
#

  • Select a log e.g. “error log”
  • Click “Show context”

Promtail Container for Containers
#

Prerequisites
#

Docker Network
# 

# Create network used for Promtail to communicate with other Docker containers
sudo docker network create promtail

Docker System User
# 

# Create a system user for Docker
sudo adduser --system --ingroup docker --shell /sbin/nologin docker-system

Promtail Docker Compose
#

Folder Structure
# 

# Create folder struture
sudo mkdir -p /opt/promtail && cd /opt/promtail

Promtail Configuration
# 

# Create Promtail configuration file
sudo vi promtail.yaml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://192.168.30.20:3101/loki/api/v1/push

scrape_configs:
  - job_name: docker_scrape
    docker_sd_configs:
      - host: unix:///var/run/docker.sock
        refresh_interval: 5s
        filters:
          - name: label
            values: ["logging=promtail"]
    relabel_configs:
      - source_labels: ['__meta_docker_container_name']
        regex: '/(.*)'
        target_label: 'container'
      - source_labels: ['__meta_docker_container_log_stream']
        target_label: 'logstream'
      - source_labels: ['__meta_docker_container_label_logging_jobname']
        target_label: 'job'

Description:

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://192.168.30.20:3101/loki/api/v1/push

scrape_configs:
  - job_name: docker-scrape
    docker_sd_configs: # Discover targets using Docker service discovery
      - host: unix:///var/run/docker.sock # Connect to Docker daemon through a Unix socket
        refresh_interval: 5s # Time definition for the checks of changes in the list of Docker containers
        filters: # Filters to limit which containers are considered for scraping
          - name: label
            values: ["logging=promtail"]
    relabel_configs: # Modify or set labels for the scraped logs based on metadata from Docker
      - source_labels: ['__meta_docker_container_name'] # Grabs metadata from the container names
        regex: '/(.*)' # Used to extract a part of the container name
        target_label: 'container' # Sets the extracted value as the label container for the log data.
      - source_labels: ['__meta_docker_container_log_stream'] # Takes the log stream type (stdout or stderr)
        target_label: 'logstream' # Sets this as the logstream label on the logs
      - source_labels: ['__meta_docker_container_label_logging_jobname'] # Fetches the value of a specific Docker label logging_jobname
        target_label: 'job' # Sets this value to label the logs with the job name
  • Docker socket is mounted to Promtail so that it’s aware of all the docker events. It’s configured to filter only containers with docker label logging=promtail.

Docker Compose File
# 

# Create Docker Compose file
sudo vi docker-compose.yml

services:
  promtail:
    image:  grafana/promtail:latest
    container_name: promtail
    volumes:
      - ./promtail.yaml:/etc/promtail/docker-config.yaml
      - /var/lib/docker/containers:/var/lib/docker/containers:ro
      - /var/run/docker.sock:/var/run/docker.sock
    command: -config.file=/etc/promtail/docker-config.yaml
    restart: unless-stopped
    networks:
      - promtail

networks:
  promtail:
    external: true

Change Ownership
# 

# Change ownership
sudo chown -R docker-system:docker /opt/promtail

Create & Start Docker Stack
# 

# Create & start containers
sudo -u docker-system docker compose up -d

Example Container
#

Folder Structure
# 

# Create folder struture
sudo mkdir -p /opt/some-containers && cd /opt/some-containers

Docker Compose File
# 

# Create Docker Compose file
sudo vi docker-compose.yml

services:
  nginx:
    container_name: container-1
    image: nginx:latest
    labels:
      logging: "promtail"
      logging_jobname: "containerlogs"
    ports:
      - 8080:80
    command: /bin/sh -c "echo 'Web-App-1 Nginx Container' > /usr/share/nginx/html/index.html && nginx -g 'daemon off;'"
    restart: unless-stopped
    networks:
      - intern
      - promtail

  apache2:
    container_name: container-2
    image: httpd:latest
    labels:
      logging: "promtail"
      logging_jobname: "containerlogs"
    ports:
      - "8081:80"
    command: >
            bash -c "echo 'Web-App-2 Apache2 Container' > /usr/local/apache2/htdocs/index.html && httpd-foreground"
    restart: unless-stopped
    networks:
      - intern
      - promtail

networks:
  promtail:
    external: true
  intern:

Change Ownership
# 

# Change ownership
sudo chown -R docker-system:docker /opt/some-containers

Create & Start Docker Stack
# 

# Create & start containers
sudo -u docker-system docker compose up -d

Promtail Container for “/var/log”
#

Prerequisites
#

Docker System User
# 

# Create a system user for Docker
sudo adduser --system --ingroup docker --shell /sbin/nologin docker-system

# Add the user to the systemd-journal group
sudo usermod -aG systemd-journal docker-system

# Reboot the system to apply the group changes
sudo reboot

Promtail Container
#

Folder Structure
# 

# Create folder struture
sudo mkdir -p /opt/promtail && cd /opt/promtail

Promtail Configuration
# 

# Create Promtail configuration file
sudo vi promtail.yaml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://192.168.30.20:3102/loki/api/v1/push

scrape_configs:
  - job_name: system
    static_configs:
      - targets: ['localhost']
        labels:
          job: 'varlogs'
          __path__: /var/log/*log
          host: 'ubuntu-01'    

      - targets: ['localhost']
        labels:
          bare_metal_webapps: 'nginx-logs'
          __path__: /var/log/nginx/*.log
          host: 'ubuntu-01'

      - targets: ['localhost']
        labels:
          bare_metal_webapps: 'apache-logs'
          __path__: /var/log/apache2/*.log
          host: 'ubuntu-01'

Note: Label names must match a specific regex pattern, which is generally alphanumeric characters and underscores.

Docker Compose File
# 

# Create Docker Compose file
sudo vi docker-compose.yml

services:
  promtail:
    image:  grafana/promtail:latest
    container_name: promtail
    volumes:
      - ./promtail.yaml:/etc/promtail/docker-config.yaml
      - /var/log:/var/log
    command: -config.file=/etc/promtail/docker-config.yaml
    restart: unless-stopped
    networks:
      - loki

networks:
  loki:
    name: loki

Change Ownership
# 

# Change ownership
sudo chown -R docker-system:docker /opt/promtail

Create & Start Docker Stack
# 

# Create & start containers
sudo -u docker-system docker compose up -d

Example Webserver App
# 

# Install Nginx
sudo apt install nginx -y

Promtail Container for Systemd-Journal
#

Prerequisites
#

Docker System User
# 

# Create a system user for Docker
sudo adduser --system --ingroup docker --shell /sbin/nologin docker-system

# Add the user to the systemd-journal & adm group
sudo usermod -aG systemd-journal docker-system

# Reboot the system to apply the group changes
sudo reboot

# Verify the user groups
groups docker-system

# Shell output
docker-system : docker systemd-journal

Promtail Container
#

Folder Structure
# 

# Create folder struture
sudo mkdir -p /opt/promtail && cd /opt/promtail

Promtail Configuration
# 

# Create Promtail configuration file
sudo vi promtail.yaml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://192.168.30.20:3103/loki/api/v1/push

scrape_configs:
  - job_name: systemd-journal
    journal:
      max_age: 12h
      labels:
        job: systemd-journal
      path: /var/log/journal
    relabel_configs:
      - source_labels: ["__journal__systemd_unit"]
        target_label: "unit"
      - source_labels: ["__journal__hostname"]
        target_label: host
      - source_labels: ["__journal_priority_keyword"]
        target_label: level
      - source_labels: ["__journal_syslog_identifier"]
        target_label: syslog_identifier

Docker Compose File
# 

# Create Docker Compose file
sudo vi docker-compose.yml

services:
  promtail:
    image:  grafana/promtail:latest
    container_name: promtail
    volumes:
      - ./promtail.yaml:/etc/promtail/docker-config.yaml
      - /var/log/journal:/var/log/journal:ro
      - /etc/machine-id:/etc/machine-id:ro
    command: -config.file=/etc/promtail/docker-config.yaml
    restart: unless-stopped
    networks:
      - loki

networks:
  loki:
    name: loki

Change Ownership
# 

# Change ownership
sudo chown -R docker-system:docker /opt/promtail

Create & Start Docker Stack
# 

# Create & start containers
sudo -u docker-system docker compose up -d

Example Webserver App
# 

# Install Nginx
sudo apt install nginx -y

Promtail Bare-Metal for Systemd-Journal
#

Prerequisites
#

Docker System User
# 

# Create a system user for Promtail
sudo adduser --system --ingroup systemd-journal --shell /sbin/nologin promtail-system

# Reboot the system to apply the group changes
sudo reboot

# Verify the user groups
groups promtail-system

# Shell output
promtail-system : systemd-journal

Install Promtail
# 

# Find latest Promtail release
https://github.com/grafana/loki/releases/

# Install curl and zip
sudo apt install curl zip -y

# Download promtail
cd /tmp && curl -O -L "https://github.com/grafana/loki/releases/download/v2.9.8/promtail-linux-amd64.zip"

# Unzip
unzip promtail-linux-amd64.zip

# Move file
sudo mv promtail-linux-amd64 /usr/local/bin/promtail

# Change permissions
sudo chmod +x /usr/local/bin/promtail

Promtail configuration
# 

# Create directory for the Promtail configuration
sudo mkdir -p /etc/promtail

# Create promtail configuration file
sudo vi /etc/promtail/config-promtail.yml

server:
  http_listen_port: 9080
  grpc_listen_port: 0

positions:
  filename: /tmp/positions.yaml

clients:
  - url: http://192.168.30.20:3103/loki/api/v1/push

scrape_configs:
  - job_name: systemd-journal
    journal:
      max_age: 12h
      labels:
        job: systemd-journal
      path: /var/log/journal
    relabel_configs:
      - source_labels: ["__journal__systemd_unit"]
        target_label: "unit"
      - source_labels: ["__journal__hostname"]
        target_label: host
      - source_labels: ["__journal_priority_keyword"]
        target_label: level
      - source_labels: ["__journal_syslog_identifier"]
        target_label: syslog_identifier

Systemd Service Unit
# 

# Create a systemd service unit for Promtail
sudo vi /etc/systemd/system/promtail.service

[Unit]
Description=Promtail Loki
After=network.target

[Service]
Type=simple
User=promtail-system
ExecStart=/usr/local/bin/promtail -config.file /etc/promtail/config-promtail.yml

[Install]
WantedBy=multi-user.target

Change Ownership
# 

# Change ownership
sudo chown -R promtail-system:root /etc/promtail &&
sudo chown promtail-system:root /usr/local/bin/promtail

Start Service
# 

# Reload systemd daemon / reload configuration files
sudo systemctl daemon-reload

# Start service and enable after boot
sudo systemctl enable promtail && sudo systemctl start promtail

# Check status
sudo systemctl status promtail

Grafana Explore
#

Select the Loki data source:

Example Loki container logging:

Example Loki systemd journal logging:

The logs can be sorted bei either the host a the log type:

Example Loki /var/log logging:

The logs can be sorted bei either the host a the log type:


Loki-Stack - This article is part of a series.
Part 1: This Article
Part 2: Loki, Grafana & Traefik - Docker Compose Stack: Deployment with GitLab CI Pipeline, Git Commands
Part 3: Loki, Grafana & Promtail Stack - Deployment with Helm Chart in a K3s Kubernetes Cluster, NodePort Service and Traefik Ingress with TLS Secret; Promtail Helm Deployment for External Loki Data Source
Part 4: Loki & Grafana - Kubernetes K8s Deployment with Several Loki Instances for Seperated Data Sources; Nginx Ingress with TLS Secret; Promtail Helm and Docker Container Deployment
Part 5: Loki: Automated Promtail Container Deployment with Ansible