Overview #
I’m using the following Ubuntu 24.04 based servers in this tuturial:
192.168.30.10 # Prometheus Docker Compose stack
192.168.30.12 # Python Flask app container
Python Flask Application #
File and Folder Structure #
# Create a folder for the Python Flask app
mkdir -p flask-app/python
The file and folder structure looks like this:
flask-app
├── docker-compose.yml
├── Dockerfile
└── python
├── app.py
└── requirements.txt
Flask App #
- python/app.py
# Import the Flask class from the Flask library
from flask import Flask
from prometheus_flask_exporter import PrometheusMetrics
# Create an instance of the Flask application
app = Flask(__name__)
# Attach Prometheus metrics exporter to the Flask app
PrometheusMetrics(app)
# Define some example endpoints
@app.route("/one")
def first_route():
return "Hi there from Python Flask application\n"
@app.route("/two")
def the_second():
return "Some text\n"
if __name__ == "__main__":
# Start Flask application server on all interfaces
app.run(host='0.0.0.0', port=8080)
Requirements #
- python/requirements.txt
# Flask Web Framework
Flask==3.1.0
# Prometheus Flask Exporter
prometheus_flask_exporter==0.23.1
Dockerfile #
- Dockerfile
# Use the latest Python-slime (Debian 12) base image
FROM python:slim AS builder
# Set the working directory inside the container to /app
WORKDIR /app
# Copy the contents of the current directory to "/app" inside the container
ADD python/. /app/
# Install the dependencies listed in "requirements.txt"
RUN pip install -r requirements.txt
# Expose port "8080" for the application
EXPOSE 8080
# Create a non-root user and group
RUN groupadd appgroup && useradd -g appgroup appuser
# Set permissions for the application directory
RUN chown -R appuser:appgroup /app
# Switch to the non-root user
USER appuser
# Run the app.py file with Python when the container starts
ENTRYPOINT ["python", "app.py"]
Docker Compose Manifest #
- docker-compose.yml
services:
flask-app:
image: python-flask-app:latest
container_name: flask-app
restart: unless-stopped
build:
context: .
dockerfile: Dockerfile
ports:
- "8080:8080"
Docker Container #
Create Container #
# Create the Docker container
docker compose up -d
Verify Container #
# Verify the Docker image
docker images
# Shell output:
REPOSITORY TAG IMAGE ID CREATED SIZE
python-flask-app latest 23a7f5ff770d 8 seconds ago 134MB
# List containers
docker ps
# Shell output:
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
39466eadfd7e python-flask-app:latest "python app.py" 26 seconds ago Up 26 seconds 0.0.0.0:8080->8080/tcp, :::8080->8080/tcp flask-app
Verify Python Flask App & Metrics #
# Verify the Python Flask app
curl localhost:8080/one
# Shell output:
Hi there from Python Flask application
# Verify the Python Flask app
curl localhost:8080/two
# Shell output:
Some text
# Verify the Python Flask app metrics
curl localhost:8080/metrics
# Shell output:
...
flask_app_request_latency_seconds_created{endpoint="/"} 1.7359880724409585e+09
Prometheus Configuration #
Prometheus Configuration: prometheus.yml #
# Adapt the Prometheus configuration
sudo vi prometheus_conf/prometheus.yml
# prometheus.yml
global:
scrape_interval: 15s
evaluation_interval: 10s
rule_files:
- rules.yml
alerting:
alertmanagers:
- scheme: http
static_configs:
- targets: [ 'alertmanager:9093' ]
scrape_configs:
# Prometheus Server
- job_name: 'Prometheus-Server'
static_configs:
- targets:
- localhost:9090
# Linux Servers / Node Exporter
- job_name: 'Linux-Server'
static_configs:
- targets:
- 192.168.30.11:9100
- 192.168.30.12:9100
# MySQL Server
- job_name: 'MySQL-Server'
params:
auth_module: [client] # Specify authentication mode
scrape_interval: 5s
static_configs:
- targets: ['192.168.30.11:3306']
relabel_configs:
- source_labels: [__address__]
target_label: __param_target
- source_labels: [__param_target]
target_label: instance
- target_label: __address__
replacement: 192.168.30.11:9104
# PostgreSQL Server
- job_name: 'Postgres-Server'
static_configs:
- targets: ["192.168.30.12:9187"]
relabel_configs:
- source_labels: [__address__]
target_label: instance
- target_label: __address__
replacement: 192.168.30.12:9187
# Nginx Webserver
- job_name: 'Nginx-Webserver'
metrics_path: /metrics
static_configs:
- targets: ['192.168.30.11:9113']
# Pyrthon Flask App
- job_name: 'Python-Flask-App'
scrape_interval: 5s
static_configs:
- targets: ['192.168.30.13:8080']
Prometheus Configuration: rules.yml #
# Adapt the rules.yml configuration
sudo vi prometheus_conf/rules.yml
# rules.yml
groups:
- name: NodeExporter
rules:
- alert: InstanceDown
expr: up{job="Linux-Server"} == 0
for: 1m
- name: MysqldExporter
rules:
- alert: MysqlDown
expr: mysql_up == 0
for: 0m
labels:
severity: critical
annotations:
summary: MySQL down (instance {{ $labels.instance }})
description: "MySQL instance is down on {{ $labels.instance }}\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- name: PostgresExporter
rules:
- alert: PostgresDown
expr: pg_up == 0
for: 0m
labels:
severity: critical
annotations:
summary: PostgreSQL down (instance {{ $labels.instance }})
description: |
PostgreSQL instance is down on {{ $labels.instance }}
VALUE = {{ $value }}
LABELS = {{ $labels }}
- name: NginxExporter
rules:
- alert: NginxInstanceDown
expr: up{job="Nginx-Webserver"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: Nginx instance down
description: "Nginx instance {{ $labels.instance }} is unreachable."
- name: PythonFlaskApp
rules:
- alert: PythonFlaskAppDown
expr: up{job="Python-Flask-App"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: Python Flask App down
description: "Python Flask App (job: {{ $labels.job }}, instance: {{ $labels.instance }}) is unreachable."
Restart Docker Container #
# Restart the Docker container
sudo docker compose restart
Prometheus Webinterface #
Open Webinterface #
# Open the Prometheus webinterface
http://192.168.30.10:9090
Verify Endpoints #
- Select “Status” > “Target health”
Grafana Dashboard #
Open Webinterface #
# Grafana webinterface
192.168.30.10:3000
Import Dashboard #
I’m using the following Grafana dashboard:
https://github.com/rycus86/prometheus_flask_exporter/blob/master/examples/sample-signals/grafana/dashboards/example.json
Replace the job name job=\"Python-Flask-App\"
with the name defined in the Prometheus configuration: prometheus.yml
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"links": [],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 4,
"w": 10,
"x": 0,
"y": 0
},
"id": 2,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sort": "avg",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"$$hashKey": "object:214",
"expr": "rate(flask_http_request_duration_seconds_count{status=\"200\"}[30s])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ path }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Requests per second",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:376",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:377",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 4,
"w": 6,
"x": 10,
"y": 0
},
"id": 4,
"legend": {
"avg": true,
"current": true,
"max": true,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:1922",
"alias": "errors",
"color": "#c15c17"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"$$hashKey": "object:766",
"expr": "sum(rate(flask_http_request_duration_seconds_count{status!=\"200\"}[30s]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "errors",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Errors per second",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:890",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:891",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 4,
"w": 8,
"x": 16,
"y": 0
},
"id": 13,
"legend": {
"avg": true,
"current": false,
"max": true,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": false,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"$$hashKey": "object:255",
"alias": "HTTP 500",
"color": "#bf1b00"
}
],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"$$hashKey": "object:140",
"expr": "increase(flask_http_request_total[1m])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "HTTP {{ status }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Total requests per minute",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:211",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"$$hashKey": "object:212",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"decimals": null,
"fill": 1,
"gridPos": {
"h": 5,
"w": 10,
"x": 0,
"y": 4
},
"id": 6,
"legend": {
"alignAsTable": true,
"avg": false,
"current": true,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sort": "avg",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"$$hashKey": "object:146",
"expr": "rate(flask_http_request_duration_seconds_sum{status=\"200\"}[30s])\n/\nrate(flask_http_request_duration_seconds_count{status=\"200\"}[30s])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ path }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Average response time [30s]",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1004",
"decimals": null,
"format": "s",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1005",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"description": "",
"fill": 1,
"gridPos": {
"h": 5,
"w": 9,
"x": 10,
"y": 4
},
"id": 15,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": true,
"rightSide": true,
"show": true,
"sort": "avg",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"$$hashKey": "object:426",
"expr": "histogram_quantile(0.5, rate(flask_http_request_duration_seconds_bucket{status=\"200\"}[30s]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ path }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Request duration [s] - p50",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1280",
"format": "none",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:1281",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 5,
"w": 5,
"x": 19,
"y": 4
},
"id": 8,
"legend": {
"avg": false,
"current": true,
"max": false,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"$$hashKey": "object:638",
"expr": "process_resident_memory_bytes{job=\"Python-Flask-App\"}",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "mem",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Memory usage",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:683",
"format": "decbytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:684",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 5,
"w": 10,
"x": 0,
"y": 9
},
"id": 11,
"legend": {
"alignAsTable": true,
"avg": false,
"current": true,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"sort": "current",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"$$hashKey": "object:1079",
"expr": "increase(flask_http_request_duration_seconds_bucket{status=\"200\",le=\"0.25\"}[30s]) \n/ ignoring (le) increase(flask_http_request_duration_seconds_count{status=\"200\"}[30s])",
"format": "time_series",
"instant": false,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ path }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Requests under 250ms",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:1137",
"decimals": null,
"format": "percentunit",
"label": null,
"logBase": 1,
"max": "1",
"min": "0",
"show": true
},
{
"$$hashKey": "object:1138",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 5,
"w": 9,
"x": 10,
"y": 9
},
"id": 16,
"legend": {
"alignAsTable": true,
"avg": true,
"current": true,
"max": true,
"min": true,
"rightSide": true,
"show": true,
"sort": "avg",
"sortDesc": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"$$hashKey": "object:426",
"expr": "histogram_quantile(0.9, rate(flask_http_request_duration_seconds_bucket{status=\"200\"}[30s]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ path }}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "Request duration [s] - p90",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "Prometheus",
"fill": 1,
"gridPos": {
"h": 5,
"w": 5,
"x": 19,
"y": 9
},
"id": 9,
"legend": {
"avg": false,
"current": true,
"max": true,
"min": false,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 1,
"links": [],
"nullPointMode": "null",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"$$hashKey": "object:638",
"expr": "rate(process_cpu_seconds_total{job=\"Python-Flask-App\"}[30s])",
"format": "time_series",
"intervalFactor": 1,
"legendFormat": "cpu",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeShift": null,
"title": "CPU usage",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:683",
"format": "percentunit",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"$$hashKey": "object:684",
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"refresh": "3s",
"schemaVersion": 16,
"style": "dark",
"tags": [],
"templating": {
"list": []
},
"time": {
"from": "now-5m",
"to": "now"
},
"timepicker": {
"refresh_intervals": [
"3s"
],
"time_options": [
"5m",
"15m",
"1h",
"6h",
"12h",
"24h",
"2d",
"7d",
"30d"
]
},
"timezone": "",
"title": "Python Flask App",
"uid": "_eX4mpl3",
"version": 1
}
Import the dashboard:
-
Go to: “Home” > “Dashboard”
-
Click “New” > “Import”
-
In the “Import via dashboard JSON model” field paste the JSON config and press “Load”
-
Click “Import”
Verify Dashboard #
The Grafana dashboard looks like this:
Example Alert #
Stop Python Flask App Container #
Stop the Python Flask app container:
# CD into the Docker Compose directory
cd flask-app
# Stop the Docker containers
docker compose down
Verify the Prometheus Alert #
# Verify the alert in the Prometheus webinterface
http://192.168.30.10:9090/alerts
Verify the Alertmanager Alert #
# Verify the alert in the Prometheus webinterface
http://192.168.30.10:9093/#/alerts
Links #
# Prometheus Flask Exporter
https://github.com/rycus86/prometheus_flask_exporter
https://github.com/rycus86/prometheus_flask_exporter/tree/master/examples/sample-signals
# Prometheus Python Client
https://github.com/prometheus/client_python/releases