Prometheus Blackbox-Exporter for VictoriaMetrics: TLS Certificate Monitoring, Ansible Installation

# Add Helm Repository
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts &&
helm repo update

# List available chart versions
helm search repo prometheus-community/prometheus-blackbox-exporter --versions

---
- name: Blackbox-Exporter for TLS Checks
  hosts: localhost
  connection: local
  gather_facts: false
  become: false
  vars:
    # Helm Repository
    helm_chart_url: "https://prometheus-community.github.io/helm-charts"
    helm_chart_name: "prometheus-community"
    # Helm Configuration
    helm_chart: "prometheus-community/prometheus-blackbox-exporter"
    helm_chart_version: "11.4.1"
    helm_release_name: "blackbox-exporter"
    kubernetes_namespace: "victoria-metrics"

  roles:
    - helm_blackbox_exporter

# Run Ansible playbook:
ansible-playbook playbooks/helm_blackbox_exporter.yml -i inventory

- name: Add Helm repository
  kubernetes.core.helm_repository:
    name: "{{ helm_chart_name }}"
    repo_url: "{{ helm_chart_url }}"
    force_update: true

- name: Install Helm Chart
  kubernetes.core.helm:
    name: "{{ helm_release_name }}"
    chart_ref: "{{ helm_chart }}"
    chart_version: "{{ helm_chart_version }}"
    release_namespace: "{{ kubernetes_namespace }}"
    create_namespace: false
    wait: false  # Ansible waits till all resources are ready
    atomic: false  # Auto-rollback on failure
    values: "{{ lookup('template', 'blackbox-exporter-values.yml') | from_yaml }}"

- name: Apply VMProbe for Blackbox-Exporter
  kubernetes.core.k8s:
    state: present
    definition: "{{ lookup('template', 'vmprobe-blackbox-exporter.yml.j2') }}"

revisionHistoryLimit: 1
replicas: 2

strategy:
  rollingUpdate:
    maxSurge: 1
    maxUnavailable: 100%
  type: RollingUpdate

config:
  modules:
    http_2xx:
      prober: http
      timeout: 5s
      http:
        valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
        follow_redirects: true
        preferred_ip_protocol: "ip4"

apiVersion: operator.victoriametrics.com/v1beta1
kind: VMProbe
metadata:
  name: tls-expiry
  namespace: "{{ kubernetes_namespace }}"
spec:
  jobName: blackbox
  vmProberSpec:
    url: "http://blackbox-exporter-prometheus-blackbox-exporter.{{ kubernetes_namespace }}.svc:9115/probe"
  module: http_2xx
  targets:
    staticConfig:
      targets:
          # DevOps
        - https://gitlab.jklug.work
        - https://gitlab-registry.jklug.work
        - https://argocd.jklug.work
        - https://harbor.jklug.work
        - https://grafana.jklug.work

# Create port forwarding
kubectl port-forward svc/blackbox-exporter-prometheus-blackbox-exporter 9115 -n victoria-metrics

# Curl the Blackbox-Exporter
curl 'http://localhost:9115/probe?module=http_2xx&target=https://grafana.jklug.work'

# Verify VMProbe
kubectl get vmprobe -n victoria-metrics

# Shell output:
NAME         AGE   STATUS        SYNC ERROR
tls-expiry   7s    operational

# Grafana Query
probe_success{job="blackbox"}

# Query output:
probe_success{instance="https://argocd.jklug.work", job="blackbox", prometheus="victoria-metrics/vm-vmks"}
1

probe_success{instance="https://grafana.jklug.work", job="blackbox", prometheus="victoria-metrics/vm-vmks"}
1

probe_success{instance="https://harbor.jklug.work", job="blackbox", prometheus="victoria-metrics/vm-vmks"}
1

# Define query and alert condition
round((probe_ssl_earliest_cert_expiry{job=~"blackbox"} - time()) / 3600 / 24, 1)

# Alert condition
WHEN QUERY > IS BELOW > 7