VictoriaMetrics Installation with Ansible: VictoriaMetrics K8s Stack, Drop-in Replacement for Prometheus

# Install dependencies
sudo apt install apache2-utils

# Create credentials
htpasswd -c authvm vmapi

# Shell output:
New password: ghVVgjkdT6aeWsnKPB3

# Print auth string
cat authvm

# Shell output:
vmapi:$apr1$uh2xZWlc$NVC2wgvhSH3kicxjh0tkk.

# Add Helm repo
helm repo add victoria-metrics https://victoriametrics.github.io/helm-charts/ &&
helm repo update

# List chart versions
helm search repo victoria-metrics/victoria-metrics-k8s-stack -l

---
# Helm Chart: VictoriaMetrics K8s Stack
- name: Helm Chart
  hosts: localhost
  connection: local
  gather_facts: false
  become: false
  vars:
    # Vault Secrets
    credentials: "{{ lookup('community.hashi_vault.vault_kv2_get', 'services/victoriametrics', engine_mount_point='homelab_prod').secret }}"
    # Helm Configuration
    helm_chart: "victoria-metrics/victoria-metrics-k8s-stack"
    helm_chart_version:  "0.66.0" # "0.63.5"
    helm_release_name: "vm"
    kubernetes_namespace: "victoria-metrics"
    # Storage & Scrap Config
    kubernetes_storage_class: "csi-nfs-2-dev"
    kubernetes_storage_size: "64Gi"
    scrape_interval: "60s"
    # Ingress Configuration
    kubernetes_ingress_class: "nginx-metallb"
    kubernetes_cluster_issuer: "cluster-issuer-dns01"
    kubernetes_ingress_url: "vm.jklug.work"
    # VM Authentication
    vmsingle_basic_auth: "{{ credentials.auth }}"
    # CRDs
    prometheus_podmonitor_crd_version: "release-0.83"
    prometheus_servicemonitor_crd_version: "release-0.83"

  roles:
    - helm_victoriametrics

# Run Ansible playbook:
ansible-playbook playbooks/helm_victoriametrics.yml -i inventory

# VictoriaMetrics K8s Stack Setup
---
- name: Add Helm repository
  kubernetes.core.helm_repository:
    name: "victoria-metrics"
    repo_url: "https://victoriametrics.github.io/helm-charts/"
    force_update: true

- name: Create namespace
  kubernetes.core.k8s:
    api_version: v1
    kind: Namespace
    name: "{{ kubernetes_namespace }}"
    state: present

- name: Create vmsingle basic auth secret
  kubernetes.core.k8s:
    state: present
    definition:
      apiVersion: v1
      kind: Secret
      metadata:
        name: vmsingle-basic-auth
        namespace: "{{ kubernetes_namespace }}"
      type: Opaque
      stringData:
        auth: "{{ vmsingle_basic_auth }}"


- name: Apply Prometheus ServiceMonitor CRD
  kubernetes.core.k8s:
    state: present
    src: "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/{{ prometheus_servicemonitor_crd_version }}/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml"

- name: Apply Prometheus PodMonitor CRD
  kubernetes.core.k8s:
    state: present
    src: "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/{{ prometheus_podmonitor_crd_version }}/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml"


- name: Install Helm Chart
  kubernetes.core.helm:
    name: "{{ helm_release_name }}"
    chart_ref: "{{ helm_chart }}"
    chart_version: "{{ helm_chart_version }}"
    release_namespace: "{{ kubernetes_namespace }}"
    create_namespace: false
    wait: true  # Ansible waits till all resources are ready
    atomic: true  # Auto-rollback on failure
    values: "{{ lookup('template', 'victoria-metrics-values.yml.j2') | from_yaml }}"

nameOverride: "vmks"

victoria-metrics-operator:
  enabled: true

# Preload Grafana Dashboards 
defaultDashboards:
  # -- Enable custom dashboards installation
  enabled: true
  defaultTimezone: Europe/Vienna
  labels: {}
  annotations: {}
  grafanaOperator:
    # -- Create dashboards as CRDs (requires grafana-operator to be installed)
    enabled: false
    spec:
      instanceSelector:
        matchLabels:
          dashboards: grafana
      allowCrossNamespaceImport: false
  # -- Create dashboards as ConfigMap despite dependency it requires is not installed
  dashboards:
    victoriametrics-vmalert:
      enabled: false
    victoriametrics-operator:
      enabled: false
    # -- In ArgoCD using client-side apply this dashboard reaches annotations size limit and causes k8s issues without server side apply
    # See [this issue](https://github.com/VictoriaMetrics/helm-charts/tree/master/charts/victoria-metrics-k8s-stack#metadataannotations-too-long-must-have-at-most-262144-bytes-on-dashboards)
    node-exporter-full:  # Kubernetes Nodes Dashboard
      enabled: true

defaultRules:
  create: false


# Optional: External Grafana Instance
external:
  grafana:
    # -- External Grafana host
    host: ""
    # -- External Grafana datasource name
    datasource: VictoriaMetrics
  # -- External VM read and write URLs
  vm:
    read:
      url: ""
      # bearerTokenSecret:
      #   name: dbaas-read-access-token
      #   key: bearerToken
    write:
      url: ""
      # bearerTokenSecret:
      #   name: dbaas-read-access-token
      #   key: bearerToken

vmsingle:
  # -- VMSingle labels
  labels: {}
  # -- VMSingle annotations
  annotations: {}
  # -- Create VMSingle CR
  enabled: true
  # -- Full spec for VMSingle CRD. Allowed values describe [here](https://docs.victoriametrics.com/operator/api#vmsinglespec)
  spec:
    port: "8429"
    # -- Data retention period. Possible units character: h(ours), d(ays), w(eeks), y(ears), if no unit character specified - month. The minimum retention period is 24h. See these [docs](https://docs.victoriametrics.com/single-server-victoriametrics/#retention)
    retentionPeriod: "1"
    replicaCount: 1
    extraArgs: {}
    storage:
      storageClassName: "{{ kubernetes_storage_class }}"
      accessModes:
        - ReadWriteOnce
      resources:
        requests:
          storage: {{ kubernetes_storage_size }}
  ingress:
    # -- Enable deployment of ingress for server component
    enabled: true
    # -- Ingress annotations
    annotations:
      cert-manager.io/cluster-issuer: "{{ kubernetes_cluster_issuer }}"
      nginx.ingress.kubernetes.io/ssl-redirect: "true"
      nginx.ingress.kubernetes.io/proxy-body-size: "0"
      # Basic auth
      nginx.ingress.kubernetes.io/auth-type: basic
      nginx.ingress.kubernetes.io/auth-secret: vmsingle-basic-auth
      nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required'
    # -- Ingress extra labels
    labels: {}
    # -- Ingress default path
    path: ""
    # -- Ingress path type
    pathType: Prefix
    # -- Ingress controller class name
    ingressClassName: "{{ kubernetes_ingress_class }}"

    # -- Array of host objects
    hosts:
      - {{ kubernetes_ingress_url}}
    # -- Extra paths to prepend to every host configuration. This is useful when working with annotation based services.
    extraPaths: []
    # - path: /*
    #   pathType: Prefix
    #   backend:
    #     service:
    #       name: ssl-redirect
    #       port:
    #         name: service

    # -- Array of TLS objects
    tls:
      - secretName: vmsingle-ingress-tls
        hosts:
          - {{ kubernetes_ingress_url}}

vmcluster:
  enabled: false

alertmanager:
  enabled: false

vmalert:
  # -- VMAlert annotations
  annotations: {}
  # -- VMAlert labels
  labels: {}
  # -- Create VMAlert CR
  enabled: false

vmauth:
  # -- Enable VMAuth CR
  enabled: false

vmagent:
  enabled: true
  spec:
    scrapeInterval: "{{ scrape_interval }}"
    selectAllByDefault: true

# Enable Grafana
grafana:
  enabled: false

  ingress:
    enabled: false

defaultDatasources:
  # you configure datasources via your separate Grafana chart
  grafanaOperator:
    enabled: false

prometheus-node-exporter:
  enabled: true

kube-state-metrics:
  enabled: true

coreDns:
  enabled: true

kubelet:
  enabled: true

kubeApiServer:
  enabled: true

kubeControllerManager:
  enabled: true

kubeScheduler:
  enabled: true

kubeEtcd:
  enabled: true

kubeProxy:
  enabled: true