Overview #
I’m using VMware Workstation Pro for virtualization, each VM got 4 Cores and 8GB RAM. The deployment want work with less then 2 CPU Cores.
My setup is based on the following Ubuntu 24.04 servers:
192.168.30.10 ubuntu1 # First / Initial Controller Node
192.168.30.11 ubuntu2 # Controller Node 2
192.168.30.12 ubuntu3 # Controller Node 3
192.168.30.13 ubuntu4 # Worker Node 1
192.168.30.14 ubuntu5 # Worker Node 2
RKE2 deploys an Nginx Ingress per default, but since I’m using a bare-metal deployment, it’s necessary to deploy MetalLB as load balancer.
Install First Server / Controller Node #
Install RKE2 #
# Run the RKE2 Server installer
curl -sfL https://get.rke2.io | sudo sh -
# Enable & start the RKE2 Server service
sudo systemctl enable rke2-server.service &&
sudo systemctl start rke2-server.service
Add Path Environment Variable #
Temporary: #
# Add RKE2 & Kubectl configuration path environment variable
export PATH="$PATH:/var/lib/rancher/rke2/bin" &&
export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
Permanent: (All users) #
# Add RKE2 & Kubectl configuration path environment variable
echo 'export PATH="$PATH:/var/lib/rancher/rke2/bin"' | sudo tee -a /etc/profile > /dev/null &&
echo 'export KUBECONFIG="/etc/rancher/rke2/rke2.yaml"' | sudo tee -a /etc/profile > /dev/null
# Apply changes
source /etc/profile
Kubeconfig Permissions #
# Add Rancher group
sudo groupadd rancher
# Add current user to Rancher group (Logout / login to apply to user)
sudo usermod -a -G rancher $(whoami)
# Change group owner of kubeconfig file to Rancher
sudo chown :rancher /etc/rancher/rke2/rke2.yaml
# Change permissionm
sudo chmod 640 /etc/rancher/rke2/rke2.yaml
Verify the Deployment #
# List Kubernetes nodes
kubectl get nodes -o wide
# Shell output
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
ubuntu1 Ready control-plane,etcd,master 86s v1.28.10+rke2r1 192.168.30.10 <none> Ubuntu 24.04 LTS 6.8.0-35-generic containerd://1.7.11-k3s2
# List pods
kubectl get pods --all-namespaces
# Shell output:
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system cloud-controller-manager-ubuntu1 1/1 Running 1 (2m15s ago) 2m16s
kube-system etcd-ubuntu1 1/1 Running 0 112s
kube-system helm-install-rke2-canal-sbnnh 0/1 Completed 0 2m4s
kube-system helm-install-rke2-coredns-qcmbg 0/1 Completed 0 2m4s
kube-system helm-install-rke2-ingress-nginx-zwwgr 0/1 Completed 0 2m4s
kube-system helm-install-rke2-metrics-server-g9sfs 0/1 Completed 0 2m4s
kube-system helm-install-rke2-snapshot-controller-crd-85w4c 0/1 Completed 0 2m4s
kube-system helm-install-rke2-snapshot-controller-qjcs9 0/1 Completed 2 2m4s
kube-system helm-install-rke2-snapshot-validation-webhook-ttwrp 0/1 Completed 0 2m4s
kube-system kube-apiserver-ubuntu1 1/1 Running 0 2m12s
kube-system kube-controller-manager-ubuntu1 1/1 Running 0 2m15s
kube-system kube-proxy-ubuntu1 1/1 Running 0 2m14s
kube-system kube-scheduler-ubuntu1 1/1 Running 0 2m8s
kube-system rke2-canal-np4nv 2/2 Running 0 115s
kube-system rke2-coredns-rke2-coredns-84b9cb946c-kctjv 1/1 Running 0 116s
kube-system rke2-coredns-rke2-coredns-autoscaler-b49765765-vqk8v 1/1 Running 0 116s
kube-system rke2-ingress-nginx-controller-z24fw 1/1 Running 0 79s
kube-system rke2-metrics-server-655477f655-7rbnk 1/1 Running 0 86s
kube-system rke2-snapshot-controller-59cc9cd8f4-r2fhx 1/1 Running 0 69s
kube-system rke2-snapshot-validation-webhook-54c5989b65-5dds8 1/1 Running 0 89s
Get Token #
# Get the node token from the initial node
sudo cat /var/lib/rancher/rke2/server/node-token
# Shell output:
K1024868341057ead9a795b2eb4314a22b94af0b9004cde2122ffd1f57732f7f3a1::server:4ce1b262a9aa72a78db0105747ba6c1d
Add Controller Nodes #
Install RKE2 #
# Run the RKE2 Server installer
curl -sfL https://get.rke2.io | sudo sh -
Create RKE2 Configuration #
Use the token and IP address from the first / initional Controller Node:
# Create the RKE2 config.yaml
sudo mkdir -p /etc/rancher/rke2 && \
sudo tee /etc/rancher/rke2/config.yaml > /dev/null << EOF
token: K1024868341057ead9a795b2eb4314a22b94af0b9004cde2122ffd1f57732f7f3a1::server:4ce1b262a9aa72a78db0105747ba6c1d
server: https://192.168.30.10:9345
EOF
Start & Enable RKE2 #
# Enable & start the RKE2 Server service
sudo systemctl enable rke2-server.service &&
sudo systemctl start rke2-server.service
Verify the Deployment #
Note: It can take a while till the status changes from “NotReady” to “Ready”.
# List Kubernetes nodes
kubectl get nodes -o wide
# Shell output
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
ubuntu1 Ready control-plane,etcd,master 11m v1.28.10+rke2r1 192.168.30.10 <none> Ubuntu 24.04 LTS 6.8.0-35-generic containerd://1.7.11-k3s2
ubuntu2 Ready control-plane,etcd,master 5m37s v1.28.10+rke2r1 192.168.30.11 <none> Ubuntu 24.04 LTS 6.8.0-35-generic containerd://1.7.11-k3s2
ubuntu3 Ready control-plane,etcd,master 30s v1.28.10+rke2r1 192.168.30.12 <none> Ubuntu 24.04 LTS 6.8.0-35-generic containerd://1.7.11-k3s2
# List pods
kubectl get pods -n kube-system
# Shell output:
NAME READY STATUS RESTARTS AGE
cloud-controller-manager-ubuntu1 1/1 Running 1 (14m ago) 14m
cloud-controller-manager-ubuntu2 1/1 Running 0 7m26s
cloud-controller-manager-ubuntu3 1/1 Running 0 2m58s
etcd-ubuntu1 1/1 Running 0 13m
etcd-ubuntu2 1/1 Running 0 7m48s
etcd-ubuntu3 1/1 Running 0 2m41s
helm-install-rke2-canal-sbnnh 0/1 Completed 0 13m
helm-install-rke2-coredns-qcmbg 0/1 Completed 0 13m
helm-install-rke2-ingress-nginx-zwwgr 0/1 Completed 0 13m
helm-install-rke2-metrics-server-g9sfs 0/1 Completed 0 13m
helm-install-rke2-snapshot-controller-crd-85w4c 0/1 Completed 0 13m
helm-install-rke2-snapshot-controller-qjcs9 0/1 Completed 2 13m
helm-install-rke2-snapshot-validation-webhook-ttwrp 0/1 Completed 0 13m
kube-apiserver-ubuntu1 1/1 Running 0 14m
kube-apiserver-ubuntu2 1/1 Running 0 7m32s
kube-apiserver-ubuntu3 1/1 Running 0 2m30s
kube-controller-manager-ubuntu1 1/1 Running 0 14m
kube-controller-manager-ubuntu2 1/1 Running 0 7m24s
kube-controller-manager-ubuntu3 1/1 Running 0 2m58s
kube-proxy-ubuntu1 1/1 Running 0 14m
kube-proxy-ubuntu2 1/1 Running 0 8m
kube-proxy-ubuntu3 1/1 Running 0 2m52s
kube-scheduler-ubuntu1 1/1 Running 0 14m
kube-scheduler-ubuntu2 1/1 Running 0 7m29s
kube-scheduler-ubuntu3 1/1 Running 0 2m58s
rke2-canal-np4nv 2/2 Running 0 13m
rke2-canal-qjnm8 2/2 Running 0 8m18s
rke2-canal-vl6cr 2/2 Running 0 3m11s
rke2-coredns-rke2-coredns-84b9cb946c-kctjv 1/1 Running 0 13m
rke2-coredns-rke2-coredns-84b9cb946c-tznmz 1/1 Running 0 8m18s
rke2-coredns-rke2-coredns-autoscaler-b49765765-vqk8v 1/1 Running 0 13m
rke2-ingress-nginx-controller-427ml 1/1 Running 0 7m50s
rke2-ingress-nginx-controller-z24fw 1/1 Running 0 13m
rke2-ingress-nginx-controller-zg4hf 1/1 Running 0 2m44s
rke2-metrics-server-655477f655-7rbnk 1/1 Running 0 13m
rke2-snapshot-controller-59cc9cd8f4-r2fhx 1/1 Running 0 13m
rke2-snapshot-validation-webhook-54c5989b65-5dds8 1/1 Running 0 13m
Add Worker Nodes #
Install RKE2 #
# Run the RKE2 Agent installer
curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sudo sh -
Create RKE2 Configuration #
Use the token and IP address from the first / initional Controller Node:
# Create the RKE2 config.yaml
sudo mkdir -p /etc/rancher/rke2 && \
sudo tee /etc/rancher/rke2/config.yaml > /dev/null << EOF
token: K1024868341057ead9a795b2eb4314a22b94af0b9004cde2122ffd1f57732f7f3a1::server:4ce1b262a9aa72a78db0105747ba6c1d
server: https://192.168.30.10:9345
EOF
Start & Enable RKE2 #
# Enable & start the RKE2 Agent service
sudo systemctl enable rke2-agent.service &&
sudo systemctl start rke2-agent.service
Label Worker Nodes #
# Label a worker nodes
kubectl label nodes ubuntu4 kubernetes.io/role=worker &&
kubectl label nodes ubuntu5 kubernetes.io/role=worker
Verify the Deployment #
Note: It can take a while till the status changes from “NotReady” to “Ready”.
# List Kubernetes nodes
kubectl get nodes -o wide
# Shell output
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
ubuntu1 Ready control-plane,etcd,master 18m v1.28.10+rke2r1 192.168.30.10 <none> Ubuntu 24.04 LTS 6.8.0-35-generic containerd://1.7.11-k3s2
ubuntu2 Ready control-plane,etcd,master 12m v1.28.10+rke2r1 192.168.30.11 <none> Ubuntu 24.04 LTS 6.8.0-35-generic containerd://1.7.11-k3s2
ubuntu3 Ready control-plane,etcd,master 7m28s v1.28.10+rke2r1 192.168.30.12 <none> Ubuntu 24.04 LTS 6.8.0-35-generic containerd://1.7.11-k3s2
ubuntu4 Ready worker 3m33s v1.28.10+rke2r1 192.168.30.13 <none> Ubuntu 24.04 LTS 6.8.0-35-generic containerd://1.7.11-k3s2
ubuntu5 Ready worker 3m v1.28.10+rke2r1 192.168.30.14 <none> Ubuntu 24.04 LTS 6.8.0-35-generic containerd://1.7.11-k3s2
# List pods
kubectl get pods -n kube-system
# Shell output:
NAME READY STATUS RESTARTS AGE
cloud-controller-manager-ubuntu1 1/1 Running 1 (20m ago) 20m
cloud-controller-manager-ubuntu2 1/1 Running 0 13m
cloud-controller-manager-ubuntu3 1/1 Running 0 8m55s
etcd-ubuntu1 1/1 Running 0 19m
etcd-ubuntu2 1/1 Running 0 13m
etcd-ubuntu3 1/1 Running 0 8m38s
helm-install-rke2-canal-sbnnh 0/1 Completed 0 19m
helm-install-rke2-coredns-qcmbg 0/1 Completed 0 19m
helm-install-rke2-ingress-nginx-zwwgr 0/1 Completed 0 19m
helm-install-rke2-metrics-server-g9sfs 0/1 Completed 0 19m
helm-install-rke2-snapshot-controller-crd-85w4c 0/1 Completed 0 19m
helm-install-rke2-snapshot-controller-qjcs9 0/1 Completed 2 19m
helm-install-rke2-snapshot-validation-webhook-ttwrp 0/1 Completed 0 19m
kube-apiserver-ubuntu1 1/1 Running 0 20m
kube-apiserver-ubuntu2 1/1 Running 0 13m
kube-apiserver-ubuntu3 1/1 Running 0 8m27s
kube-controller-manager-ubuntu1 1/1 Running 0 20m
kube-controller-manager-ubuntu2 1/1 Running 0 13m
kube-controller-manager-ubuntu3 1/1 Running 0 8m55s
kube-proxy-ubuntu1 1/1 Running 0 20m
kube-proxy-ubuntu2 1/1 Running 0 13m
kube-proxy-ubuntu3 1/1 Running 0 8m49s
kube-proxy-ubuntu4 1/1 Running 0 5m13s
kube-proxy-ubuntu5 1/1 Running 0 4m40s
kube-scheduler-ubuntu1 1/1 Running 0 19m
kube-scheduler-ubuntu2 1/1 Running 0 13m
kube-scheduler-ubuntu3 1/1 Running 0 8m55s
rke2-canal-8wq2k 2/2 Running 0 4m40s
rke2-canal-c29rg 2/2 Running 0 5m13s
rke2-canal-np4nv 2/2 Running 0 19m
rke2-canal-qjnm8 2/2 Running 0 14m
rke2-canal-vl6cr 2/2 Running 0 9m8s
rke2-coredns-rke2-coredns-84b9cb946c-kctjv 1/1 Running 0 19m
rke2-coredns-rke2-coredns-84b9cb946c-tznmz 1/1 Running 0 14m
rke2-coredns-rke2-coredns-autoscaler-b49765765-vqk8v 1/1 Running 0 19m
rke2-ingress-nginx-controller-427ml 1/1 Running 0 13m
rke2-ingress-nginx-controller-ntbd2 1/1 Running 0 4m42s
rke2-ingress-nginx-controller-z24fw 1/1 Running 0 19m
rke2-ingress-nginx-controller-zg4hf 1/1 Running 0 8m41s
rke2-ingress-nginx-controller-zlc8s 1/1 Running 0 3m49s
rke2-metrics-server-655477f655-7rbnk 1/1 Running 0 19m
rke2-snapshot-controller-59cc9cd8f4-r2fhx 1/1 Running 0 19m
rke2-snapshot-validation-webhook-54c5989b65-5dds8 1/1 Running 0 19m
More #
Logs #
# List RKE2 Server logs
sudo journalctl -u rke2-server -f
# List RKE2 Agent logs
sudo journalctl -u rke2-agent -f
Shutdown the Cluster #
# Drain the Kubernetes nodes
kubectl drain ubuntu5 --ignore-daemonsets --delete-emptydir-data
kubectl drain ubuntu4 --ignore-daemonsets --delete-emptydir-data
kubectl drain ubuntu3 --ignore-daemonsets --delete-emptydir-data
kubectl drain ubuntu2 --ignore-daemonsets --delete-emptydir-data
kubectl drain ubuntu1 --ignore-daemonsets --delete-emptydir-data
# Shut down the nodes
sudo shutdown now
MetalLB #
Install Helm #
# Install Helm with script
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 &&
chmod +x get_helm.sh &&
./get_helm.sh
# Verify the installation / check version
helm version
Add Helm Repository #
# Add the MetalLB repository
helm repo add metallb https://metallb.github.io/metallb
# Update index
helm repo update
# Optional: Save & adopt the MetalLB chart values
helm show values metallb/metallb > metallb-values.yaml
Install MetalLB #
# Install MetalLB
helm install --create-namespace --namespace metallb-system metallb metallb/metallb
# Shell output:
NAME: metallb
LAST DEPLOYED: Thu Jun 20 10:47:27 2024
NAMESPACE: metallb-system
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
MetalLB is now running in the cluster.
Now you can configure it via its CRs. Please refer to the metallb official docs
on how to use the CRs.
# Verify the resources
kubectl get pods -n metallb-system
# Shell output:
NAME READY STATUS RESTARTS AGE
metallb-controller-665d96757f-2bxkk 1/1 Running 0 9m10s
metallb-speaker-qc4b2 4/4 Running 0 9m10s
metallb-speaker-rldrw 4/4 Running 0 9m10s
metallb-speaker-rrm2s 4/4 Running 0 9m10s
metallb-speaker-vd88k 4/4 Running 0 9m10s
metallb-speaker-x6nzg 4/4 Running 0 9m10s
MetalLB Configuration #
# Create a configuration for MetalLB
vi metallb-configuration.yaml
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: main-pool
namespace: metallb-system
spec:
addresses:
- 192.168.30.200-192.168.30.254
---
apiVersion: metallb.io/v1beta1
kind: L2Advertisement
metadata:
name: main-advertisement
namespace: metallb-system
spec:
ipAddressPools:
- main-pool
# Deploy the MetalLB configuration
kubectl apply -f metallb-configuration.yaml
Verify the Configuration #
# Verify the MetalLB IP pools
kubectl get IPAddressPool -n metallb-system
# Shell output:
NAME AUTO ASSIGN AVOID BUGGY IPS ADDRESSES
main-pool true false ["192.168.30.200-192.168.30.254"]
# Verify the L2Advertisement
kubectl get L2Advertisement -n metallb-system
# Shell output:
NAME IPADDRESSPOOLS IPADDRESSPOOL SELECTORS INTERFACES
main-advertisement ["main-pool"]
Test Deployment #
Deploy a pod with LoadBalancer service to verify the LoadBalancer assigns an IP address.
Deploy Pod and LoadBalancer #
# Run container: Example
kubectl run my-container --image=jueklu/container-2 --port=8080 --restart=Never --labels app=testing
# Create a LoadBalancer service to expose the pod "my-container"
kubectl expose pod/my-container --port=8080 --target-port=8080 --type=LoadBalancer --name=my-container-service
Verify the Deployment #
# List the pods
kubectl get pods
# Shell output
NAME READY STATUS RESTARTS AGE
my-container 1/1 Running 0 14s
# List LoadBalancer service details
kubectl get svc my-container-service
# Shell output
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
my-container-service LoadBalancer 10.43.223.17 192.168.30.200 8080:32306/TCP 14s
Access the Deployment #
# Access the deployment from a browser
192.168.30.200:8080
Delete the Deployment #
# Delete the deployment
kubectl delete pod my-container
# Delete the LoadBalancer service
kubectl delete svc my-container-service
Links #
# Official Documentation: Requirements
https://docs.rke2.io/install/requirements
# Official Documentation: Installation
https://docs.rke2.io/install/quickstart
# Official Documentation: Installation Details
https://docs.rke2.io/install/methods
# MetalLB Configuration
https://metallb.universe.tf/configuration/
# MetalLB Configuration Examples
https://github.com/metallb/metallb/tree/v0.14.5/configsamples