Kubectl #
Kubectl Installation #
# Install Kubectl
curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl &&
chmod +x kubectl &&
sudo mv kubectl /usr/local/bin/
# List kubectl commands
kubectl -h
# List help for specific command: "get" command
kubectl get -h
Kubectl Autocomplete #
Insall Bash-Completion & Verify Installation #
# Search installed packages on Deb based system
dpkg -l | grep bash-completion
# Shell output:
ii bash-completion 1:2.11-8 all programmable completion for the bash shell
Install the bash-completion package if it’s not yet installed:
# Install bash-completion package
sudo apt update &&
sudo apt install bash-completion -y
Enable Kubectl Autocomplete #
Current Shell:
# Enable kubectl autocomplete: Current shell
source <(kubectl completion bash)
Permanent:
# Append the command to the .bashrc file
echo "source <(kubectl completion bash)" >> ~/.bashrc
# Apply changes
source ~/.bashrc
Kubeconfig Context #
# Switch Kubeconfig context
kubectl config use-context <context-name>
# List current Kubeconfig context
kubectl auth whoami
# List Kubeconfig permissions in specific namespace
kubectl auth can-i --list -n <namespace-name>
Useful Pods #
Maintenance Pods #
# Create netshoot pod
kubectl run netshoot \
--image=nicolaka/netshoot \
--restart=Never \
-it --rm -- /bin/bash
# Utils available inside the pod
ping google.com
nslookup google.com
dig google.com
curl google.com
# Run pod for network troubleshooting
kubectl run busybox --image=busybox --restart=Never --stdin --tty
# Utils available inside the pod
ping google.com
nslookup google.com
# Delete pod
kubectl delete pod busybox
# Run curl pod
kubectl run curltest --image=curlimages/curl:latest --rm -it --restart=Never -- sh
# Delete pod
kubectl delete pod curltest
Postgres Client Pod #
# Create Postgres client pod
kubectl run postgres-client \
--rm -it \
--restart=Never \
--image=postgres:17 \
-- bash
List Time on Kubernetes Node #
# List system time and date
kubectl run time-check \
--rm \
--attach \
--restart=Never \
--image=busybox:1.36 \
--command -- date
Access Pod / Container Shell #
Access Shell #
# Access an container shell: Single container pod
kubectl exec -it <pod-name> -- /bin/bash
# Access an container shell: Multi container pod
kubectl exec -it <pod-name> -c <container-name> -- /bin/bash
List Environment Variables #
# List environment variables: Single container pod
kubectl exec <pod-name> -- printenv
# List environment variables: Multi container pod
kubectl exec <pod-name> -c <container-name> -- printenv
Pod / Container Logs #
# Container logs: Single container pod
kubectl logs <pod-name>
# Container logs: Multi container pod
kubectl logs <pod-name> -c <container-name>
# Container logs: Follow the logs
kubectl logs -f <pod-name>
# Container logs: Define timeframe
kubectl logs <pod-name> --since=30m
# Container logs, previous pod
kubectl logs <pod-name> --previous
Port Forwarding #
Kubernetes will forward to the correct container based on the destination port:
# Forward container port: Access from other hosts
kubectl -n <namespace-name> port-forward --address 0.0.0.0 <pod-name> 8080:80
# Curl the local port
curl localhost:8080
# Stop port forwarding
Strg + C
List Pods #
List Pods via Label #
# List pods via label
kubectl -n kube-system get pods -l k8s-app=kube-dns
List Pods On Specific Node #
# List all pods that run on specific node
kubectl get pods -A -o wide --field-selector spec.nodeName=k8s-master-1
List Pods That Mount A PVC #
# List pods that mount a specific PVC
kubectl get pods -n dagster -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{range .spec.volumes[*]}{.persistentVolumeClaim.claimName}{" "}{end}{"\n"}{end}' | grep <pvc-name>
List Pod Resource Usage #
# List pods regarding RAM usage
kubectl -n <namespace> top pod
List Pods / Sort by Resource Usage #
# List pods regarding RAM usage
kubectl top pods -A --sort-by=memory
# Shell output:
NAMESPACE NAME CPU(cores) MEMORY(bytes)
loki loki-chunks-cache-0 4m 4141Mi
loki loki-ingester-0 56m 3229Mi
loki loki-ingester-1 64m 2686Mi
vmks vmstorage-vmks-victoria-metrics-k8s-stack-1 94m 2420Mi
vmks vmstorage-vmks-victoria-metrics-k8s-stack-0 103m 2311Mi
grafana grafana-8548c57ccc-rssmv 37m 1062Mi
# List pods regarding RAM usage, namespace specific, containers
kubectl top pods -n loki --containers --sort-by=memory
# Shell output:
POD NAME CPU(cores) MEMORY(bytes)
loki-chunks-cache-0 memcached 28m 4124Mi
loki-chunks-cache-0 exporter 3m 17Mi
loki-ingester-0 ingester 95m 3293Mi
loki-ingester-1 ingester 79m 2890Mi
loki-ingester-2 ingester 68m 738Mi
loki-distributor-54f8484489-hqmcv distributor 59m 491Mi
loki-query-frontend-889cdfc8b-lc66q query-frontend 40m 392Mi
loki-distributor-54f8484489-xgxlc distributor 56m 361Mi
loki-querier-698dc6fb99-qspkw querier 119m 320Mi
loki-querier-698dc6fb99-7625j querier 11m 276Mi
loki-querier-698dc6fb99-hslgs querier 125m 148Mi
loki-distributor-54f8484489-6s877 distributor 52m 99Mi
loki-query-frontend-889cdfc8b-tqr67 query-frontend 3m 87Mi
loki-index-gateway-0 index-gateway 3m 80Mi
loki-compactor-0 compactor 10m 71Mi
loki-index-gateway-1 index-gateway 2m 53Mi
loki-query-scheduler-757fd976d9-m2v46 query-scheduler 2m 34Mi
loki-query-scheduler-757fd976d9-zsh4j query-scheduler 2m 32Mi
loki-results-cache-0 exporter 2m 13Mi
loki-results-cache-0 memcached 1m 11Mi
Pod Troubleshooting #
Delete Pod with Force #
# Delete pod with force
kubectl -n <namespace> delete pod <pod-name> --grace-period=0 --force
Patch Finalizer #
# Patch finalizer when resource is stuck in deleting
kubectl patch chi -n <namespace> <pod-name> --type=merge -p '{"metadata":{"finalizers":[]}}'
Scale Deployment #
# Set the number of replicas for a deployment
kubectl scale deployment <deployment-name> --replicas=2
Restart Rollout #
# Set the number of replicas for a deployment
kubectl rollout restart <deployment-name>
Certificates #
List Certificates #
# List certificates
kubectl get certificate -n <namespace>
# List certificate details
kubectl describe certificate <certificate-name>
List Certificate Request #
# Check certificate request
kubectl get certificaterequest -n <namespace>
kubectl get certificaterequest -A
# List Cert request details
kubectl describe certificaterequest <certificate-request-name>
# Delete certificate request
kubectl -n <namespace> delete certificaterequest <certificate-request-name>
List Orders #
# List Orders
kubectl get orders.acme.cert-manager.io -n <namespace>
# List Orders details
kubectl -n <namespace> describe orders.acme.cert-manager.io <ordername>
# Delete Order
kubectl -n <namespace> delete order <ordername>
List Challenges #
# List Challenges
kubectl get challenges.acme.cert-manager.io -n <namespace>
# List Challgenge details
kubectl -n <namespace> describe challenges.acme.cert-manager.io <challenge-name>
Manually Renew Certificate #
# Install cmctl
curl -LO https://github.com/cert-manager/cmctl/releases/latest/download/cmctl_linux_amd64
chmod +x cmctl_linux_amd64
sudo mv cmctl_linux_amd64 /usr/local/bin/cmctl
# Verify installation
cmctl version
# Trigger renewal of certificate
cmctl -n <namespace> renew >certificate-name>
# Shell output:
Manually triggered issuance of Certificate <namespace>/<cert-name>
Traces public DNS resolution for the TXT record #
# Run test pod
kubectl run -it --rm dns-debug \
--image=ghcr.io/nicolaka/netshoot \
--restart=Never -- bash
# Test DNS resolution
dig +trace _acme-challenge.jklug.work TXT
CertManager Logs #
# List CertManager logs
kubectl logs -n cert-manager deploy/cert-manager
# List CertManager logs
kubectl logs -n cert-manager deploy/cert-manager --since=24h | grep -iE \
'authoritative|nameserver|self check|propagation|error'
Update CertManager Nameserver #
# Update CertManager DNS servers
helm upgrade cert-manager jetstack/cert-manager \
-n cert-manager \
--reuse-values \
--set "extraArgs={--dns01-recursive-nameservers=1.1.1.1:53\,8.8.8.8:53,--dns01-recursive-nameservers-only}"
Nodes #
List Nodes & Check Cluster Status #
# List nodes
kubectl get nodes
# List nodes: List node IP and Kubernetes version
kubectl get nodes -o wide
# List full Node details
kubectl describe node <node-name>
List Node Resource Usage #
# List Node resource usage
kubectl top node
- Divides the usage by the node’s
allocatablememory. That is the part Kubernetes lets pods use.
# List Node resource usage
kubectl top node --show-capacity
- Divides the usage by the node’s capacity, which is the total RAM of the VM.
Label Worker Nodes #
# Label a worker node
kubectl label nodes <node-name> kubernetes.io/role=worker
List Node Taints #
# List node taints
kubectl describe node <node> | grep -A5 Taints
Kubelet Logs #
# List Kubelet logs of a Kubernetes Node: Run on desired node
sudo journalctl -u kubelet
Kubernetes Version #
# List Kubernetes version: Client (Kubectl) & Server (Kubernetes Cluster)
kubectl version
Shutdown Node / Maintenance Mode #
# drain each node, which safely evicts all pods from the node in preparation for maintenance
kubectl drain node3 --ignore-daemonsets --delete-emptydir-data
# Verify the status
kubectl get nodes
# Shell output
NAME STATUS ROLES AGE VERSION
node1 Ready control-plane 6d1h v1.29.5
node2 Ready control-plane 6d1h v1.29.5
node3 Ready,SchedulingDisabled <none> 6d1h v1.29.5
node4 Ready,SchedulingDisabled <none> 6d1h v1.29.5
# Uncord node / mark it as schedulable
kubectl uncordon node3
# Verify the status
kubectl get nodes
# Shell output
NAME STATUS ROLES AGE VERSION
node1 Ready control-plane 6d1h v1.29.5
node2 Ready control-plane 6d1h v1.29.5
node3 Ready <none> 6d1h v1.29.5
node4 Ready <none> 6d1h v1.29.5
Switch the Default Namespace #
List Current Cluster #
# List the current kubectl context: Cluster & user
kubectl config view | grep current-context
# Shell output:
current-context: kubernetes-admin@k8s.jkw.local
Set new Default Namespace #
# Set the namespace "new-namespace" as the default namespace
kubectl config set-context kubernetes-admin@k8s.jkw.local --namespace new-namespace
# Set the namespace "default" as the default namespace
kubectl config set-context kubernetes-admin@k8s.jkw.local --namespace default
Verify the default Namespace #
# Verify the configuration
kubectl config view
# Shell output:
apiVersion: v1
clusters:
- cluster:
certificate-authority-data: DATA+OMITTED
server: https://127.0.0.1:6443
name: k8s.jkw.local
contexts:
- context:
cluster: k8s.jkw.local
namespace: new-namespace # Default namespace
user: kubernetes-admin
name: kubernetes-admin@k8s.jkw.local
current-context: kubernetes-admin@k8s.jkw.local
kind: Config
preferences: {}
users:
- name: kubernetes-admin
user:
client-certificate-data: DATA+OMITTED
client-key-data: DATA+OMITTED
LimitRange Resource Limits #
Create LimitRange Resource #
# Create a LimitRange configuration
vi namespace-limits.yaml
apiVersion: v1
kind: LimitRange
metadata:
name: example-namespace-limits
namespace: example-namespace
spec:
limits:
- type: Pod
max:
cpu: "2"
memory: "1Gi"
min:
cpu: "200m"
memory: "100Mi"
- type: Container
default:
memory: "300Mi"
cpu: "200m"
defaultRequest:
memory: "200Mi"
cpu: "100m"
max:
memory: "600Mi"
cpu: "400m"
min:
memory: "100Mi"
cpu: "100m"
# Deploy the LimitRange resource
kubectl create -f namespace-limits.yaml
List LimitRange Resources #
# List the LimitRange resource for the namespace
kubectl get LimitRange --namespace example-namespace
# Shell output:
NAME CREATED AT
example-namespace-limits 2024-06-01T11:33:15Z
List Namespace Resource Limits #
# List the namespace details
kubectl describe namespace example-namespace
# Shell output:
Name: example-namespace
Labels: kubernetes.io/metadata.name=example-namespace
Annotations: <none>
Status: Active
No resource quota.
Resource Limits
Type Resource Min Max Default Request Default Limit Max Limit/Request Ratio
---- -------- --- --- --------------- ------------- -----------------------
Pod cpu 200m 2 - - -
Pod memory 100Mi 1Gi - - -
Container memory 100Mi 600Mi 200Mi 300Mi -
Container cpu 100m 400m 100m 200m -
Delete LimitRange Resource #
# Delete the LimitRange resource
kubectl delete LimitRange example-namespace-limits --namespace example-namespace
YAML Resources #
Apply / Update Resources from YAML Config #
# Deploy resources updates from YAML configuration: Current namespace
kubectl apply -f filename.yaml
# Deploy resources updates from YAML configuration: Specific namespace
kubectl apply -f filename.yaml -n <namespace-name>
Delete Resources from YAML Config #
# Delete resources from YAML configuration
kubectl delete -f filename.yaml
Create YAML Manifest from Deployment #
# Save YAML Manifest from existing deployment
kubectl -n <namespace> get deployment <deployment-name> -o yaml > <filename>.yaml
Example: Backup & Restore Deployment #
This is an example for CoreDNS deployment backup and restore:
# Create a backup of a Deployment
kubectl get deployment coredns -n kube-system -o yaml > coredns-deployment.yaml
# Delete the Deployment
kubectl delete deployment coredns -n kube-system
# Restore the Deployment
kubectl apply -f coredns-deployment.yaml
Services #
Service Types #
-
ClusterIPDefault service type when no service type is defined -
--type=NodePortCreates a NodePort service -
--type=LoadBalancerCreates a LoadBalancer service
Access NodePort #
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
testservice NodePort 10.233.13.17 192.168.30.100 8000:32007/TCP 2s
A NodePort service exposes a static port (NodePort) randomly assigned, in this example port “32007” on all nodes external IPs in the cluster, and internal on the cluster IP, in this example on port “8000”. Optional an external IP can be defined.
Create Service #
Service for Deployment #
ClusterIP Service:
# Create service type "ClusterIP" for a deployment
kubectl expose deployment <deployment-name> --port=8000 --target-port=80 --name=<service-name>
# Create service type "ClusterIP" for a deployment: Add external IP to access the service
kubectl expose deployment <deployment-name> --port=8000 --target-port=80 --name=<service-name> --external-ip="192.168.30.100"
NodePort Service:
# Create service type "NodePort" for a deployment
kubectl expose deployment <deployment-name> --type=NodePort --port=8000 --target-port=80 --name=<service-name>
# Create service type "NodePort" for a deployment: Add external IP to access the service
kubectl expose deployment <deployment-name> --type=NodePort --port=8000 --target-port=80 --name=<service-name> --external-ip="192.168.30.100"
LoadBalancer Service:
# Create service type "LoadBalancer" for a deployment: Random LoadBalancer IP
kubectl expose deployment <deployment-name> --type=LoadBalancer --port=8000 --target-port=80 --name=<service-name>
# Create service type "LoadBalancer" for a deployment: Define a specific LoadBalancer IP
kubectl expose deployment <deployment-name> --type=LoadBalancer --port=8000 --target-port=80 --name=<service-name> --external-ip="192.168.30.111"
Service for Pod #
# Create service type "ClusterIP" for a pod
kubectl expose pod <pod-name> --port=8000 --target-port=80 --name=<service-name>
# Create service type "ClusterIP" for a pod: Add external IP to access the service
kubectl expose pod <pod-name> --port=8000 --target-port=80 --name=<service-name> --external-ip="192.168.30.100"
Service for Replication Controller #
# Create service type "ClusterIP" for a Replication Controller
kubectl expose rc <rc-name> --port=8000 --target-port=80 --name=<service-name>
# Create service type "ClusterIP" for a Replication Controller: Add external IP to access the service
kubectl expose rc <rc-name> --port=8000 --target-port=80 --name=<service-name> --external-ip="192.168.30.100"
List Services #
# List all services / service details
kubectl get svc
List Service Details #
# List service details
kubectl get svc <service-name>
# List more service details
kubectl describe svc <service-name>
Delete Service #
# Delete service
kubectl delete service <service-name>
StorageClass, PV, PVC #
StorageClasses #
# List storage clases
kubectl get storageclasses
# Delete storage class
kubectl delete storageclass storageclass-name
Persistent Volume Claims #
# List PVC
kubectl get pvc
# Delete PVC
kubectl delete pvc pvc-name
Persistent Volumes #
# List PV
kubectl get pv
# Delete PV
kubectl delete pv pv-name
Horizontal Pod Autoscaler (HPA) #
Deploy HPA #
# Deploy a pod autoscaler: Maintains between 1 and 10 pod replicas
kubectl autoscale deployment deployment-name --cpu-percent=50 --min=1 --max=10 --name=hpa-name
List & Watch HPA / Check Status #
# List the HPA status: Wait till the HPA gets a target output from the metrics server
kubectl get hpa
# Wacht the load of the HPA
kubectl get hpa hpa-name --watch
List HPA Details #
# List HPA details
kubectl describe hpa hpa-name
Delete HPA #
# Wacht the HPA
kubectl delete hpa hpa-name
ConfigMaps #
# List ConfigMaps
kubectl get cm
# Edit ConfigMap
kubectl edit cm <configmap-name>
# List ConfigMap details
kubectl describe cm <configmap-name>
# Delete ConfigMap
kubectl delete cm <configmap-name>
Secrets #
# List secrets
kubectl get secrets
# List secret details
kubectl describe secret <secret-name>
# Delete secret
kubectl delete secret <secret-name>
Cronjobs #
YAML Configuration Example #
# Create cronjob configuration
vi example-conjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
name: example-cronjob
spec:
schedule: "*/1 * * * *"
jobTemplate: # Defines the job to be created on each schedule occurrence
spec:
template: # The Pod template
spec:
containers:
- name: cronjob-container
image: busybox
args:
- /bin/sh
- -c
- date; echo Hello from the Kubernetes cluster
restartPolicy: OnFailure
# Deploy cronjob
kubectl apply -f example-conjob.yaml
List Cronjobs #
Example Details:
# List cronjobs
kubectl get cronjobs
# Shell output:
NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE
example-cronjob */1 * * * * False 0 16s 40s
-
SUSPEND = FalseCronJob is active and will trigger executions according to its defined schedule -
SUSPEND = TrueCronJob is currently suspended, and no new jobs will be started until it is unsuspended
Command Overview:
# List cronjobs: In current namespace
kubectl get cronjobs
# List cronjobs: In a specific namespace
kubectl get cronjobs -n namespace-name
# List cronjobs: In all namespaces
kubectl get cronjobs --all-namespaces
Suspend & Resume Conjob #
Example Details:
# Suspend cronjob: Example
kubectl patch cronjob example-cronjob -p '{"spec":{"suspend": true}}'
# Verify the cronjob status
kubectl get cronjobs
# Shell output:
NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE
example-cronjob */1 * * * * True 0 43s 67s
# Resume cronjob: Example
kubectl patch cronjob example-cronjob -p '{"spec":{"suspend": false}}'
Command Overview:
# Suspend cronjob: Syntax
kubectl patch cronjob cronjob-name -p '{"spec":{"suspend": true}}'
# Resume cronjob: Syntax
kubectl patch cronjob cronjob-name -p '{"spec":{"suspend": false}}'
Check Job Execution #
# List jobs
kubectl get jobs
# Shell output:
NAME COMPLETIONS DURATION AGE
example-cronjob-28621260 1/1 5s 4m56s
example-cronjob-28621264 1/1 5s 49s
List Cronjob Details #
# List cronjobs: In current namespace
kubectl describe cronjob cronjob-name
# List cronjobs: In a specific namespace
kubectl describe cronjob cronjob-name -n namespace-name
Delete Cronjob #
# Delete cronjob: In current namespace
kubectl delete cronjob cronjob-name
# Delete cronjob: In a specific namespace
kubectl delete cronjob cronjob-name -n namespace-name
Utilities #
K9s #
Install K9s #
Find latest release:
https://github.com/derailed/k9s/tags
# Download package (Debian based)
wget https://github.com/derailed/k9s/releases/download/v0.32.5/k9s_linux_amd64.deb
# Install package
sudo dpkg -i k9s_linux_amd64.deb
# Install missing dependencies
sudo apt install -f
Verify K9s Installation #
# Verify installation / check version
k9s version
Start K9s #
# Start k9s
k9s
# Exit k9s
Strg + C
K9s Commands #
# List nodes
:node
# List namespaces
:ns
# List deployments
:deploy
# List pods
:pod
# List services
:svc
# Go back
ESC
# List logs
l
CRICTL ( Container Runtime Interface commandline) #
List Pods #
# List pods
sudo crictl pods
# Shelloutput:
POD ID CREATED STATE NAME NAMESPACE ATTEMPT RUNTIME
01af3ad77e53f 7 minutes ago Ready nginx default 0 (default)
286b4244a3c6d 14 minutes ago Ready svclb-traefik-e17c3435-j5gl2 kube-system 2 (default)
51e1265e2dfa5 15 minutes ago Ready loki-stack-promtail-dh7b5 default 2 (default)
31e1a65143a40 15 minutes ago Ready loki-stack-0 default 2 (default)
List Containers #
# List containers
sudo crictl ps -a
# Shell output:
CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
8fbd470d59191 1e5f3c5b981a9 8 minutes ago Running nginx 0 01af3ad77e53f nginx
80452d57546a6 edc812b8e25d0 15 minutes ago Running lb-tcp-443 2 286b4244a3c6d svclb-traefik-e17c3435-j5gl2
e55811e88e9ba edc812b8e25d0 15 minutes ago Running lb-tcp-80 2 286b4244a3c6d svclb-traefik-e17c3435-j5gl2
d4e3ff82fb9b7 6860eccd97258 15 minutes ago Running promtail 2 51e1265e2dfa5 loki-stack-promtail-dh7b5
9649420c8fb15 42dd246360519 15 minutes ago Running loki 2 31e1a65143a40 loki-stack-0
List Images #
# List images
sudo crictl images
sudo k3s crictl images
# Shell output:
IMAGE TAG IMAGE ID SIZE
docker.io/grafana/loki 2.6.1 42dd246360519 20.1MB
docker.io/grafana/promtail 2.9.3 6860eccd97258 75.1MB
docker.io/library/nginx latest 1e5f3c5b981a9 72.4MB
docker.io/rancher/klipper-lb v0.4.7 edc812b8e25d0 4.78MB
docker.io/rancher/mirrored-pause 3.6 6270bb605e12e 301kB
Delete Unused Images #
# Delete all unused images
sudo crictl rmi --prune
sudo k3s crictl rmi --prune
# Cronjob
0 5 * * 1 /usr/local/bin/crictl rmi --prune
# If a timeout error like this appears:
E0224 14:13:54.431447 2408301 log.go:32] "RemoveImage from image service failed" err="rpc error: code = DeadlineExceeded desc = context deadline exceeded" image="sha256:4f2ddb5d593f740f63deb711c27181d968147e15315026184f710e346dfb4a0c"
# Just run the prune command several times
sudo k3s crictl rmi --prune
# If necessary restart K3s agent service first
sudo systemctl restart k3s-agent