275 lines
7.3 KiB
Plaintext
275 lines
7.3 KiB
Plaintext
Deployment with Ressource Constraints
|
|
|
|
https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
|
|
|
|
It's a good practice to assign a CPU request and a CPU limit to a container. Containers cannot use more CPU than the configured limit. Provided the system has CPU time free, a container is guaranteed to be allocated as much CPU as it requests.
|
|
|
|
mkdir ~/stress && cd ~/stress
|
|
kubectl create namespace stress
|
|
kubectl config set-context --current --namespace stress
|
|
|
|
cat >deployment2.yaml <<EOF
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
creationTimestamp: null
|
|
labels:
|
|
app: web
|
|
name: web
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: web
|
|
strategy: {}
|
|
template:
|
|
metadata:
|
|
creationTimestamp: null
|
|
labels:
|
|
app: web
|
|
spec:
|
|
containers:
|
|
- image: bee42/whoami:2.2.0
|
|
name: whoami
|
|
ports:
|
|
- containerPort: 80
|
|
resources:
|
|
requests:
|
|
memory: "64Mi"
|
|
cpu: "250m"
|
|
limits:
|
|
memory: "128Mi"
|
|
cpu: "500m"
|
|
EOF
|
|
|
|
kubectl apply -f deployment2.yaml
|
|
kubectl get pods
|
|
kubectl get all
|
|
kubectl describe deployment web
|
|
kubectl rollout history deployment web
|
|
# set change cause
|
|
kubectl annotate deployment web kubernetes.io/change-cause='set cpu limit 500'
|
|
kubectl rollout history deployment web
|
|
|
|
https://kubernetes.io/docs/tasks/configure-pod-container/quality-service-pod/
|
|
QoS
|
|
https://kubernetes.io/docs/tasks/administer-cluster/manage-resources/quota-pod-namespace/
|
|
ResoureQuota
|
|
|
|
Generate Stress
|
|
|
|
https://linux.die.net/man/1/stress
|
|
|
|
create your own stress tool
|
|
|
|
mkdir -p ~/stress && cd ~/stress
|
|
cat >Dockerfile <<EOF
|
|
FROM ubuntu
|
|
|
|
RUN apt-get update && apt-get install -y stress && rm -rf /var/lib/apt/lists/*
|
|
|
|
ENTRYPOINT ["/usr/bin/stress", "--verbose"]
|
|
CMD []
|
|
EOF
|
|
docker build -t 127.0.0.1:5000/bee42/stress .
|
|
docker push 127.0.0.1:5000/bee42/stress
|
|
|
|
Todo:
|
|
|
|
apply the pod
|
|
check with kubectl top pods
|
|
|
|
cat >pod.yaml <<EOF
|
|
apiVersion: v1
|
|
kind: Pod
|
|
metadata:
|
|
name: cpu-demo
|
|
spec:
|
|
containers:
|
|
- name: cpu
|
|
image: cnbc-registry:5000/bee42/stress
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
requests:
|
|
cpu: "0.5"
|
|
args:
|
|
- --cpu
|
|
- "2"
|
|
EOF
|
|
kubectl apply -f pod.yaml
|
|
kubectl describe pod cpu-demo
|
|
# wait a minute
|
|
kubectl top pods
|
|
NAME CPU(cores) MEMORY(bytes)
|
|
cpu-demo 1962m 1Mi
|
|
hey 0m 0Mi
|
|
load 0m 1Mi
|
|
web-7cdc98f947-2n2dp 0m 3Mi
|
|
web-7cdc98f947-gn2cg 0m 4Mi
|
|
|
|
# Consume more cpu the limits set!
|
|
|
|
# PLEASE KILL this pod
|
|
kubectl delete pod cpu-demo --force=true --grace-period=0
|
|
|
|
stress-deploy.yaml
|
|
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: cpu-demo
|
|
labels:
|
|
app: cpu-demo
|
|
spec:
|
|
replicas: 4
|
|
selector:
|
|
matchLabels:
|
|
app: cpu-demo
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: cpu-demo
|
|
spec:
|
|
containers:
|
|
- name: stress
|
|
image: cnbc-registry:5000/bee42/stress
|
|
resources:
|
|
limits:
|
|
cpu: "1"
|
|
requests:
|
|
cpu: "0.5"
|
|
args:
|
|
- --cpu
|
|
- "2"
|
|
nodeName: k3d-cnbc-agent-1
|
|
|
|
kubectl apply -f stress-deploy.yaml
|
|
kubectl top pods
|
|
kubectl top nodes
|
|
NAME CPU(cores) MEMORY(bytes)
|
|
cpu-demo-595664dd8-lschw 1022m 1Mi
|
|
cpu-demo-595664dd8-npx4w 975m 1Mi
|
|
cpu-demo-595664dd8-p249m 993m 1Mi
|
|
cpu-demo-595664dd8-twn84 940m 1Mi
|
|
web-588896757f-52gx9 0m 1Mi
|
|
|
|
kubectl scale deploy cpu-demo --replicas=3
|
|
|
|
sleep 45
|
|
|
|
kubectl top nodes
|
|
kubectl top pods
|
|
NAME CPU(cores) MEMORY(bytes)
|
|
cpu-demo-595664dd8-npx4w 1017m 1Mi
|
|
cpu-demo-595664dd8-p249m 988m 1Mi
|
|
cpu-demo-595664dd8-twn84 980m 1Mi
|
|
web-588896757f-52gx9 1m 1Mi
|
|
|
|
https://kubernetes.io/docs/tasks/administer-cluster/cpu-management-policies/
|
|
Kubelet parameter
|
|
|
|
Tipp:
|
|
|
|
use imagePullSecerts to auth container registries
|
|
|
|
https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
|
|
|
|
Memory Overload
|
|
|
|
https://www.cyberciti.biz/faq/stress-test-linux-unix-server-with-stress-ng/
|
|
https://sysdig.com/blog/troubleshoot-kubernetes-oom/
|
|
|
|
k3d node create --cluster cnbc --role agent --memory 2G k3d-cnbc-agent-2
|
|
kubectl get nodes
|
|
# wait for ready
|
|
kubectl apply -f mem-deploy.yaml
|
|
|
|
kubectl top nodes
|
|
# OK!
|
|
kubectl scale deploy mem-demo --replicas=4
|
|
|
|
# Hups...
|
|
kubectl scale deploy mem-demo --replicas=5
|
|
# wait 60 secs
|
|
kubectl get pods -w
|
|
NAME READY STATUS RESTARTS AGE
|
|
web-588896757f-52gx9 1/1 Running 0 60m
|
|
mem-demo-67998d7456-4ntqw 1/1 Running 0 4m20s
|
|
mem-demo-67998d7456-htvnn 1/1 Running 0 4m20s
|
|
mem-demo-67998d7456-4f2wd 1/1 Running 0 2m2s
|
|
mem-demo-67998d7456-kx4xv 1/1 Running 2 (49s ago) 4m20s
|
|
mem-demo-67998d7456-d86cs 0/1 OOMKilled 2 (25s ago) 58s
|
|
mem-demo-67998d7456-d86cs 0/1 CrashLoopBackOff 2 (12s ago) 65s
|
|
mem-demo-67998d7456-d86cs 1/1 Running 3 (28s ago) 81s
|
|
mem-demo-67998d7456-4ntqw 0/1 OOMKilled 0 4m44s
|
|
mem-demo-67998d7456-4ntqw 1/1 Running 1 (2s ago) 4m45s
|
|
mem-demo-67998d7456-4f2wd 0/1 OOMKilled 0 2m29s
|
|
mem-demo-67998d7456-4f2wd 1/1 Running 1 (2s ago) 2m30s
|
|
mem-demo-67998d7456-d86cs 0/1 OOMKilled 3 (35s ago) 88s
|
|
mem-demo-67998d7456-d86cs 0/1 CrashLoopBackOff 3 (12s ago) 99s
|
|
|
|
# fix: scale down
|
|
kubectl scale deploy mem-demo --replicas=4
|
|
|
|
# cleanup deploy
|
|
kubectl delete -f mem-deploy.yaml
|
|
|
|
# drain node
|
|
|
|
kubectl drain k3d-k3d-cnbc-agent-2-0
|
|
k3d node delete k3d-k3d-cnbc-agent-2-0
|
|
kubectl delete node k3d-k3d-cnbc-agent-2-0
|
|
|
|
mem-deploy.yaml
|
|
|
|
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: mem-demo
|
|
labels:
|
|
app: mem-demo
|
|
spec:
|
|
replicas: 3
|
|
selector:
|
|
matchLabels:
|
|
app: mem-demo
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: mem-demo
|
|
spec:
|
|
containers:
|
|
- name: stress
|
|
image: cnbc-registry:5000/bee42/stress
|
|
resources:
|
|
limits:
|
|
memory: "500Mi"
|
|
requests:
|
|
memory: "128Mi"
|
|
args:
|
|
- "-m"
|
|
- "1"
|
|
- "--vm-bytes"
|
|
- "490M"
|
|
nodeName: k3d-k3d-cnbc-agent-2-0
|
|
|
|
Kubernetes Failures
|
|
|
|
https://codeberg.org/hjacobs/kubernetes-failure-stories
|
|
https://medium.com/omio-engineering/cpu-limits-and-aggressive-throttling-in-kubernetes-c5b20bd8a718
|
|
https://home.robusta.dev/blog/stop-using-cpu-limits
|
|
|
|
Best practices for CPU limits and requests on Kubernetes
|
|
|
|
Use CPU requests for everything
|
|
Make sure they are accurate
|
|
Do not use CPU limits.
|
|
|
|
What about memory limits and requests?
|
|
|
|
Always use memory limits
|
|
Always use memory requests
|
|
Always set your memory requests equal to your limits
|
|
|