Kubernetes sections in docs
Documentation regarding k8s technologies.
ArgoCD is a declarative, GitOps continuous delivery tool for Kubernetes.
Your Git repository is the single source of truth; ArgoCD watches it and continuously syncs the cluster to match the desired state.
1Git repo โโโบ ArgoCD โโโบ Kubernetes cluster
1kubectl create namespace argocd
2kubectl apply -n argocd \
3 -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
4
5# expose the UI
6kubectl port-forward svc/argocd-server -n argocd 8080:443
1argocd login localhost:8080
2
3argocd app create myapp \
4 --repo https://github.com/org/repo.git \
5 --path deploy \
6 --dest-server https://kubernetes.default.svc \
7 --dest-namespace default
8
9argocd app list
10argocd app sync myapp
11argocd app get myapp
syncPolicy.automated set, ArgoCD re-applies any drift on its own.1images=$(helm template $helm |yq -N '..|.image? | select(. == "*" and . != null)'|sort|uniq|grep ":"|egrep -v '*:[[:blank:]]' || echo "")
1load_helm_images(){
2 # look in helm charts
3 for helm in $(ls ../../roles/*/files/helm/*.tgz); do
4 printf "\e[1;34m[INFO]\e[m Look for images in ${helm}...\n"
5
6 images=$(helm template $helm |yq -N '..|.image? | select(. == "*" and . != null)'|sort|uniq|grep ":"|egrep -v '*:[[:blank:]]' || echo "")
7
8 dir=$( dirname $helm | xargs dirname )
9
10 echo "####"
11
12 if [ "$images" != "" ]; then
13 printf "\e[1;34m[INFO]\e[m Images found in the helm charts: ${images}\n"
14 printf "\e[1;34m[INFO]\e[m Create ${dir}/images images...\n"
15
16 mkdir -p ${dir}/images
17
18 while i= read -r image_name; do
19 archive_name=$(basename -a $(awk -F : '{print $1}'<<<${image_name}));
20 printf "\e[1;34m[INFO]\e[m Pull images...\n"
21 podman pull ${image_name};
22 printf "\e[1;34m[INFO]\e[m Push ${image_name} in ${dir}/images/${archive_name}\n"
23 podman save ${image_name} --format oci-archive -o ${dir}/images/${archive_name};
24 done <<< ${images}
25 else
26 printf "\e[1;34m[INFO]\e[m No Images found in the helm charts: $helm\n"
27 fi
28 done
29}
1function checkComponentsInstall() {
2 componentsArray=("kubectl" "helm")
3 for i in "${componentsArray[@]}"; do
4 command -v "${i}" >/dev/null 2>&1 ||
5 { echo "[ERROR] ${i} is required, but it's not installed. Aborting." >&2; exit 1; }
6 done
7}
1function checkK8sVersion() {
2 currentK8sVersion=$(kubectl version --short | grep "Server Version" | awk '{gsub(/v/,$5)}1 {print $3}')
3 testVersionComparator 1.20 "$currentK8sVersion" '<'
4 if [[ $k8sVersion == "ok" ]]; then
5 echo "current kubernetes version is ok"
6 else
7 minikube start --kubernetes-version=v1.22.4;
8 fi
9}
10
11
12# the comparator based on https://stackoverflow.com/a/4025065
13versionComparator () {
14 if [[ $1 == $2 ]]
15 then
16 return 0
17 fi
18 local IFS=.
19 local i ver1=($1) ver2=($2)
20 # fill empty fields in ver1 with zeros
21 for ((i=${#ver1[@]}; i<${#ver2[@]}; i++))
22 do
23 ver1[i]=0
24 done
25 for ((i=0; i<${#ver1[@]}; i++))
26 do
27 if [[ -z ${ver2[i]} ]]
28 then
29 # fill empty fields in ver2 with zeros
30 ver2[i]=0
31 fi
32 if ((10#${ver1[i]} > 10#${ver2[i]}))
33 then
34 return 1
35 fi
36 if ((10#${ver1[i]} < 10#${ver2[i]}))
37 then
38 return 2
39 fi
40 done
41 return 0
42}
43
44testVersionComparator () {
45 versionComparator $1 $2
46 case $? in
47 0) op='=';;
48 1) op='>';;
49 2) op='<';;
50 esac
51 if [[ $op != "$3" ]]
52 then
53 echo "Kubernetes test fail: Expected '$3', Actual '$op', Arg1 '$1', Arg2 '$2'"
54 k8sVersion="not ok"
55 else
56 echo "Kubernetes test pass: '$1 $op $2'"
57 k8sVersion="ok"
58 fi
59}
cert-manager automates the management of X.509 certificates inside Kubernetes via CRDs โ it requests, issues, renews and rotates certificates automatically.
Supported backends: ACME (Let’s Encrypt), self-signed, CA, Vault, Venafi, โฆ
1helm repo add jetstack https://charts.jetstack.io
2helm repo update
3helm upgrade --install cert-manager jetstack/cert-manager \
4 --namespace cert-manager --create-namespace \
5 --set installCRDs=true
1apiVersion: cert-manager.io/v1
2kind: ClusterIssuer
3metadata:
4 name: selfsigned
5spec:
6 selfSigned: {}
1apiVersion: cert-manager.io/v1
2kind: Certificate
3metadata:
4 name: example-tls
5spec:
6 secretName: example-tls
7 dnsNames:
8 - example.com
9 issuerRef:
10 name: selfsigned
11 kind: ClusterIssuer
1kubectl get certificate -A
2kubectl get certificaterequest -A
3kubectl describe certificate <name> -n <ns>
4
5# manual renewal (normally automatic)
6cmctl renew <name> -n <ns>
<name> secret, ready for Ingress.kubectl get challenges is the first place to look for ACME/Let’s Encrypt
failures.1helm list -A
2NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION
3nesux3 default 1 2022-08-12 20:01:16.0982324 +0200 CEST deployed nexus3-1.0.6 3.37.3
1helm status nesux3
2helm uninstall nesux3
3helm install nexus3 <chart> # chart URL or path
4helm history nexus3
5
6# work even if already installed
7helm upgrade --install ingress-nginx ${DIR}/helm/ingress-nginx \
8 --namespace=ingress-nginx \
9 --create-namespace \
10 -f ${DIR}/helm/ingress-values.yml
11
12#Make helm unsee an apps (it does not delete the apps)
13kubectl delete secret -l owner=helm,name=argo-cd
1#Handle repo
2helm repo list
3helm repo add gitlab https://charts.gitlab.io/
4helm repo update
5
6#Pretty usefull to configure
7helm show values elastic/eck-operator
8helm show values grafana/grafana --version 8.5.1
9
10#See different version available
11helm search repo hashicorp/vault
12helm search repo hashicorp/vault -l
13
14# download a chart
15helm fetch ingress/ingress-nginx --untar
K3D equal k3s in a container. a tools to create single- and multi-node k3s clusters.
Our favorite use case, is with podman and rootless. So there is some customization upstream to do.
One downside Iโve found with k3d is that the Kubernetes version it uses is behind the current k3s release.
Note for ARM PC:
1sudo apt install qemu-user-static
2podman run --rm --privileged multiarch/qemu-user-static --reset -p yes
1# Manual way
2curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
3
4# or with arkade:
5arkade get k3d
6
7# Auto-completion
8k3d completion zsh > "$ZSH/completions/_k3d"
1k3d cluster create test
2
3ERRO[0000] Failed to get nodes for cluster 'test': docker failed to get containers with labels 'map[k3d.cluster:test]': failed to list containers: permission denied while trying to connect to the Docker daemon socket at unix:///var/run/docker.sock: Get "http://%2Fvar%2Frun%2Fdocker.sock/v1.46/containers/json?all=1&filters=%7B%22label%22%3A%7B%22app%3Dk3d%22%3Atrue%2C%22k3d.cluster%3Dtest%22%3Atrue%7D%7D": dial unix /var/run/docker.sock: connect: permission denied
1# TODO
2loginctl enable-linger $(whoami)
3
4# Either reload terminal or do below:
5export XDG_RUNTIME_DIR=/tmp/run-$(id -u)
6mkdir -p $XDG_RUNTIME_DIR
7chmod 700 $XDG_RUNTIME_DIR
8
9sudo mkdir -p /etc/containers/containers.conf.d
10sudo sh -c "echo 'service_timeout=0' > /etc/containers/containers.conf.d/timeout.conf"
11
12sudo ln -s /run/podman/podman.sock /var/run/docker.sock
13
14XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR:-/run/user/$(id -u)}
15export DOCKER_HOST=unix://$XDG_RUNTIME_DIR/podman/podman.sock
16export DOCKER_SOCK=$XDG_RUNTIME_DIR/podman/podman.sock
17
18systemctl --user enable --now podman.socket
If /sys/fs/cgroup/cgroup.controllers is present on your system, you are using v2, otherwise you are using v1.
1# Create a trust zone for the two interconnect
2sudo firewall-cmd --permanent --zone=trusted --add-source=10.42.0.0/16 #pods
3sudo firewall-cmd --permanent --zone=trusted --add-source=10.43.0.0/16 #services
4sudo firewall-cmd --reload
5sudo firewall-cmd --list-all-zones
6
7# on Master
8sudo rm -f /var/lib/cni/networks/cbr0/lock
9sudo /usr/local/bin/k3s-killall.sh
10sudo systemctl restart k3s
11sudo systemctl status k3s
12
13# on Worker
14sudo rm -f /var/lib/cni/networks/cbr0/lock
15sudo /usr/local/bin/k3s-killall.sh
16sudo systemctl restart k3s-agent
17sudo systemctl status k3s-agent
1# Get CA from K3s master
2openssl s_client -connect localhost:6443 -showcerts < /dev/null 2>&1 | openssl x509 -noout -enddate
3openssl s_client -showcerts -connect 193.168.51.103:6443 < /dev/null 2>/dev/null|openssl x509 -outform PEM
4openssl s_client -showcerts -connect 193.168.51.103:6443 < /dev/null 2>/dev/null|openssl x509 -outform PEM | base64 | tr -d '\n'
5
6# Check end date:
7for i in `ls /var/lib/rancher/k3s/server/tls/*.crt`; do echo $i; openssl x509 -enddate -noout -in $i; done
8
9# More efficient:
10cd /var/lib/rancher/k3s/server/tls/
11for crt in *.crt; do printf '%s: %s\n' "$(date --date="$(openssl x509 -enddate -noout -in "$crt"|cut -d= -f 2)" --iso-8601)" "$crt"; done | sort
12
13# Check CA issuer
14for i in $(find . -maxdepth 1 -type f -name "*.crt"); do openssl x509 -in ${i} -noout -issuer; done
1# Rancher local install - for example on WSL
2sudo podman run --privileged -d --restart=unless-stopped -p 80:80 -p 443:443 rancher/rancher
3sudo podman ps
4sudo podman logs 74533d50d991 2>&1 | grep "Bootstrap Password:"
Nice gist to troubleshoot etcd link
1journalctl -u rke2-server.service -f
2
3tail -f /var/lib/rancher/rke2/agent/containerd/containerd.log
4
5tail -f /var/lib/rancher/rke2/agent/logs/kubelet.log
6
7# crictl
8export CRI_CONFIG_FILE=/var/lib/rancher/rke2/agent/etc/crictl.yaml
9/var/lib/rancher/rke2/bin/crictl ps
10
11/var/lib/rancher/rke2/bin/crictl --config /var/lib/rancher/rke2/agent/etc/crictl.yaml ps
12
13/var/lib/rancher/rke2/bin/crictl --runtime-endpoint unix:///run/k3s/containerd/containerd.sock ps -a
14
15/var/lib/rancher/rke2/bin/ctr --address /run/k3s/containerd/containerd.sock --namespace k8s.io container ls
16
17# Kubectl
18export KUBECONFIG=/etc/rancher/rke2/rke2.yaml
19export PATH=$PATH:/usr/local/bin/:/var/lib/rancher/rke2/bin/
20kubectl get addon -A
1export CRI_CONFIG_FILE=/var/lib/rancher/rke2/agent/etc/crictl.yaml
2etcdcontainer=$(/var/lib/rancher/rke2/bin/crictl ps --label io.kubernetes.container.name=etcd --quiet)
3/var/lib/rancher/rke2/bin/crictl exec $etcdcontainer sh -c "ETCDCTL_ENDPOINTS='https://127.0.0.1:2379' ETCDCTL_CACERT='/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt' ETCDCTL_CERT='/var/lib/rancher/rke2/server/tls/etcd/server-client.crt' ETCDCTL_KEY='/var/lib/rancher/rke2/server/tls/etcd/server-client.key' ETCDCTL_API=3 etcdctl endpoint status --cluster --write-out=table"
1export CRI_CONFIG_FILE=/var/lib/rancher/rke2/agent/etc/crictl.yaml
2etcdcontainer=$(/var/lib/rancher/rke2/bin/crictl ps --label io.kubernetes.container.name=etcd --quiet)
3/var/lib/rancher/rke2/bin/crictl exec $etcdcontainer sh -c "ETCDCTL_ENDPOINTS='https://127.0.0.1:2379' ETCDCTL_CACERT='/var/lib/rancher/rke2/server/tls/etcd/server-ca.crt' ETCDCTL_CERT='/var/lib/rancher/rke2/server/tls/etcd/server-client.crt' ETCDCTL_KEY='/var/lib/rancher/rke2/server/tls/etcd/server-client.key' ETCDCTL_API=3 etcdctl endpoint health --cluster --write-out=table"
1# Default one
2KUBECONFIG=~/.kube/config
3
4# Several context - to keep splited
5KUBECONFIG=~/.kube/k3sup-lab:~/.kube/k3s-dev
6
7# Or can be specified in command
8kubectl get pods --kubeconfig=admin-kube-config
1kubectl config view
2kubectl config current-context
3
4kubectl config set-context \
5dev-context \
6--namespace=dev-namespace \
7--cluster=docker-desktop \
8--user=dev-user
9
10kubectl config use-context lab
1#set Namespace
2kubectl config set-context --current --namespace=nexus3
3kubectl config get-contexts
The problem with the kubeconfig is that it get nexted in one kubeconfig and difficult to manage on long term.
The best way to install it, is with Arkade arkade get kubecm - see arkade.
vi dns.yml 1apiVersion: v1
2kind: Pod
3metadata:
4 name: dnsutils
5 namespace: default
6spec:
7 containers:
8 - name: dnsutils
9 image: registry.k8s.io/e2e-test-images/jessie-dnsutils:1.3
10 command:
11 - sleep
12 - "infinity"
13 imagePullPolicy: IfNotPresent
14 restartPolicy: Always
1k apply -f dns.yml
2pod/dnsutils created
3
4kubectl get pods dnsutils
5NAME READY STATUS RESTARTS AGE
6dnsutils 1/1 Running 0 36s
1kubectl exec -i -t dnsutils -- nslookup kubernetes.default
2;; connection timed out; no servers could be reached
3command terminated with exit code 1
4
5kubectl exec -ti dnsutils -- cat /etc/resolv.conf
6search default.svc.cluster.local svc.cluster.local cluster.local example.local
7nameserver 10.43.0.10
8options ndots:5
9
10kubectl get endpoints kube-dns --namespace=kube-system
11NAME ENDPOINTS AGE
12kube-dns 10.42.0.6:53,10.42.0.6:53,10.42.0.6:9153 5d1h
13
14kubectl get svc kube-dns --namespace=kube-system
15NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
16kube-dns ClusterIP 10.43.0.10 <none> 53/UDP,53/TCP,9153/TCP 5d1h
1cat << EOF > curl.yml
2apiVersion: v1
3kind: Pod
4metadata:
5 name: curl
6 namespace: default
7spec:
8 containers:
9 - name: curl
10 image: curlimages/curl
11 command:
12 - sleep
13 - "infinity"
14 imagePullPolicy: IfNotPresent
15 restartPolicy: Always
16EOF
17
18k apply -f curl.yml
19
20#Test du DNS
21kubectl exec -i -t curl -- curl -v telnet://10.43.0.10:53
22kubectl exec -i -t curl -- curl -v telnet://kube-dns.kube-system.svc.cluster.local:53
23kubectl exec -i -t curl -- nslookup kube-dns.kube-system.svc.cluster.local
24
25curl -k -I --resolve subdomain.domain.com:52.165.230.62 https:/subdomain.domain.com/
They are essentially the same Kubernetes distribution.
OKD โ short for “The Community Distribution of Kubernetes that powers Red Hat OpenShift” โ is the free, upstream/community edition: the same codebase, community support, and images pulled from quay.io.
OpenShift is Red Hat’s enterprise product: OKD plus commercial support, certifications, a longer support lifecycle, Red Hat registries (registry.redhat.io) and OperatorHub access.
Day-to-day they are nearly interchangeable โ both use the same oc, openshift-install and oc-mirror tooling, and the same install-config.yaml layout. Unless stated otherwise, the notes below apply to both.
A controller is a loop that watches Kubernetes objects and reconciles their desired state (the spec) with the actual state (the cluster).
An Operator is a controller with operational knowledge baked in: it knows how to install, configure, scale, back up and upgrade an application โ the jobs a human admin used to do by hand.
1desired state (CR) โโโบ reconcile loop โโโบ actual state (cluster)
Classic examples: etcd-operator, prometheus-operator.
Some time ago, I made a small shell script to handle Vault on a cluster kubernetes. For documentation purpose.
1#!/bin/bash
2
3## Variables
4DIRNAME=$(dirname $0)
5DEFAULT_VALUE="vault/values-override.yaml"
6NewAdminPasswd="PASSWORD"
7PRIVATE_REGISTRY_USER="registry-admin"
8PRIVATE_REGISTRY_PASSWORD="PASSWORD"
9PRIVATE_REGISTRY_ADDRESS="registry.example.com"
10DOMAIN="example.com"
11INGRESS="vault.${DOMAIN}"
12
13if [ -z ${CM_NS+x} ];then
14 CM_NS='your-namespace'
15fi
16
17if [ -z ${1+x} ]; then
18 VALUES_FILE="${DIRNAME}/${DEFAULT_VALUE}"
19 echo -e "\n[INFO] Using default values file '${DEFAULT_VALUE}'"
20else
21 if [ -f $1 ]; then
22 echo -e "\n[INFO] Using values file $1"
23 VALUES_FILE=$1
24 else
25 echo -e "\n[ERROR] No file exist $1"
26 exit 1
27 fi
28fi
29
30## Functions
31function checkComponentsInstall() {
32 componentsArray=("kubectl" "helm")
33 for i in "${componentsArray[@]}"; do
34 command -v "${i}" >/dev/null 2>&1 ||
35 { echo "${i} is required, but it's not installed. Aborting." >&2; exit 1; }
36 done
37}
38
39function createSecret() {
40kubectl get secret -n ${CM_NS} registry-pull-secret --no-headers 2> /dev/null \
41|| \
42kubectl create secret docker-registry -n ${CM_NS} registry-pull-secret \
43 --docker-server=${PRIVATE_REGISTRY_ADDRESS} \
44 --docker-username=${PRIVATE_REGISTRY_USER} \
45 --docker-password=${PRIVATE_REGISTRY_ADDRESS}
46}
47
48function installWithHelm() {
49helm dep update ${DIRNAME}/helm
50
51helm upgrade --install vault ${DIRNAME}/helm \
52--namespace=${CM_NS} --create-namespace \
53--set global.imagePullSecrets.[0]=registry-pull-secret \
54--set global.image.repository=${PRIVATE_REGISTRY_ADDRESS}/hashicorp/vault-k8s \
55--set global.agentImage.repository=${PRIVATE_REGISTRY_ADDRESS}/hashicorp/vault \
56--set ingress.hosts.[0]=${INGRESS} \
57--set ingress.enabled=true \
58--set global.leaderElection.namespace=${CM_NS}
59
60echo -e "\n[INFO] sleep 30s" && sleep 30
61}
62
63checkComponentsInstall
64createSecret
65installWithHelm
Allow local kubernetes to create and reach secret on the Vault