Refactor ansible inventory to add dev server

ansible/02-dev-inventory.yml

@@ -1,4 +1,41 @@
 devservers:
   hosts:
-    dev1:
-      ansible_host: 192.168.50.58
+    node09:
+      ansible_host: 101.100.172.55
+      ansible_port: 9991
+    node00:
+      ansible_host: 101.100.172.55
+      ansible_port: 9999
+    node01:
+      ansible_host: 101.100.172.55
+      ansible_port: 9998
+
+debian:
+  hosts:
+    node00:
+      ansible_host: 101.100.172.55
+      ansible_port: 9999
+    node01:
+      ansible_host: 101.100.172.55
+      ansible_port: 9998
+
+local:
+  hosts:
+    node00:
+      ansible_host: 192.168.50.114
+      ansible_port: 22
+    node01:
+      ansible_host: 192.168.50.68
+      ansible_port: 22
+    node02:
+      ansible_host: 192.168.50.56
+      ansible_port: 22
+
+test:
+  hosts:
+    node05:
+      ansible_host: 192.168.50.205
+      ansible_port: 22
+    node09:
+      ansible_host: 192.168.50.209
+      ansible_port: 22

ansible/playbook.yaml

@@ -0,0 +1,10 @@
+- name: My first play
+  hosts: devservers
+  tasks:
+   - name: Ping my hosts
+     ansible.builtin.ping:
+
+   - name: Print message
+     ansible.builtin.debug:
+      msg: Hello world
+

ansible/rke2/collections/requirements.yaml

@@ -0,0 +1,6 @@
+---
+collections:
+  - name: ansible.utils
+  - name: community.general
+  - name: ansible.posix
+  - name: kubernetes.core

ansible/rke2/config.yaml

@@ -0,0 +1,8 @@
+tls-san:
+  - 192.168.50.190
+  - 192.168.50.209
+  - 
+  - 
+write-kubeconfig-mode: 0644
+disable:
+  - rke2-ingress-nginx

ansible/rke2/inventory/group_vars/all.yaml

@@ -0,0 +1,20 @@
+os: "linux"
+# arch: "amd64"
+# Set your timezone
+system_timezone: "Asia/Singapore"
+
+kube_vip_version: "v0.8.1"
+vip_interface: eth0
+vip: 192.168.50.210
+
+metallb_version: v0.14.5
+lb_range: 192.168.50.190-192.168.50.199
+lb_pool_name: first-pool
+
+rke2_version: "v1.28.10+rke2r1"
+rke2_install_dir: "/usr/local/bin"
+rke2_binary_url: "https://github.com/rancher/rke2/releases/download/{{ rke2_version }}"
+
+ansible_user: furyhawk
+ansible_become: true
+ansible_become_method: sudo

ansible/rke2/inventory/hosts.ini

@@ -0,0 +1,8 @@
+# Make sure Ansible host has access to these devices
+# Good idea to snapshot all machines and deploy uing cloud-init
+
+[servers]
+server1 ansible_host=192.168.50.209
+
+[agents]
+agent1 ansible_host=192.168.50.205

ansible/rke2/kube-vip

@@ -0,0 +1,89 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: kube-vip-ds
+    app.kubernetes.io/version: v0.6.3
+  name: kube-vip-ds
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-vip-ds
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/name: kube-vip-ds
+        app.kubernetes.io/version: v0.6.3
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+      containers:
+      - args:
+        - manager
+        env:
+        - name: vip_arp
+          value: "true"
+        - name: port
+          value: "6443"
+        - name: vip_interface
+          value: $interface
+        - name: vip_cidr
+          value: "32"
+        - name: cp_enable
+          value: "true"
+        - name: cp_namespace
+          value: kube-system
+        - name: vip_ddns
+          value: "false"
+        - name: svc_enable
+          value: "false"
+        - name: svc_leasename
+          value: plndr-svcs-lock
+        - name: vip_leaderelection
+          value: "true"
+        - name: vip_leasename
+          value: plndr-cp-lock
+        - name: vip_leaseduration
+          value: "5"
+        - name: vip_renewdeadline
+          value: "3"
+        - name: vip_retryperiod
+          value: "1"
+        - name: address
+          value: $vip
+        - name: prometheus_server
+          value: :2112
+        image: ghcr.io/kube-vip/kube-vip:v0.6.3
+        imagePullPolicy: Always
+        name: kube-vip
+        resources: {}
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+      hostNetwork: true
+      serviceAccountName: kube-vip
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+  updateStrategy: {}
+status:
+  currentNumberScheduled: 0
+  desiredNumberScheduled: 0
+  numberMisscheduled: 0
+  numberReady: 0
+

ansible/rke2/longhorn.sh

@@ -0,0 +1,100 @@
+#!/bin/bash
+
+echo -e " \033[33;2m    __  _          _        ___                            \033[0m"
+echo -e " \033[33;2m    \ \(_)_ __ ___( )__    / _ \__ _ _ __ __ _  __ _  ___  \033[0m"
+echo -e " \033[33;2m     \ \ | '_ \` _ \/ __|  / /_\/ _\` | '__/ _\` |/ _\` |/ _ \ \033[0m"
+echo -e " \033[33;2m  /\_/ / | | | | | \__ \ / /_\\  (_| | | | (_| | (_| |  __/ \033[0m"
+echo -e " \033[33;2m  \___/|_|_| |_| |_|___/ \____/\__,_|_|  \__,_|\__, |\___| \033[0m"
+echo -e " \033[33;2m                                               |___/       \033[0m"
+echo -e " \033[35;2m          __                   _                          \033[0m"
+echo -e " \033[35;2m         / /  ___  _ __   __ _| |__   ___  _ __ _ __      \033[0m"
+echo -e " \033[35;2m        / /  / _ \| '_ \ / _\` | '_ \ / _ \| '__| '_ \     \033[0m"
+echo -e " \033[35;2m       / /__| (_) | | | | (_| | | | | (_) | |  | | | |    \033[0m"
+echo -e " \033[35;2m       \____/\___/|_| |_|\__, |_| |_|\___/|_|  |_| |_|    \033[0m"
+echo -e " \033[35;2m                         |___/                            \033[0m"
+echo -e " \033[36;2m                                                          \033[0m"
+echo -e " \033[32;2m             https://youtube.com/@jims-garage              \033[0m"
+echo -e " \033[32;2m                                                           \033[0m"
+
+#############################################
+# YOU SHOULD ONLY NEED TO EDIT THIS SECTION #
+#############################################
+
+# THIS SCRIPT IS FOR RKE2, NOT K3S!
+# THIS SCRIPT IS FOR RKE2, NOT K3S!
+# THIS SCRIPT IS FOR RKE2, NOT K3S!
+
+# Set the IP addresses of master1
+master1=192.168.50.209
+
+# Set the IP addresses of your Longhorn nodes
+longhorn1=192.168.50.205
+
+# User of remote machines
+user=furyhawk
+
+# Interface used on remotes
+interface=eth0
+
+# Set the virtual IP address (VIP)
+vip=192.168.50.210
+
+# Array of longhorn nodes
+storage=($longhorn1)
+
+#ssh certificate name variable
+certName=id_rsa
+
+#############################################
+#            DO NOT EDIT BELOW              #
+#############################################
+# For testing purposes - in case time is wrong due to VM snapshots
+sudo timedatectl set-ntp off
+sudo timedatectl set-ntp on
+
+# add ssh keys for all nodes
+for node in "${storage[@]}"; do
+  ssh-copy-id $user@$node
+done
+
+# add open-iscsi - needed for Debian and non-cloud Ubuntu
+if ! command -v sudo service open-iscsi status &> /dev/null
+then
+    echo -e " \033[31;5mOpen-ISCSI not found, installing\033[0m"
+    sudo apt install open-iscsi
+else
+    echo -e " \033[32;5mOpen-ISCSI already installed\033[0m"
+fi
+
+# Step 1: Add new longhorn nodes to cluster (note: label added)
+# Set token variable needed for RKE2 (not required for K3S)
+token=`cat token`
+for newnode in "${storage[@]}"; do
+  ssh -tt $user@$newnode -i ~/.ssh/$certName sudo su <<EOF
+  mkdir -p /etc/rancher/rke2
+  touch /etc/rancher/rke2/config.yaml
+  echo "token: $token" >> /etc/rancher/rke2/config.yaml
+  echo "server: https://$vip:9345" >> /etc/rancher/rke2/config.yaml
+  echo "node-label:" >> /etc/rancher/rke2/config.yaml
+  echo "  - longhorn=true" >> /etc/rancher/rke2/config.yaml
+  curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh -
+  systemctl enable rke2-agent.service
+  systemctl start rke2-agent.service
+  exit
+EOF
+  echo -e " \033[32;5mLonghorn node joined successfully!\033[0m"
+done
+
+# Step 2: Install Longhorn (using modified Official to pin to Longhorn Nodes)
+# kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.6.2/deploy/longhorn.yaml
+kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/Longhorn/longhorn.yaml
+kubectl get pods \
+--namespace longhorn-system \
+--watch
+
+# Step 3: Print out confirmation
+
+kubectl get nodes
+kubectl get svc -n longhorn-system
+
+echo -e " \033[32;5mHappy Kubing! Access Longhorn through Rancher UI\033[0m"

ansible/rke2/rke2.sh

@@ -0,0 +1,262 @@
+#!/bin/bash
+
+echo -e " \033[33;5m    __  _          _        ___                            \033[0m"
+echo -e " \033[33;5m    \ \(_)_ __ ___( )__    / _ \__ _ _ __ __ _  __ _  ___  \033[0m"
+echo -e " \033[33;5m     \ \ | '_ \` _ \/ __|  / /_\/ _\` | '__/ _\` |/ _\` |/ _ \ \033[0m"
+echo -e " \033[33;5m  /\_/ / | | | | | \__ \ / /_\\  (_| | | | (_| | (_| |  __/ \033[0m"
+echo -e " \033[33;5m  \___/|_|_| |_| |_|___/ \____/\__,_|_|  \__,_|\__, |\___| \033[0m"
+echo -e " \033[33;5m                                               |___/       \033[0m"
+
+echo -e " \033[36;5m                      ___ _  _____ ___                     \033[0m"
+echo -e " \033[36;5m                     | _ \ |/ / __|_  )                    \033[0m"
+echo -e " \033[36;5m                     |   / ' <| _| / /                     \033[0m"
+echo -e " \033[36;5m                     |_|_\_|\_\___/___|                    \033[0m"
+echo -e " \033[36;5m                                                           \033[0m"
+echo -e " \033[32;5m             https://youtube.com/@jims-garage              \033[0m"
+echo -e " \033[32;5m                                                           \033[0m"
+
+
+#############################################
+# YOU SHOULD ONLY NEED TO EDIT THIS SECTION #
+#############################################
+
+# Version of Kube-VIP to deploy
+KVVERSION="v0.6.3"
+
+# Set the IP addresses of the admin, masters, and workers nodes
+admin=192.168.50.143
+master1=192.168.50.209
+worker1=192.168.50.205
+
+# User of remote machines
+user=furyhawk
+
+# Interface used on remotes
+interface=eth0
+
+# Set the virtual IP address (VIP)
+vip=192.168.50.190
+
+# Array of all master nodes
+allmasters=($master1)
+
+# Array of master nodes
+masters=($master2 $master3)
+
+# Array of worker nodes
+workers=($worker1)
+
+# Array of all
+all=($master1 $worker1)
+
+# Array of all minus master1
+allnomaster1=($master2 $master3 $worker1)
+
+#Loadbalancer IP range
+lbrange=192.168.3.191-192.168.3.199
+
+#ssh certificate name variable
+certName=id_rsa
+
+#############################################
+#            DO NOT EDIT BELOW              #
+#############################################
+# For testing purposes - in case time is wrong due to VM snapshots
+sudo timedatectl set-ntp off
+sudo timedatectl set-ntp on
+
+# Move SSH certs to ~/.ssh and change permissions
+cp /home/$user/{$certName,$certName.pub} /home/$user/.ssh
+chmod 600 /home/$user/.ssh/$certName 
+chmod 644 /home/$user/.ssh/$certName.pub
+
+# Install Kubectl if not already present
+if ! command -v kubectl version &> /dev/null
+then
+    echo -e " \033[31;5mKubectl not found, installing\033[0m"
+    curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+    sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+else
+    echo -e " \033[32;5mKubectl already installed\033[0m"
+fi
+
+# Create SSH Config file to ignore checking (don't use in production!)
+sed -i '1s/^/StrictHostKeyChecking no\n/' ~/.ssh/config
+
+#add ssh keys for all nodes
+for node in "${all[@]}"; do
+  ssh-copy-id $user@$node
+done
+
+# Step 1: Create Kube VIP
+# create RKE2's self-installing manifest dir
+sudo mkdir -p /var/lib/rancher/rke2/server/manifests
+# Install the kube-vip deployment into rke2's self-installing manifest folder
+curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/kube-vip
+cat kube-vip | sed 's/$interface/'$interface'/g; s/$vip/'$vip'/g' > $HOME/kube-vip.yaml
+sudo mv kube-vip.yaml /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
+
+# Find/Replace all k3s entries to represent rke2
+sudo sed -i 's/k3s/rke2/g' /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
+# copy kube-vip.yaml to home directory
+sudo cp /var/lib/rancher/rke2/server/manifests/kube-vip.yaml ~/kube-vip.yaml
+# change owner
+sudo chown $user:$user kube-vip.yaml
+# make kube folder to run kubectl later
+mkdir ~/.kube
+
+# create the rke2 config file
+sudo mkdir -p /etc/rancher/rke2
+touch config.yaml
+echo "tls-san:" >> config.yaml 
+echo "  - $vip" >> config.yaml
+echo "  - $master1" >> config.yaml
+echo "  - $master2" >> config.yaml
+echo "  - $master3" >> config.yaml
+echo "write-kubeconfig-mode: 0644" >> config.yaml
+echo "disable:" >> config.yaml
+echo "  - rke2-ingress-nginx" >> config.yaml
+# copy config.yaml to rancher directory
+sudo cp ~/config.yaml /etc/rancher/rke2/config.yaml
+
+# update path with rke2-binaries
+echo 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml' >> ~/.bashrc ; echo 'export PATH=${PATH}:/var/lib/rancher/rke2/bin' >> ~/.bashrc ; echo 'alias k=kubectl' >> ~/.bashrc ; source ~/.bashrc ;
+
+# Step 2: Copy kube-vip.yaml and certs to all masters
+for newnode in "${allmasters[@]}"; do
+  scp -i ~/.ssh/$certName $HOME/kube-vip.yaml $user@$newnode:~/kube-vip.yaml
+  scp -i ~/.ssh/$certName $HOME/config.yaml $user@$newnode:~/config.yaml
+  scp -i ~/.ssh/$certName ~/.ssh/{$certName,$certName.pub} $user@$newnode:~/.ssh
+  echo -e " \033[32;5mCopied successfully!\033[0m"
+done
+
+# Step 3: Connect to Master1 and move kube-vip.yaml and config.yaml. Then install RKE2, copy token back to admin machine. We then use the token to bootstrap additional masternodes
+ssh -tt $user@$master1 -i ~/.ssh/$certName sudo su <<EOF
+mkdir -p /var/lib/rancher/rke2/server/manifests
+mv kube-vip.yaml /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
+mkdir -p /etc/rancher/rke2
+mv config.yaml /etc/rancher/rke2/config.yaml
+echo 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml' >> ~/.bashrc ; echo 'export PATH=${PATH}:/var/lib/rancher/rke2/bin' >> ~/.bashrc ; echo 'alias k=kubectl' >> ~/.bashrc ; source ~/.bashrc ;
+curl -sfL https://get.rke2.io | sh -
+systemctl enable rke2-server.service
+systemctl start rke2-server.service
+echo "StrictHostKeyChecking no" > ~/.ssh/config
+ssh-copy-id -i /home/$user/.ssh/$certName $user@$admin
+scp -i /home/$user/.ssh/$certName /var/lib/rancher/rke2/server/token $user@$admin:~/token
+scp -i /home/$user/.ssh/$certName /etc/rancher/rke2/rke2.yaml $user@$admin:~/.kube/rke2.yaml
+exit
+EOF
+echo -e " \033[32;5mMaster1 Completed\033[0m"
+
+# Step 4: Set variable to the token we just extracted, set kube config location
+token=`cat token`
+sudo cat ~/.kube/rke2.yaml | sed 's/127.0.0.1/'$master1'/g' > $HOME/.kube/config
+sudo chown $(id -u):$(id -g) $HOME/.kube/config
+export KUBECONFIG=${HOME}/.kube/config
+sudo cp ~/.kube/config /etc/rancher/rke2/rke2.yaml
+kubectl get nodes
+
+# Step 5: Install kube-vip as network LoadBalancer - Install the kube-vip Cloud Provider
+kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
+kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
+
+# Step 6: Add other Masternodes, note we import the token we extracted from step 3
+# for newnode in "${masters[@]}"; do
+#   ssh -tt $user@$newnode -i ~/.ssh/$certName sudo su <<EOF
+#   mkdir -p /etc/rancher/rke2
+#   touch /etc/rancher/rke2/config.yaml
+#   echo "token: $token" >> /etc/rancher/rke2/config.yaml
+#   echo "server: https://$master1:9345" >> /etc/rancher/rke2/config.yaml
+#   echo "tls-san:" >> /etc/rancher/rke2/config.yaml
+#   echo "  - $vip" >> /etc/rancher/rke2/config.yaml
+#   echo "  - $master1" >> /etc/rancher/rke2/config.yaml
+#   echo "  - $master2" >> /etc/rancher/rke2/config.yaml
+#   echo "  - $master3" >> /etc/rancher/rke2/config.yaml
+#   curl -sfL https://get.rke2.io | sh -
+#   systemctl enable rke2-server.service
+#   systemctl start rke2-server.service
+#   exit
+# EOF
+#   echo -e " \033[32;5mMaster node joined successfully!\033[0m"
+# done
+
+kubectl get nodes
+
+# Step 7: Add Workers
+for newnode in "${workers[@]}"; do
+  ssh -tt $user@$newnode -i ~/.ssh/$certName sudo su <<EOF
+  mkdir -p /etc/rancher/rke2
+  touch /etc/rancher/rke2/config.yaml
+  echo "token: $token" >> /etc/rancher/rke2/config.yaml
+  echo "server: https://$vip:9345" >> /etc/rancher/rke2/config.yaml
+  echo "node-label:" >> /etc/rancher/rke2/config.yaml
+  echo "  - worker=true" >> /etc/rancher/rke2/config.yaml
+  echo "  - longhorn=true" >> /etc/rancher/rke2/config.yaml
+  curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh -
+  systemctl enable rke2-agent.service
+  systemctl start rke2-agent.service
+  exit
+EOF
+  echo -e " \033[32;5mWorker node joined successfully!\033[0m"
+done
+
+kubectl get nodes
+
+# Step 8: Install Metallb
+echo -e " \033[32;5mDeploying Metallb\033[0m"
+kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml
+kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml
+# Download ipAddressPool and configure using lbrange above
+curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/ipAddressPool
+cat ipAddressPool | sed 's/$lbrange/'$lbrange'/g' > $HOME/ipAddressPool.yaml
+
+# Step 9: Deploy IP Pools and l2Advertisement
+echo -e " \033[32;5mAdding IP Pools, waiting for Metallb to be available first. This can take a long time as we're likely being rate limited for container pulls...\033[0m"
+kubectl wait --namespace metallb-system \
+                --for=condition=ready pod \
+                --selector=component=controller \
+                --timeout=1800s
+kubectl apply -f ipAddressPool.yaml
+kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/l2Advertisement.yaml
+
+# Step 10: Install Rancher (Optional - Delete if not required)
+#Install Helm
+echo -e " \033[32;5mInstalling Helm\033[0m"
+curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+chmod 700 get_helm.sh
+./get_helm.sh
+
+# Add Rancher Helm Repo & create namespace
+helm repo add rancher-latest https://releases.rancher.com/server-charts/latest
+kubectl create namespace cattle-system
+
+# Install Cert-Manager
+echo -e " \033[32;5mDeploying Cert-Manager\033[0m"
+kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.crds.yaml
+helm repo add jetstack https://charts.jetstack.io
+helm repo update
+helm install cert-manager jetstack/cert-manager \
+--namespace cert-manager \
+--create-namespace \
+--version v1.13.2
+kubectl get pods --namespace cert-manager
+
+# Install Rancher
+echo -e " \033[32;5mDeploying Rancher\033[0m"
+helm install rancher rancher-latest/rancher \
+ --namespace cattle-system \
+ --set hostname=rancher.my.org \
+ --set bootstrapPassword=admin
+kubectl -n cattle-system rollout status deploy/rancher
+kubectl -n cattle-system get deploy rancher
+
+# Add Rancher LoadBalancer
+kubectl get svc -n cattle-system
+kubectl expose deployment rancher --name=rancher-lb --port=443 --type=LoadBalancer -n cattle-system
+while [[ $(kubectl get svc -n cattle-system 'jsonpath={..status.conditions[?(@.type=="Pending")].status}') = "True" ]]; do
+   sleep 5
+   echo -e " \033[32;5mWaiting for LoadBalancer to come online\033[0m" 
+done
+kubectl get svc -n cattle-system
+
+echo -e " \033[32;5mAccess Rancher from the IP above - Password is admin!\033[0m"

ansible/rke2/roles/add-agent/tasks/main.yaml

@@ -0,0 +1,17 @@
+# Copy agent config to all agents - we need to change agent2 & 3 later with the token
+- name: Deploy RKE2 Agent Configuration
+  ansible.builtin.template:
+    src: templates/rke2-agent-config.j2
+    dest: /etc/rancher/rke2/config.yaml
+    owner: root
+    group: root
+    mode: '0644'
+  when: inventory_hostname in groups['agents']
+
+# Check agents have restarted to pick up config
+- name: Ensure RKE2 agents are enabled and running
+  ansible.builtin.systemd:
+    name: rke2-agent
+    enabled: true
+    state: restarted
+    daemon_reload: true

ansible/rke2/roles/add-agent/templates/rke2-agent-config.j2

@@ -0,0 +1,6 @@
+write-kubeconfig-mode: "0644"
+token: {{ hostvars['server1']['token'] }}
+server: https://{{ hostvars['server1']['ansible_host'] }}:9345
+node-label:
+  - "agent=true"
+  - "longhorn=true"

ansible/rke2/roles/add-server/tasks/main.yaml

@@ -0,0 +1,53 @@
+# Copy server config with token to all servers except server 1 (this has token)
+- name: Deploy RKE2 server Configuration
+  ansible.builtin.template:
+    src: templates/rke2-server-config.j2
+    dest: /etc/rancher/rke2/config.yaml
+    owner: root
+    group: root
+    mode: '0644'
+  when: inventory_hostname != groups['servers'][0]
+
+# Keep checking the cluster API until it's functioning (deployed)
+- name: Wait for cluster API to be ready (can take 5-10 mins depending on internet/hardware)
+  ansible.builtin.command:
+    cmd: "kubectl get nodes"
+  register: kubectl_output
+  until: "'connection refused' not in kubectl_output.stderr"
+  retries: 120
+  delay: 10
+  changed_when: true
+  become_user: "{{ ansible_user }}"
+  when: inventory_hostname == groups['servers'][0]
+
+# Use kubectl to deploy yaml. Perhaps this can be added to the manifest folder initially
+- name: Apply kube vip configuration file
+  ansible.builtin.command:
+    cmd: kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml apply -f https://kube-vip.io/manifests/rbac.yaml
+  changed_when: true
+  when: inventory_hostname == groups['servers'][0]
+
+# Apply the kube-vip configration. Perhaps this can be added to the manifest folder initially
+- name: Apply kube vip configuration file
+  ansible.builtin.command:
+    cmd: kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
+  changed_when: true
+  when: inventory_hostname == groups['servers'][0]
+
+# Check that additional servers are restarted
+- name: Ensure additional RKE2 servers are enabled and running
+  ansible.builtin.systemd:
+    name: rke2-server
+    enabled: true
+    state: restarted
+    daemon_reload: true
+  when: inventory_hostname != groups['servers'][0]
+
+# enable additional servers
+- name: Ensure RKE2 server is enabled and running
+  ansible.builtin.systemd:
+    name: rke2-server
+    enabled: true
+    state: restarted
+    daemon_reload: true
+  when: inventory_hostname != groups['servers'][0]

ansible/rke2/roles/add-server/templates/rke2-server-config.j2

@@ -0,0 +1,8 @@
+write-kubeconfig-mode: "0644"
+token: {{ hostvars['server1']['token'] }}
+server: https://{{ hostvars['server1']['ansible_host'] }}:9345
+tls-san:
+  - {{ vip }}
+  - {{ hostvars['server1']['ansible_host'] }}
+node-label:
+  - server=true

ansible/rke2/roles/apply-manifests/tasks/main.yaml

@@ -0,0 +1,60 @@
+# Wait for Server 1 to be ready before continuing with metallb deployment
+- name: Wait for k8s nodes with node label 'server=true' to be ready, otherwise we cannot start metallb deployment
+  ansible.builtin.command:
+    cmd: "kubectl wait --for=condition=Ready nodes --selector server=true --timeout=600s"
+  register: nodes_ready
+  retries: 120
+  delay: 10
+  changed_when: true
+  become_user: "{{ ansible_user }}"
+  when: inventory_hostname == groups['servers'][0]
+
+# Create namespace so that we can deploy metallb
+- name: Apply metallb namespace
+  ansible.builtin.command:
+    cmd: kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml
+  become_user: "{{ ansible_user }}"
+  changed_when: true
+  when: inventory_hostname == groups['servers'][0]
+
+# Apply metallb manifest
+- name: Apply metallb manifest
+  ansible.builtin.command:
+    cmd: kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/{{ metallb_version }}/config/manifests/metallb-native.yaml
+  become_user: "{{ ansible_user }}"
+  changed_when: true
+  when: inventory_hostname == groups['servers'][0]
+
+# Wait for metallb deployment pods to be alive before deploying metallb manifests
+- name: Wait for metallb pods to be ready, otherwise we cannot start metallb deployment
+  ansible.builtin.command:
+    cmd: "kubectl wait --namespace metallb-system --for=condition=ready pod --selector=component=controller --timeout=1800s"
+  changed_when: true
+  become_user: "{{ ansible_user }}"
+  when: inventory_hostname == groups['servers'][0]
+
+# Apply L2 Advertisement for metallb
+- name: Apply metallb L2 Advertisement
+  ansible.builtin.command:
+    cmd: kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/l2Advertisement.yaml
+  become_user: "{{ ansible_user }}"
+  changed_when: true
+  when: inventory_hostname == groups['servers'][0]
+
+# Deploy metal IP Pool to Server 1
+- name: Copy metallb IPPool to server 1
+  ansible.builtin.template:
+    src: templates/metallb-ippool.j2
+    dest: /home/{{ ansible_user }}/ippool.yaml
+    owner: "{{ ansible_user }}"
+    group: "{{ ansible_user }}"
+    mode: '0755'
+  when: inventory_hostname == groups['servers'][0]
+
+# don't think this will work as nodes are no execute, might need agents first
+- name: Apply metallb ipppool
+  ansible.builtin.command:
+    cmd: kubectl apply -f /home/{{ ansible_user }}/ippool.yaml
+  become_user: "{{ ansible_user }}"
+  changed_when: true
+  when: inventory_hostname == groups['servers'][0]

ansible/rke2/roles/apply-manifests/templates/metallb-ippool.j2

@@ -0,0 +1,8 @@
+apiVersion: metallb.io/v1beta1
+kind: IPAddressPool
+metadata:
+  name: {{ lb_pool_name }}
+  namespace: metallb-system
+spec:
+  addresses:
+  - {{ lb_range }}

ansible/rke2/roles/kube-vip/tasks/main.yaml

@@ -0,0 +1,17 @@
+# Create directory to deploy kube-vip manifest
+- name: Create directory for Kube VIP Manifest
+  ansible.builtin.file:
+    path: "/var/lib/rancher/rke2/server/manifests"
+    state: directory
+    mode: '0644'
+  when: inventory_hostname in groups['servers']
+
+# Copy kube-vip to server 1 manifest folder for auto deployment at bootstrap
+- name: Deploy Kube VIP Configuration
+  ansible.builtin.template:
+    src: templates/kube-vip-config.j2
+    dest: /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
+    owner: root
+    group: root
+    mode: '0644'
+  when: inventory_hostname == groups['servers'][0]

ansible/rke2/roles/kube-vip/templates/kube-vip-config.j2

@@ -0,0 +1,88 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  creationTimestamp: null
+  labels:
+    app.kubernetes.io/name: kube-vip-ds
+    app.kubernetes.io/version: {{ kube_vip_version }}
+  name: kube-vip-ds
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: kube-vip-ds
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app.kubernetes.io/name: kube-vip-ds
+        app.kubernetes.io/version: {{ kube_vip_version }}
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: node-role.kubernetes.io/master
+                operator: Exists
+            - matchExpressions:
+              - key: node-role.kubernetes.io/control-plane
+                operator: Exists
+      containers:
+      - args:
+        - manager
+        env:
+        - name: vip_arp
+          value: "true"
+        - name: port
+          value: "6443"
+        - name: vip_interface
+          value: {{ vip_interface }}
+        - name: vip_cidr
+          value: "32"
+        - name: cp_enable
+          value: "true"
+        - name: cp_namespace
+          value: kube-system
+        - name: vip_ddns
+          value: "false"
+        - name: svc_enable
+          value: "false"
+        - name: svc_leasename
+          value: plndr-svcs-lock
+        - name: vip_leaderelection
+          value: "true"
+        - name: vip_leasename
+          value: plndr-cp-lock
+        - name: vip_leaseduration
+          value: "5"
+        - name: vip_renewdeadline
+          value: "3"
+        - name: vip_retryperiod
+          value: "1"
+        - name: address
+          value: {{ vip }}
+        - name: prometheus_server
+          value: :2112
+        image: ghcr.io/kube-vip/kube-vip:{{ kube_vip_version }}
+        imagePullPolicy: Always
+        name: kube-vip
+        resources: {}
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+      hostNetwork: true
+      serviceAccountName: kube-vip
+      tolerations:
+      - effect: NoSchedule
+        operator: Exists
+      - effect: NoExecute
+        operator: Exists
+  updateStrategy: {}
+status:
+  currentNumberScheduled: 0
+  desiredNumberScheduled: 0
+  numberMisscheduled: 0
+  numberReady: 0

ansible/rke2/roles/prepare-nodes/tasks/main.yaml

@@ -0,0 +1,15 @@
+- name: Enable IPv4 forwarding
+  ansible.posix.sysctl:
+    name: net.ipv4.ip_forward
+    value: "1"
+    state: present
+    reload: true
+  tags: sysctl
+
+- name: Enable IPv6 forwarding
+  ansible.posix.sysctl:
+    name: net.ipv6.conf.all.forwarding
+    value: "1"
+    state: present
+    reload: true
+  tags: sysctl

ansible/rke2/roles/rke2-download/tasks/main.yaml

@@ -0,0 +1,31 @@
+# Create a directory to download RKE2 binary to
+- name: Create directory for RKE2 binary
+  ansible.builtin.file:
+    path: "{{ rke2_install_dir }}"
+    state: directory
+    mode: '0755'
+
+# Download the RKE2 binary
+- name: Download RKE2 amd64 binary
+  ansible.builtin.get_url:
+    url: "{{ rke2_binary_url }}/rke2.linux-amd64"
+    dest: "{{ rke2_install_dir }}/rke2"
+    mode: '0755'
+  when: ansible_facts.architecture == "x86_64"
+
+- name: Download RKE2 arm64 binary
+  ansible.builtin.get_url:
+    url: "{{ rke2_binary_url }}/rke2.linux-arm64"
+    dest: "{{ rke2_install_dir }}/rke2"
+    mode: '0755'
+  when:
+    - ( ansible_facts.architecture is search("arm") and
+        ansible_facts.userspace_bits == "64" ) or
+      ansible_facts.architecture is search("aarch64")
+
+# Set permissions on the RKE2 binary
+- name: Set executable permissions on the RKE2 binary
+  ansible.builtin.file:
+    path: "{{ rke2_install_dir }}/rke2"
+    mode: '0755'
+    state: file

ansible/rke2/roles/rke2-prepare/tasks/main.yaml

@@ -0,0 +1,134 @@
+- name: Create directory for RKE2 config
+  ansible.builtin.file:
+    path: "/etc/rancher/rke2"
+    state: directory
+    mode: '0644'
+
+- name: Create directory for RKE2 token
+  ansible.builtin.file:
+    path: "/var/lib/rancher/rke2/server"
+    state: directory
+    mode: '0644'
+
+# Copy server config to server 1 for bootstrap - we need to change server2 & 3 later with the token
+- name: Deploy RKE2 server Configuration
+  ansible.builtin.template:
+    src: templates/rke2-server-config.j2
+    dest: /etc/rancher/rke2/config.yaml
+    owner: root
+    group: root
+    mode: '0644'
+  when: inventory_hostname in groups['servers']
+
+- name: Create systemd service file for RKE2 server
+  ansible.builtin.template:
+    src: templates/rke2-server.service.j2
+    dest: /etc/systemd/system/rke2-server.service
+    owner: root
+    group: root
+    mode: '0644'
+  when: inventory_hostname in groups['servers']
+
+- name: Create systemd service file for RKE2 agent
+  ansible.builtin.template:
+    src: templates/rke2-agent.service.j2
+    dest: /etc/systemd/system/rke2-agent.service
+    owner: root
+    group: root
+    mode: '0644'
+  when: inventory_hostname in groups['agents']
+
+# we enable the first server to generate tokens etc, copy this afterwards to other servers
+- name: Ensure RKE2 server is enabled and running
+  ansible.builtin.systemd:
+    name: rke2-server
+    enabled: true
+    state: restarted
+    daemon_reload: true
+  when: inventory_hostname in groups['servers'][0]
+
+# wait for node token to be availale so that we can copy it, we need this to join other nodes
+- name: Wait for node-token
+  ansible.builtin.wait_for:
+    path: /var/lib/rancher/rke2/server/node-token
+  when: inventory_hostname == groups['servers'][0]
+
+# wait for kubectl to be downloaded, part of the rke2 installation
+- name: Wait for kubectl
+  ansible.builtin.wait_for:
+    path: /var/lib/rancher/rke2/bin/kubectl
+  when: inventory_hostname == groups['servers'][0]
+
+# copy kubectl to usr bin so that all users can run kubectl commands
+- name: Copy kubectl to user bin
+  ansible.builtin.copy:
+    src: /var/lib/rancher/rke2/bin/kubectl
+    dest: /usr/local/bin/kubectl
+    mode: '0755'
+    remote_src: true
+  become: true
+  when: inventory_hostname == groups['servers'][0]
+
+# wait for the kubectl copy to complete
+- name: Wait for kubectl
+  ansible.builtin.wait_for:
+    path: /usr/local/bin/kubectl
+  when: inventory_hostname == groups['servers'][0]
+
+# modify token access
+- name: Register node-token file access mode
+  ansible.builtin.stat:
+    path: /var/lib/rancher/rke2/server
+  register: p
+
+- name: Change file access for node-token
+  ansible.builtin.file:
+    path: /var/lib/rancher/rke2/server
+    mode: "g+rx,o+rx"
+  when: inventory_hostname == groups['servers'][0]
+
+# Save token as variable
+- name: Fetch the token from the first server node
+  ansible.builtin.slurp:
+    src: /var/lib/rancher/rke2/server/token
+  register: rke2_token
+  when: inventory_hostname == groups['servers'][0]
+  run_once: true
+
+# convert token to fact
+- name: Save Master node-token for later
+  ansible.builtin.set_fact:
+    token: "{{ rke2_token.content | b64decode | regex_replace('\n', '') }}"
+
+# revert token file access
+- name: Restore node-token file access
+  ansible.builtin.file:
+    path: /var/lib/rancher/rke2/server
+    mode: "{{ p.stat.mode }}"
+  when: inventory_hostname == groups['servers'][0]
+
+# check .kube folder exists so that we can use kubectl (config resides here)
+- name: Ensure .kube directory exists in user's home
+  ansible.builtin.file:
+    path: "/home/{{ ansible_user }}/.kube"
+    state: directory
+    mode: '0755'
+  become: true
+
+# copy kubectl config file to .kube folder
+- name: Copy config file to user home directory
+  ansible.builtin.copy:
+    src: /etc/rancher/rke2/rke2.yaml
+    dest: "/home/{{ ansible_user }}/.kube/config"
+    remote_src: true
+    owner: "{{ ansible_user }}"
+    mode: "u=rw,g=,o="
+  when: inventory_hostname == groups['servers'][0]
+
+# change IP from local to server 1 IP
+- name: Replace IP address with server1
+  ansible.builtin.replace:
+    path: /home/{{ ansible_user }}/.kube/config
+    regexp: '127.0.0.1'
+    replace: "{{ hostvars['server1']['ansible_host'] }}"
+  when: inventory_hostname == groups['servers'][0]

ansible/rke2/roles/rke2-prepare/templates/rke2-agent.service.j2

@@ -0,0 +1,13 @@
+# rke2-agent.service.j2
+[Unit]
+Description=RKE2 Agent
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/rke2 agent
+KillMode=process
+Restart=on-failure
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target

ansible/rke2/roles/rke2-prepare/templates/rke2-server-config.j2

@@ -0,0 +1,8 @@
+write-kubeconfig-mode: "0644"
+tls-san:
+  - {{ vip }}
+  - {{ hostvars['server1']['ansible_host'] }}
+node-label:
+  - server=true
+disable:
+  - rke2-ingress-nginx

ansible/rke2/roles/rke2-prepare/templates/rke2-server.service.j2

@@ -0,0 +1,13 @@
+# rke2-server.service.j2
+[Unit]
+Description=RKE2 server
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/rke2 server
+KillMode=process
+Restart=on-failure
+RestartSec=5s
+
+[Install]
+WantedBy=multi-user.target

ansible/rke2/site.yaml

@@ -0,0 +1,61 @@
+# Hello, thanks for using my playbook, hopefully you can help to improve it.
+# Things that need adding: (there are many more)
+# 1) Support different OS & architectures
+# 2) Support multiple CNIs
+# 3) Improve the wait logic
+# 4) Use kubernetes Ansible plugins more sensibly
+# 5) Optimise flow logic
+# 6) Clean up
+
+###############################################################
+# MAKE SURE YOU CHANGE group_vars/all.yaml VARIABLES!!!!!!!!!!!
+###############################################################
+
+# bootstraps first server and copies configs for others/agents
+- name: Prepare all nodes
+  hosts: servers,agents
+  gather_facts: true # enables us to gather lots of useful variables: https://docs.ansible.com/ansible/latest/collections/ansible/builtin/setup_module.html
+  roles:
+    - prepare-nodes
+
+# creates directories for download and then downloads RKE2 and changes permissions
+- name: Download RKE2
+  hosts: servers,agents
+  gather_facts: true
+  roles:
+    - rke2-download
+
+# Creates RKE2 bootstrap manifests folder and copies kube-vip template over (configured with variables)
+- name: Deploy Kube VIP
+  hosts: servers
+  gather_facts: true
+  roles:
+    - kube-vip
+
+# bootstraps the first server, copies configs to nodes, saves token to use later
+- name: Prepare RKE2 on Servers and Agents
+  hosts: servers,agents
+  gather_facts: true
+  roles:
+    - rke2-prepare
+
+# Adds additional servers using the token from the previous task
+- name: Add additional RKE2 Servers
+  hosts: servers
+  gather_facts: true
+  roles:
+    - add-server
+
+# Adds agents to the cluster
+- name: Add additional RKE2 Agents
+  hosts: agents
+  gather_facts: true
+  roles:
+    - add-agent
+
+# Finish kube-vip, add metallb
+- name: Apply manifests after cluster is created
+  hosts: servers
+  gather_facts: true
+  roles:
+    - apply-manifests

ansible/update-apt.yml

@@ -0,0 +1,21 @@
+- name: Upgrade everything on the cluster.
+  hosts: test
+  become: true
+  tasks:
+    - name: Upgrade all software.
+      ansible.builtin.apt:
+        update_cache: true
+        upgrade: dist
+
+    - name: Check if a reboot is required.
+      stat:
+        path: /var/run/reboot-required
+      register: reboot_required_file
+
+    - name: Reboot the server (if required).
+      reboot:
+      when: reboot_required_file.stat.exists == true
+
+    - name: Remove dependencies that are no longer required.
+      apt:
+        autoremove: yes