diff --git a/ansible/02-dev-inventory.yml b/ansible/02-dev-inventory.yml index 47d1d1e..66e094d 100644 --- a/ansible/02-dev-inventory.yml +++ b/ansible/02-dev-inventory.yml @@ -1,4 +1,41 @@ devservers: hosts: - dev1: - ansible_host: 192.168.50.58 \ No newline at end of file + node09: + ansible_host: 101.100.172.55 + ansible_port: 9991 + node00: + ansible_host: 101.100.172.55 + ansible_port: 9999 + node01: + ansible_host: 101.100.172.55 + ansible_port: 9998 + +debian: + hosts: + node00: + ansible_host: 101.100.172.55 + ansible_port: 9999 + node01: + ansible_host: 101.100.172.55 + ansible_port: 9998 + +local: + hosts: + node00: + ansible_host: 192.168.50.114 + ansible_port: 22 + node01: + ansible_host: 192.168.50.68 + ansible_port: 22 + node02: + ansible_host: 192.168.50.56 + ansible_port: 22 + +test: + hosts: + node05: + ansible_host: 192.168.50.205 + ansible_port: 22 + node09: + ansible_host: 192.168.50.209 + ansible_port: 22 diff --git a/ansible/playbook.yaml b/ansible/playbook.yaml new file mode 100644 index 0000000..872b298 --- /dev/null +++ b/ansible/playbook.yaml @@ -0,0 +1,10 @@ +- name: My first play + hosts: devservers + tasks: + - name: Ping my hosts + ansible.builtin.ping: + + - name: Print message + ansible.builtin.debug: + msg: Hello world + diff --git a/ansible/rke2/collections/requirements.yaml b/ansible/rke2/collections/requirements.yaml new file mode 100644 index 0000000..3ffb535 --- /dev/null +++ b/ansible/rke2/collections/requirements.yaml @@ -0,0 +1,6 @@ +--- +collections: + - name: ansible.utils + - name: community.general + - name: ansible.posix + - name: kubernetes.core \ No newline at end of file diff --git a/ansible/rke2/config.yaml b/ansible/rke2/config.yaml new file mode 100644 index 0000000..9e7fd15 --- /dev/null +++ b/ansible/rke2/config.yaml @@ -0,0 +1,8 @@ +tls-san: + - 192.168.50.190 + - 192.168.50.209 + - + - +write-kubeconfig-mode: 0644 +disable: + - rke2-ingress-nginx diff --git a/ansible/rke2/inventory/group_vars/all.yaml b/ansible/rke2/inventory/group_vars/all.yaml new file mode 100644 index 0000000..e826e42 --- /dev/null +++ b/ansible/rke2/inventory/group_vars/all.yaml @@ -0,0 +1,20 @@ +os: "linux" +# arch: "amd64" +# Set your timezone +system_timezone: "Asia/Singapore" + +kube_vip_version: "v0.8.1" +vip_interface: eth0 +vip: 192.168.50.210 + +metallb_version: v0.14.5 +lb_range: 192.168.50.190-192.168.50.199 +lb_pool_name: first-pool + +rke2_version: "v1.28.10+rke2r1" +rke2_install_dir: "/usr/local/bin" +rke2_binary_url: "https://github.com/rancher/rke2/releases/download/{{ rke2_version }}" + +ansible_user: furyhawk +ansible_become: true +ansible_become_method: sudo diff --git a/ansible/rke2/inventory/hosts.ini b/ansible/rke2/inventory/hosts.ini new file mode 100644 index 0000000..f9cefe9 --- /dev/null +++ b/ansible/rke2/inventory/hosts.ini @@ -0,0 +1,8 @@ +# Make sure Ansible host has access to these devices +# Good idea to snapshot all machines and deploy uing cloud-init + +[servers] +server1 ansible_host=192.168.50.209 + +[agents] +agent1 ansible_host=192.168.50.205 diff --git a/ansible/rke2/kube-vip b/ansible/rke2/kube-vip new file mode 100644 index 0000000..83dcbc5 --- /dev/null +++ b/ansible/rke2/kube-vip @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.6.3 + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: v0.6.3 + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: $interface + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "false" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: $vip + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.6.3 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 + diff --git a/ansible/rke2/longhorn.sh b/ansible/rke2/longhorn.sh new file mode 100644 index 0000000..5c67602 --- /dev/null +++ b/ansible/rke2/longhorn.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +echo -e " \033[33;2m __ _ _ ___ \033[0m" +echo -e " \033[33;2m \ \(_)_ __ ___( )__ / _ \__ _ _ __ __ _ __ _ ___ \033[0m" +echo -e " \033[33;2m \ \ | '_ \` _ \/ __| / /_\/ _\` | '__/ _\` |/ _\` |/ _ \ \033[0m" +echo -e " \033[33;2m /\_/ / | | | | | \__ \ / /_\\ (_| | | | (_| | (_| | __/ \033[0m" +echo -e " \033[33;2m \___/|_|_| |_| |_|___/ \____/\__,_|_| \__,_|\__, |\___| \033[0m" +echo -e " \033[33;2m |___/ \033[0m" +echo -e " \033[35;2m __ _ \033[0m" +echo -e " \033[35;2m / / ___ _ __ __ _| |__ ___ _ __ _ __ \033[0m" +echo -e " \033[35;2m / / / _ \| '_ \ / _\` | '_ \ / _ \| '__| '_ \ \033[0m" +echo -e " \033[35;2m / /__| (_) | | | | (_| | | | | (_) | | | | | | \033[0m" +echo -e " \033[35;2m \____/\___/|_| |_|\__, |_| |_|\___/|_| |_| |_| \033[0m" +echo -e " \033[35;2m |___/ \033[0m" +echo -e " \033[36;2m \033[0m" +echo -e " \033[32;2m https://youtube.com/@jims-garage \033[0m" +echo -e " \033[32;2m \033[0m" + +############################################# +# YOU SHOULD ONLY NEED TO EDIT THIS SECTION # +############################################# + +# THIS SCRIPT IS FOR RKE2, NOT K3S! +# THIS SCRIPT IS FOR RKE2, NOT K3S! +# THIS SCRIPT IS FOR RKE2, NOT K3S! + +# Set the IP addresses of master1 +master1=192.168.50.209 + +# Set the IP addresses of your Longhorn nodes +longhorn1=192.168.50.205 + +# User of remote machines +user=furyhawk + +# Interface used on remotes +interface=eth0 + +# Set the virtual IP address (VIP) +vip=192.168.50.210 + +# Array of longhorn nodes +storage=($longhorn1) + +#ssh certificate name variable +certName=id_rsa + +############################################# +# DO NOT EDIT BELOW # +############################################# +# For testing purposes - in case time is wrong due to VM snapshots +sudo timedatectl set-ntp off +sudo timedatectl set-ntp on + +# add ssh keys for all nodes +for node in "${storage[@]}"; do + ssh-copy-id $user@$node +done + +# add open-iscsi - needed for Debian and non-cloud Ubuntu +if ! command -v sudo service open-iscsi status &> /dev/null +then + echo -e " \033[31;5mOpen-ISCSI not found, installing\033[0m" + sudo apt install open-iscsi +else + echo -e " \033[32;5mOpen-ISCSI already installed\033[0m" +fi + +# Step 1: Add new longhorn nodes to cluster (note: label added) +# Set token variable needed for RKE2 (not required for K3S) +token=`cat token` +for newnode in "${storage[@]}"; do + ssh -tt $user@$newnode -i ~/.ssh/$certName sudo su <> /etc/rancher/rke2/config.yaml + echo "server: https://$vip:9345" >> /etc/rancher/rke2/config.yaml + echo "node-label:" >> /etc/rancher/rke2/config.yaml + echo " - longhorn=true" >> /etc/rancher/rke2/config.yaml + curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh - + systemctl enable rke2-agent.service + systemctl start rke2-agent.service + exit +EOF + echo -e " \033[32;5mLonghorn node joined successfully!\033[0m" +done + +# Step 2: Install Longhorn (using modified Official to pin to Longhorn Nodes) +# kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.6.2/deploy/longhorn.yaml +kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/Longhorn/longhorn.yaml +kubectl get pods \ +--namespace longhorn-system \ +--watch + +# Step 3: Print out confirmation + +kubectl get nodes +kubectl get svc -n longhorn-system + +echo -e " \033[32;5mHappy Kubing! Access Longhorn through Rancher UI\033[0m" diff --git a/ansible/rke2/rke2.sh b/ansible/rke2/rke2.sh new file mode 100755 index 0000000..01fa957 --- /dev/null +++ b/ansible/rke2/rke2.sh @@ -0,0 +1,262 @@ +#!/bin/bash + +echo -e " \033[33;5m __ _ _ ___ \033[0m" +echo -e " \033[33;5m \ \(_)_ __ ___( )__ / _ \__ _ _ __ __ _ __ _ ___ \033[0m" +echo -e " \033[33;5m \ \ | '_ \` _ \/ __| / /_\/ _\` | '__/ _\` |/ _\` |/ _ \ \033[0m" +echo -e " \033[33;5m /\_/ / | | | | | \__ \ / /_\\ (_| | | | (_| | (_| | __/ \033[0m" +echo -e " \033[33;5m \___/|_|_| |_| |_|___/ \____/\__,_|_| \__,_|\__, |\___| \033[0m" +echo -e " \033[33;5m |___/ \033[0m" + +echo -e " \033[36;5m ___ _ _____ ___ \033[0m" +echo -e " \033[36;5m | _ \ |/ / __|_ ) \033[0m" +echo -e " \033[36;5m | / ' <| _| / / \033[0m" +echo -e " \033[36;5m |_|_\_|\_\___/___| \033[0m" +echo -e " \033[36;5m \033[0m" +echo -e " \033[32;5m https://youtube.com/@jims-garage \033[0m" +echo -e " \033[32;5m \033[0m" + + +############################################# +# YOU SHOULD ONLY NEED TO EDIT THIS SECTION # +############################################# + +# Version of Kube-VIP to deploy +KVVERSION="v0.6.3" + +# Set the IP addresses of the admin, masters, and workers nodes +admin=192.168.50.143 +master1=192.168.50.209 +worker1=192.168.50.205 + +# User of remote machines +user=furyhawk + +# Interface used on remotes +interface=eth0 + +# Set the virtual IP address (VIP) +vip=192.168.50.190 + +# Array of all master nodes +allmasters=($master1) + +# Array of master nodes +masters=($master2 $master3) + +# Array of worker nodes +workers=($worker1) + +# Array of all +all=($master1 $worker1) + +# Array of all minus master1 +allnomaster1=($master2 $master3 $worker1) + +#Loadbalancer IP range +lbrange=192.168.3.191-192.168.3.199 + +#ssh certificate name variable +certName=id_rsa + +############################################# +# DO NOT EDIT BELOW # +############################################# +# For testing purposes - in case time is wrong due to VM snapshots +sudo timedatectl set-ntp off +sudo timedatectl set-ntp on + +# Move SSH certs to ~/.ssh and change permissions +cp /home/$user/{$certName,$certName.pub} /home/$user/.ssh +chmod 600 /home/$user/.ssh/$certName +chmod 644 /home/$user/.ssh/$certName.pub + +# Install Kubectl if not already present +if ! command -v kubectl version &> /dev/null +then + echo -e " \033[31;5mKubectl not found, installing\033[0m" + curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl +else + echo -e " \033[32;5mKubectl already installed\033[0m" +fi + +# Create SSH Config file to ignore checking (don't use in production!) +sed -i '1s/^/StrictHostKeyChecking no\n/' ~/.ssh/config + +#add ssh keys for all nodes +for node in "${all[@]}"; do + ssh-copy-id $user@$node +done + +# Step 1: Create Kube VIP +# create RKE2's self-installing manifest dir +sudo mkdir -p /var/lib/rancher/rke2/server/manifests +# Install the kube-vip deployment into rke2's self-installing manifest folder +curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/kube-vip +cat kube-vip | sed 's/$interface/'$interface'/g; s/$vip/'$vip'/g' > $HOME/kube-vip.yaml +sudo mv kube-vip.yaml /var/lib/rancher/rke2/server/manifests/kube-vip.yaml + +# Find/Replace all k3s entries to represent rke2 +sudo sed -i 's/k3s/rke2/g' /var/lib/rancher/rke2/server/manifests/kube-vip.yaml +# copy kube-vip.yaml to home directory +sudo cp /var/lib/rancher/rke2/server/manifests/kube-vip.yaml ~/kube-vip.yaml +# change owner +sudo chown $user:$user kube-vip.yaml +# make kube folder to run kubectl later +mkdir ~/.kube + +# create the rke2 config file +sudo mkdir -p /etc/rancher/rke2 +touch config.yaml +echo "tls-san:" >> config.yaml +echo " - $vip" >> config.yaml +echo " - $master1" >> config.yaml +echo " - $master2" >> config.yaml +echo " - $master3" >> config.yaml +echo "write-kubeconfig-mode: 0644" >> config.yaml +echo "disable:" >> config.yaml +echo " - rke2-ingress-nginx" >> config.yaml +# copy config.yaml to rancher directory +sudo cp ~/config.yaml /etc/rancher/rke2/config.yaml + +# update path with rke2-binaries +echo 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml' >> ~/.bashrc ; echo 'export PATH=${PATH}:/var/lib/rancher/rke2/bin' >> ~/.bashrc ; echo 'alias k=kubectl' >> ~/.bashrc ; source ~/.bashrc ; + +# Step 2: Copy kube-vip.yaml and certs to all masters +for newnode in "${allmasters[@]}"; do + scp -i ~/.ssh/$certName $HOME/kube-vip.yaml $user@$newnode:~/kube-vip.yaml + scp -i ~/.ssh/$certName $HOME/config.yaml $user@$newnode:~/config.yaml + scp -i ~/.ssh/$certName ~/.ssh/{$certName,$certName.pub} $user@$newnode:~/.ssh + echo -e " \033[32;5mCopied successfully!\033[0m" +done + +# Step 3: Connect to Master1 and move kube-vip.yaml and config.yaml. Then install RKE2, copy token back to admin machine. We then use the token to bootstrap additional masternodes +ssh -tt $user@$master1 -i ~/.ssh/$certName sudo su <> ~/.bashrc ; echo 'export PATH=${PATH}:/var/lib/rancher/rke2/bin' >> ~/.bashrc ; echo 'alias k=kubectl' >> ~/.bashrc ; source ~/.bashrc ; +curl -sfL https://get.rke2.io | sh - +systemctl enable rke2-server.service +systemctl start rke2-server.service +echo "StrictHostKeyChecking no" > ~/.ssh/config +ssh-copy-id -i /home/$user/.ssh/$certName $user@$admin +scp -i /home/$user/.ssh/$certName /var/lib/rancher/rke2/server/token $user@$admin:~/token +scp -i /home/$user/.ssh/$certName /etc/rancher/rke2/rke2.yaml $user@$admin:~/.kube/rke2.yaml +exit +EOF +echo -e " \033[32;5mMaster1 Completed\033[0m" + +# Step 4: Set variable to the token we just extracted, set kube config location +token=`cat token` +sudo cat ~/.kube/rke2.yaml | sed 's/127.0.0.1/'$master1'/g' > $HOME/.kube/config +sudo chown $(id -u):$(id -g) $HOME/.kube/config +export KUBECONFIG=${HOME}/.kube/config +sudo cp ~/.kube/config /etc/rancher/rke2/rke2.yaml +kubectl get nodes + +# Step 5: Install kube-vip as network LoadBalancer - Install the kube-vip Cloud Provider +kubectl apply -f https://kube-vip.io/manifests/rbac.yaml +kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + +# Step 6: Add other Masternodes, note we import the token we extracted from step 3 +# for newnode in "${masters[@]}"; do +# ssh -tt $user@$newnode -i ~/.ssh/$certName sudo su <> /etc/rancher/rke2/config.yaml +# echo "server: https://$master1:9345" >> /etc/rancher/rke2/config.yaml +# echo "tls-san:" >> /etc/rancher/rke2/config.yaml +# echo " - $vip" >> /etc/rancher/rke2/config.yaml +# echo " - $master1" >> /etc/rancher/rke2/config.yaml +# echo " - $master2" >> /etc/rancher/rke2/config.yaml +# echo " - $master3" >> /etc/rancher/rke2/config.yaml +# curl -sfL https://get.rke2.io | sh - +# systemctl enable rke2-server.service +# systemctl start rke2-server.service +# exit +# EOF +# echo -e " \033[32;5mMaster node joined successfully!\033[0m" +# done + +kubectl get nodes + +# Step 7: Add Workers +for newnode in "${workers[@]}"; do + ssh -tt $user@$newnode -i ~/.ssh/$certName sudo su <> /etc/rancher/rke2/config.yaml + echo "server: https://$vip:9345" >> /etc/rancher/rke2/config.yaml + echo "node-label:" >> /etc/rancher/rke2/config.yaml + echo " - worker=true" >> /etc/rancher/rke2/config.yaml + echo " - longhorn=true" >> /etc/rancher/rke2/config.yaml + curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh - + systemctl enable rke2-agent.service + systemctl start rke2-agent.service + exit +EOF + echo -e " \033[32;5mWorker node joined successfully!\033[0m" +done + +kubectl get nodes + +# Step 8: Install Metallb +echo -e " \033[32;5mDeploying Metallb\033[0m" +kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml +kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml +# Download ipAddressPool and configure using lbrange above +curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/ipAddressPool +cat ipAddressPool | sed 's/$lbrange/'$lbrange'/g' > $HOME/ipAddressPool.yaml + +# Step 9: Deploy IP Pools and l2Advertisement +echo -e " \033[32;5mAdding IP Pools, waiting for Metallb to be available first. This can take a long time as we're likely being rate limited for container pulls...\033[0m" +kubectl wait --namespace metallb-system \ + --for=condition=ready pod \ + --selector=component=controller \ + --timeout=1800s +kubectl apply -f ipAddressPool.yaml +kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/l2Advertisement.yaml + +# Step 10: Install Rancher (Optional - Delete if not required) +#Install Helm +echo -e " \033[32;5mInstalling Helm\033[0m" +curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 +chmod 700 get_helm.sh +./get_helm.sh + +# Add Rancher Helm Repo & create namespace +helm repo add rancher-latest https://releases.rancher.com/server-charts/latest +kubectl create namespace cattle-system + +# Install Cert-Manager +echo -e " \033[32;5mDeploying Cert-Manager\033[0m" +kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.crds.yaml +helm repo add jetstack https://charts.jetstack.io +helm repo update +helm install cert-manager jetstack/cert-manager \ +--namespace cert-manager \ +--create-namespace \ +--version v1.13.2 +kubectl get pods --namespace cert-manager + +# Install Rancher +echo -e " \033[32;5mDeploying Rancher\033[0m" +helm install rancher rancher-latest/rancher \ + --namespace cattle-system \ + --set hostname=rancher.my.org \ + --set bootstrapPassword=admin +kubectl -n cattle-system rollout status deploy/rancher +kubectl -n cattle-system get deploy rancher + +# Add Rancher LoadBalancer +kubectl get svc -n cattle-system +kubectl expose deployment rancher --name=rancher-lb --port=443 --type=LoadBalancer -n cattle-system +while [[ $(kubectl get svc -n cattle-system 'jsonpath={..status.conditions[?(@.type=="Pending")].status}') = "True" ]]; do + sleep 5 + echo -e " \033[32;5mWaiting for LoadBalancer to come online\033[0m" +done +kubectl get svc -n cattle-system + +echo -e " \033[32;5mAccess Rancher from the IP above - Password is admin!\033[0m" diff --git a/ansible/rke2/roles/add-agent/tasks/main.yaml b/ansible/rke2/roles/add-agent/tasks/main.yaml new file mode 100644 index 0000000..edd2aed --- /dev/null +++ b/ansible/rke2/roles/add-agent/tasks/main.yaml @@ -0,0 +1,17 @@ +# Copy agent config to all agents - we need to change agent2 & 3 later with the token +- name: Deploy RKE2 Agent Configuration + ansible.builtin.template: + src: templates/rke2-agent-config.j2 + dest: /etc/rancher/rke2/config.yaml + owner: root + group: root + mode: '0644' + when: inventory_hostname in groups['agents'] + +# Check agents have restarted to pick up config +- name: Ensure RKE2 agents are enabled and running + ansible.builtin.systemd: + name: rke2-agent + enabled: true + state: restarted + daemon_reload: true diff --git a/ansible/rke2/roles/add-agent/templates/rke2-agent-config.j2 b/ansible/rke2/roles/add-agent/templates/rke2-agent-config.j2 new file mode 100644 index 0000000..9125bc3 --- /dev/null +++ b/ansible/rke2/roles/add-agent/templates/rke2-agent-config.j2 @@ -0,0 +1,6 @@ +write-kubeconfig-mode: "0644" +token: {{ hostvars['server1']['token'] }} +server: https://{{ hostvars['server1']['ansible_host'] }}:9345 +node-label: + - "agent=true" + - "longhorn=true" diff --git a/ansible/rke2/roles/add-server/tasks/main.yaml b/ansible/rke2/roles/add-server/tasks/main.yaml new file mode 100644 index 0000000..3acb216 --- /dev/null +++ b/ansible/rke2/roles/add-server/tasks/main.yaml @@ -0,0 +1,53 @@ +# Copy server config with token to all servers except server 1 (this has token) +- name: Deploy RKE2 server Configuration + ansible.builtin.template: + src: templates/rke2-server-config.j2 + dest: /etc/rancher/rke2/config.yaml + owner: root + group: root + mode: '0644' + when: inventory_hostname != groups['servers'][0] + +# Keep checking the cluster API until it's functioning (deployed) +- name: Wait for cluster API to be ready (can take 5-10 mins depending on internet/hardware) + ansible.builtin.command: + cmd: "kubectl get nodes" + register: kubectl_output + until: "'connection refused' not in kubectl_output.stderr" + retries: 120 + delay: 10 + changed_when: true + become_user: "{{ ansible_user }}" + when: inventory_hostname == groups['servers'][0] + +# Use kubectl to deploy yaml. Perhaps this can be added to the manifest folder initially +- name: Apply kube vip configuration file + ansible.builtin.command: + cmd: kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml apply -f https://kube-vip.io/manifests/rbac.yaml + changed_when: true + when: inventory_hostname == groups['servers'][0] + +# Apply the kube-vip configration. Perhaps this can be added to the manifest folder initially +- name: Apply kube vip configuration file + ansible.builtin.command: + cmd: kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml + changed_when: true + when: inventory_hostname == groups['servers'][0] + +# Check that additional servers are restarted +- name: Ensure additional RKE2 servers are enabled and running + ansible.builtin.systemd: + name: rke2-server + enabled: true + state: restarted + daemon_reload: true + when: inventory_hostname != groups['servers'][0] + +# enable additional servers +- name: Ensure RKE2 server is enabled and running + ansible.builtin.systemd: + name: rke2-server + enabled: true + state: restarted + daemon_reload: true + when: inventory_hostname != groups['servers'][0] diff --git a/ansible/rke2/roles/add-server/templates/rke2-server-config.j2 b/ansible/rke2/roles/add-server/templates/rke2-server-config.j2 new file mode 100644 index 0000000..8b505ee --- /dev/null +++ b/ansible/rke2/roles/add-server/templates/rke2-server-config.j2 @@ -0,0 +1,8 @@ +write-kubeconfig-mode: "0644" +token: {{ hostvars['server1']['token'] }} +server: https://{{ hostvars['server1']['ansible_host'] }}:9345 +tls-san: + - {{ vip }} + - {{ hostvars['server1']['ansible_host'] }} +node-label: + - server=true \ No newline at end of file diff --git a/ansible/rke2/roles/apply-manifests/tasks/main.yaml b/ansible/rke2/roles/apply-manifests/tasks/main.yaml new file mode 100644 index 0000000..bf5ea47 --- /dev/null +++ b/ansible/rke2/roles/apply-manifests/tasks/main.yaml @@ -0,0 +1,60 @@ +# Wait for Server 1 to be ready before continuing with metallb deployment +- name: Wait for k8s nodes with node label 'server=true' to be ready, otherwise we cannot start metallb deployment + ansible.builtin.command: + cmd: "kubectl wait --for=condition=Ready nodes --selector server=true --timeout=600s" + register: nodes_ready + retries: 120 + delay: 10 + changed_when: true + become_user: "{{ ansible_user }}" + when: inventory_hostname == groups['servers'][0] + +# Create namespace so that we can deploy metallb +- name: Apply metallb namespace + ansible.builtin.command: + cmd: kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml + become_user: "{{ ansible_user }}" + changed_when: true + when: inventory_hostname == groups['servers'][0] + +# Apply metallb manifest +- name: Apply metallb manifest + ansible.builtin.command: + cmd: kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/{{ metallb_version }}/config/manifests/metallb-native.yaml + become_user: "{{ ansible_user }}" + changed_when: true + when: inventory_hostname == groups['servers'][0] + +# Wait for metallb deployment pods to be alive before deploying metallb manifests +- name: Wait for metallb pods to be ready, otherwise we cannot start metallb deployment + ansible.builtin.command: + cmd: "kubectl wait --namespace metallb-system --for=condition=ready pod --selector=component=controller --timeout=1800s" + changed_when: true + become_user: "{{ ansible_user }}" + when: inventory_hostname == groups['servers'][0] + +# Apply L2 Advertisement for metallb +- name: Apply metallb L2 Advertisement + ansible.builtin.command: + cmd: kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/l2Advertisement.yaml + become_user: "{{ ansible_user }}" + changed_when: true + when: inventory_hostname == groups['servers'][0] + +# Deploy metal IP Pool to Server 1 +- name: Copy metallb IPPool to server 1 + ansible.builtin.template: + src: templates/metallb-ippool.j2 + dest: /home/{{ ansible_user }}/ippool.yaml + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: '0755' + when: inventory_hostname == groups['servers'][0] + +# don't think this will work as nodes are no execute, might need agents first +- name: Apply metallb ipppool + ansible.builtin.command: + cmd: kubectl apply -f /home/{{ ansible_user }}/ippool.yaml + become_user: "{{ ansible_user }}" + changed_when: true + when: inventory_hostname == groups['servers'][0] diff --git a/ansible/rke2/roles/apply-manifests/templates/metallb-ippool.j2 b/ansible/rke2/roles/apply-manifests/templates/metallb-ippool.j2 new file mode 100644 index 0000000..6673b63 --- /dev/null +++ b/ansible/rke2/roles/apply-manifests/templates/metallb-ippool.j2 @@ -0,0 +1,8 @@ +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: {{ lb_pool_name }} + namespace: metallb-system +spec: + addresses: + - {{ lb_range }} \ No newline at end of file diff --git a/ansible/rke2/roles/kube-vip/tasks/main.yaml b/ansible/rke2/roles/kube-vip/tasks/main.yaml new file mode 100644 index 0000000..aaa500b --- /dev/null +++ b/ansible/rke2/roles/kube-vip/tasks/main.yaml @@ -0,0 +1,17 @@ +# Create directory to deploy kube-vip manifest +- name: Create directory for Kube VIP Manifest + ansible.builtin.file: + path: "/var/lib/rancher/rke2/server/manifests" + state: directory + mode: '0644' + when: inventory_hostname in groups['servers'] + +# Copy kube-vip to server 1 manifest folder for auto deployment at bootstrap +- name: Deploy Kube VIP Configuration + ansible.builtin.template: + src: templates/kube-vip-config.j2 + dest: /var/lib/rancher/rke2/server/manifests/kube-vip.yaml + owner: root + group: root + mode: '0644' + when: inventory_hostname == groups['servers'][0] diff --git a/ansible/rke2/roles/kube-vip/templates/kube-vip-config.j2 b/ansible/rke2/roles/kube-vip/templates/kube-vip-config.j2 new file mode 100644 index 0000000..c0731fc --- /dev/null +++ b/ansible/rke2/roles/kube-vip/templates/kube-vip-config.j2 @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: {{ kube_vip_version }} + name: kube-vip-ds + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-vip-ds + template: + metadata: + creationTimestamp: null + labels: + app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/version: {{ kube_vip_version }} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: {{ vip_interface }} + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "false" + - name: svc_leasename + value: plndr-svcs-lock + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "5" + - name: vip_renewdeadline + value: "3" + - name: vip_retryperiod + value: "1" + - name: address + value: {{ vip }} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:{{ kube_vip_version }} + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + hostNetwork: true + serviceAccountName: kube-vip + tolerations: + - effect: NoSchedule + operator: Exists + - effect: NoExecute + operator: Exists + updateStrategy: {} +status: + currentNumberScheduled: 0 + desiredNumberScheduled: 0 + numberMisscheduled: 0 + numberReady: 0 diff --git a/ansible/rke2/roles/prepare-nodes/tasks/main.yaml b/ansible/rke2/roles/prepare-nodes/tasks/main.yaml new file mode 100644 index 0000000..400f4b0 --- /dev/null +++ b/ansible/rke2/roles/prepare-nodes/tasks/main.yaml @@ -0,0 +1,15 @@ +- name: Enable IPv4 forwarding + ansible.posix.sysctl: + name: net.ipv4.ip_forward + value: "1" + state: present + reload: true + tags: sysctl + +- name: Enable IPv6 forwarding + ansible.posix.sysctl: + name: net.ipv6.conf.all.forwarding + value: "1" + state: present + reload: true + tags: sysctl \ No newline at end of file diff --git a/ansible/rke2/roles/rke2-download/tasks/main.yaml b/ansible/rke2/roles/rke2-download/tasks/main.yaml new file mode 100644 index 0000000..aba9780 --- /dev/null +++ b/ansible/rke2/roles/rke2-download/tasks/main.yaml @@ -0,0 +1,31 @@ +# Create a directory to download RKE2 binary to +- name: Create directory for RKE2 binary + ansible.builtin.file: + path: "{{ rke2_install_dir }}" + state: directory + mode: '0755' + +# Download the RKE2 binary +- name: Download RKE2 amd64 binary + ansible.builtin.get_url: + url: "{{ rke2_binary_url }}/rke2.linux-amd64" + dest: "{{ rke2_install_dir }}/rke2" + mode: '0755' + when: ansible_facts.architecture == "x86_64" + +- name: Download RKE2 arm64 binary + ansible.builtin.get_url: + url: "{{ rke2_binary_url }}/rke2.linux-arm64" + dest: "{{ rke2_install_dir }}/rke2" + mode: '0755' + when: + - ( ansible_facts.architecture is search("arm") and + ansible_facts.userspace_bits == "64" ) or + ansible_facts.architecture is search("aarch64") + +# Set permissions on the RKE2 binary +- name: Set executable permissions on the RKE2 binary + ansible.builtin.file: + path: "{{ rke2_install_dir }}/rke2" + mode: '0755' + state: file diff --git a/ansible/rke2/roles/rke2-download/vars/main.yaml b/ansible/rke2/roles/rke2-download/vars/main.yaml new file mode 100644 index 0000000..e69de29 diff --git a/ansible/rke2/roles/rke2-prepare/tasks/main.yaml b/ansible/rke2/roles/rke2-prepare/tasks/main.yaml new file mode 100644 index 0000000..f1fbccc --- /dev/null +++ b/ansible/rke2/roles/rke2-prepare/tasks/main.yaml @@ -0,0 +1,134 @@ +- name: Create directory for RKE2 config + ansible.builtin.file: + path: "/etc/rancher/rke2" + state: directory + mode: '0644' + +- name: Create directory for RKE2 token + ansible.builtin.file: + path: "/var/lib/rancher/rke2/server" + state: directory + mode: '0644' + +# Copy server config to server 1 for bootstrap - we need to change server2 & 3 later with the token +- name: Deploy RKE2 server Configuration + ansible.builtin.template: + src: templates/rke2-server-config.j2 + dest: /etc/rancher/rke2/config.yaml + owner: root + group: root + mode: '0644' + when: inventory_hostname in groups['servers'] + +- name: Create systemd service file for RKE2 server + ansible.builtin.template: + src: templates/rke2-server.service.j2 + dest: /etc/systemd/system/rke2-server.service + owner: root + group: root + mode: '0644' + when: inventory_hostname in groups['servers'] + +- name: Create systemd service file for RKE2 agent + ansible.builtin.template: + src: templates/rke2-agent.service.j2 + dest: /etc/systemd/system/rke2-agent.service + owner: root + group: root + mode: '0644' + when: inventory_hostname in groups['agents'] + +# we enable the first server to generate tokens etc, copy this afterwards to other servers +- name: Ensure RKE2 server is enabled and running + ansible.builtin.systemd: + name: rke2-server + enabled: true + state: restarted + daemon_reload: true + when: inventory_hostname in groups['servers'][0] + +# wait for node token to be availale so that we can copy it, we need this to join other nodes +- name: Wait for node-token + ansible.builtin.wait_for: + path: /var/lib/rancher/rke2/server/node-token + when: inventory_hostname == groups['servers'][0] + +# wait for kubectl to be downloaded, part of the rke2 installation +- name: Wait for kubectl + ansible.builtin.wait_for: + path: /var/lib/rancher/rke2/bin/kubectl + when: inventory_hostname == groups['servers'][0] + +# copy kubectl to usr bin so that all users can run kubectl commands +- name: Copy kubectl to user bin + ansible.builtin.copy: + src: /var/lib/rancher/rke2/bin/kubectl + dest: /usr/local/bin/kubectl + mode: '0755' + remote_src: true + become: true + when: inventory_hostname == groups['servers'][0] + +# wait for the kubectl copy to complete +- name: Wait for kubectl + ansible.builtin.wait_for: + path: /usr/local/bin/kubectl + when: inventory_hostname == groups['servers'][0] + +# modify token access +- name: Register node-token file access mode + ansible.builtin.stat: + path: /var/lib/rancher/rke2/server + register: p + +- name: Change file access for node-token + ansible.builtin.file: + path: /var/lib/rancher/rke2/server + mode: "g+rx,o+rx" + when: inventory_hostname == groups['servers'][0] + +# Save token as variable +- name: Fetch the token from the first server node + ansible.builtin.slurp: + src: /var/lib/rancher/rke2/server/token + register: rke2_token + when: inventory_hostname == groups['servers'][0] + run_once: true + +# convert token to fact +- name: Save Master node-token for later + ansible.builtin.set_fact: + token: "{{ rke2_token.content | b64decode | regex_replace('\n', '') }}" + +# revert token file access +- name: Restore node-token file access + ansible.builtin.file: + path: /var/lib/rancher/rke2/server + mode: "{{ p.stat.mode }}" + when: inventory_hostname == groups['servers'][0] + +# check .kube folder exists so that we can use kubectl (config resides here) +- name: Ensure .kube directory exists in user's home + ansible.builtin.file: + path: "/home/{{ ansible_user }}/.kube" + state: directory + mode: '0755' + become: true + +# copy kubectl config file to .kube folder +- name: Copy config file to user home directory + ansible.builtin.copy: + src: /etc/rancher/rke2/rke2.yaml + dest: "/home/{{ ansible_user }}/.kube/config" + remote_src: true + owner: "{{ ansible_user }}" + mode: "u=rw,g=,o=" + when: inventory_hostname == groups['servers'][0] + +# change IP from local to server 1 IP +- name: Replace IP address with server1 + ansible.builtin.replace: + path: /home/{{ ansible_user }}/.kube/config + regexp: '127.0.0.1' + replace: "{{ hostvars['server1']['ansible_host'] }}" + when: inventory_hostname == groups['servers'][0] diff --git a/ansible/rke2/roles/rke2-prepare/templates/rke2-agent.service.j2 b/ansible/rke2/roles/rke2-prepare/templates/rke2-agent.service.j2 new file mode 100644 index 0000000..b032208 --- /dev/null +++ b/ansible/rke2/roles/rke2-prepare/templates/rke2-agent.service.j2 @@ -0,0 +1,13 @@ +# rke2-agent.service.j2 +[Unit] +Description=RKE2 Agent +After=network.target + +[Service] +ExecStart=/usr/local/bin/rke2 agent +KillMode=process +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/ansible/rke2/roles/rke2-prepare/templates/rke2-server-config.j2 b/ansible/rke2/roles/rke2-prepare/templates/rke2-server-config.j2 new file mode 100644 index 0000000..327e4d9 --- /dev/null +++ b/ansible/rke2/roles/rke2-prepare/templates/rke2-server-config.j2 @@ -0,0 +1,8 @@ +write-kubeconfig-mode: "0644" +tls-san: + - {{ vip }} + - {{ hostvars['server1']['ansible_host'] }} +node-label: + - server=true +disable: + - rke2-ingress-nginx \ No newline at end of file diff --git a/ansible/rke2/roles/rke2-prepare/templates/rke2-server.service.j2 b/ansible/rke2/roles/rke2-prepare/templates/rke2-server.service.j2 new file mode 100644 index 0000000..a091ebd --- /dev/null +++ b/ansible/rke2/roles/rke2-prepare/templates/rke2-server.service.j2 @@ -0,0 +1,13 @@ +# rke2-server.service.j2 +[Unit] +Description=RKE2 server +After=network.target + +[Service] +ExecStart=/usr/local/bin/rke2 server +KillMode=process +Restart=on-failure +RestartSec=5s + +[Install] +WantedBy=multi-user.target diff --git a/ansible/rke2/roles/rke2-prepare/vars/main.yaml b/ansible/rke2/roles/rke2-prepare/vars/main.yaml new file mode 100644 index 0000000..e69de29 diff --git a/ansible/rke2/site.yaml b/ansible/rke2/site.yaml new file mode 100644 index 0000000..d885cbb --- /dev/null +++ b/ansible/rke2/site.yaml @@ -0,0 +1,61 @@ +# Hello, thanks for using my playbook, hopefully you can help to improve it. +# Things that need adding: (there are many more) +# 1) Support different OS & architectures +# 2) Support multiple CNIs +# 3) Improve the wait logic +# 4) Use kubernetes Ansible plugins more sensibly +# 5) Optimise flow logic +# 6) Clean up + +############################################################### +# MAKE SURE YOU CHANGE group_vars/all.yaml VARIABLES!!!!!!!!!!! +############################################################### + +# bootstraps first server and copies configs for others/agents +- name: Prepare all nodes + hosts: servers,agents + gather_facts: true # enables us to gather lots of useful variables: https://docs.ansible.com/ansible/latest/collections/ansible/builtin/setup_module.html + roles: + - prepare-nodes + +# creates directories for download and then downloads RKE2 and changes permissions +- name: Download RKE2 + hosts: servers,agents + gather_facts: true + roles: + - rke2-download + +# Creates RKE2 bootstrap manifests folder and copies kube-vip template over (configured with variables) +- name: Deploy Kube VIP + hosts: servers + gather_facts: true + roles: + - kube-vip + +# bootstraps the first server, copies configs to nodes, saves token to use later +- name: Prepare RKE2 on Servers and Agents + hosts: servers,agents + gather_facts: true + roles: + - rke2-prepare + +# Adds additional servers using the token from the previous task +- name: Add additional RKE2 Servers + hosts: servers + gather_facts: true + roles: + - add-server + +# Adds agents to the cluster +- name: Add additional RKE2 Agents + hosts: agents + gather_facts: true + roles: + - add-agent + +# Finish kube-vip, add metallb +- name: Apply manifests after cluster is created + hosts: servers + gather_facts: true + roles: + - apply-manifests diff --git a/ansible/update-apt.yml b/ansible/update-apt.yml new file mode 100644 index 0000000..53c4c55 --- /dev/null +++ b/ansible/update-apt.yml @@ -0,0 +1,21 @@ +- name: Upgrade everything on the cluster. + hosts: test + become: true + tasks: + - name: Upgrade all software. + ansible.builtin.apt: + update_cache: true + upgrade: dist + + - name: Check if a reboot is required. + stat: + path: /var/run/reboot-required + register: reboot_required_file + + - name: Reboot the server (if required). + reboot: + when: reboot_required_file.stat.exists == true + + - name: Remove dependencies that are no longer required. + apt: + autoremove: yes \ No newline at end of file