furyhawk/cloudy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor ansible inventory to add dev server

Browse Source
This commit is contained in:
Teck Meng
2024-06-30 16:46:04 +08:00
parent a896598eb0
commit d3ef4cd14a
27 changed files with 1095 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ansible/02-dev-inventory.yml
+39 -2
View File
@@ -1,4 +1,41 @@
devservers:
hosts:
dev1:
ansible_host: 192.168.50.58
node09:
ansible_host: 101.100.172.55
ansible_port: 9991
node00:
ansible_host: 101.100.172.55
ansible_port: 9999
node01:
ansible_host: 101.100.172.55
ansible_port: 9998
debian:
hosts:
node00:
ansible_host: 101.100.172.55
ansible_port: 9999
node01:
ansible_host: 101.100.172.55
ansible_port: 9998
local:
hosts:
node00:
ansible_host: 192.168.50.114
ansible_port: 22
node01:
ansible_host: 192.168.50.68
ansible_port: 22
node02:
ansible_host: 192.168.50.56
ansible_port: 22
test:
hosts:
node05:
ansible_host: 192.168.50.205
ansible_port: 22
node09:
ansible_host: 192.168.50.209
ansible_port: 22
ansible/playbook.yaml
+10
View File
@@ -0,0 +1,10 @@
- name: My first play
hosts: devservers
tasks:
- name: Ping my hosts
ansible.builtin.ping:
- name: Print message
ansible.builtin.debug:
msg: Hello world
ansible/rke2/collections/requirements.yaml
+6
View File
@@ -0,0 +1,6 @@
---
collections:
- name: ansible.utils
- name: community.general
- name: ansible.posix
- name: kubernetes.core
ansible/rke2/config.yaml
+8
View File
@@ -0,0 +1,8 @@
tls-san:
- 192.168.50.190
- 192.168.50.209
-
-
write-kubeconfig-mode: 0644
disable:
- rke2-ingress-nginx
ansible/rke2/inventory/group_vars/all.yaml
+20
View File
@@ -0,0 +1,20 @@
os: "linux"
# arch: "amd64"
# Set your timezone
system_timezone: "Asia/Singapore"
kube_vip_version: "v0.8.1"
vip_interface: eth0
vip: 192.168.50.210
metallb_version: v0.14.5
lb_range: 192.168.50.190-192.168.50.199
lb_pool_name: first-pool
rke2_version: "v1.28.10+rke2r1"
rke2_install_dir: "/usr/local/bin"
rke2_binary_url: "https://github.com/rancher/rke2/releases/download/{{ rke2_version }}"
ansible_user: furyhawk
ansible_become: true
ansible_become_method: sudo
ansible/rke2/inventory/hosts.ini
+8
View File
@@ -0,0 +1,8 @@
# Make sure Ansible host has access to these devices
# Good idea to snapshot all machines and deploy uing cloud-init
[servers]
server1 ansible_host=192.168.50.209
[agents]
agent1 ansible_host=192.168.50.205
ansible/rke2/kube-vip
+89
View File
@@ -0,0 +1,89 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/name: kube-vip-ds
app.kubernetes.io/version: v0.6.3
name: kube-vip-ds
namespace: kube-system
spec:
selector:
matchLabels:
app.kubernetes.io/name: kube-vip-ds
template:
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/name: kube-vip-ds
app.kubernetes.io/version: v0.6.3
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/master
operator: Exists
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "6443"
- name: vip_interface
value: $interface
- name: vip_cidr
value: "32"
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_ddns
value: "false"
- name: svc_enable
value: "false"
- name: svc_leasename
value: plndr-svcs-lock
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: $vip
- name: prometheus_server
value: :2112
image: ghcr.io/kube-vip/kube-vip:v0.6.3
imagePullPolicy: Always
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
hostNetwork: true
serviceAccountName: kube-vip
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
updateStrategy: {}
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0
ansible/rke2/longhorn.sh
+100
View File
@@ -0,0 +1,100 @@
#!/bin/bash
echo -e " \033[33;2m __ _ _ ___ \033[0m"
echo -e " \033[33;2m \ \(_)_ __ ___( )__ / _ \__ _ _ __ __ _ __ _ ___ \033[0m"
echo -e " \033[33;2m \ \ | '_ \` _ \/ __| / /_\/ _\` | '__/ _\` |/ _\` |/ _ \ \033[0m"
echo -e " \033[33;2m /\_/ / | | | | | \__ \ / /_\\ (_| | | | (_| | (_| | __/ \033[0m"
echo -e " \033[33;2m \___/|_|_| |_| |_|___/ \____/\__,_|_| \__,_|\__, |\___| \033[0m"
echo -e " \033[33;2m |___/ \033[0m"
echo -e " \033[35;2m __ _ \033[0m"
echo -e " \033[35;2m / / ___ _ __ __ _| |__ ___ _ __ _ __ \033[0m"
echo -e " \033[35;2m / / / _ \| '_ \ / _\` | '_ \ / _ \| '__| '_ \ \033[0m"
echo -e " \033[35;2m / /__| (_) | | | | (_| | | | | (_) | | | | | | \033[0m"
echo -e " \033[35;2m \____/\___/|_| |_|\__, |_| |_|\___/|_| |_| |_| \033[0m"
echo -e " \033[35;2m |___/ \033[0m"
echo -e " \033[36;2m \033[0m"
echo -e " \033[32;2m https://youtube.com/@jims-garage \033[0m"
echo -e " \033[32;2m \033[0m"
#############################################
# YOU SHOULD ONLY NEED TO EDIT THIS SECTION #
#############################################
# THIS SCRIPT IS FOR RKE2, NOT K3S!
# THIS SCRIPT IS FOR RKE2, NOT K3S!
# THIS SCRIPT IS FOR RKE2, NOT K3S!
# Set the IP addresses of master1
master1=192.168.50.209
# Set the IP addresses of your Longhorn nodes
longhorn1=192.168.50.205
# User of remote machines
user=furyhawk
# Interface used on remotes
interface=eth0
# Set the virtual IP address (VIP)
vip=192.168.50.210
# Array of longhorn nodes
storage=($longhorn1)
#ssh certificate name variable
certName=id_rsa
#############################################
# DO NOT EDIT BELOW #
#############################################
# For testing purposes - in case time is wrong due to VM snapshots
sudo timedatectl set-ntp off
sudo timedatectl set-ntp on
# add ssh keys for all nodes
for node in "${storage[@]}"; do
ssh-copy-id $user@$node
done
# add open-iscsi - needed for Debian and non-cloud Ubuntu
if ! command -v sudo service open-iscsi status &> /dev/null
then
echo -e " \033[31;5mOpen-ISCSI not found, installing\033[0m"
sudo apt install open-iscsi
else
echo -e " \033[32;5mOpen-ISCSI already installed\033[0m"
fi
# Step 1: Add new longhorn nodes to cluster (note: label added)
# Set token variable needed for RKE2 (not required for K3S)
token=`cat token`
for newnode in "${storage[@]}"; do
ssh -tt $user@$newnode -i ~/.ssh/$certName sudo su <<EOF
mkdir -p /etc/rancher/rke2
touch /etc/rancher/rke2/config.yaml
echo "token: $token" >> /etc/rancher/rke2/config.yaml
echo "server: https://$vip:9345" >> /etc/rancher/rke2/config.yaml
echo "node-label:" >> /etc/rancher/rke2/config.yaml
echo " - longhorn=true" >> /etc/rancher/rke2/config.yaml
curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh -
systemctl enable rke2-agent.service
systemctl start rke2-agent.service
exit
EOF
echo -e " \033[32;5mLonghorn node joined successfully!\033[0m"
done
# Step 2: Install Longhorn (using modified Official to pin to Longhorn Nodes)
# kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/v1.6.2/deploy/longhorn.yaml
kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/Longhorn/longhorn.yaml
kubectl get pods \
--namespace longhorn-system \
--watch
# Step 3: Print out confirmation
kubectl get nodes
kubectl get svc -n longhorn-system
echo -e " \033[32;5mHappy Kubing! Access Longhorn through Rancher UI\033[0m"
ansible/rke2/rke2.sh
Executable
+262
View File
@@ -0,0 +1,262 @@
#!/bin/bash
echo -e " \033[33;5m __ _ _ ___ \033[0m"
echo -e " \033[33;5m \ \(_)_ __ ___( )__ / _ \__ _ _ __ __ _ __ _ ___ \033[0m"
echo -e " \033[33;5m \ \ | '_ \` _ \/ __| / /_\/ _\` | '__/ _\` |/ _\` |/ _ \ \033[0m"
echo -e " \033[33;5m /\_/ / | | | | | \__ \ / /_\\ (_| | | | (_| | (_| | __/ \033[0m"
echo -e " \033[33;5m \___/|_|_| |_| |_|___/ \____/\__,_|_| \__,_|\__, |\___| \033[0m"
echo -e " \033[33;5m |___/ \033[0m"
echo -e " \033[36;5m ___ _ _____ ___ \033[0m"
echo -e " \033[36;5m | _ \ |/ / __|_ ) \033[0m"
echo -e " \033[36;5m | / ' <| _| / / \033[0m"
echo -e " \033[36;5m |_|_\_|\_\___/___| \033[0m"
echo -e " \033[36;5m \033[0m"
echo -e " \033[32;5m https://youtube.com/@jims-garage \033[0m"
echo -e " \033[32;5m \033[0m"
#############################################
# YOU SHOULD ONLY NEED TO EDIT THIS SECTION #
#############################################
# Version of Kube-VIP to deploy
KVVERSION="v0.6.3"
# Set the IP addresses of the admin, masters, and workers nodes
admin=192.168.50.143
master1=192.168.50.209
worker1=192.168.50.205
# User of remote machines
user=furyhawk
# Interface used on remotes
interface=eth0
# Set the virtual IP address (VIP)
vip=192.168.50.190
# Array of all master nodes
allmasters=($master1)
# Array of master nodes
masters=($master2 $master3)
# Array of worker nodes
workers=($worker1)
# Array of all
all=($master1 $worker1)
# Array of all minus master1
allnomaster1=($master2 $master3 $worker1)
#Loadbalancer IP range
lbrange=192.168.3.191-192.168.3.199
#ssh certificate name variable
certName=id_rsa
#############################################
# DO NOT EDIT BELOW #
#############################################
# For testing purposes - in case time is wrong due to VM snapshots
sudo timedatectl set-ntp off
sudo timedatectl set-ntp on
# Move SSH certs to ~/.ssh and change permissions
cp /home/$user/{$certName,$certName.pub} /home/$user/.ssh
chmod 600 /home/$user/.ssh/$certName
chmod 644 /home/$user/.ssh/$certName.pub
# Install Kubectl if not already present
if ! command -v kubectl version &> /dev/null
then
echo -e " \033[31;5mKubectl not found, installing\033[0m"
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
else
echo -e " \033[32;5mKubectl already installed\033[0m"
fi
# Create SSH Config file to ignore checking (don't use in production!)
sed -i '1s/^/StrictHostKeyChecking no\n/' ~/.ssh/config
#add ssh keys for all nodes
for node in "${all[@]}"; do
ssh-copy-id $user@$node
done
# Step 1: Create Kube VIP
# create RKE2's self-installing manifest dir
sudo mkdir -p /var/lib/rancher/rke2/server/manifests
# Install the kube-vip deployment into rke2's self-installing manifest folder
curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/kube-vip
cat kube-vip | sed 's/$interface/'$interface'/g; s/$vip/'$vip'/g' > $HOME/kube-vip.yaml
sudo mv kube-vip.yaml /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
# Find/Replace all k3s entries to represent rke2
sudo sed -i 's/k3s/rke2/g' /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
# copy kube-vip.yaml to home directory
sudo cp /var/lib/rancher/rke2/server/manifests/kube-vip.yaml ~/kube-vip.yaml
# change owner
sudo chown $user:$user kube-vip.yaml
# make kube folder to run kubectl later
mkdir ~/.kube
# create the rke2 config file
sudo mkdir -p /etc/rancher/rke2
touch config.yaml
echo "tls-san:" >> config.yaml
echo " - $vip" >> config.yaml
echo " - $master1" >> config.yaml
echo " - $master2" >> config.yaml
echo " - $master3" >> config.yaml
echo "write-kubeconfig-mode: 0644" >> config.yaml
echo "disable:" >> config.yaml
echo " - rke2-ingress-nginx" >> config.yaml
# copy config.yaml to rancher directory
sudo cp ~/config.yaml /etc/rancher/rke2/config.yaml
# update path with rke2-binaries
echo 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml' >> ~/.bashrc ; echo 'export PATH=${PATH}:/var/lib/rancher/rke2/bin' >> ~/.bashrc ; echo 'alias k=kubectl' >> ~/.bashrc ; source ~/.bashrc ;
# Step 2: Copy kube-vip.yaml and certs to all masters
for newnode in "${allmasters[@]}"; do
scp -i ~/.ssh/$certName $HOME/kube-vip.yaml $user@$newnode:~/kube-vip.yaml
scp -i ~/.ssh/$certName $HOME/config.yaml $user@$newnode:~/config.yaml
scp -i ~/.ssh/$certName ~/.ssh/{$certName,$certName.pub} $user@$newnode:~/.ssh
echo -e " \033[32;5mCopied successfully!\033[0m"
done
# Step 3: Connect to Master1 and move kube-vip.yaml and config.yaml. Then install RKE2, copy token back to admin machine. We then use the token to bootstrap additional masternodes
ssh -tt $user@$master1 -i ~/.ssh/$certName sudo su <<EOF
mkdir -p /var/lib/rancher/rke2/server/manifests
mv kube-vip.yaml /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
mkdir -p /etc/rancher/rke2
mv config.yaml /etc/rancher/rke2/config.yaml
echo 'export KUBECONFIG=/etc/rancher/rke2/rke2.yaml' >> ~/.bashrc ; echo 'export PATH=${PATH}:/var/lib/rancher/rke2/bin' >> ~/.bashrc ; echo 'alias k=kubectl' >> ~/.bashrc ; source ~/.bashrc ;
curl -sfL https://get.rke2.io | sh -
systemctl enable rke2-server.service
systemctl start rke2-server.service
echo "StrictHostKeyChecking no" > ~/.ssh/config
ssh-copy-id -i /home/$user/.ssh/$certName $user@$admin
scp -i /home/$user/.ssh/$certName /var/lib/rancher/rke2/server/token $user@$admin:~/token
scp -i /home/$user/.ssh/$certName /etc/rancher/rke2/rke2.yaml $user@$admin:~/.kube/rke2.yaml
exit
EOF
echo -e " \033[32;5mMaster1 Completed\033[0m"
# Step 4: Set variable to the token we just extracted, set kube config location
token=`cat token`
sudo cat ~/.kube/rke2.yaml | sed 's/127.0.0.1/'$master1'/g' > $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
export KUBECONFIG=${HOME}/.kube/config
sudo cp ~/.kube/config /etc/rancher/rke2/rke2.yaml
kubectl get nodes
# Step 5: Install kube-vip as network LoadBalancer - Install the kube-vip Cloud Provider
kubectl apply -f https://kube-vip.io/manifests/rbac.yaml
kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
# Step 6: Add other Masternodes, note we import the token we extracted from step 3
# for newnode in "${masters[@]}"; do
# ssh -tt $user@$newnode -i ~/.ssh/$certName sudo su <<EOF
# mkdir -p /etc/rancher/rke2
# touch /etc/rancher/rke2/config.yaml
# echo "token: $token" >> /etc/rancher/rke2/config.yaml
# echo "server: https://$master1:9345" >> /etc/rancher/rke2/config.yaml
# echo "tls-san:" >> /etc/rancher/rke2/config.yaml
# echo " - $vip" >> /etc/rancher/rke2/config.yaml
# echo " - $master1" >> /etc/rancher/rke2/config.yaml
# echo " - $master2" >> /etc/rancher/rke2/config.yaml
# echo " - $master3" >> /etc/rancher/rke2/config.yaml
# curl -sfL https://get.rke2.io | sh -
# systemctl enable rke2-server.service
# systemctl start rke2-server.service
# exit
# EOF
# echo -e " \033[32;5mMaster node joined successfully!\033[0m"
# done
kubectl get nodes
# Step 7: Add Workers
for newnode in "${workers[@]}"; do
ssh -tt $user@$newnode -i ~/.ssh/$certName sudo su <<EOF
mkdir -p /etc/rancher/rke2
touch /etc/rancher/rke2/config.yaml
echo "token: $token" >> /etc/rancher/rke2/config.yaml
echo "server: https://$vip:9345" >> /etc/rancher/rke2/config.yaml
echo "node-label:" >> /etc/rancher/rke2/config.yaml
echo " - worker=true" >> /etc/rancher/rke2/config.yaml
echo " - longhorn=true" >> /etc/rancher/rke2/config.yaml
curl -sfL https://get.rke2.io | INSTALL_RKE2_TYPE="agent" sh -
systemctl enable rke2-agent.service
systemctl start rke2-agent.service
exit
EOF
echo -e " \033[32;5mWorker node joined successfully!\033[0m"
done
kubectl get nodes
# Step 8: Install Metallb
echo -e " \033[32;5mDeploying Metallb\033[0m"
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml
kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.13.12/config/manifests/metallb-native.yaml
# Download ipAddressPool and configure using lbrange above
curl -sO https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/ipAddressPool
cat ipAddressPool | sed 's/$lbrange/'$lbrange'/g' > $HOME/ipAddressPool.yaml
# Step 9: Deploy IP Pools and l2Advertisement
echo -e " \033[32;5mAdding IP Pools, waiting for Metallb to be available first. This can take a long time as we're likely being rate limited for container pulls...\033[0m"
kubectl wait --namespace metallb-system \
--for=condition=ready pod \
--selector=component=controller \
--timeout=1800s
kubectl apply -f ipAddressPool.yaml
kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/l2Advertisement.yaml
# Step 10: Install Rancher (Optional - Delete if not required)
#Install Helm
echo -e " \033[32;5mInstalling Helm\033[0m"
curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
chmod 700 get_helm.sh
./get_helm.sh
# Add Rancher Helm Repo & create namespace
helm repo add rancher-latest https://releases.rancher.com/server-charts/latest
kubectl create namespace cattle-system
# Install Cert-Manager
echo -e " \033[32;5mDeploying Cert-Manager\033[0m"
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.13.2/cert-manager.crds.yaml
helm repo add jetstack https://charts.jetstack.io
helm repo update
helm install cert-manager jetstack/cert-manager \
--namespace cert-manager \
--create-namespace \
--version v1.13.2
kubectl get pods --namespace cert-manager
# Install Rancher
echo -e " \033[32;5mDeploying Rancher\033[0m"
helm install rancher rancher-latest/rancher \
--namespace cattle-system \
--set hostname=rancher.my.org \
--set bootstrapPassword=admin
kubectl -n cattle-system rollout status deploy/rancher
kubectl -n cattle-system get deploy rancher
# Add Rancher LoadBalancer
kubectl get svc -n cattle-system
kubectl expose deployment rancher --name=rancher-lb --port=443 --type=LoadBalancer -n cattle-system
while [[ $(kubectl get svc -n cattle-system 'jsonpath={..status.conditions[?(@.type=="Pending")].status}') = "True" ]]; do
sleep 5
echo -e " \033[32;5mWaiting for LoadBalancer to come online\033[0m"
done
kubectl get svc -n cattle-system
echo -e " \033[32;5mAccess Rancher from the IP above - Password is admin!\033[0m"
ansible/rke2/roles/add-agent/tasks/main.yaml
+17
View File
@@ -0,0 +1,17 @@
# Copy agent config to all agents - we need to change agent2 & 3 later with the token
- name: Deploy RKE2 Agent Configuration
ansible.builtin.template:
src: templates/rke2-agent-config.j2
dest: /etc/rancher/rke2/config.yaml
owner: root
group: root
mode: '0644'
when: inventory_hostname in groups['agents']
# Check agents have restarted to pick up config
- name: Ensure RKE2 agents are enabled and running
ansible.builtin.systemd:
name: rke2-agent
enabled: true
state: restarted
daemon_reload: true
ansible/rke2/roles/add-agent/templates/rke2-agent-config.j2
+6
View File
@@ -0,0 +1,6 @@
write-kubeconfig-mode: "0644"
token: {{ hostvars['server1']['token'] }}
server: https://{{ hostvars['server1']['ansible_host'] }}:9345
node-label:
- "agent=true"
- "longhorn=true"
ansible/rke2/roles/add-server/tasks/main.yaml
+53
View File
@@ -0,0 +1,53 @@
# Copy server config with token to all servers except server 1 (this has token)
- name: Deploy RKE2 server Configuration
ansible.builtin.template:
src: templates/rke2-server-config.j2
dest: /etc/rancher/rke2/config.yaml
owner: root
group: root
mode: '0644'
when: inventory_hostname != groups['servers'][0]
# Keep checking the cluster API until it's functioning (deployed)
- name: Wait for cluster API to be ready (can take 5-10 mins depending on internet/hardware)
ansible.builtin.command:
cmd: "kubectl get nodes"
register: kubectl_output
until: "'connection refused' not in kubectl_output.stderr"
retries: 120
delay: 10
changed_when: true
become_user: "{{ ansible_user }}"
when: inventory_hostname == groups['servers'][0]
# Use kubectl to deploy yaml. Perhaps this can be added to the manifest folder initially
- name: Apply kube vip configuration file
ansible.builtin.command:
cmd: kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml apply -f https://kube-vip.io/manifests/rbac.yaml
changed_when: true
when: inventory_hostname == groups['servers'][0]
# Apply the kube-vip configration. Perhaps this can be added to the manifest folder initially
- name: Apply kube vip configuration file
ansible.builtin.command:
cmd: kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml
changed_when: true
when: inventory_hostname == groups['servers'][0]
# Check that additional servers are restarted
- name: Ensure additional RKE2 servers are enabled and running
ansible.builtin.systemd:
name: rke2-server
enabled: true
state: restarted
daemon_reload: true
when: inventory_hostname != groups['servers'][0]
# enable additional servers
- name: Ensure RKE2 server is enabled and running
ansible.builtin.systemd:
name: rke2-server
enabled: true
state: restarted
daemon_reload: true
when: inventory_hostname != groups['servers'][0]
ansible/rke2/roles/add-server/templates/rke2-server-config.j2
+8
View File
@@ -0,0 +1,8 @@
write-kubeconfig-mode: "0644"
token: {{ hostvars['server1']['token'] }}
server: https://{{ hostvars['server1']['ansible_host'] }}:9345
tls-san:
- {{ vip }}
- {{ hostvars['server1']['ansible_host'] }}
node-label:
- server=true
ansible/rke2/roles/apply-manifests/tasks/main.yaml
+60
View File
@@ -0,0 +1,60 @@
# Wait for Server 1 to be ready before continuing with metallb deployment
- name: Wait for k8s nodes with node label 'server=true' to be ready, otherwise we cannot start metallb deployment
ansible.builtin.command:
cmd: "kubectl wait --for=condition=Ready nodes --selector server=true --timeout=600s"
register: nodes_ready
retries: 120
delay: 10
changed_when: true
become_user: "{{ ansible_user }}"
when: inventory_hostname == groups['servers'][0]
# Create namespace so that we can deploy metallb
- name: Apply metallb namespace
ansible.builtin.command:
cmd: kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/v0.12.1/manifests/namespace.yaml
become_user: "{{ ansible_user }}"
changed_when: true
when: inventory_hostname == groups['servers'][0]
# Apply metallb manifest
- name: Apply metallb manifest
ansible.builtin.command:
cmd: kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/{{ metallb_version }}/config/manifests/metallb-native.yaml
become_user: "{{ ansible_user }}"
changed_when: true
when: inventory_hostname == groups['servers'][0]
# Wait for metallb deployment pods to be alive before deploying metallb manifests
- name: Wait for metallb pods to be ready, otherwise we cannot start metallb deployment
ansible.builtin.command:
cmd: "kubectl wait --namespace metallb-system --for=condition=ready pod --selector=component=controller --timeout=1800s"
changed_when: true
become_user: "{{ ansible_user }}"
when: inventory_hostname == groups['servers'][0]
# Apply L2 Advertisement for metallb
- name: Apply metallb L2 Advertisement
ansible.builtin.command:
cmd: kubectl apply -f https://raw.githubusercontent.com/JamesTurland/JimsGarage/main/Kubernetes/RKE2/l2Advertisement.yaml
become_user: "{{ ansible_user }}"
changed_when: true
when: inventory_hostname == groups['servers'][0]
# Deploy metal IP Pool to Server 1
- name: Copy metallb IPPool to server 1
ansible.builtin.template:
src: templates/metallb-ippool.j2
dest: /home/{{ ansible_user }}/ippool.yaml
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: '0755'
when: inventory_hostname == groups['servers'][0]
# don't think this will work as nodes are no execute, might need agents first
- name: Apply metallb ipppool
ansible.builtin.command:
cmd: kubectl apply -f /home/{{ ansible_user }}/ippool.yaml
become_user: "{{ ansible_user }}"
changed_when: true
when: inventory_hostname == groups['servers'][0]
ansible/rke2/roles/apply-manifests/templates/metallb-ippool.j2
+8
View File
@@ -0,0 +1,8 @@
apiVersion: metallb.io/v1beta1
kind: IPAddressPool
metadata:
name: {{ lb_pool_name }}
namespace: metallb-system
spec:
addresses:
- {{ lb_range }}
ansible/rke2/roles/kube-vip/tasks/main.yaml
+17
View File
@@ -0,0 +1,17 @@
# Create directory to deploy kube-vip manifest
- name: Create directory for Kube VIP Manifest
ansible.builtin.file:
path: "/var/lib/rancher/rke2/server/manifests"
state: directory
mode: '0644'
when: inventory_hostname in groups['servers']
# Copy kube-vip to server 1 manifest folder for auto deployment at bootstrap
- name: Deploy Kube VIP Configuration
ansible.builtin.template:
src: templates/kube-vip-config.j2
dest: /var/lib/rancher/rke2/server/manifests/kube-vip.yaml
owner: root
group: root
mode: '0644'
when: inventory_hostname == groups['servers'][0]
ansible/rke2/roles/kube-vip/templates/kube-vip-config.j2
+88
View File
@@ -0,0 +1,88 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/name: kube-vip-ds
app.kubernetes.io/version: {{ kube_vip_version }}
name: kube-vip-ds
namespace: kube-system
spec:
selector:
matchLabels:
app.kubernetes.io/name: kube-vip-ds
template:
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/name: kube-vip-ds
app.kubernetes.io/version: {{ kube_vip_version }}
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/master
operator: Exists
- matchExpressions:
- key: node-role.kubernetes.io/control-plane
operator: Exists
containers:
- args:
- manager
env:
- name: vip_arp
value: "true"
- name: port
value: "6443"
- name: vip_interface
value: {{ vip_interface }}
- name: vip_cidr
value: "32"
- name: cp_enable
value: "true"
- name: cp_namespace
value: kube-system
- name: vip_ddns
value: "false"
- name: svc_enable
value: "false"
- name: svc_leasename
value: plndr-svcs-lock
- name: vip_leaderelection
value: "true"
- name: vip_leasename
value: plndr-cp-lock
- name: vip_leaseduration
value: "5"
- name: vip_renewdeadline
value: "3"
- name: vip_retryperiod
value: "1"
- name: address
value: {{ vip }}
- name: prometheus_server
value: :2112
image: ghcr.io/kube-vip/kube-vip:{{ kube_vip_version }}
imagePullPolicy: Always
name: kube-vip
resources: {}
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
hostNetwork: true
serviceAccountName: kube-vip
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
updateStrategy: {}
status:
currentNumberScheduled: 0
desiredNumberScheduled: 0
numberMisscheduled: 0
numberReady: 0
ansible/rke2/roles/prepare-nodes/tasks/main.yaml
+15
View File
@@ -0,0 +1,15 @@
- name: Enable IPv4 forwarding
ansible.posix.sysctl:
name: net.ipv4.ip_forward
value: "1"
state: present
reload: true
tags: sysctl
- name: Enable IPv6 forwarding
ansible.posix.sysctl:
name: net.ipv6.conf.all.forwarding
value: "1"
state: present
reload: true
tags: sysctl
ansible/rke2/roles/rke2-download/tasks/main.yaml
+31
View File
@@ -0,0 +1,31 @@
# Create a directory to download RKE2 binary to
- name: Create directory for RKE2 binary
ansible.builtin.file:
path: "{{ rke2_install_dir }}"
state: directory
mode: '0755'
# Download the RKE2 binary
- name: Download RKE2 amd64 binary
ansible.builtin.get_url:
url: "{{ rke2_binary_url }}/rke2.linux-amd64"
dest: "{{ rke2_install_dir }}/rke2"
mode: '0755'
when: ansible_facts.architecture == "x86_64"
- name: Download RKE2 arm64 binary
ansible.builtin.get_url:
url: "{{ rke2_binary_url }}/rke2.linux-arm64"
dest: "{{ rke2_install_dir }}/rke2"
mode: '0755'
when:
- ( ansible_facts.architecture is search("arm") and
ansible_facts.userspace_bits == "64" ) or
ansible_facts.architecture is search("aarch64")
# Set permissions on the RKE2 binary
- name: Set executable permissions on the RKE2 binary
ansible.builtin.file:
path: "{{ rke2_install_dir }}/rke2"
mode: '0755'
state: file
ansible/rke2/roles/rke2-download/vars/main.yaml
View File
ansible/rke2/roles/rke2-prepare/tasks/main.yaml
+134
View File
@@ -0,0 +1,134 @@
- name: Create directory for RKE2 config
ansible.builtin.file:
path: "/etc/rancher/rke2"
state: directory
mode: '0644'
- name: Create directory for RKE2 token
ansible.builtin.file:
path: "/var/lib/rancher/rke2/server"
state: directory
mode: '0644'
# Copy server config to server 1 for bootstrap - we need to change server2 & 3 later with the token
- name: Deploy RKE2 server Configuration
ansible.builtin.template:
src: templates/rke2-server-config.j2
dest: /etc/rancher/rke2/config.yaml
owner: root
group: root
mode: '0644'
when: inventory_hostname in groups['servers']
- name: Create systemd service file for RKE2 server
ansible.builtin.template:
src: templates/rke2-server.service.j2
dest: /etc/systemd/system/rke2-server.service
owner: root
group: root
mode: '0644'
when: inventory_hostname in groups['servers']
- name: Create systemd service file for RKE2 agent
ansible.builtin.template:
src: templates/rke2-agent.service.j2
dest: /etc/systemd/system/rke2-agent.service
owner: root
group: root
mode: '0644'
when: inventory_hostname in groups['agents']
# we enable the first server to generate tokens etc, copy this afterwards to other servers
- name: Ensure RKE2 server is enabled and running
ansible.builtin.systemd:
name: rke2-server
enabled: true
state: restarted
daemon_reload: true
when: inventory_hostname in groups['servers'][0]
# wait for node token to be availale so that we can copy it, we need this to join other nodes
- name: Wait for node-token
ansible.builtin.wait_for:
path: /var/lib/rancher/rke2/server/node-token
when: inventory_hostname == groups['servers'][0]
# wait for kubectl to be downloaded, part of the rke2 installation
- name: Wait for kubectl
ansible.builtin.wait_for:
path: /var/lib/rancher/rke2/bin/kubectl
when: inventory_hostname == groups['servers'][0]
# copy kubectl to usr bin so that all users can run kubectl commands
- name: Copy kubectl to user bin
ansible.builtin.copy:
src: /var/lib/rancher/rke2/bin/kubectl
dest: /usr/local/bin/kubectl
mode: '0755'
remote_src: true
become: true
when: inventory_hostname == groups['servers'][0]
# wait for the kubectl copy to complete
- name: Wait for kubectl
ansible.builtin.wait_for:
path: /usr/local/bin/kubectl
when: inventory_hostname == groups['servers'][0]
# modify token access
- name: Register node-token file access mode
ansible.builtin.stat:
path: /var/lib/rancher/rke2/server
register: p
- name: Change file access for node-token
ansible.builtin.file:
path: /var/lib/rancher/rke2/server
mode: "g+rx,o+rx"
when: inventory_hostname == groups['servers'][0]
# Save token as variable
- name: Fetch the token from the first server node
ansible.builtin.slurp:
src: /var/lib/rancher/rke2/server/token
register: rke2_token
when: inventory_hostname == groups['servers'][0]
run_once: true
# convert token to fact
- name: Save Master node-token for later
ansible.builtin.set_fact:
token: "{{ rke2_token.content | b64decode | regex_replace('\n', '') }}"
# revert token file access
- name: Restore node-token file access
ansible.builtin.file:
path: /var/lib/rancher/rke2/server
mode: "{{ p.stat.mode }}"
when: inventory_hostname == groups['servers'][0]
# check .kube folder exists so that we can use kubectl (config resides here)
- name: Ensure .kube directory exists in user's home
ansible.builtin.file:
path: "/home/{{ ansible_user }}/.kube"
state: directory
mode: '0755'
become: true
# copy kubectl config file to .kube folder
- name: Copy config file to user home directory
ansible.builtin.copy:
src: /etc/rancher/rke2/rke2.yaml
dest: "/home/{{ ansible_user }}/.kube/config"
remote_src: true
owner: "{{ ansible_user }}"
mode: "u=rw,g=,o="
when: inventory_hostname == groups['servers'][0]
# change IP from local to server 1 IP
- name: Replace IP address with server1
ansible.builtin.replace:
path: /home/{{ ansible_user }}/.kube/config
regexp: '127.0.0.1'
replace: "{{ hostvars['server1']['ansible_host'] }}"
when: inventory_hostname == groups['servers'][0]
ansible/rke2/roles/rke2-prepare/templates/rke2-agent.service.j2
+13
View File
@@ -0,0 +1,13 @@
# rke2-agent.service.j2
[Unit]
Description=RKE2 Agent
After=network.target
[Service]
ExecStart=/usr/local/bin/rke2 agent
KillMode=process
Restart=on-failure
RestartSec=5s
[Install]
WantedBy=multi-user.target
ansible/rke2/roles/rke2-prepare/templates/rke2-server-config.j2
+8
View File
@@ -0,0 +1,8 @@
write-kubeconfig-mode: "0644"
tls-san:
- {{ vip }}
- {{ hostvars['server1']['ansible_host'] }}
node-label:
- server=true
disable:
- rke2-ingress-nginx
ansible/rke2/roles/rke2-prepare/templates/rke2-server.service.j2
+13
View File
@@ -0,0 +1,13 @@
# rke2-server.service.j2
[Unit]
Description=RKE2 server
After=network.target
[Service]
ExecStart=/usr/local/bin/rke2 server
KillMode=process
Restart=on-failure
RestartSec=5s
[Install]
WantedBy=multi-user.target
ansible/rke2/roles/rke2-prepare/vars/main.yaml
View File
ansible/rke2/site.yaml
+61
View File
@@ -0,0 +1,61 @@
# Hello, thanks for using my playbook, hopefully you can help to improve it.
# Things that need adding: (there are many more)
# 1) Support different OS & architectures
# 2) Support multiple CNIs
# 3) Improve the wait logic
# 4) Use kubernetes Ansible plugins more sensibly
# 5) Optimise flow logic
# 6) Clean up
###############################################################
# MAKE SURE YOU CHANGE group_vars/all.yaml VARIABLES!!!!!!!!!!!
###############################################################
# bootstraps first server and copies configs for others/agents
- name: Prepare all nodes
hosts: servers,agents
gather_facts: true # enables us to gather lots of useful variables: https://docs.ansible.com/ansible/latest/collections/ansible/builtin/setup_module.html
roles:
- prepare-nodes
# creates directories for download and then downloads RKE2 and changes permissions
- name: Download RKE2
hosts: servers,agents
gather_facts: true
roles:
- rke2-download
# Creates RKE2 bootstrap manifests folder and copies kube-vip template over (configured with variables)
- name: Deploy Kube VIP
hosts: servers
gather_facts: true
roles:
- kube-vip
# bootstraps the first server, copies configs to nodes, saves token to use later
- name: Prepare RKE2 on Servers and Agents
hosts: servers,agents
gather_facts: true
roles:
- rke2-prepare
# Adds additional servers using the token from the previous task
- name: Add additional RKE2 Servers
hosts: servers
gather_facts: true
roles:
- add-server
# Adds agents to the cluster
- name: Add additional RKE2 Agents
hosts: agents
gather_facts: true
roles:
- add-agent
# Finish kube-vip, add metallb
- name: Apply manifests after cluster is created
hosts: servers
gather_facts: true
roles:
- apply-manifests
ansible/update-apt.yml
+21
View File
@@ -0,0 +1,21 @@
- name: Upgrade everything on the cluster.
hosts: test
become: true
tasks:
- name: Upgrade all software.
ansible.builtin.apt:
update_cache: true
upgrade: dist
- name: Check if a reboot is required.
stat:
path: /var/run/reboot-required
register: reboot_required_file
- name: Reboot the server (if required).
reboot:
when: reboot_required_file.stat.exists == true
- name: Remove dependencies that are no longer required.
apt:
autoremove: yes