commit b117a375b888d7574b167e1b7701a2cf854aa296
parent 26bc2325b6790070f5b435798167d04e06567c43
Author: Vincent Demeester <vincent@sbr.pm>
Date: Tue, 28 Sep 2021 19:26:21 +0200
tools/k8s.infra: bootstrap script…
… and update configuration & co
Signed-off-by: Vincent Demeester <vincent@sbr.pm>
Diffstat:
6 files changed, 38 insertions(+), 28 deletions(-)
diff --git a/shell.nix b/shell.nix
@@ -21,6 +21,7 @@ pkgs.mkShell
nixos-generators
nixpkgs-fmt
sops
+ libguestfs-with-appliance
];
shellHook = ''
export NIX_PATH="nixpkgs=${pkgs.path}:nixos=${nixos.path}:nixos-unstable=${nixos-unstable.path}"
diff --git a/systems/hosts/k8sn1.nix b/systems/hosts/k8sn1.nix
@@ -12,9 +12,9 @@ in
imports = [
<nixpkgs/nixos/modules/profiles/qemu-guest.nix>
(import ../../nix).home-manager-stable
- ../../systems/modules
+ ../modules
# FIXME Need to refactor vincent user as.. it's adding way to much by default...
- # (import ../../users).vincent
+ # (import ../../../users).vincent
(import ../../users).root
];
@@ -24,12 +24,6 @@ in
autoResize = true;
};
- fileSystems."/var" = {
- device = "/dev/vdb1"; # /dev/vdb ?
- fsType = "ext4";
- # autoResize = true; # Is this needed ?
- };
-
boot.growPartition = true;
boot.kernelParams = [ "console=ttyS0" ];
boot.loader.grub.device = "/dev/vda";
diff --git a/systems/hosts/k8sn2.nix b/systems/hosts/k8sn2.nix
@@ -1,4 +1,8 @@
-{ pkgs, lib, ... }:
+{ sources ? import ../../nix
+, lib ? sources.lib
+, pkgs ? sources.pkgs { }
+, ...
+}:
let
hostname = "k8sn2";
@@ -8,9 +12,9 @@ in
imports = [
<nixpkgs/nixos/modules/profiles/qemu-guest.nix>
(import ../../nix).home-manager-stable
- ../../systems/modules
+ ../modules
# FIXME Need to refactor vincent user as.. it's adding way to much by default...
- # (import ../../users).vincent
+ # (import ../../../users).vincent
(import ../../users).root
];
@@ -20,12 +24,6 @@ in
autoResize = true;
};
- fileSystems."/var" = {
- device = "/dev/vdb1";
- fsType = "ext4";
- # autoResize = true; # Is this needed ?
- };
-
boot.growPartition = true;
boot.kernelParams = [ "console=ttyS0" ];
boot.loader.grub.device = "/dev/vda";
diff --git a/systems/hosts/k8sn3.nix b/systems/hosts/k8sn3.nix
@@ -1,4 +1,8 @@
-{ pkgs, lib, ... }:
+{ sources ? import ../../nix
+, lib ? sources.lib
+, pkgs ? sources.pkgs { }
+, ...
+}:
let
hostname = "k8sn3";
@@ -8,9 +12,9 @@ in
imports = [
<nixpkgs/nixos/modules/profiles/qemu-guest.nix>
(import ../../nix).home-manager-stable
- ../../systems/modules
+ ../modules
# FIXME Need to refactor vincent user as.. it's adding way to much by default...
- # (import ../../users).vincent
+ # (import ../../../users).vincent
(import ../../users).root
];
@@ -19,11 +23,6 @@ in
fsType = "ext4";
autoResize = true;
};
- fileSystems."/var" = {
- device = "/dev/vdb1";
- fsType = "ext4";
- # autoResize = true; # Is this needed ?
- };
boot.growPartition = true;
boot.kernelParams = [ "console=ttyS0" ];
diff --git a/systems/modules/profiles/kubernetes.nix b/systems/modules/profiles/kubernetes.nix
@@ -48,6 +48,7 @@ in
securePort = cfg.master.port;
advertiseAddress = cfg.master.ip;
};
+ # TODO: Remove this when switching to 21.11
controllerManager.extraOpts = "--horizontal-pod-autoscaler-use-rest-clients=false";
# use coredns
addons.dns.enable = true;
diff --git a/tools/k8s.infra/k8s.infra.sh b/tools/k8s.infra/k8s.infra.sh
@@ -32,9 +32,14 @@ build() {
logs=$(mktemp)
output=$(mktemp)
echo "Build ${n} node (logs: ${logs})…"
- nixos-generate -f qcow -c ./systems/hosts/${n}.nix 2>${logs} 1>${output}
+ nixos-generate -I nixpkgs=channel:nixos-21.05 -f qcow -c ./systems/hosts/${n}.nix 2>${logs} 1>${output}
+ echo "Resize ${n} image"
+ qemu-img create -f qcow2 -o preallocation=metadata ${n}.qcow2 40G
+ virt-resize --expand /dev/vda1 $(cat ${output} | tr -d '\n') ${n}.qcow2
echo "Syncthing image to ${HOST}…"
- ${RSYNC_COMMAND} $(cat ${output} | tr -d '\n') root@${HOST}:/var/lib/libvirt/images/${n}.qcow2
+ ${RSYNC_COMMAND} ${n}.qcow2 root@${HOST}:/var/lib/libvirt/images/${n}.qcow2
+ echo "Remove ${n} (local) image"
+ rm -f ${n}.qcow2
done
}
@@ -63,11 +68,23 @@ bootstrap() {
--name="${n}" --vcpus=4 --ram=8192 \
--network bridge=br1,mac.address=${!mac_addr} \
--disk path=/var/lib/libvirt/images/${n}.qcow2,bus=virtio,size=10 \
+ --disk path=/var/lib/libvirt/images/${n}-data.qcow2,bus=virtio,size=40 \
--print-xml > ${folder}/${n}.xml
- # --disk path=/var/lib/libvirt/images/${n}-data.qcow2,bus=virtio,size=40 \
echo "Node ${n} : ${folder}/${n}.xml"
${VIRSH_COMMAND} define --file ${folder}/${n}.xml
done
+ # Start the nodes
+ for n in ${NODES[@]}; do
+ ${VIRSH_COMMAND} start ${n}
+ done
+ # Wait for.. long time..
+ # Not sure how to ensure k8s is running on the master
+ token=$(ssh root@k8sn1.home cat /var/lib/kubernetes/secrets/apitoken.secret)
+ echo $token | ssh root@k8sn2.home nixos-kubernetes-node-join
+ echo $token | ssh root@k8sn3.home nixos-kubernetes-node-join
+ mkdir -p $HOME/.kube
+ # TODO: Copy cluster-admin configuration and sed the certs
+ scp root@k8sn1.home:/etc/kubernetes/cluster-admin.kubeconfig $HOME/home.cluster-admin.config
}
status() {
