Enable longhorn in cluster

nix/nix-lab/configuration.nix

@@ -1,11 +1,10 @@
 { pkgs, hostConf, inputs, ... }: {
-  #imports = [ inputs.sops-nix.nixosModules.sops ];
-  # sops = {
-  #   defaultSopsFile = ./secrets/secrets.yaml;
-  #   defaultSopsFormat = "yaml";
-  #   age.keyFile = "../../../.sops/nix.key";
-  #   secrets."k3s.token" = { };
-  # };
+  imports = [
+    "${inputs.nixpkgs-unstable}/nixos/modules/services/cluster/k3s/default.nix"
+  ];
+  disabledModules = [
+    "services/cluster/k3s/default.nix"
+  ];
   networking.firewall = {
     allowedTCPPorts = [
       6443 # k3s API
@@ -33,7 +32,16 @@
     ];
     clusterInit = (hostConf.name == "bard");
     serverAddr = (if hostConf.name == "bard" then "" else "https://192.168.1.31:6443");
+    manifests = { longhorn-nixos-path.source = ./manifests/longhorn-nixos-path.yaml; };
   };
+  services.openiscsi = {
+    enable = false;
+    name = "iqn.2020-03.net.jafner:${hostConf.name}-initiatorhost";
+  };
+  systemd.tmpfiles.rules = [
+    "L+ /usr/local/bin - - - - /run/current-system-sw/bin/"
+  ];
+  virtualisation.docker.logDriver = "json-file";
   environment.systemPackages = with pkgs; [
     vim 
     fastfetch
@@ -47,6 +55,8 @@
     cifs-utils
     nfs-utils
     git
+    dig
+    openiscsi
   ];
   security.sudo = {
     enable = true;
@@ -65,6 +75,7 @@
     settings.PasswordAuthentication = false;
     settings.KbdInteractiveAuthentication = false;
   };
+  services.rpcbind.enable = true;
   users.users = {
     admin = {
       isNormalUser = true;
@@ -86,11 +97,15 @@
       macAddress = "${hostConf.nic.mac}";
       ipv4.addresses = [ { address = "${hostConf.nic.ip}"; prefixLength = 24; } ];
     };
+    nameservers = [
+      "10.0.0.1"
+    ];
   };
   nix.settings.experimental-features = [ "nix-command" "flakes" ];
   time.timeZone = "America/Los_Angeles";
   nix.settings.trusted-users = [ "root" "admin" ];
   boot.loader.systemd-boot.enable = true;
   boot.loader.efi.canTouchEfiVariables = true;
+  boot.supportedFilesystems = [ "nfs" ];
   system.stateVersion = "24.05";
 }

nix/nix-lab/flake.nix

@@ -31,6 +31,8 @@
         inherit system;
         specialArgs = { 
           inherit pkgs; 
+          inherit pkgs-unstable; 
+          inherit inputs;
           hostConf = {
             name = "bard";
             nic.mac = "6c:2b:59:37:89:40";
@@ -44,6 +46,8 @@
         inherit system;
         specialArgs = { 
           inherit pkgs; 
+          inherit pkgs-unstable; 
+          inherit inputs;
           hostConf = {
             name = "ranger";
             nic.mac = "6c:2b:59:37:9e:91";
@@ -57,6 +61,8 @@
         inherit system;
         specialArgs = { 
           inherit pkgs; 
+          inherit pkgs-unstable; 
+          inherit inputs;
           hostConf = {
             name = "cleric";
             nic.mac = "6c:2b:59:37:9e:00";

nix/nix-lab/helm/helmfile.yaml

@@ -0,0 +1,17 @@
+repositories:
+  - name: longhorn
+    url: https://charts.longhorn.io
+  - name: kyverno
+    url: https://kyverno.github.io/kyverno
+---
+releases:
+  # Distributed storage
+  - name: longhorn
+    namespace: longhorn-system
+    chart: longhorn/longhorn
+    version: 1.6.2
+  # Policy management; required fix for longhorn on nixos
+  - name: kyverno
+    namespace: kyverno
+    chart: kyverno/kyverno
+    version: 3.2.7

nix/nix-lab/manifests/longhorn-nixos-path.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: longhorn-nixos-path
+  namespace: longhorn-system
+data:
+  PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/run/wrappers/bin:/nix/var/nix/profiles/default/bin:/run/current-system/sw/bin
+---
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: longhorn-add-nixos-path
+  annotations:
+    policies.kyverno.io/title: Add Environment Variables from ConfigMap
+    policies.kyverno.io/subject: Pod
+    policies.kyverno.io/category: Other
+    policies.kyverno.io/description: >-
+      Longhorn invokes executables on the host system, and needs
+      to be aware of the host systems PATH. This modifies all
+      deployments such that the PATH is explicitly set to support
+      NixOS based systems.
+spec:
+  rules:
+    - name: add-env-vars
+      match:
+        resources:
+          kinds:
+            - Pod
+          namespaces:
+            - longhorn-system
+      mutate:
+        patchStrategicMerge:
+          spec:
+            initContainers:
+              - (name): "*"
+                envFrom:
+                  - configMapRef:
+                      name: longhorn-nixos-path
+            containers:
+              - (name): "*"
+                envFrom:
+                  - configMapRef:
+                      name: longhorn-nixos-path
+---