WIP: configure rook

badhouseplants/helmfile.yaml

@@ -5,42 +5,42 @@ releases:
   - <<: *drone
     installed: true
     namespace: drone-service
-    createNamespace: false
+    createNamespace: true
 
   - <<: *drone-runner-docker
     installed: true
     namespace: drone-service
-    createNamespace: false
+    createNamespace: true
 
   - <<: *longhorn
     installed: true
     namespace: longhorn-system
-    createNamespace: false
+    createNamespace: true
 
   - <<: *argocd
     installed: true
     namespace: argo-system
-    createNamespace: false
+    createNamespace: true
 
   - <<: *nrodionov
     installed: true
     namespace: nrodionov-application
-    createNamespace: false
+    createNamespace: true
 
   - <<: *minecraft
     installed: true
     namespace: minecraft-application
-    createNamespace: false
+    createNamespace: true
 
   - <<: *gitea
     installed: true
     namespace: gitea-service
-    createNamespace: false
+    createNamespace: true
 
   - <<: *funkwhale
     installed: true
     namespace: funkwhale-application
-    createNamespace: false
+    createNamespace: true
 
   - <<: *prometheus
     installed: true
@@ -50,16 +50,11 @@ releases:
   - <<: *loki
     installed: true
     namespace: monitoring-system
-    createNamespace: false
+    createNamespace: true
 
   - <<: *promtail
     installed: true
     namespace: monitoring-system
-    createNamespace: false
-
-  - <<: *bitwarden
-    installed: false
-    namespace: bitwarden-application
     createNamespace: true
 
   - <<: *redis
@@ -82,11 +77,6 @@ releases:
     namespace: database-service
     createNamespace: true
 
-  - <<: *mysql
-    installed: false
-    namespace: database-service
-    createNamespace: true
-
   - <<: *docker-mailserver
     installed: true
     namespace: mail-service
@@ -95,7 +85,7 @@ releases:
   - <<: *istio-gateway-resources
     installed: true
     namespace: istio-system
-    createNamespace: false
+    createNamespace: true
 
   - <<: *vaultwarden
     createNamespace: true
@@ -107,6 +97,16 @@ releases:
     namespace: woodpecker-ci
     createNamespace: true
 
+  - <<: *rook-ceph
+    installed: true
+    namespace: rook-ceph
+    createNamespace: true
+
+  - <<: *rook-ceph-cluster
+    installed: true
+    namespace: rook-ceph-cluster
+    createNamespace: true
+
 bases:
   - ../environments.yaml
   - ../repositories.yaml

badhouseplants/values/values.istio-gateway-resources.yaml

@@ -76,7 +76,7 @@ istio-gateway:
           - '*'
           port:
             name: ssh
-            number: 22
+            number: 2022
             protocol: TCP
     - name: badhouseplants-minecraft
       servers:

badhouseplants/values/values.istio-ingressgateway.yaml

@@ -6,7 +6,7 @@ service:
       protocol: TCP
       targetPort: 25565
     - name: ssh-gitea
-      port: 22
+      port: 2222
       protocol: TCP
       targetPort: 22
     - name: http2
@@ -21,10 +21,6 @@ service:
       port: 1194
       protocol: TCP
       targetPort: 1194
-    - name: tcp
-      port: 25
-      protocol: TCP
-      targetPort: 25
     # -----------
     # -- Email
     # -----------

badhouseplants/values/values.rook-ceph-cluster.yaml

@@ -0,0 +1,96 @@
+cephFileSystems:
+  - name: ceph-filesystem
+    spec:
+      metadataPool:
+        replicated:
+          size: 3
+      dataPools:
+        - failureDomain: host
+          replicated:
+            size: 3
+          name: data0
+      metadataServer:
+        activeCount: 1
+        activeStandby: true
+        resources:
+          limits:
+            cpu: "200m"
+            memory: "256Mi"
+          requests:
+            cpu: "50m"
+            memory: "128Mi"
+        priorityClassName: system-cluster-critical
+    storageClass:
+      enabled: true
+      isDefault: false
+      name: ceph-filesystem
+      pool: data0
+      reclaimPolicy: Delete
+      allowVolumeExpansion: true
+      volumeBindingMode: "Immediate"
+      mountOptions: []
+      parameters:
+        csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+        csi.storage.k8s.io/provisioner-secret-namespace: "{{ .Release.Namespace }}"
+        csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+        csi.storage.k8s.io/controller-expand-secret-namespace: "{{ .Release.Namespace }}"
+        csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+        csi.storage.k8s.io/node-stage-secret-namespace: "{{ .Release.Namespace }}"
+        csi.storage.k8s.io/fstype: ext4
+
+cephObjectStores: []
+  #  - name: ceph-objectstore
+  #    spec:
+  #      metadataPool:
+  #        failureDomain: host
+  #        replicated:
+  #          size: 3
+  #      dataPool:
+  #        failureDomain: host
+  #        erasureCoded:
+  #          dataChunks: 2
+  #          codingChunks: 1
+  #      preservePoolsOnDelete: true
+  #      gateway:
+  #        port: 80
+  #        resources:
+  #          limits:
+  #            cpu: "150m"
+  #            memory: "256Mi"
+  #          requests:
+  #            cpu: "50m"
+  #            memory: "128Mi"
+  #        instances: 1
+  #        priorityClassName: system-cluster-critical
+  #    storageClass:
+  #      enabled: true
+  #      name: ceph-bucket
+  #      reclaimPolicy: Delete
+  #      volumeBindingMode: "Immediate"
+  #      parameters:
+  #        region: us-east-1
+  #    ingress:
+  #      enabled: false
+cephClusterSpec:
+  resources:
+    mgr:
+      limits:
+        cpu: "200m"
+        memory: "512Mi"
+      requests:
+        cpu: "100m"
+        memory: "128Mi"
+    mon:
+      limits:
+        cpu: "200m"
+        memory: "512Mi"
+      requests:
+        cpu: "100m"
+        memory: "128Mi"
+    osd:
+      limits:
+        cpu: "200m"
+        memory: "2Gi"
+      requests:
+        cpu: "100m"
+        memory: "256Mi"

badhouseplants/values/values.rook-ceph.yaml

@@ -0,0 +1,215 @@
+---
+csi:
+  csiRBDProvisionerResource: |
+    - name : csi-provisioner
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-resizer
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-attacher
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-snapshotter
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-rbdplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-omap-generator
+      resource:
+        requests:
+          memory: 12Mi
+          cpu: 250m
+        limits:
+          memory: 1Gi
+          cpu: 500m
+    - name : liveness-prometheus
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+
+  # -- CEPH CSI RBD plugin resource requirement list
+  # @default -- see values.yaml
+  csiRBDPluginResource: |
+    - name : driver-registrar
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : csi-rbdplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : liveness-prometheus
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+
+  # -- CEPH CSI CephFS provisioner resource requirement list
+  # @default -- see values.yaml
+  csiCephFSProvisionerResource: |
+    - name : csi-provisioner
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-resizer
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-attacher
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-snapshotter
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-cephfsplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : liveness-prometheus
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+
+  # -- CEPH CSI CephFS plugin resource requirement list
+  # @default -- see values.yaml
+  csiCephFSPluginResource: |
+    - name : driver-registrar
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : csi-cephfsplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : liveness-prometheus
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+
+  # -- CEPH CSI NFS provisioner resource requirement list
+  # @default -- see values.yaml
+  csiNFSProvisionerResource: |
+    - name : csi-provisioner
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 200m
+    - name : csi-nfsplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : csi-attacher
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+
+  # -- CEPH CSI NFS plugin resource requirement list
+  # @default -- see values.yaml
+  csiNFSPluginResource: |
+    - name : driver-registrar
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m
+    - name : csi-nfsplugin
+      resource:
+        requests:
+          memory: 128Mi
+          cpu: 50m
+        limits:
+          memory: 256Mi
+          cpu: 50m

environments.yaml

@@ -1,5 +1,5 @@
 environments:
   badhouseplants:
-    kubeContext: badhouseplants
+    kubeContext: badhouseplants-arm
   etersoft:
     kubeContext: etersoft

helmfile.yaml

@@ -8,35 +8,43 @@ bases:
 releases:
   - <<: *metrics-server
     installed: true
+    namespace: kube-system
+    createNamespace: true
 
   - <<: *istio-base
     installed: true
+    namespace: istio-system
+    createNamespace: true
   
   - <<: *istio-gateway
     installed: true
     namespace: istio-system
-    createNamespace: false
+    createNamespace: true
 
   - <<: *istiod
     installed: true
     namespace: istio-system
-    createNamespace: false
+    createNamespace: true
 
   - <<: *cert-manager
     installed: true
+    namespace: cert-manager
+    createNamespace: true
 
   - <<: *minio
     installed: true
     namespace: minio-service
-    createNamespace: false
+    createNamespace: true
 
   - <<: *openvpn
     installed: true
     namespace: openvpn-service
-    createNamespace: false
+    createNamespace: true
   
   - <<: *metallb
     installed: true
+    namespace: metallb-system
+    createNamespace: true
 
   - <<: *reflector
     installed: true

releases.yaml

@@ -96,8 +96,6 @@ templates:
     name: metrics-server
     chart: metrics-server/metrics-server
     version: 3.11.0
-    namespace: kube-system
-    createNamespace: true
     values:
       - common/values.{{ .Release.Name }}.yaml
 
@@ -105,19 +103,14 @@ templates:
     name: metallb
     chart: metallb/metallb
     version: 0.13.12
-    namespace: metallb-system
-    createNamespace: true
 
   cert-manager: &cert-manager
     name: cert-manager
     chart: jetstack/cert-manager
     version: 1.13.3
-    namespace: cert-manager
-    createNamespace: true
     set:
       - name: installCRDs
         value: true
-
   longhorn: &longhorn
     name: longhorn
     chart: longhorn/longhorn
@@ -133,9 +126,7 @@ templates:
       - template: default-env-values
       - template: default-env-secrets
       - template: ext-istio-resource
-  # -------------------------------------------------------------------
-  # -- Monitoring
-  # -------------------------------------------------------------------
+
   monitoring-common:
     labels:
       bundle: monitoring
@@ -170,11 +161,9 @@ templates:
   # -- Istio
   # ----------------------------
   istio-common:
-    version: 1.20.1
     labels:
       bundle: istio
-    namespace: istio-system
-    createNamespace: true
+    version: 1.20.1
 
   istio-base: &istio-base
     name: istio-base
@@ -186,8 +175,6 @@ templates:
   istio-gateway: &istio-gateway
     name: istio-ingressgateway
     chart: istio/gateway
-    needs:
-      - istio-system/istio-base
     inherit:
       - template: istio-common
       - template: default-env-values
@@ -204,8 +191,6 @@ templates:
   istiod: &istiod
     name: istiod
     chart: istio/istiod
-    needs:
-      - istio-system/istio-base
     inherit:
       - template: istio-common
       - template: default-env-values
@@ -372,3 +357,17 @@ templates:
     name: reflector
     chart: emberstack/reflector
     version: 7.1.216
+
+  rook-ceph: &rook-ceph
+    name: rook-ceph
+    chart: rook/rook-ceph
+    version: v1.13.1
+    inherit:
+      - template: default-env-values
+
+  rook-ceph-cluster: &rook-ceph-cluster
+    name: rook-ceph-cluster
+    chart: rook/rook-ceph-cluster
+    version: v1.13.1
+    inherit:
+      - template: default-env-values

repositories.yaml

@@ -43,3 +43,5 @@ repositories:
     url: https://firefly-iii.github.io/kubernetes/
   - name: emberstack
     url: https://emberstack.github.io/helm-charts
+  - name: rook
+    url: https://charts.rook.io/release

test/test.yaml

@@ -0,0 +1,30 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pvc-test
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 5Gi
+
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pvc-test
+spec:
+  restartPolicy: Never
+  volumes:
+  - name: vol
+    persistentVolumeClaim:
+      claimName: pvc-test
+  containers:
+  - name: pv-recycler
+    image: ubuntu
+    command: ["/bin/sh", "-c", "sleep 10000"]
+    volumeMounts:
+    - name: vol
+      mountPath: /data