Install loki and promtail

installations/monitoring/helmfile.yaml

@@ -19,9 +19,21 @@ releases:
       - template: crd-management-hook
   - name: grafana
     chart: grafana/grafana
-    namespace: applications
+    namespace: observability
     version: 8.5.2
     installed: true
     inherit:
       - template: default-env-values
       - template: default-env-secrets
+  - name: loki
+    chart: grafana/loki
+    namespace: observability
+    version: 6.16.0
+    inherit:
+      - template: default-env-values
+  - name: promtail
+    chart: grafana/promtail
+    namespace: observability
+    version: 6.16.6
+    inherit:
+      - template: default-env-values

values/badhouseplants/secrets.minio.yaml

@@ -1,27 +1,27 @@
-rootPassword: ENC[AES256_GCM,data:O3sU0X3qFeTiodPdVdlnpxBV6Dw=,iv:leHInQACT7j69t8tcCoD1W3i8hulAdVgKhJ833UtvuE=,tag:iRgiVHaS1H0jMm1q0rjiqw==,type:str]
+rootPassword: ENC[AES256_GCM,data:gvRRtUdNK95j5Q0fmeLWAzYB/zs=,iv:ywFhwBx4jRl0pBnafbqrd0JP4vAIUm277H0YdCaDobY=,tag:9f2O1+gCsF7E/pAFCiuQiQ==,type:str]
 users:
-  - accessKey: ENC[AES256_GCM,data:3j/8y1M3pU0=,iv:YpBhagbOxV8M3ouAUb0h51ePEjpkS8cYrkoFFhvYVfo=,tag:PswvP4au0x3pn5nsurmR2w==,type:str]
+  - accessKey: ENC[AES256_GCM,data:34UOVAjuQZk=,iv:hJfgCO3v8JJDZYbBnOU/jENrnKKVybrftqh2BDLLsCI=,tag:/pLuwmnA09fWBz0QKH/YGQ==,type:str]
-    secretKey: ENC[AES256_GCM,data:IO7KhFwDXIH2r/iMsliqrlVzWg==,iv:mZtFE5+LDldoicjfibG9XaSeu0q7FuHWEJ67eqYuHSw=,tag:E2py0sEDWDpV7EthtnB6ow==,type:str]
+    secretKey: ENC[AES256_GCM,data:TMC5oaxXscBa+8zstIc8Pgwg5Q==,iv:dUNyYvOyFmwuoElo+CyZ2p3Fhv3119HSl/krzFUeHDE=,tag:F2mLLc/c+QJ+f9S5rxzb8w==,type:str]
-    policy: ENC[AES256_GCM,data:T+WB4uR3ITw=,iv:9aKnBcXrk8B7kzkqFcICiYutmTizndisGBagargPAnQ=,tag:m4IGYwlXjTet1c3AAkqFoA==,type:str]
+    policy: ENC[AES256_GCM,data:u66022KoFFM=,iv:IJNFj5PClOyKJawN9mXxcGqXHqU1XqcbwvLJo4P05d0=,tag:BNNmuSPb407nurtgXzy7XQ==,type:str]
-  - accessKey: ENC[AES256_GCM,data:W4FQ,iv:GMBbAwEo7fDVEZi2W4r6ODYMFflA7N5BybUIb+7XpEw=,tag:K1ZbwnBUUWcjApoVI8ueCg==,type:str]
+  - accessKey: ENC[AES256_GCM,data:g2My,iv:PiI0qN3U48u3Eel8ivY1sNNiEHGX2UgXitTCg58rGrc=,tag:kaJKPm0XIsHseg1AuUhWNw==,type:str]
-    secretKey: ENC[AES256_GCM,data:9JZre6QOEGrA9Xm1UFnmOR93VB/Et8XdbQ==,iv:0qRpqKaZ2e2I9KcLJP2hU3KykCIBu3oDRvqnIcADc7U=,tag:ltARnaSEnAxp9hmZms8F/w==,type:str]
+    secretKey: ENC[AES256_GCM,data:ZXyyrbgm7ys72jbjy/sYH1IY8Fkx5eKwEw==,iv:YJlxXejRn5L6vrdb88FmyQZyRmXuNco5I5ssxJ+3HyA=,tag:yG2c1ibTQGQNbF0/do2vbQ==,type:str]
-    policy: ENC[AES256_GCM,data:51Oj,iv:gdBAKYRNeXModDcJBmz1hapHxIs5ubVBxAIoxJMhufE=,tag:wwUP1CFGqLUKW8lfkuaSpQ==,type:str]
+    policy: ENC[AES256_GCM,data:rI8C,iv:hBTRPM6I+bIByKVJ1NGZUtDiFf7v1LG4aD/Z+fjQrpQ=,tag:ULD0b4kBpr4KrGbIx7Lqtg==,type:str]
-  - accessKey: ENC[AES256_GCM,data:vuuzRTQGIPrJ,iv:8u8QSmF604i1XQcEE+rJiDTeGdkqk6Y1fpnGFFRRoFI=,tag:CAFV/1teBBZXV0+XKBfbMA==,type:str]
+  - accessKey: ENC[AES256_GCM,data:WUryEG3UsKcL,iv:3mPdzR52NnQrmz/cSk0NvzIOCL9dBvENnDgoOqSzIXk=,tag:YRHLkTwhbXEcN+Z8LAN9bA==,type:str]
-    secretKey: ENC[AES256_GCM,data:kKXPljfQ6S1bd1e0zMVZ3IFCSRI=,iv:fIN7GOVjztfHk+jIpcvuWGRVqNENrWV2f05d+TdHyhI=,tag:VVX0zzRD91+CzxFwROshCw==,type:str]
+    secretKey: ENC[AES256_GCM,data:PG3EC4noUzclRu1wp2L60QipWbM=,iv:HS4Qt/1hsDabs29EFLd8CtPNCSyyr29HaO6UQJcVqWs=,tag:wCsCfUb6ZE4kDoI7ZcnlSg==,type:str]
-    policy: ENC[AES256_GCM,data:aMHwISW3LUEN,iv:PAIOUKrIJuucOIAAnOradrce6LJl6JN5o+LdVQ4WY00=,tag:5Pb+Zc4ZrYnS0pNIzNWT9A==,type:str]
+    policy: ENC[AES256_GCM,data:XbA1JS1VxqEg,iv:NzcMS3KlDxnMnb4LvtL/Y2BtHPgYdcRPfvfpNei6qfA=,tag:0AjhrbZK/wbISfv9CNPLUQ==,type:str]
-  - accessKey: ENC[AES256_GCM,data:jbqbh2w=,iv:qdu2+wD53+tC41J6ulShkAhVw6H0J1pUfyqm7rYGVpY=,tag:nykfL4W+Tjsx3mHEOANwEg==,type:str]
+  - accessKey: ENC[AES256_GCM,data:av5X5Jc=,iv:0DjnV21W8kVnp8LXAqy2rcA6iK1g1ZogK66VfcQbfNc=,tag:2RkaEH9WRfPzlME9kmQFnw==,type:str]
-    secretKey: ENC[AES256_GCM,data:pCumHuYMRkL92Kh1BtkTFpTmalmQ2KqXbw==,iv:6lx1fXDrgCAEl15SmwhphUzUpfI2PE0Oc+180u4qyAw=,tag:4qdzozmME/Lda8Hi0DObYg==,type:str]
+    secretKey: ENC[AES256_GCM,data:mv9dRUN/a93HyqojgG0Ct0kDbXTBNcWfwQ==,iv:S0i/JxnOaFS/MQl5qP2WNRGtmaif+reHu3onj0Z238k=,tag:NvnR/8RaiukX0xV+8Gpibg==,type:str]
-    policy: ENC[AES256_GCM,data:u/S47qs=,iv:pGy2Fui/NNr335BX1BOOGGjuIHy03oE2lOvxhy5iDEU=,tag:ax+45aF/i1Rcm3NF2+n2vw==,type:str]
+    policy: ENC[AES256_GCM,data:YPoC00g=,iv:Ezfch2+T5MRgvRHhnl29OGq4nuepIv7AS8EqQCz6xo8=,tag:1QGJiHUxUnqFDBkpHw8Nfg==,type:str]
 oidc:
-  enabled: ENC[AES256_GCM,data:1QLtsg==,iv:byOxT1fJsjp3TYxa88yKrE5qNXvIy1ACr9go10e3BeU=,tag:vBkXanfsoV4CCEkSgJq7NQ==,type:bool]
+  enabled: ENC[AES256_GCM,data:OvHIQA==,iv:/Ekf/hcOQiMX6vUKMWDzqS1GvTXdE8LAOW6MdYnz8Sk=,tag:Z+2roA12aAYfETRlt0v6Lw==,type:bool]
-  configUrl: ENC[AES256_GCM,data:FtU0Oa2UcAW/24FocfIrQuSrMPn/hv6MP9GI8fLgIyv7Z7tDmnJh9VEHcpnWMBjUCv0cCv0p1jNOaAfJVrETZ+GX9Jm4J9KHOYIWrf355SahmrhrHBSJ79A=,iv:B49pmuz07qkOa+dmJqkOvEsQKXjA5rHr85TwDoM3rl4=,tag:e6wCBVKi4qBwIkSZaL1v0g==,type:str]
+  configUrl: ENC[AES256_GCM,data:ZI01kc1RO2Q2cPbQIgoN62lZrgjH5jM2mTHF2WLtLO+7k5/k18LTlBesVLCW4ocaInqK9dgQCAtfcesMP6IEFcJhL0YmOdvyiQPM528hC0rbCValPK4SsO0=,iv:HKEz0hIS34tNnsEBXtdEWMHpTqRc9vdM3/ERPxY0hAM=,tag:dAsPFQkqMwxd8E5fCOHjsg==,type:str]
-  clientId: ENC[AES256_GCM,data:LOCR2x4=,iv:4eGhAz1FpFbpvRByrhIGX4SXDA4cgnBai1jChc/VgcU=,tag:0lEKGDKM27zkrFruL2tn9w==,type:str]
+  clientId: ENC[AES256_GCM,data:ifdcXPs=,iv:eq83QALMzSZsz6XZzzeQOTrYikjEr4e4jMOq/iyS2eY=,tag:3i6hTRlCaeMcRuImV87RGA==,type:str]
-  clientSecret: ENC[AES256_GCM,data:x77rftDKcITMk+DVQhgPRpRGycw8x7BcHOaqR9GpM8vLP3G/Vx8io6c5OZBRqwBBsN9vshAej3vegvMmCHn8H2W6RYi6Qtk5faKFW0eKdRks4PwLbxrdVcjdVVQEFJXcwl0Zg9Qb1UH99OCl+HNgqCam1KKBc1IYVMMQkkY2VKo=,iv:zcWdl0bEJ9R8Lu3hwxDmRQ53biyh9s+0r/q0p7HDlX0=,tag:7BXwQlOIGnI1/L1RelLmXw==,type:str]
+  clientSecret: ENC[AES256_GCM,data:B2XczA/Ja1ic/6DbuihYeBPfyi9Pp4HAAAcSzTASyH3NTJjWstfhl0drTQ3LRYH2SAJmkwgA+OJaj7a8Xjuac5C7OTtMVq9m0BdYZiFe9G4Unwpb4Aj2M45bbgR7PjM9Q/nF7M5H4g72MaIz50Zity81nqnJhUanAdqOCqacZPU=,iv:SXXKkxABtvuUjETU3bcO1JdcXXNxUKs5xkdR6nl5Kgc=,tag:n4xxOPfcDMsrUM/TiQyM6Q==,type:str]
-  claimName: ENC[AES256_GCM,data:MlOxqMeB,iv:URG7IG2KV1Ef02K9VvDv6nGBHv4psoBea5B1Sv7WIb0=,tag:UO09riMtp2JjGOVl5QNiKQ==,type:str]
+  claimName: ENC[AES256_GCM,data:gGig+aKf,iv:lpuzl/Jph6wEY6qSFFs7DhMEzf4zDb2QQNGf84Mcdyw=,tag:RdE0fHO9S1XHPEEvL5zY6A==,type:str]
-  redirectUri: ENC[AES256_GCM,data:S0vQObd2gMBYlEj9O8xllLhS8dmf015av7rXcRJJJzqB3LI5hm6elIycvwBo+kE=,iv:OxVhUax+L37XFXYs8uXU90/hWqjxEHYtcrQyCgO0REQ=,tag:oRvPJ8O2z5IJ8cP97dkqAg==,type:str]
+  redirectUri: ENC[AES256_GCM,data:VSrfQsvV9QZnkAeknnGq6R0ZvQbJcCT3IZcoxXgV0+DctKKDigSpz5rjtpnU/mQ=,iv:IvLb95SFh6e+dvIbdaEmwjD6bQ1HyVDyMM773RYO3bE=,tag:g1AW/2hclPyjVD1UL5g1Ag==,type:str]
-  comment: ENC[AES256_GCM,data:FbAal8KkDHLQzsvOghDgpfohlK4=,iv:jzlt2wWf0Fn07fpFRlK8rABHq75orAoQla60RDciErg=,tag:CaTdTlsWBR8HgTyq8t0fNQ==,type:str]
+  comment: ENC[AES256_GCM,data:jZrlWictBAa+5p+FKbNNBcf6tzA=,iv:Z06T+TnlD3Y8e1HxtkYaqhpxuBPqTJEgPM559MnIg/o=,tag:B1G1W3za7IVI6e+P15FShw==,type:str]
   claimPrefix: ""
-  scopes: ENC[AES256_GCM,data:pyL2d9r0nhYSscXKcMwUphGEinYaT+lYrINI,iv:B5yKsgprBI9Gsdhzxmz2gg3cc6wvJFUIys6Z41RvSZk=,tag:mSPFtfR59h5Qj8VN27dBlQ==,type:str]
+  scopes: ENC[AES256_GCM,data:O5fv8qY45snUKYKBVePHzVeozzjoOEYKdIl4,iv:B2igPkg/Rs686KushjILiMgfr91KxE/TkN+YD3YYUUo=,tag:cQP48UEa/EYciijpMZIasg==,type:str]
 sops:
   kms: []
   gcp_kms: []
@@ -31,14 +31,14 @@ sops:
     - recipient: age1vzkv97n2p7gfkw8dyx8ctz2kumattz89th2jq47zyjyrarmnssysdkw9v8
       enc: |
         -----BEGIN AGE ENCRYPTED FILE-----
-        YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZdjN5SHRLVW10dDJQM2Jw
+        YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxSm5Pamo5NmJJYW5qVlZJ
-        d2NJdkt0OTdpanJieWFiczBGK3pudVVZUEVrCk5zR1pZSWc4bzBZOTVvYXNjMDVs
+        RzA5R0V3U0ZmUVlXa1ZUd0V6Y21hK1g0aHpNClRUeDZOaEw3eG1SeUQ5OFdUbVVF
-        eklLMWp0blNuV0JEamxMTi9ZSzdYaWMKLS0tIEd2M0tGTDdZSytOUlJNbVdWVFdF
+        K2ZUaGhxUlpOSkMrbHlwTXNpSjJtUXcKLS0tIHJnTi8xWVE2clJSSEhCblpMWXRl
-        ZmZRZ1UrNTNLYkZiWUtOMEN4VUZ1STAKnY672aGUgQzE7r89vK6bP/nSZu8E0LTZ
+        WEttYkkwZmdRTzNYY1RLbzl5VUx2ZzQKQqIbYt23ASWjCCvBH/ktF9LJt0qPHIaR
-        0dk1iBy2Bde8Bjs9bSuAVI+B5hABHmS2gnyw6etodWmzhTxpEADdKA==
+        Trh3IP67jfwe86brNn5ZoZE776OOr6q7I87eYImwEwFQezbrAJ6c5g==
         -----END AGE ENCRYPTED FILE-----
-  lastmodified: "2024-10-01T10:20:14Z"
+  lastmodified: "2024-10-10T04:12:28Z"
-  mac: ENC[AES256_GCM,data:GhJBNo4wtEExHw+XIUHuMaJv6oZTYCJ52+Hcyhf6DMp6t/AxUL2sWPLVfM8RZRV1SHUDDJbmOQAP0EM4Lpsu3n9OsQxQ3t6nd9oOs6FbYguJ/vwaHrOnC9GmvCqmuDjWfArGW/u/bIa1U5KRWYUheG7856IprClLtiGsHv/5bo0=,iv:rE8r0vDHSKDTA9kyQnDhqHyX6lc1HrUYWtPP4O/zOIE=,tag:3ofp8NT+vMA4wu4eQFToWQ==,type:str]
+  mac: ENC[AES256_GCM,data:r/bukTjjTxEmg5YHNE9eM+Ur97bcBTSEfTHYPsMsysN0Y8cnUy/QykaeQFiaO4aKnRatMK2aNBqoit4uuqLLAAsU8fU/4hIyH15LgEk1kx2cGQCFl/nwWlm6cs+EpDhDjFA9rJIggjguLbZxcUr+jJK+d75dbWDcE9/yBPMH438=,iv:Nj+6Ch5srcyH7xcybvc6MOfaS3IkLXqwZAU7WbrTcsk=,tag:akyFyT16cr0HoZLTJUqMZQ==,type:str]
   pgp: []
   unencrypted_suffix: _unencrypted
-  version: 3.9.0
+  version: 3.9.1

values/badhouseplants/values.grafana.yaml

@@ -26,6 +26,15 @@ datasources:
         url: http://prometheus-operated.observability.svc.cluster.local:9090
         access: proxy
         isDefault: true
+      - name: Loki
+        type: loki
+        url: http://loki-gateway.observability.svc.cluster.local:80
+        access: proxy
+        isDefault: false
+        jsonData:
+          httpHeaderName1: 'X-Scope-OrgID'
+        secureJsonData:
+          httpHeaderValue1: 'badhouseplants'
 grafana.ini:
   server:
     root_url: https://grafana.badhouseplants.net

values/badhouseplants/values.loki.yaml

@@ -1,84 +1,42 @@
----
 global:
   dnsService: "coredns"
-
+deploymentMode: SingleBinary
 loki:
-  auth_enabled: false
   commonConfig:
     replication_factor: 1
   storage:
     type: 'filesystem'
-  commonConfig:
-    replication_factor: 1
   schemaConfig:
     configs:
-      - from: 2024-04-01
+      - from: "2024-01-01"
         store: tsdb
-        object_store: s3
-        schema: v13
         index:
           prefix: loki_index_
           period: 24h
-  ingester:
+        object_store: filesystem # we're storing on filesystem so there's no real persistence here.
-    chunk_encoding: snappy
+        schema: v13
-  tracing:
+  memcached:
-    enabled: true
+    chunk_cache:
-  querier:
+      enabled: false
-    # Default is 4, if you have enough memory and CPU you can increase, reduce if OOMing
+      host: chunk-cache-memcached.loki.svc
-    max_concurrent: 2
+      service: memcached-client
-
+      batch_size: 256
-compactor:
+      parallelism: 10
-  retention_enabled: true
+    results_cache:
-limits_config:
+      enabled: false
-  retention_period: 14d
+      host: results-cache-memcached.loki.svc
-
+      service: memcached-client
-monitoring:
+      default_validity: 12h
-  selfMonitoring:
-    enabled: false
-  lokiCanary:
-    enabled: false
-
-#gateway:
-#  ingress:
-#    enabled: true
-#    hosts:
-#      - host: FIXME
-#        paths:
-#          - path: /
-#            pathType: Prefix
-
-deploymentMode: SingleBinary
 singleBinary:
-  persistence:
-    size: 5Gi
   replicas: 1
-  resources:
-    limits:
-      cpu: 1
-      memory: 1Gi
-    requests:
-      cpu: 0.5
-      memory: 512Mi
-  extraEnv:
-    # Keep a little bit lower than memory limits
-    - name: GOMEMLIMIT
-      value: 3750MiB
-
-chunksCache:
-  # default is 500MB, with limited memory keep this smaller
-  writebackSizeLimit: 10MB
-
-minio:
-  enabled: false
-
-# Zero out replica counts of other deployment modes
-backend:
-  replicas: 0
 read:
   replicas: 0
+backend:
+  replicas: 0
 write:
   replicas: 0
-
+chunksCache:
+  enabled: false
 ingester:
   replicas: 0
 querier:

values/badhouseplants/values.minio.yaml

@@ -111,6 +111,10 @@ buckets:
     policy: none
     purge: false
     versioning: false
+  - name: xray-public
+    policy: download
+    purge: false
+    versioning: false
 metrics:
   serviceMonitor:
     enabled: false

values/badhouseplants/values.promtail.yaml

@@ -1,11 +1,12 @@
----
 config:
   clients:
     #    - url: http://loki.monitoring-system:3100
     - url: http://loki-gateway/loki/api/v1/push
+      tenant_id: badhouseplants
+  # publish data to loki
   snippets:
     pipelineStages:
       - match:
           pipeline_name: "drop-all"
-          selector: '{namespace!~"mail-service|woodpecker|minecraft-application"}'
+          selector: '{namespace!~"public-xray"}'
           action: drop