build: Refactor the Minio sync logic and separate the helm chart release

.dockerignore

@@ -1,4 +1,17 @@
+.git/**/*
+.git
+
+node_modules/**/*
 node_modules
+
+static/**/*
 static
+
+scripts/**/*
 scripts
 
+chart/**/*
+chart
+
+kube/**/*
+kube

.drone.yml

@@ -1,5 +1,28 @@
 ---
 # ----------------------------------------------
+# -- Update the helm chart
+# ----------------------------------------------
+kind: pipeline
+type: kubernetes
+name: Publish the helm chart
+
+trigger:
+  event:
+    - push
+
+steps:
+  - name: Publish the Helm chart
+    image: alpine/helm
+    environment:
+      GITEA_TOKEN:
+        from_secret: GITEA_TOKEN
+    commands:
+      - helm plugin install https://github.com/chartmuseum/helm-push
+      - helm package chart -d chart-package
+      - helm repo add  --username allanger --password $GITEA_TOKEN badhouseplants-net https://git.badhouseplants.net/api/packages/badhouseplants/helm
+      - helm cm-push "./chart-package/$(ls chart-package)" badhouseplants-net
+---
+# ----------------------------------------------
 # -- Build the site and push it to the registry
 # ----------------------------------------------
 kind: pipeline
@@ -8,137 +31,166 @@ name: Build badhouseplants.net
 
 trigger:
   event:
-  - push
+    - push
 
 steps:
-- name: Publish the Helm chart
+  - name: Download submodules with themes
-  image: alpine/helm
+    image: alpine/git
-  environment:
+    commands:
-    GITEA_TOKEN:
+      - git submodule update --init --recursive
-      from_secret: GITEA_TOKEN
-  commands:
-    - helm plugin install https://github.com/chartmuseum/helm-push
-    - helm package chart -d chart-package
-    - helm repo add  --username allanger --password $GITEA_TOKEN badhouseplants-net https://git.badhouseplants.net/api/packages/badhouseplants/helm
-    - helm cm-push "./chart-package/$(ls chart-package)" badhouseplants-net
 
-- name: Init git submodules with themes
+  - name: Test a build
-  image: alpine/git
+    image: git.badhouseplants.net/badhouseplants/hugo-builder
-  commands:
+    commands:
-  - git submodule update --init --recursive
+      - hugo
 
-- name: Test a build
+  - name: Build and push the docker image
-  image: git.badhouseplants.net/badhouseplants/hugo-builder
+    image: plugins/docker
-  commands:
+    when:
-    - hugo
+      branch:
+        exclude:
+          - main
+    settings:
+      registry: git.badhouseplants.net
+      username: allanger
+      password:
+        from_secret: GITEA_TOKEN
+      repo: git.badhouseplants.net/${DRONE_REPO}
+      tags: ${DRONE_COMMIT_SHA}
 
-- name: Build and push the docker image 
+  - name: Build and push the docker image with latest
-  image: plugins/docker
+    image: plugins/docker
-  when: 
+    when:
-    branch:
+      branch:
-      exclude:
         - main
-  settings: 
+    settings:
-    registry: git.badhouseplants.net
+      registry: git.badhouseplants.net
-    username: allanger
+      username: allanger
-    password: 
+      password:
-      from_secret: GITEA_TOKEN
+        from_secret: GITEA_TOKEN
-    repo: git.badhouseplants.net/${DRONE_REPO}
+      repo: git.badhouseplants.net/${DRONE_REPO}
-    tags: ${DRONE_COMMIT_SHA}
+      tags:
+        - ${DRONE_COMMIT_SHA}
+        - latest
 
-- name: Build and push the docker image with latest 
+  - name: Download media files
-  image: plugins/docker
+    image: alpine/git
-  when: 
+    commands:
-    branch:
+      - git lfs pull
-      - main
-  settings: 
-    registry: git.badhouseplants.net
-    username: allanger
-    password: 
-      from_secret: GITEA_TOKEN
-    repo: git.badhouseplants.net/${DRONE_REPO}
-    tags: 
-      - ${DRONE_COMMIT_SHA}
-      - latest
 
-- name: Deploy a preview ApplicationSet
+  - name: Sync pictures from lfs to Minio
-  image: alpine/k8s:1.24.10
+    image: rclone/rclone:latest
-  when: 
+    when:
-    branch:
+      branch:
-      exclude:
+        exclude:
+          - main
+    environment:
+      RCLONE_CONFIG_CONTENT:
+        from_secret: RCLONE_CONFIG_CONTENT_PRIVATE
+      RCLONE_CONFIG: /tmp/rclone.conf
+    commands:
+      - echo "$RCLONE_CONFIG_CONTENT" > $RCLONE_CONFIG
+      - apk update
+      - apk add git make
+      - make upload_static
+
+  - name: Sync pictures to the main Minio bucket
+    image: rclone/rclone:latest
+    when:
+      branch:
         - main
-  environment:
+    environment:
-    KUBECONFIG_CONTENT:
+      RCLONE_CONFIG_CONTENT:
-      from_secret: KUBECONFIG_CONTENT
+        from_secret: RCLONE_CONFIG_CONTENT_PRIVATE
-  commands:
+      RCLONE_CONFIG: /tmp/rclone.conf
-    - mkdir $HOME/.kube
+    commands:
-    - echo $KUBECONFIG_CONTENT | base64 -d > $HOME/.kube/config
+      - echo "$RCLONE_CONFIG_CONTENT" > $RCLONE_CONFIG
-    - apk update --no-cache && apk add yq gettext openssl
+      - apk update
-    - export ARGO_APP_CHART_VERSION=`cat chart/Chart.yaml | yq '.version'`
+      - apk add git make
-    - export ARGO_APP_BRANCH=$DRONE_BRANCH
+      - make sync_static_with_main
-    - export ARGO_APP_HOSTNAME="${DRONE_BRANCH}-dev.badhouseplants.net"
-    - export ARGO_APP_IMAGE_TAG=$DRONE_COMMIT_SHA
-    - export ARGO_REMARK42_SECRET=$(openssl rand -hex 12)
-    - kubectl get -f ./kube/applicationset.yaml -o yaml  > /tmp/appset.yaml
-    - yq -i "del(.metadata.resourceVersion)"  /tmp/appset.yaml
-    - yq -i "del(.metadata.generation)"  /tmp/appset.yaml
-    - yq -i "del(.metadata.uid)"  /tmp/appset.yaml
-    - yq -i "del(.status)"  /tmp/appset.yaml
-    - yq -i "del(.spec.generators[].list.elements[] | select(.branch == \"$ARGO_APP_BRANCH\"))" /tmp/appset.yaml
-    - envsubst  < ./kube/template.yaml > /tmp/elements.yaml
-    - yq -i '.spec.generators[].list.elements += load("/tmp/elements.yaml")' /tmp/appset.yaml
-    - kubectl apply -f /tmp/appset.yaml
 
-- name: Deploy a main ApplicationSet
+  - name: Deploy a preview ApplicationSet
-  image: alpine/k8s:1.24.10
+    image: alpine/k8s:1.24.10
-  when: 
+    when:
-    branch:
+      branch:
-      - main
+        exclude:
-  environment:
+          - main
-    KUBECONFIG_CONTENT:
+    environment:
-      from_secret: KUBECONFIG_CONTENT
+      KUBECONFIG_CONTENT:
-    ARGO_GITHUB_OAUTH_KEY:
+        from_secret: KUBECONFIG_CONTENT
-      from_secret: GITHUB_OAUTH_KEY
+    commands:
-    ARGO_GOOGLE_OAUTH_KEY:
+      - mkdir $HOME/.kube
-      from_secret: GOOGLE_OAUTH_KEY
+      - echo $KUBECONFIG_CONTENT | base64 -d > $HOME/.kube/config
-  commands: 
+      - apk update --no-cache && apk add yq gettext openssl
-    - mkdir $HOME/.kube
+      - export ARGO_APP_CHART_VERSION=`cat chart/Chart.yaml | yq '.version'`
-    - echo $KUBECONFIG_CONTENT | base64 -d > $HOME/.kube/config
+      - export ARGO_APP_BRANCH=$DRONE_BRANCH
-    - apk update --no-cache && apk add yq gettext
+      - export ARGO_APP_HOSTNAME="${DRONE_BRANCH}-dev.badhouseplants.net"
-    - export ARGO_APP_CHART_VERSION=`cat chart/Chart.yaml | yq '.version'`
+      - export ARGO_APP_IMAGE_TAG=$DRONE_COMMIT_SHA
-    - export ARGO_APP_BRANCH=$DRONE_BRANCH
+      - export ARGO_REMARK42_SECRET=$(openssl rand -hex 12)
-    - export ARGO_APP_IMAGE_TAG=$DRONE_COMMIT_SHA
+      - kubectl get -f ./kube/applicationset.yaml -o yaml  > /tmp/appset.yaml
-    - kubectl get -f ./kube/applicationset.yaml -o yaml  > /tmp/appset.yaml
+      - yq -i "del(.metadata.resourceVersion)"  /tmp/appset.yaml
-    - yq -i "del(.metadata.resourceVersion)"  /tmp/appset.yaml
+      - yq -i "del(.metadata.generation)"  /tmp/appset.yaml
-    - yq -i "del(.metadata.generation)"  /tmp/appset.yaml
+      - yq -i "del(.metadata.uid)"  /tmp/appset.yaml
-    - yq -i "del(.metadata.uid)"  /tmp/appset.yaml
+      - yq -i "del(.status)"  /tmp/appset.yaml
-    - yq -i "del(.status)"  /tmp/appset.yaml
+      - yq -i "del(.spec.generators[].list.elements[] | select(.branch == \"$ARGO_APP_BRANCH\"))" /tmp/appset.yaml
-    - yq -i "del(.spec.generators[].list.elements[] | select(.branch == \"$ARGO_APP_BRANCH\"))" /tmp/appset.yaml
+      - envsubst  < ./kube/template.yaml > /tmp/elements.yaml
-    - yq -i "del(.spec.generators[].list.elements[] | select(.commit_sha == \"$ARGO_APP_IMAGE_TAG\"))" /tmp/appset.yaml
+      - yq -i '.spec.generators[].list.elements += load("/tmp/elements.yaml")' /tmp/appset.yaml
-    - envsubst  < ./kube/main.yaml > /tmp/elements.yaml
+      - kubectl apply -f /tmp/appset.yaml
-    - yq -i '.spec.generators[].list.elements += load("/tmp/elements.yaml")' /tmp/appset.yaml
-    - kubectl apply -f /tmp/appset.yaml
 
-- name: Sync application 
+  - name: Deploy a main ApplicationSet
-  image: argoproj/argocd
+    image: alpine/k8s:1.24.10
-  environment:
+    when:
-    ARGOCD_SERVER: 
+      branch:
-      from_secret: ARGOCD_SERVER
+        - main
-    ARGOCD_AUTH_TOKEN:
+    environment:
-      from_secret: ARGOCD_AUTH_TOKEN
+      KUBECONFIG_CONTENT:
-  commands:
+        from_secret: KUBECONFIG_CONTENT
-    - argocd app sync -l app=badhouseplants -l branch=$DRONE_BRANCH
+      ARGO_GITHUB_OAUTH_KEY:
-    - argocd app wait -l app=badhouseplants -l branch=$DRONE_BRANCH
+        from_secret: GITHUB_OAUTH_KEY
+      ARGO_GOOGLE_OAUTH_KEY:
+        from_secret: GOOGLE_OAUTH_KEY
+    commands: 
+      - mkdir $HOME/.kube
+      - echo $KUBECONFIG_CONTENT | base64 -d > $HOME/.kube/config
+      - apk update --no-cache && apk add yq gettext
+      - export ARGO_APP_CHART_VERSION=`cat chart/Chart.yaml | yq '.version'`
+      - export ARGO_APP_BRANCH=$DRONE_BRANCH
+      - export ARGO_APP_IMAGE_TAG=$DRONE_COMMIT_SHA
+      - kubectl get -f ./kube/applicationset.yaml -o yaml  > /tmp/appset.yaml
+      - yq -i "del(.metadata.resourceVersion)"  /tmp/appset.yaml
+      - yq -i "del(.metadata.generation)"  /tmp/appset.yaml
+      - yq -i "del(.metadata.uid)"  /tmp/appset.yaml
+      - yq -i "del(.status)"  /tmp/appset.yaml
+      - yq -i "del(.spec.generators[].list.elements[] | select(.branch == \"$ARGO_APP_BRANCH\"))" /tmp/appset.yaml
+      - yq -i "del(.spec.generators[].list.elements[] | select(.commit_sha == \"$ARGO_APP_IMAGE_TAG\"))" /tmp/appset.yaml
+      - envsubst  < ./kube/main.yaml > /tmp/elements.yaml
+      - yq -i '.spec.generators[].list.elements += load("/tmp/elements.yaml")' /tmp/appset.yaml
+      - kubectl apply -f /tmp/appset.yaml
 
-- name: Cleanup container registry
+  - name: Sync application
-  image: alpine
+    image: argoproj/argocd
-  environment:
+    environment:
-    GITEA_TOKEN:
+      ARGOCD_SERVER:
-      from_secret: GITEA_TOKEN
+        from_secret: ARGOCD_SERVER
-  commands:
+      ARGOCD_AUTH_TOKEN:
-    - apk update
+        from_secret: ARGOCD_AUTH_TOKEN
-    - apk add curl jq perl git
+    commands:
-    - ./scripts/cleanup.pl
+      - argocd app sync -l app=badhouseplants -l branch=$DRONE_BRANCH
+      - argocd app wait -l app=badhouseplants -l branch=$DRONE_BRANCH
+
+  - name: Cleanup everything
+    image: rclone/rclone:latest
+    environment:
+      RCLONE_CONFIG_CONTENT:
+        from_secret: RCLONE_CONFIG_CONTENT_PRIVATE
+      RCLONE_CONFIG: /tmp/rclone.conf
+      GITEA_TOKEN:
+        from_secret: GITEA_TOKEN
+    commands:
+      - echo "$RCLONE_CONFIG_CONTENT" > $RCLONE_CONFIG
+      - apk update
+      - apk add curl jq perl git
+      - ./scripts/cleanup.pl
 
 ---
 # ----------------------------------------------

.gitattributes

@@ -0,0 +1 @@
+static/* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,5 +1,5 @@
 node_modules
-static
+#static
 content/cv/index.pdf
 public
 resources

Dockerfile

@@ -1,4 +1,5 @@
 FROM git.badhouseplants.net/badhouseplants/hugo-builder
 WORKDIR /src
 COPY . /src
+RUN ls -la /src
 ENTRYPOINT ["hugo"]

Makefile

@@ -1,9 +1,15 @@
 upload_static:
-	rclone copy -P static badhouseplants-minio:/badhouseplants-static
+	rclone copy -P static badhouseplants-minio:/badhouseplants-net/$(shell git rev-parse HEAD)/
+
+sync_static_with_main:
+	rclone sync -P badhouseplants-minio:/badhouseplants-net/$(shell git rev-parse HEAD)/ badhouseplants-minio:/badhouseplants-net-main/
 
 get_static:
 	rclone copy -P badhouseplants-public:/badhouseplants-static static
 
+init_static: 
+	rclone copy -P badhouseplants-public:/badhouseplants-static badhouseplants-minio:/badhouseplants-net/$(shell git rev-parse HEAD)/
+
 submodules:
 	git submodule update --init --recursive
 

chart/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: badhouseplants-net
 description: A Helm chart for Kubernetes
 type: application
-version: 0.5.8
+version: 0.6.0
 appVersion: "4.20.0"
 dependencies:
   - name: remark42

chart/templates/deployment.yaml

@@ -35,7 +35,7 @@ spec:
             - sh
           args:
             - "-c"
-            - "rclone copy -P badhouseplants-public:/badhouseplants-static /static"
+            - {{ .Values.rclone.command }}
           volumeMounts:
             - name: rclone-config
               mountPath: "/tmp"

chart/values.yaml

@@ -17,6 +17,7 @@ nginx:
     tag: latest
 
 rclone:
+  command: "rclone copy -P badhouseplants-public:/badhouseplants-net-main /static"
   container:
     name: rclone
     resources: {}

content/posts/design-a-scalable-system/index.md

@@ -0,0 +1,26 @@
+---
+title: "Design a Scalable System"
+date: 2023-03-06T21:18:23+01:00
+draft: true
+ShowToc: true
+cover:
+    image: "cover.png"
+    caption: "Design a Scalable System"
+    relative: false
+    responsiveImages: false
+---
+
+Well, after I've posted my argo serie, I've found out that I couldn't really make myself understood. So now I want to talk more not about the way of implementation, but rather about the consequinces of different implementations. And maybe I will e able to finally make a point about why I don't like Terraform and why I think that ArgoCD is mostly mis-used by almost any SRE team I know. 
+
+But first I'll try to describe how I see myself as a part of a team, the team as a part of a bigger team, and all the teams across different companies as links in the bigger chain.
+
+This is how I used to see development teams before:
+
+![Chain](/posts/design-a-scalable-system/chain-1.png)
+
+The whole team is using something as a service, for example `AWS`, the whole team is working together and producing something that is passed to a customer. But apparently this approach is only applicable to small teams, and I think it's working just fine. But there is a problem. Teams tend to grow without an understanding that they are growing, hence they keep acting like they're small but in the same time they don't change the workflow, and brick-by-brick they are building something that eventually is something unscalable at first and later unmaintanble. 
+
+Example of an evolution like this:
+
+Let's take a regular Java(backend) + JS(frontend) web application
+

kube/template.yaml

@@ -34,3 +34,5 @@
         auth:
           anonymous: true
         secretKey: $ARGO_REMARK42_SECRET
+    rclone:
+      command: "rclone copy -P badhouseplants-public:/badhouseplants-net/$ARGO_APP_IMAGE_TAG /static"

scripts/cleanup.pl

@@ -30,18 +30,40 @@ my $commits = "git fetch && git log --format=format:%H --all";
 my @commits_out = `$commits`;
 chomp @commits_out;
 push @commits_out, 'latest';
+# --------------------------------------
+# -- Rclone variables
+# -------------------------------------
+my $dirs = "rclone lsf badhouseplants-minio:/badhouseplants-net";
+my @dirs_out = `$dirs`;
+chomp @dirs_out;
 # ---------------------------------------
 # -- Compare builds to commits 
 # -- And remove obsolete imgages from
 # --  registry
 # ---------------------------------------
+print "Cleaning up the container registry\n";
 foreach my $line (@builds_out)
 {
-    if ( ! grep( /^$line$/, @commits_out ) ) {
+  print "Checking if $line is in @commits_out\n\n";
-      my $cmd = "curl -X 'DELETE' -s \"$gitea_api/$line\"  -H 'accept: application/json' -u $gitea_user:$gitea_token || true";
+  if ( ! grep( /^$line$/, @commits_out ) ) {
-      print "Removing ${line}\n\n";
+    my $cmd = "curl -X 'DELETE' -s \"$gitea_api/$line\"  -H 'accept: application/json' -u $gitea_user:$gitea_token || true";
-      my $output = `$cmd`;
+    print "Removing ${line}\n\n";
-      print "$output \n";
+    my $output = `$cmd`;
-    } 
+    print "$output \n";
+  } 
 }
 
+print "Cleaning up Minio\n";
+foreach my $line (@dirs_out)
+{
+  print "Checking if $line is in @commits_out\n\n";
+  chop($line);
+  if ( ! grep( /^$line$/, @commits_out ) ) {
+    my $cmd = "rclone purge badhouseplants-minio:/badhouseplants-net/$line";
+    print "Removing ${line}\n\n";
+    my $output = `$cmd`;
+    print "$output \n";
+  }
+}
+
+

scripts/upload-media.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -e
+
+#=== 'prev-commit' solution by o_O Tync
+#commit_hash=$(git rev-parse --verify HEAD)
+commit=$(git log -1 --pretty="%H%n%ci") # hash \n date
+commit_date=$(echo "$commit" | head -2 | tail -1) # 2010-12-28 05:16:23 +0300
+commit_hash=$(echo "$commit" | head -1)
+echo "$(git rev-parse HEAD)" > /tmp/test
+echo "prev_commit='$commit_hash'\ndate='$commit_date'\nbranch='$branch'\n" > /tmp/test.txt

static/android-chrome-192x192.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5da923b06b69780de08c3450e99793a266c615e535a1053c718d6ae9eb8cb35
+size 20377

static/android-chrome-512x512.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:193f9db492edcc5a1202f85998675cc23dcf82a6666fcebe8a492c8c1dbe2673
+size 44441

static/argocd-vs-helmfile/diff-in-ui.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60603706ad018d3c73ed54eb475058424043ce9cf87de76f236fb9be6124d048
+size 465450

static/argocd-vs-helmfile/goldilocks-ui.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e42129f54056e12affbbb81ac2597c4f07fe08c3acc9c5879c0e1b40e8b879c8
+size 385843

static/vst-on-linux-1/glitchmachines.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d23698e02dd5412beeba2edbae158464cf723a65f82849b5ba5e79b034197f40
+size 263703

static/vst-on-linux-2/fire.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d7cc2ea5c4515e101f09ee82b3294af580b95ac6d7a30c4f80ed7746c897179
+size 382513

static/vst-on-linux-3/eventide-pendulate.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aef76ca940fb050a6d26e75f27b9ec5effb87af8c10b5d8ba30152f1f42151a2
+size 6126500

static/vst-on-linux-3/labs-example.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:566decff07913546a463f35bb9c0e47945b0366c9294b1afbc4bf89c04bd5eea
+size 6973433

static/vst-on-linux-3/mdrummer-example.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73b29483feaf48cb9baf01946f9da926ac25d8367110958de00a5a8fce1bc996
+size 7935948

static/vst-on-linux-3/mdrummer-lfo-carla.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af8e673858ef7bb279a89cf574ef29d56484f4258891e10716078072d001485b
+size 321828

static/vst-on-linux-3/native-access.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9247d5488be4c562b51b51db9eceb26afbfefd0c79d6802eaeeb39f5885bd4a7
+size 49413