Initial logic is implemented

internal/build/build.go

@@ -0,0 +1,11 @@
+package build
+
+/*
+* Build time variables, if you don't want to use Makefile for building,
+*  you still might have a look at to see how they should be configured
+ */
+var (
+	Version    = "dev-0.0.0"
+	CommitHash = "n/a"
+	BuildTime  = "n/a"
+)

internal/controller/controller.go

@@ -0,0 +1,178 @@
+package controller
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"git.badhouseplants.net/allanger/shoebill/internal/providers"
+	"git.badhouseplants.net/allanger/shoebill/internal/utils/diff"
+	"git.badhouseplants.net/allanger/shoebill/internal/utils/githelper"
+	"git.badhouseplants.net/allanger/shoebill/internal/utils/helmhelper"
+	"git.badhouseplants.net/allanger/shoebill/internal/utils/kustomize"
+	"git.badhouseplants.net/allanger/shoebill/internal/utils/sopshelper"
+	"git.badhouseplants.net/allanger/shoebill/internal/utils/workdir"
+	"git.badhouseplants.net/allanger/shoebill/pkg/config"
+	"git.badhouseplants.net/allanger/shoebill/pkg/lockfile"
+	"git.badhouseplants.net/allanger/shoebill/pkg/release"
+)
+
+func ReadTheConfig(path string) (*config.Config, error) {
+	conf, err := config.NewConfigFromFile(path)
+	if err != nil {
+		return nil, err
+	}
+	return conf, nil
+}
+
+// func cloneSnapshoot(gh githelper.Githelper, snapshotDir, snapshotBranch string) error {
+	// if err := gh.CloneRepo(snapshotBranch, snapshotUrl, false); err != nil {
+		// return err
+	// }
+	// return nil
+// }
+
+func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool, diffArg string) error {
+	// Start by creating a directory where everything should be happening
+	configPath := filepath.Dir(conf.ConfigPath)
+	workdirPath, err := workdir.CreateWorkdir(definedWorkdirPath)
+	if err != nil {
+		return err
+	}
+
+	// Prepare helm repositories
+	for _, repository := range conf.Repositories {
+		if err := repository.KindFromUrl(); err != nil {
+			return err
+		}
+	}
+
+	// Configure a git client
+	gh := githelper.NewGit(sshKeyPath)
+	// if len(diffArg) > 0 {
+		// snapshotDir := fmt.Sprint("%s/.snapshot", workdirPath)
+		// cloneSnapshoot(gh, snapshotDir, diffArg)
+	// }
+
+	// The main logic starts here
+	for _, cluster := range conf.Clusters {
+		// Create a dir for the cluster git repo
+		clusterWorkdirPath := fmt.Sprintf("%s/%s", workdirPath, cluster.Name)
+
+		// Init a gitops provider (Currently onle flux is supported)
+		provider, err := providers.NewProvider(cluster.Provider, clusterWorkdirPath, conf.SopsBin, gh)
+		if err != nil {
+			return err
+		}
+
+		if err := cluster.CloneRepo(gh, clusterWorkdirPath, dry); err != nil {
+			return err
+		}
+
+		if err := cluster.BootstrapRepo(gh, clusterWorkdirPath, dry); err != nil {
+			return err
+		}
+
+		// Read the lockfile generated by the shoebill
+		lockfileData, err := lockfile.NewFromFile(clusterWorkdirPath)
+		if err != nil {
+			return err
+		}
+
+		currentRepositories, err := lockfileData.ReposFromLockfile()
+		if err != nil {
+			return err
+		}
+
+		if err := conf.Releases.PopulateRepositories(conf.Repositories); err != nil {
+			return err
+		}
+
+		// Init the helm client
+		hh := helmhelper.NewHelm()
+
+		// Init the sops client
+		sops := sopshelper.NewSops()
+
+		for _, release := range conf.Releases {
+			err := release.VersionHandler(workdirPath, hh)
+			if err != nil {
+				return err
+			}
+
+			if len(diffArg) > 0 {
+				_, err := hh.PullChart(workdirPath, release.ToHelmReleaseData())
+				if err != nil {
+					return err
+				}
+				if err := hh.RenderChart(workdirPath, release.ToHelmReleaseData()); err != nil {
+					return err
+				}
+
+			}
+
+			if err := release.ValuesHandler(configPath); err != nil {
+				return err
+			}
+
+			if err := release.SecretsHandler(configPath, sops); err != nil {
+				return err
+			}
+		}
+
+		releaseObj := release.FindReleaseByNames(cluster.Releases, conf.Releases)
+		cluster.PopulateReleases(releaseObj)
+
+		releasesCurrent, err := release.ReleasesFromLockfile(lockfileData, conf.Repositories)
+		if err != nil {
+			return err
+		}
+
+		if len(diffArg) > 0 {
+			for _, releaseCurrent := range releasesCurrent {
+				hh.PullChart(workdirPath, releaseCurrent.ToHelmReleaseData())
+			}
+		}
+
+		// Compare releases from the lockfile to ones from the current cluster config
+		diffReleases, err := diff.DiffReleases(releasesCurrent, cluster.ReleasesObj)
+		if err != nil {
+			return err
+		}
+
+		lockfile, diffRepos, err := diffReleases.Resolve(currentRepositories, clusterWorkdirPath)
+		if err != nil {
+			return err
+		}
+
+		hashesPerRelease, err := provider.SyncState(diffReleases, diffRepos)
+		if err != nil {
+			return err
+		}
+
+		if err := kustomize.Generate(clusterWorkdirPath, gh); err != nil {
+			return err
+		}
+
+		lockfile.AddHashes(hashesPerRelease)
+
+		if err := lockfile.WriteToFile(clusterWorkdirPath); err != nil {
+			return err
+		}
+		if _, err := gh.AddAllAndCommit(clusterWorkdirPath, "Update the lockfile"); err != nil {
+			return err
+		}
+		if !dry {
+			if err := gh.Push(clusterWorkdirPath); err != nil {
+				return err
+			}
+		}
+
+	}
+	if !dry {
+		if err := workdir.RemoveWorkdir(workdirPath); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

internal/providers/flux.go

@@ -0,0 +1,426 @@
+package providers
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"git.badhouseplants.net/allanger/shoebill/internal/utils/diff"
+	"git.badhouseplants.net/allanger/shoebill/internal/utils/githelper"
+	"git.badhouseplants.net/allanger/shoebill/pkg/lockfile"
+	"git.badhouseplants.net/allanger/shoebill/pkg/release"
+	"git.badhouseplants.net/allanger/shoebill/pkg/repository"
+	release_v2beta1 "github.com/fluxcd/helm-controller/api/v2beta1"
+	helmrepo_v1beta2 "github.com/fluxcd/source-controller/api/v1beta2"
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
+)
+
+type Flux struct {
+	path    string
+	sopsBin string
+	gh      githelper.Githelper
+}
+
+func FluxProvider(path, sopsBin string, gh githelper.Githelper) Provider {
+	return &Flux{
+		path:    path,
+		sopsBin: sopsBin,
+		gh:      gh,
+	}
+}
+
+// TODO: This function is ugly as hell, I need to do something about it
+func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.RepositoriesDiffs) (lockfile.HashesPerReleases, error) {
+	entity := "repository"
+	srcDirPath := fmt.Sprintf("%s/src", f.path)
+	// It should containe either release or repository as a prefix, because it's how files are called
+	entiryFilePath := fmt.Sprintf("%s/%s-", srcDirPath, entity)
+
+	for _, repository := range repoDiffs {
+		switch repository.Action {
+		case diff.ACTION_ADD:
+			manifest, err := GenerateRepository(repository.Wished)
+			if err != nil {
+				return nil, err
+			}
+			file, err := os.Create(entiryFilePath + repository.Wished.Name + ".yaml")
+			if err != nil {
+				return nil, err
+			}
+			if _, err := file.Write(manifest); err != nil {
+				return nil, err
+			}
+			message := `chore(repository): Add a repo: %s
+		
+		A new repo added to the cluster:
+		  Name: %s 
+		  URL: %s
+		`
+			if _, err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Wished.Name, repository.Wished.Name, repository.Wished.URL)); err != nil {
+				return nil, err
+			}
+
+		case diff.ACTION_PRESERVE:
+		case diff.ACTION_UPDATE:
+			manifest, err := GenerateRepository(repository.Wished)
+			if err != nil {
+				return nil, err
+			}
+			if err := os.WriteFile(entiryFilePath+repository.Wished.Name+".yaml", manifest, os.ModeExclusive); err != nil {
+				return nil, err
+			}
+			message := `chore(repository): Update a repo: %s
+		
+		A repo has been updated:
+		  Name: %s 
+		  URL: %s
+		`
+			if _, err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Wished.Name, repository.Wished.Name, repository.Wished.URL)); err != nil {
+				return nil, err
+			}
+
+		case diff.ACTION_DELETE:
+			if err := os.Remove(entiryFilePath + repository.Current.Name + ".yaml"); err != nil {
+				return nil, err
+			}
+
+			message := `chore(repository): Removed a repo: %s
+		A repo has been removed from the cluster:
+		  Name: %s 
+		  URL: %s
+		`
+			if _, err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Current.Name, repository.Current.Name, repository.Current.URL)); err != nil {
+				return nil, err
+			}
+
+		default:
+			return nil, fmt.Errorf("unknown action is requests: %s", repository.Action)
+		}
+
+	}
+	hashesPerReleases := lockfile.HashesPerReleases{}
+	entity = "release"
+	entiryFilePath = fmt.Sprintf("%s/%s-", srcDirPath, entity)
+	for _, release := range releasesDiffs {
+		var hash string
+		var err error
+		if err := SyncValues(release.Current, release.Wished, srcDirPath); err != nil {
+			return nil, err
+		}
+
+		if err := SyncSecrets(release.Current, release.Wished, f.path, f.sopsBin); err != nil {
+			return nil, err
+		}
+
+		switch release.Action {
+		case diff.ACTION_ADD:
+			manifest, err := GenerateRelease(release.Wished)
+			if err != nil {
+				return nil, err
+			}
+			file, err := os.Create(entiryFilePath + release.Wished.Release + ".yaml")
+			if err != nil {
+				return nil, err
+
+			}
+			if _, err := file.Write(manifest); err != nil {
+				return nil, err
+			}
+			message := `chore(release): Add a new release: %s
+		
+		A new release is added to the cluster:
+		  Name: %s 
+		  Namespace: %s
+		  Version: %s
+		  Chart: %s/%s
+		`
+			hash, err = f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Wished.Release, release.Wished.Release, release.Wished.Namespace, release.Wished.Version, release.Wished.Repository, release.Wished.Release))
+			if err != nil {
+				return nil, err
+			}
+
+		case diff.ACTION_UPDATE:
+			manifest, err := GenerateRelease(release.Wished)
+			if err != nil {
+				return nil, err
+			}
+
+			if err := os.WriteFile(entiryFilePath+release.Wished.Release+".yaml", manifest, os.ModeExclusive); err != nil {
+				return nil, err
+			}
+
+			message := `chore(release): Update a release: %s
+	
+		A release has been updated:
+			Name: %s 
+			Namespace: %s
+			Version: %s
+			Chart: %s/%s
+		  `
+			hash, err = f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Wished.Release, release.Wished.Release, release.Wished.Namespace, release.Wished.Version, release.Wished.Repository, release.Wished.Release))
+			if err != nil {
+				return nil, err
+			}
+
+		case diff.ACTION_DELETE:
+			if err := os.Remove(entiryFilePath + release.Current.Release + ".yaml"); err != nil {
+				return nil, err
+			}
+			message := `chore(release): Remove a release: %s
+	
+			A release has been removed from the cluster:
+			Name: %s 
+			Namespace: %s
+			Version: %s
+			Chart: %s/%s
+		  `
+			hash, err = f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Current.Release, release.Current.Release, release.Current.Namespace, release.Current.Version, release.Current.Repository, release.Current.Release))
+			if err != nil {
+				return nil, err
+			}
+
+		default:
+			return nil, fmt.Errorf("unknown action is requests: %s", release.Action)
+		}
+		hashPerRelease := &lockfile.HashPerRelease{
+			Release:    release.Wished.Release,
+			Namespace:  release.Wished.Namespace,
+			CommitHash: hash,
+		}
+		hashesPerReleases = append(hashesPerReleases, hashPerRelease)
+
+	}
+
+	return hashesPerReleases, nil
+
+}
+
+func GenerateRepository(repo *repository.Repository) ([]byte, error) {
+	fluxRepo := &helmrepo_v1beta2.HelmRepository{
+		TypeMeta: v1.TypeMeta{
+			Kind:       helmrepo_v1beta2.HelmRepositoryKind,
+			APIVersion: helmrepo_v1beta2.GroupVersion.String(),
+		},
+		ObjectMeta: v1.ObjectMeta{
+			Name:      repo.Name,
+			Namespace: "flux-system",
+		},
+		Spec: helmrepo_v1beta2.HelmRepositorySpec{
+			URL:  repo.URL,
+			Type: repo.Kind,
+		},
+	}
+	return yaml.Marshal(&fluxRepo)
+}
+
+// GenerateRelease and put
+func GenerateRelease(release *release.Release) ([]byte, error) {
+	fluxRelease := &release_v2beta1.HelmRelease{
+		TypeMeta: v1.TypeMeta{
+			Kind:       release_v2beta1.HelmReleaseKind,
+			APIVersion: release_v2beta1.GroupVersion.String(),
+		},
+		ObjectMeta: v1.ObjectMeta{
+			Name:      release.Release,
+			Namespace: "flux-system",
+		},
+		Spec: release_v2beta1.HelmReleaseSpec{
+			Chart: release_v2beta1.HelmChartTemplate{
+				Spec: release_v2beta1.HelmChartTemplateSpec{
+					Chart:   release.Chart,
+					Version: release.Version,
+					SourceRef: release_v2beta1.CrossNamespaceObjectReference{
+						Kind:      helmrepo_v1beta2.HelmRepositoryKind,
+						Name:      release.RepositoryObj.Name,
+						Namespace: "flux-system",
+					},
+				},
+			},
+			ReleaseName: release.Release,
+			Install: &release_v2beta1.Install{
+				CRDs:            release_v2beta1.Create,
+				CreateNamespace: true,
+			},
+			TargetNamespace: release.Namespace,
+			ValuesFrom:      []release_v2beta1.ValuesReference{},
+		},
+	}
+	for _, v := range release.Values {
+		filename := fmt.Sprintf("%s-%s", release.Release, filepath.Base(v))
+		fluxRelease.Spec.ValuesFrom = append(fluxRelease.Spec.ValuesFrom, release_v2beta1.ValuesReference{
+			Kind:      "ConfigMap",
+			Name:      filename,
+			ValuesKey: filename,
+		})
+	}
+
+	for _, v := range release.Secrets {
+		filename := fmt.Sprintf("%s-%s", release.Release, filepath.Base(v))
+		fluxRelease.Spec.ValuesFrom = append(fluxRelease.Spec.ValuesFrom, release_v2beta1.ValuesReference{
+			Kind:      "Secret",
+			Name:      filename,
+			ValuesKey: filename,
+		})
+	}
+
+	return yaml.Marshal(&fluxRelease)
+}
+
+func SyncValues(currentRelease, wishedRelease *release.Release, secDirPath string) error {
+	valuesDirPath := fmt.Sprintf("%s/values", secDirPath)
+	if currentRelease != nil {
+		for _, value := range currentRelease.DestValues {
+			valuesFilePath := fmt.Sprintf("%s/%s", valuesDirPath, value.DestPath)
+			logrus.Infof("trying to remove values file: %s", valuesFilePath)
+			if err := os.RemoveAll(valuesFilePath); err != nil {
+				return err
+			}
+		}
+	}
+	if wishedRelease != nil {
+		for _, value := range wishedRelease.DestValues {
+			// Prepare a dir for values
+			valuesPath := fmt.Sprintf("%s/%s", secDirPath, "values")
+			valuesFilePath := fmt.Sprintf("%s/%s", valuesDirPath, value.DestPath)
+			logrus.Infof("trying to create values file: %s", valuesFilePath)
+			if err := os.MkdirAll(valuesPath, os.ModePerm); err != nil {
+				return err
+			}
+			var valuesFile *os.File
+			if _, err := os.Stat(valuesFilePath); err == nil {
+				valuesFile, err = os.Open(valuesFilePath)
+				if err != nil {
+					return err
+				}
+				defer valuesFile.Close()
+			} else if errors.Is(err, os.ErrNotExist) {
+				valuesFile, err = os.Create(valuesFilePath)
+				if err != nil {
+					return nil
+				}
+				defer valuesFile.Close()
+			} else {
+				return err
+			}
+
+			k8sConfigMapObj := corev1.ConfigMap{
+				TypeMeta: v1.TypeMeta{
+					Kind:       "ConfigMap",
+					APIVersion: "v1",
+				},
+				ObjectMeta: v1.ObjectMeta{
+					Name:      value.DestPath,
+					Namespace: "flux-system",
+					Labels: map[string]string{
+						"shoebill-release": wishedRelease.Release,
+						"shoebill-chart":   wishedRelease.Chart,
+					},
+				},
+				Data: map[string]string{
+					value.DestPath: string(value.Data),
+				},
+			}
+
+			valuesFileData, err := yaml.Marshal(k8sConfigMapObj)
+			if err != nil {
+				return err
+			}
+
+			if err := os.WriteFile(valuesFilePath, valuesFileData, os.ModeAppend); err != nil {
+				return nil
+			}
+		}
+	}
+	return nil
+}
+
+func SyncSecrets(currentRelease, wishedRelease *release.Release, workdirPath, sopsBin string) error {
+	secretsDirPath := fmt.Sprintf("%s/src/secrets", workdirPath)
+	if currentRelease != nil {
+		for _, secrets := range currentRelease.DestSecrets {
+			secretsFilePath := fmt.Sprintf("%s/%s", secretsDirPath, secrets.DestPath)
+			logrus.Infof("trying to remove secrets file: %s", secretsFilePath)
+			if err := os.RemoveAll(secretsFilePath); err != nil {
+				return err
+			}
+		}
+	}
+	if wishedRelease != nil {
+		for _, secrets := range wishedRelease.DestSecrets {
+			// Prepare a dir for secrets
+			secretsFilePath := fmt.Sprintf("%s/%s", secretsDirPath, secrets.DestPath)
+			logrus.Infof("trying to create secrets file: %s", secretsFilePath)
+			if err := os.MkdirAll(secretsDirPath, os.ModePerm); err != nil {
+				return err
+			}
+			var secretsFile *os.File
+			if _, err := os.Stat(secretsFilePath); err == nil {
+				secretsFile, err = os.Open(secretsFilePath)
+				if err != nil {
+					return err
+				}
+				defer secretsFile.Close()
+			} else if errors.Is(err, os.ErrNotExist) {
+				secretsFile, err = os.Create(secretsFilePath)
+				if err != nil {
+					return nil
+				}
+				defer secretsFile.Close()
+			} else {
+				return err
+			}
+
+			k8sSecretObj := corev1.Secret{
+				TypeMeta: v1.TypeMeta{
+					Kind:       "Secret",
+					APIVersion: "v1",
+				},
+				ObjectMeta: v1.ObjectMeta{
+					Name:      secrets.DestPath,
+					Namespace: "flux-system",
+					Labels: map[string]string{
+						"shoebill-release": wishedRelease.Release,
+						"shoebill-chart":   wishedRelease.Chart,
+					},
+				},
+				Data: map[string][]byte{
+					secrets.DestPath: secrets.Data,
+				},
+			}
+
+			secretsFileData, err := yaml.Marshal(k8sSecretObj)
+			if err != nil {
+				return err
+			}
+
+			if err := os.WriteFile(secretsFilePath, secretsFileData, os.ModeAppend); err != nil {
+				return nil
+			}
+
+			// I have to use the sops binary here, because they do not provide a go package that can be used for encryption :(
+			sopsConfPath := fmt.Sprintf("%s/.sops.yaml", workdirPath)
+			cmd := exec.Command(sopsBin, "--encrypt", "--in-place", "--config", sopsConfPath, secretsFilePath)
+			stderr, err := cmd.StderrPipe()
+			if err != nil {
+				return err
+			}
+
+			if err := cmd.Start(); err != nil {
+				return err
+			}
+
+			errMsg, _ := io.ReadAll(stderr)
+			if err := cmd.Wait(); err != nil {
+				err := fmt.Errorf("%s - %s", err, errMsg)
+				return err
+			}
+		}
+	}
+	return nil
+}

internal/providers/types.go

@@ -0,0 +1,22 @@
+package providers
+
+import (
+	"fmt"
+
+	"git.badhouseplants.net/allanger/shoebill/internal/utils/diff"
+	"git.badhouseplants.net/allanger/shoebill/internal/utils/githelper"
+	"git.badhouseplants.net/allanger/shoebill/pkg/lockfile"
+)
+
+type Provider interface {
+	SyncState(diff.ReleasesDiffs, diff.RepositoriesDiffs) (lockfile.HashesPerReleases, error)
+}
+
+func NewProvider(provider, path, sopsBin string, gh githelper.Githelper) (Provider, error) {
+	switch provider {
+	case "flux":
+		return FluxProvider(path, sopsBin, gh), nil
+	default:
+		return nil, fmt.Errorf("provider is not supported: %s", provider)
+	}
+}

internal/utils/diff/diff.go

@@ -0,0 +1,179 @@
+package diff
+
+import (
+	"fmt"
+	"reflect"
+
+	"git.badhouseplants.net/allanger/shoebill/pkg/lockfile"
+	"git.badhouseplants.net/allanger/shoebill/pkg/release"
+	"git.badhouseplants.net/allanger/shoebill/pkg/repository"
+	"github.com/sirupsen/logrus"
+)
+
+type ReleasesDiff struct {
+	Action  string
+	Current *release.Release
+	Wished  *release.Release
+}
+
+type ReleasesDiffs []*ReleasesDiff
+
+type RepositoriesDiff struct {
+	Action  string
+	Current *repository.Repository
+	Wished  *repository.Repository
+}
+
+type RepositoriesDiffs []*RepositoriesDiff
+
+const (
+	ACTION_PRESERVE = "preserve"
+	ACTION_ADD      = "add"
+	ACTION_UPDATE   = "update"
+	ACTION_DELETE   = "delete"
+)
+
+// TODO(@allanger): Naming should be better
+func DiffReleases(currentReleases, wishedReleases release.Releases) (ReleasesDiffs, error) {
+	newDiff := ReleasesDiffs{}
+
+	for _, currentRelease := range currentReleases {
+		found := false
+		for _, wishedRelease := range wishedReleases {
+			if currentRelease.Release == wishedRelease.Release {
+				found = true
+				if reflect.DeepEqual(currentRelease, wishedRelease) {
+					newDiff = append(newDiff, &ReleasesDiff{
+						Action:  ACTION_PRESERVE,
+						Current: currentRelease,
+						Wished:  wishedRelease,
+					})
+
+					continue
+				} else {
+					if err := wishedRelease.RepositoryObj.KindFromUrl(); err != nil {
+						return nil, err
+					}
+					newDiff = append(newDiff, &ReleasesDiff{
+						Action:  ACTION_UPDATE,
+						Current: currentRelease,
+						Wished:  wishedRelease,
+					})
+				}
+			}
+		}
+
+		if !found {
+			newDiff = append(newDiff, &ReleasesDiff{
+				Action:  ACTION_DELETE,
+				Current: currentRelease,
+				Wished:  nil,
+			})
+
+		}
+	}
+
+	for _, wishedRelease := range wishedReleases {
+		found := false
+		for _, rSrc := range currentReleases {
+			if rSrc.Release == wishedRelease.Release {
+				found = true
+				continue
+			}
+		}
+		if !found {
+			if err := wishedRelease.RepositoryObj.KindFromUrl(); err != nil {
+				return nil, err
+			}
+			newDiff = append(newDiff, &ReleasesDiff{
+				Action:  ACTION_ADD,
+				Current: nil,
+				Wished:  wishedRelease,
+			})
+
+		}
+	}
+	return newDiff, nil
+}
+
+func (diff ReleasesDiffs) Resolve(currentRepositories repository.Repositories, path string) (lockfile.LockFile, RepositoriesDiffs, error) {
+	lockfile := lockfile.LockFile{}
+	wishedRepos := repository.Repositories{}
+	repoDiffs := RepositoriesDiffs{}
+
+	for _, diff := range diff {
+		switch diff.Action {
+		case ACTION_ADD:
+			logrus.Infof("adding %s", diff.Wished.Release)
+			lockfile = append(lockfile, diff.Wished.LockEntry())
+			wishedRepos = append(wishedRepos, diff.Wished.RepositoryObj)
+		case ACTION_PRESERVE:
+			logrus.Infof("preserving %s", diff.Wished.Release)
+			lockfile = append(lockfile, diff.Wished.LockEntry())
+			wishedRepos = append(wishedRepos, diff.Wished.RepositoryObj)
+		case ACTION_UPDATE:
+			logrus.Infof("updating %s", diff.Wished.Release)
+			lockfile = append(lockfile, diff.Wished.LockEntry())
+			wishedRepos = append(wishedRepos, diff.Wished.RepositoryObj)
+		case ACTION_DELETE:
+			logrus.Infof("removing %s", diff.Current.Release)
+		default:
+			return nil, nil, fmt.Errorf("unknown action is requests: %s", diff.Action)
+		}
+	}
+	// Repo Wished is the list of all repos that are required by the current setup
+
+	// Existing repos are all the repos in the lockfile
+	for _, currentRepo := range currentRepositories {
+		found := false
+		i := 0
+		for _, wishedRepo := range wishedRepos {
+			// If there is the same repo in the wished repos and in the lockfile
+			// We need either to udpate, or preserve. If it can't be found, just remove
+			// from the reposWished slice
+			if wishedRepo.Name == currentRepo.Name {
+				// If !found, should be gone from the repo
+				found = true
+				if err := wishedRepo.ValidateURL(); err != nil {
+					return nil, nil, err
+				}
+				if err := wishedRepo.KindFromUrl(); err != nil {
+					return nil, nil, err
+				}
+				if !reflect.DeepEqual(wishedRepos, currentRepo) {
+					repoDiffs = append(repoDiffs, &RepositoriesDiff{
+						Action:  ACTION_UPDATE,
+						Current: currentRepo,
+						Wished:  wishedRepo,
+					})
+				} else {
+					repoDiffs = append(repoDiffs, &RepositoriesDiff{
+						Action:  ACTION_PRESERVE,
+						Current: currentRepo,
+						Wished:  wishedRepo,
+					})
+				}
+			} else {
+				wishedRepos[i] = wishedRepo
+				i++
+			}
+		}
+		wishedRepos = wishedRepos[:i]
+		if !found {
+			repoDiffs = append(repoDiffs, &RepositoriesDiff{
+				Action:  ACTION_DELETE,
+				Current: currentRepo,
+				Wished:  nil,
+			})
+		}
+	}
+	for _, addedRepo := range wishedRepos {
+		repoDiffs = append(repoDiffs, &RepositoriesDiff{
+			Action:  ACTION_ADD,
+			Current: nil,
+			Wished:  addedRepo,
+		})
+	}
+
+	return lockfile, repoDiffs, nil
+}

internal/utils/githelper/git.go

@@ -0,0 +1,115 @@
+package githelper
+
+import (
+	"errors"
+	"os"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
+	"github.com/go-git/go-git/v5/plumbing"
+	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
+	"github.com/sirupsen/logrus"
+)
+
+type Git struct {
+	SshPrivateKeyPath string
+}
+
+func NewGit(sshPrivateKeyPath string) Githelper {
+	return &Git{
+		SshPrivateKeyPath: sshPrivateKeyPath,
+	}
+}
+
+func (g *Git) CloneRepo(workdir, gitURL string, dry bool) error {
+	// TODO(@allanger): Support ssh keys with passwords
+	publicKeys, err := ssh.NewPublicKeysFromFile("git", g.SshPrivateKeyPath, "")
+	if err != nil {
+		return err
+	}
+	_, err = git.PlainClone(workdir, false, &git.CloneOptions{URL: gitURL, Auth: publicKeys})
+	if err != nil && !errors.Is(err, git.ErrEmptyUrls) {
+		logrus.Info("the repo seems to be empty, I'll try to bootsrap it")
+		// Initialize the repo
+		err := os.Mkdir(workdir, 0077700)
+		if err != nil {
+			return err
+		}
+		r, err := git.PlainInit(workdir, false)
+		if err != nil {
+			return err
+		}
+
+		logrus.Infof("adding an origin remote: %s", gitURL)
+		if _, err := r.CreateRemote(&config.RemoteConfig{Name: "origin", URLs: []string{gitURL}}); err != nil {
+			return err
+		}
+
+		logrus.Info("getting the worktree")
+		w, err := r.Worktree()
+		if err != nil {
+			return err
+		}
+
+		if err := r.Storer.SetReference(plumbing.NewHashReference(plumbing.Main, plumbing.ZeroHash)); err != nil {
+			return err
+		}
+
+		logrus.Info("creating an empty 'Init Commit'")
+		if _, err := w.Commit("Init Commit", &git.CommitOptions{
+			AllowEmptyCommits: true,
+		}); err != nil {
+			return err
+		}
+		if !dry {
+			if err := r.Push(&git.PushOptions{RemoteName: "origin"}); err != nil {
+				return err
+			}
+		}
+	} else if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (g *Git) AddAllAndCommit(workdir, message string) (string, error) {
+	r, err := git.PlainOpen(workdir)
+	if err != nil {
+		return "", err
+	}
+
+	w, err := r.Worktree()
+	if err != nil {
+		return "", err
+	}
+	if _, err := w.Add("."); err != nil {
+		return "", err
+	}
+
+	sha, err := w.Commit(message, &git.CommitOptions{})
+	if err != nil {
+		return "", err
+	}
+
+	return sha.String(), nil
+}
+
+func (g *Git) Push(workdir string) error {
+	r, err := git.PlainOpen(workdir)
+	if err != nil {
+		return err
+	}
+	publicKeys, err := ssh.NewPublicKeysFromFile("git", g.SshPrivateKeyPath, "")
+	if err != nil {
+		return err
+	}
+
+	if err := r.Push(&git.PushOptions{
+		RemoteName: "origin",
+		Auth:       publicKeys,
+	}); err != nil {
+		return err
+	}
+	return nil
+}

internal/utils/githelper/mock.go

@@ -0,0 +1,18 @@
+package githelper
+
+type Mock struct{}
+
+func NewGitMock() Githelper {
+	return &Mock{}
+}
+
+func (m *Mock) CloneRepo(workdir, gitURL string, dry bool) error {
+	return nil
+}
+
+func (g *Mock) AddAllAndCommit(workdir, message string) (string, error) {
+	return "HASH", nil
+}
+func (g *Mock) Push(workdir string) error {
+	return nil
+}

internal/utils/githelper/types.go

@@ -0,0 +1,7 @@
+package githelper
+
+type Githelper interface {
+	CloneRepo(workdir, gitURL string, dry bool) error
+	AddAllAndCommit(workdir, message string) (string, error)
+	Push(workdir string) error
+}

internal/utils/helmhelper/helm.go

@@ -0,0 +1,180 @@
+package helmhelper
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/cli"
+	"helm.sh/helm/v3/pkg/engine"
+	"helm.sh/helm/v3/pkg/getter"
+	"helm.sh/helm/v3/pkg/registry"
+	"helm.sh/helm/v3/pkg/repo"
+)
+
+type Helm struct{}
+
+func NewHelm() Helmhelper {
+	return &Helm{}
+}
+
+func getDownloadDirPath(workdirPath string) string {
+	return fmt.Sprintf("%s/.charts", workdirPath)
+}
+
+func getChartDirPath(downloadDirPath string, release *ReleaseData) string {
+	return fmt.Sprintf("%s/%s-%s-%s", downloadDirPath, release.RepositoryName, release.Chart, release.Version)
+
+}
+
+func (h *Helm) PullChart(workdirPath string, release *ReleaseData) (path string, err error) {
+	downloadDirPath := getDownloadDirPath(workdirPath)
+	if err := os.MkdirAll(downloadDirPath, 0777); err != nil {
+		return "", err
+	}
+
+	config := new(action.Configuration)
+	cl := cli.New()
+	chartDir := getChartDirPath(downloadDirPath, release)
+	_, err = os.Stat(chartDir)
+
+	if err != nil && !os.IsNotExist(err) {
+		return "", nil
+	} else if os.IsNotExist(err) {
+		if err := os.Mkdir(chartDir, 0777); err != nil {
+			return "", err
+		}
+		registry, err := registry.NewClient()
+		if err != nil {
+			return "", err
+		}
+
+		var path string
+		// Download the chart to the workdir
+		if release.RepositoryKind != "oci" {
+			r, err := repo.NewChartRepository(&repo.Entry{
+				Name: release.RepositoryName,
+				URL:  release.RepositoryURL,
+			}, getter.All(cl))
+			if err != nil {
+				return "", err
+			}
+			path = r.Config.Name
+
+		} else {
+			path = release.RepositoryURL
+		}
+
+		client := action.NewPullWithOpts(action.WithConfig(config))
+		client.SetRegistryClient(registry)
+		client.DestDir = chartDir
+		client.Settings = cl
+
+		chartRemote := fmt.Sprintf("%s/%s", path, release.Chart)
+		logrus.Infof("trying to pull: %s", chartRemote)
+		if _, err = client.Run(chartRemote); err != nil {
+			return "", err
+		}
+	}
+	path, err = getChartPathFromDir(chartDir)
+	if err != nil {
+		return "", err
+	}
+
+	return path, nil
+}
+
+func (h *Helm) FindLatestVersion(workdirPath string, release *ReleaseData) (version string, err error) {
+	downloadDirPath := getDownloadDirPath(workdirPath)
+	if err := os.MkdirAll(downloadDirPath, 0777); err != nil {
+		return "", err
+	}
+
+	config := new(action.Configuration)
+	cl := cli.New()
+	chartDir := getChartDirPath(downloadDirPath, release)
+	chartPath, err := h.PullChart(workdirPath, release)
+	if err != nil {
+		return "", err
+	}
+
+	showAction := action.NewShowWithConfig(action.ShowChart, config)
+
+	res, err := showAction.LocateChart(fmt.Sprintf("%s/%s", chartDir, chartPath), cl)
+	if err != nil {
+		return "", err
+	}
+	res, err = showAction.Run(res)
+	if err != nil {
+		return "", nil
+	}
+
+	chartData, err := chartFromString(res)
+	if err != nil {
+		return "", err
+	}
+	logrus.Infof("the latest version of %s is %s", release.Chart, chartData.Version)
+	versionedChartDir := getChartDirPath(downloadDirPath, release)
+	os.Rename(chartDir, versionedChartDir)
+	return chartData.Version, err
+}
+
+func (h *Helm) RenderChart(workdirPath string, release *ReleaseData) error {
+	downloadDirPath := getDownloadDirPath(workdirPath)
+	chartDirPath := getChartDirPath(downloadDirPath, release)
+	chartPath, err := getChartPathFromDir(chartDirPath)
+	if err != nil {
+		return err
+	}
+	logrus.Info(fmt.Sprintf("%s/%s", chartDirPath, chartPath))
+	chartObj, err := loader.Load(fmt.Sprintf("%s/%s", chartDirPath, chartPath))
+	if err != nil {
+		return err
+	}
+	values := chartutil.Values{}
+	values["Values"] = chartObj.Values
+	values["Release"] = map[string]string{
+		"Name":      release.Name,
+		"Namespace": release.Namespace,
+	}
+	values["Capabilities"] = map[string]map[string]string{
+		"KubeVersion": {
+			"Version":    "v1.27.9",
+			"GitVersion": "v1.27.9",
+		},
+	}
+	files, err := engine.Engine{Strict: false}.Render(chartObj, values)
+	if err != nil {
+		return err
+	}
+	logrus.Info(files)
+	for file, data := range files {
+		logrus.Infof("%s - %s", file, data)
+	}
+	logrus.Info("I'm here")
+	return nil
+}
+
+func getChartPathFromDir(downloadDir string) (file string, err error) {
+	files, err := os.ReadDir(downloadDir)
+	if err != nil {
+		return "", err
+	} else if len(files) == 0 {
+		return "", fmt.Errorf("expected to have one file, got zero in a dir %s", downloadDir)
+	} else if len(files) > 1 {
+		return "", fmt.Errorf("expected to have only one file in a dir %s", downloadDir)
+	}
+	return files[0].Name(), nil
+}
+
+func chartFromString(info string) (*ReleaseData, error) {
+	releaseData := new(ReleaseData)
+	if err := yaml.Unmarshal([]byte(info), &releaseData); err != nil {
+		return nil, err
+	}
+	return releaseData, nil
+}

internal/utils/helmhelper/mock.go

@@ -0,0 +1,24 @@
+package helmhelper
+
+const (
+	MOCK_LATEST_VERSION = "v1.12.1"
+	MOCK_CHART_PATH     = ".charts/repo-release-latest/release-latest.gz"
+)
+
+type Mock struct{}
+
+func NewHelmMock() Helmhelper {
+	return &Mock{}
+}
+
+func (h *Mock) FindLatestVersion(workdir string, release *ReleaseData) (version string, err error) {
+	return MOCK_LATEST_VERSION, nil
+}
+
+func (h *Mock) PullChart(workdirPath string, release *ReleaseData) (path string, err error) {
+	return MOCK_CHART_PATH, nil
+}
+
+func (h *Mock) RenderChart(workdirPath string, release *ReleaseData) error {
+	return nil
+}

internal/utils/helmhelper/types.go

@@ -0,0 +1,18 @@
+package helmhelper
+
+type Helmhelper interface {
+	FindLatestVersion(workdirPath string, release *ReleaseData) (string, error)
+	PullChart(workdirPath string, release *ReleaseData) (string, error)
+	RenderChart(workdirPath string, release *ReleaseData) error
+}
+
+type ReleaseData struct {
+	Name           string
+	Chart          string
+	Namespace      string
+	Version        string
+	RepositoryName string
+	RepositoryURL  string
+	RepositoryKind string
+	ValuesData     string
+}

internal/utils/kustomize/kustomize.go

@@ -0,0 +1,179 @@
+package kustomize
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"html/template"
+	"os"
+	"path/filepath"
+
+	"git.badhouseplants.net/allanger/shoebill/internal/utils/githelper"
+	"github.com/sirupsen/logrus"
+	kustomize_types "sigs.k8s.io/kustomize/api/types"
+	"sigs.k8s.io/yaml"
+)
+
+type Kusmtomize struct {
+	Files      []string
+	ConfigMaps []string
+	Secrets    []string
+}
+
+func (k *Kusmtomize) PopulateResources(path string) error {
+	// Main sources
+	files, err := os.ReadDir(fmt.Sprintf("%s/src", path))
+	if err != nil {
+		return err
+	}
+	for _, file := range files {
+		if file.Name() != ".gitkeep" && !file.IsDir() {
+			k.Files = append(k.Files, fmt.Sprintf("src/%s", file.Name()))
+		}
+	}
+	// Values
+	files, err = os.ReadDir(fmt.Sprintf("%s/src/values", path))
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		k.ConfigMaps = append(k.ConfigMaps, fmt.Sprintf("src/values/%s", file.Name()))
+	}
+
+	// Secrets
+	files, err = os.ReadDir(fmt.Sprintf("%s/src/secrets", path))
+	if err != nil {
+		return err
+	}
+
+	for _, file := range files {
+		k.Secrets = append(k.Secrets, fmt.Sprintf("src/secrets/%s", file.Name()))
+	}
+
+	return nil
+}
+
+func (k *Kusmtomize) SecGeneratorCreate(path string) error {
+	logrus.Info("preparing the secret generator file")
+	genFileTmpl := `---
+apiVersion: viaduct.ai/v1
+kind: ksops
+metadata:
+  name: shoebill-secret-gen
+files:
+{{- range $val := . }}
+  - {{ $val }}
+{{- end }}
+`
+
+	destFileName := fmt.Sprintf("%s/sec-generator.yaml", path)
+	t := template.Must(template.New("tmpl").Parse(genFileTmpl))
+	var genFileData bytes.Buffer
+	t.Execute(&genFileData, k.Secrets)
+	var genFile *os.File
+	if _, err := os.Stat(destFileName); err == nil {
+		genFile, err := os.Open(destFileName)
+		if err != nil {
+			return err
+		}
+		defer genFile.Close()
+	} else if errors.Is(err, os.ErrNotExist) {
+		genFile, err = os.Create(destFileName)
+		if err != nil {
+			return nil
+		}
+		defer genFile.Close()
+	} else {
+		return err
+	}
+	if err := os.WriteFile(destFileName, genFileData.Bytes(), os.ModeExclusive); err != nil {
+		return nil
+	}
+
+	return nil
+}
+
+func (k *Kusmtomize) CmGeneratorFromFiles() []kustomize_types.ConfigMapArgs {
+	cmGens := []kustomize_types.ConfigMapArgs{}
+	for _, cm := range k.ConfigMaps {
+		cmName := filepath.Base(cm)
+		cmGen := &kustomize_types.ConfigMapArgs{
+			GeneratorArgs: kustomize_types.GeneratorArgs{
+				Namespace: "flux-system",
+				Name:      cmName,
+				KvPairSources: kustomize_types.KvPairSources{
+					FileSources: []string{cm},
+				},
+			},
+		}
+		cmGens = append(cmGens, *cmGen)
+	}
+
+	return cmGens
+}
+
+func Generate(path string, gh githelper.Githelper) error {
+	kustomize := &Kusmtomize{}
+	if err := kustomize.PopulateResources(path); err != nil {
+		return err
+	}
+
+	kustomization := kustomize_types.Kustomization{
+		TypeMeta: kustomize_types.TypeMeta{
+			Kind:       kustomize_types.KustomizationKind,
+			APIVersion: kustomize_types.KustomizationVersion,
+		},
+		MetaData: &kustomize_types.ObjectMeta{
+			Name:      "helm-root",
+			Namespace: "flux-system",
+		},
+		Resources: append(kustomize.Files, kustomize.ConfigMaps...),
+		GeneratorOptions: &kustomize_types.GeneratorOptions{
+			DisableNameSuffixHash: true,
+		},
+	}
+
+	if len(kustomize.Secrets) > 0 {
+		kustomization.Generators = []string{"sec-generator.yaml"}
+		if err := kustomize.SecGeneratorCreate(path); err != nil {
+			return err
+		}
+	} else {
+		if err := os.RemoveAll(fmt.Sprintf("%s/sec-generator.yaml", path)); err != nil {
+			return err
+		}
+	}
+
+	manifest, err := yaml.Marshal(kustomization)
+	if err != nil {
+		return err
+	}
+	dstFilePath := path + "/kustomization.yaml"
+	var dstFile *os.File
+	if _, err = os.Stat(dstFilePath); err == nil {
+		dstFile, err = os.Open(dstFilePath)
+		if err != nil {
+			return err
+		}
+		defer dstFile.Close()
+	} else if errors.Is(err, os.ErrNotExist) {
+		dstFile, err = os.Create(dstFilePath)
+		if err != nil {
+			return nil
+		}
+		defer dstFile.Close()
+	} else {
+		return err
+	}
+
+	if err := os.WriteFile(dstFilePath, manifest, os.ModeExclusive); err != nil {
+		return nil
+	}
+
+	if _, err := gh.AddAllAndCommit(path, "Update the root kustomization"); err != nil {
+		return err
+	}
+
+	return nil
+}

internal/utils/sopshelper/mock.go

@@ -0,0 +1,11 @@
+package sopshelper
+
+type SopsMock struct{}
+
+func NewSopsMock() SopsHelper {
+	return &SopsMock{}
+}
+
+func (sops *SopsMock) Decrypt(filepath string) ([]byte, error) {
+	return nil, nil
+}

internal/utils/sopshelper/sops.go

@@ -0,0 +1,27 @@
+package sopshelper
+
+import (
+	// "go.mozilla.org/sops/v3/decrypt"
+	"os"
+
+	"github.com/getsops/sops/v3/decrypt"
+	"github.com/sirupsen/logrus"
+)
+
+type Sops struct{}
+
+func NewSops() SopsHelper {
+	return &Sops{}
+}
+func (sops Sops) Decrypt(filepath string) ([]byte, error) {
+	logrus.Infof("trying to decrypt: %s", filepath)
+	encFile, err := os.ReadFile(filepath)
+	if err != nil {
+		return nil, err
+	}
+	res, err := decrypt.Data(encFile, "yaml")
+	if err != nil {
+		return nil, err
+	}
+	return res, nil
+}

internal/utils/sopshelper/types.go

@@ -0,0 +1,5 @@
+package sopshelper
+
+type SopsHelper interface {
+	Decrypt(filepath string) ([]byte, error)
+}

internal/utils/workdir/workdir.go

@@ -0,0 +1,26 @@
+package workdir
+
+import "os"
+
+func CreateWorkdir(path string) (workdir string, err error) {
+	if len(path) > 0 {
+		// Create a dir using the path
+		if err := os.Mkdir(path, 0777); err != nil {
+			return path, err
+		}
+		// TODO(@allanger): I've got a feeling that it doesn't have to look that bad
+		workdir = path
+	} else {
+		// Create a temporary dir
+		workdir, err = os.MkdirTemp("", "shoebill")
+		if err != nil {
+			return workdir, err
+		}
+
+	}
+	return workdir, nil
+}
+
+func RemoveWorkdir(path string) (err error) {
+	return os.RemoveAll(path)
+}