WIP: Good progress on better sync

internal/providers/flux.go

@@ -14,6 +14,7 @@ import (
 	"git.badhouseplants.net/allanger/shoebill/pkg/repository"
 	release_v2beta1 "github.com/fluxcd/helm-controller/api/v2beta1"
 	helmrepo_v1beta2 "github.com/fluxcd/source-controller/api/v1beta2"
+	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/yaml"
@@ -34,168 +35,153 @@ func FluxProvider(path, sopsBin string, gh githelper.Githelper) Provider {
 }
 
 // TODO: This function is ugly as hell, I need to do something about it
-func (f *Flux) SyncState(diff diff.Diff) error {
+func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.RepositoriesDiffs) error {
 	entity := "repository"
 	srcDirPath := fmt.Sprintf("%s/src", f.path)
 	// It should containe either release or repository as a prefix, because it's how files are called
 	entiryFilePath := fmt.Sprintf("%s/%s-", srcDirPath, entity)
 
-	for _, repo := range diff.DeletedRepositories {
-		if err := os.Remove(entiryFilePath + repo.Name + ".yaml"); err != nil {
-			return err
+	for _, repository := range repoDiffs {
+		switch repository.Action {
+		case diff.ACTION_ADD:
+			manifest, err := GenerateRepository(repository.Wished)
+			if err != nil {
+				return err
+			}
+			file, err := os.Create(entiryFilePath + repository.Wished.Name + ".yaml")
+			if err != nil {
+				return err
+			}
+			if _, err := file.Write(manifest); err != nil {
+				return err
+			}
+			message := `chore(repository): Add a repo: %s
+		
+		A new repo added to the cluster:
+		  Name: %s 
+		  URL: %s
+		`
+			if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Wished.Name, repository.Wished.Name, repository.Wished.URL)); err != nil {
+				return err
+			}
+
+		case diff.ACTION_PRESERVE:
+		case diff.ACTION_UPDATE:
+			manifest, err := GenerateRepository(repository.Wished)
+			if err != nil {
+				return err
+			}
+			if err := os.WriteFile(entiryFilePath+repository.Wished.Name+".yaml", manifest, os.ModeExclusive); err != nil {
+				return err
+			}
+			message := `chore(repository): Update a repo: %s
+		
+		A repo has been updated:
+		  Name: %s 
+		  URL: %s
+		`
+			if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Wished.Name, repository.Wished.Name, repository.Wished.URL)); err != nil {
+				return err
+			}
+
+		case diff.ACTION_DELETE:
+			if err := os.Remove(entiryFilePath + repository.Current.Name + ".yaml"); err != nil {
+				return err
+			}
+
+			message := `chore(repository): Removed a repo: %s
+		A repo has been removed from the cluster:
+		  Name: %s 
+		  URL: %s
+		`
+			if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Current.Name, repository.Current.Name, repository.Current.URL)); err != nil {
+				return err
+			}
+
+		default:
+			return fmt.Errorf("unknown action is requests: %s", repository.Action)
 		}
 
-		message := `chore(repository): Removed a repo: %s
-	A repo has been removed from the cluster:
-	  Name: %s 
-	  URL: %s
-	`
-		if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repo.Name, repo.Name, repo.URL)); err != nil {
-			return err
-		}
-	}
-
-	for _, repo := range diff.UpdatedRepositories {
-		manifest, err := GenerateRepository(repo)
-		if err != nil {
-			return err
-		}
-		if err := os.WriteFile(entiryFilePath+repo.Name+".yaml", manifest, os.ModeExclusive); err != nil {
-			return err
-		}
-		message := `chore(repository): Update a repo: %s
-	
-	A repo has been updated:
-	  Name: %s 
-	  URL: %s
-	`
-		if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repo.Name, repo.Name, repo.URL)); err != nil {
-			return err
-		}
-	}
-	for _, repo := range diff.AddedRepositories {
-		manifest, err := GenerateRepository(repo)
-		if err != nil {
-			return err
-		}
-		file, err := os.Create(entiryFilePath + repo.Name + ".yaml")
-		if err != nil {
-			return err
-		}
-		if _, err := file.Write(manifest); err != nil {
-			return err
-		}
-		message := `chore(repository): Add a repo: %s
-	
-	A new repo added to the cluster:
-	  Name: %s 
-	  URL: %s
-	`
-		if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repo.Name, repo.Name, repo.URL)); err != nil {
-			return err
-		}
 	}
 
 	entity = "release"
 	entiryFilePath = fmt.Sprintf("%s/%s-", srcDirPath, entity)
+	for _, release := range releasesDiffs {
+		if err := SyncValues(release.Current, release.Wished, srcDirPath); err != nil {
+			return err
+		}
 
-	// Added are simply copying all the values
-	for _, release := range diff.AddedReleases {
-		if err := SyncValues(release, srcDirPath); err != nil {
+		if err := SyncSecrets(release.Current, release.Wished, f.path, f.sopsBin); err != nil {
 			return err
 		}
-		if err := SyncSecrets(release, srcDirPath, f.path, f.sopsBin); err != nil {
-			return err
-		}
-		manifest, err := GenerateRelease(release)
-		if err != nil {
-			return err
-		}
-		file, err := os.Create(entiryFilePath + release.Release + ".yaml")
-		if err != nil {
-			return err
 
-		}
-		if _, err := file.Write(manifest); err != nil {
-			return err
-		}
-		message := `chore(release): Add a new release: %s
+		switch release.Action {
+		case diff.ACTION_ADD:
+			manifest, err := GenerateRelease(release.Wished)
+			if err != nil {
+				return err
+			}
+			file, err := os.Create(entiryFilePath + release.Wished.Release + ".yaml")
+			if err != nil {
+				return err
+
+			}
+			if _, err := file.Write(manifest); err != nil {
+				return err
+			}
+			message := `chore(release): Add a new release: %s
+		
+		A new release is added to the cluster:
+		  Name: %s 
+		  Namespace: %s
+		  Version: %s
+		  Chart: %s/%s
+		`
+			if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Wished.Release, release.Wished.Release, release.Wished.Namespace, release.Wished.Version, release.Wished.Repository, release.Wished.Release)); err != nil {
+				return err
+			}
+
+		case diff.ACTION_UPDATE:
+			manifest, err := GenerateRelease(release.Wished)
+			if err != nil {
+				return err
+			}
+
+			if err := os.WriteFile(entiryFilePath+release.Wished.Release+".yaml", manifest, os.ModeExclusive); err != nil {
+				return err
+			}
+
+			message := `chore(release): Update a release: %s
 	
-	A new release is added to the cluster:
-	  Name: %s 
-	  Namespace: %s
-	  Version: %s
-	  Chart: %s/%s
-	`
-		if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Release, release.Release, release.Namespace, release.Version, release.Repository, release.Release)); err != nil {
-			return err
-		}
-	}
-
-	for _, release := range diff.UpdatedReleases {
-		SyncValues(release, srcDirPath)
-
-		if err := SyncSecrets(release, srcDirPath, f.path, f.sopsBin); err != nil {
-			return err
-		}
-
-		manifest, err := GenerateRelease(release)
-		if err != nil {
-			return err
-		}
-
-		if err := os.WriteFile(entiryFilePath+release.Release+".yaml", manifest, os.ModeExclusive); err != nil {
-			return err
-		}
-
-		message := `chore(release): Update a release: %s
-
-	A release has been updated:
-		Name: %s 
-		Namespace: %s
-		Version: %s
-		Chart: %s/%s
-	  `
-		if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Release, release.Release, release.Namespace, release.Version, release.Repository, release.Release)); err != nil {
-			return err
-		}
-	}
-
-	for _, release := range diff.DeletedReleases {
-		if err := os.Remove(entiryFilePath + release.Release + ".yaml"); err != nil {
-			return err
-		}
-		files, err := filepath.Glob(fmt.Sprintf("%s/values/%s*", srcDirPath, release.Release))
-		if err != nil {
-			return err
-		}
-		for _, f := range files {
-			if err := os.Remove(f); err != nil {
+		A release has been updated:
+			Name: %s 
+			Namespace: %s
+			Version: %s
+			Chart: %s/%s
+		  `
+			if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Wished.Release, release.Wished.Release, release.Wished.Namespace, release.Wished.Version, release.Wished.Repository, release.Wished.Release)); err != nil {
 				return err
 			}
-		}
 
-		files, err = filepath.Glob(fmt.Sprintf("%s/secrets/%s*", srcDirPath, release.Release))
-		if err != nil {
-			return err
-		}
-		for _, f := range files {
-			if err := os.Remove(f); err != nil {
+		case diff.ACTION_DELETE:
+			if err := os.Remove(entiryFilePath + release.Current.Release + ".yaml"); err != nil {
+				return err
+			}
+			message := `chore(release): Remove a release: %s
+	
+			A release has been removed from the cluster:
+			Name: %s 
+			Namespace: %s
+			Version: %s
+			Chart: %s/%s
+		  `
+			if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Current.Release, release.Current.Release, release.Current.Namespace, release.Current.Version, release.Current.Repository, release.Current.Release)); err != nil {
 				return err
 			}
-		}
-		message := `chore(release): Remove a release: %s
 
-		A release has been removed from the cluster:
-		Name: %s 
-		Namespace: %s
-		Version: %s
-		Chart: %s/%s
-	  `
-		if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Release, release.Release, release.Namespace, release.Version, release.Repository, release.Release)); err != nil {
-			return err
+		default:
+			return fmt.Errorf("unknown action is requests: %s", release.Action)
 		}
-
 	}
 
 	return nil
@@ -248,7 +234,7 @@ func GenerateRelease(release *release.Release) ([]byte, error) {
 				CRDs:            release_v2beta1.Create,
 				CreateNamespace: true,
 			},
-			TargetNamespace: "release-namespace",
+			TargetNamespace: release.Namespace,
 			ValuesFrom:      []release_v2beta1.ValuesReference{},
 		},
 	}
@@ -273,130 +259,155 @@ func GenerateRelease(release *release.Release) ([]byte, error) {
 	return yaml.Marshal(&fluxRelease)
 }
 
-func SyncValues(release *release.Release, path string) error {
-	for values := range release.DestValues {
-		
-	}
-	for _, valueFile := range release.Values {
-		// Prepare a dir for values
-		valuesPath := fmt.Sprintf("%s/%s", path, "values")
-		if err := os.Mkdir(valuesPath, os.ModePerm); err != nil {
-			return err
-		}
-
-		destFileName := fmt.Sprintf("%s/%s-%s", valuesPath, release.Release, filepath.Base(valueFile))
-		var dstValues *os.File
-		var srcValues *os.File
-		var err error
-		valueData, err := os.ReadFile(valueFile)
-
-		if err != nil {
-			return err
-		}
-		defer srcValues.Close()
-		if _, err = os.Stat(destFileName); err == nil {
-			dstValues, err = os.Open(destFileName)
-			if err != nil {
+func SyncValues(currentRelease, wishedRelease *release.Release, secDirPath string) error {
+	valuesDirPath := fmt.Sprintf("%s/values", secDirPath)
+	if currentRelease != nil {
+		for _, value := range currentRelease.DestValues {
+			valuesFilePath := fmt.Sprintf("%s/%s", valuesDirPath, value.DestPath)
+			logrus.Infof("trying to remove values file: %s", valuesFilePath)
+			if err := os.RemoveAll(valuesFilePath); err != nil {
 				return err
 			}
-			defer dstValues.Close()
-		} else if errors.Is(err, os.ErrNotExist) {
-			dstValues, err = os.Create(destFileName)
-			if err != nil {
-				return nil
-			}
-			defer dstValues.Close()
-		} else {
-			return err
-		}
-		if err := os.WriteFile(destFileName, valueData, os.ModeExclusive); err != nil {
-			return nil
-		}
-
-		_, err = io.Copy(dstValues, srcValues)
-		if err != nil {
-			return err
 		}
 	}
-	return nil
-}
-
-func SyncSecrets(release *release.Release, destPath, path, sopsBin string) error {
-	secretsPath := fmt.Sprintf("%s/%s", destPath, "secrets")
-	// Prepare a dir for secrets
-	if err := os.RemoveAll(secretsPath); err != nil {
-		return err
-	}
-	if err := os.Mkdir(secretsPath, os.ModePerm); err != nil {
-		return err
-	}
-
-	for srcPath, data := range release.UnencryptedSecrets {
-		destFileName := fmt.Sprintf("%s/%s-%s", secretsPath, release.Release, filepath.Base(srcPath))
-		var dstSecrets *os.File
-		var err error
-
-		if _, err = os.Stat(destFileName); err == nil {
-			dstSecrets, err = os.Open(destFileName)
-			if err != nil {
+	if wishedRelease != nil {
+		for _, value := range wishedRelease.DestValues {
+			// Prepare a dir for values
+			valuesPath := fmt.Sprintf("%s/%s", secDirPath, "values")
+			valuesFilePath := fmt.Sprintf("%s/%s", valuesDirPath, value.DestPath)
+			logrus.Infof("trying to create values file: %s", valuesFilePath)
+			if err := os.MkdirAll(valuesPath, os.ModePerm); err != nil {
 				return err
 			}
-			defer dstSecrets.Close()
-		} else if errors.Is(err, os.ErrNotExist) {
-			dstSecrets, err = os.Create(destFileName)
-			if err != nil {
-				return nil
+			var valuesFile *os.File
+			if _, err := os.Stat(valuesFilePath); err == nil {
+				valuesFile, err = os.Open(valuesFilePath)
+				if err != nil {
+					return err
+				}
+				defer valuesFile.Close()
+			} else if errors.Is(err, os.ErrNotExist) {
+				valuesFile, err = os.Create(valuesFilePath)
+				if err != nil {
+					return nil
+				}
+				defer valuesFile.Close()
+			} else {
+				return err
 			}
-			defer dstSecrets.Close()
-		} else {
-			return err
-		}
-		filename := fmt.Sprintf("%s-%s", release.Release, filepath.Base(srcPath))
-		k8sSecretObj := corev1.Secret{
-			TypeMeta: v1.TypeMeta{
-				Kind:       "Secret",
-				APIVersion: "v1",
-			},
-			ObjectMeta: v1.ObjectMeta{
-				Name:      filename,
-				Namespace: "flux-system",
-				Labels: map[string]string{
-					"shoebill-release": release.Release,
-					"shoebill-chart":   release.Chart,
+
+			k8sConfigMapObj := corev1.ConfigMap{
+				TypeMeta: v1.TypeMeta{
+					Kind:       "ConfigMap",
+					APIVersion: "v1",
 				},
-			},
-			Data: map[string][]byte{
-				filename: data,
-			},
-		}
-		secretFile, err := yaml.Marshal(k8sSecretObj)
-		if err != nil {
-			return err
-		}
+				ObjectMeta: v1.ObjectMeta{
+					Name:      value.DestPath,
+					Namespace: "flux-system",
+					Labels: map[string]string{
+						"shoebill-release": wishedRelease.Release,
+						"shoebill-chart":   wishedRelease.Chart,
+					},
+				},
+				Data: map[string]string{
+					value.DestPath: string(value.Data),
+				},
+			}
 
-		if err := os.WriteFile(destFileName, secretFile, os.ModeExclusive); err != nil {
-			return nil
-		}
+			valuesFileData, err := yaml.Marshal(k8sConfigMapObj)
+			if err != nil {
+				return err
+			}
 
-		if err != nil {
-			return err
-		}
-		// I have to use the sops binary here, because they do not provide a go package that can be used for encryption :(
-		sopsConfPath := fmt.Sprintf("%s/.sops.yaml", path)
-		cmd := exec.Command(sopsBin, "--encrypt", "--in-place", "--config", sopsConfPath, destFileName)
-		stderr, err := cmd.StderrPipe()
-		if err != nil {
-			return err
-		}
-
-		if err := cmd.Start(); err != nil {
-			return err
-		}
-
-		errMsg, _ := io.ReadAll(stderr)
-		if err := cmd.Wait(); err != nil {
-			err := fmt.Errorf("%s - %s", err, errMsg)
-			return err
+			if err := os.WriteFile(valuesFilePath, valuesFileData, os.ModeAppend); err != nil {
+				return nil
+			}
+		}
+	}
+	return nil
+}
+
+func SyncSecrets(currentRelease, wishedRelease *release.Release, workdirPath, sopsBin string) error {
+	secretsDirPath := fmt.Sprintf("%s/src/secrets", workdirPath)
+	if currentRelease != nil {
+		for _, secrets := range currentRelease.DestSecrets {
+			secretsFilePath := fmt.Sprintf("%s/%s", secretsDirPath, secrets.DestPath)
+			logrus.Infof("trying to remove secrets file: %s", secretsFilePath)
+			if err := os.RemoveAll(secretsFilePath); err != nil {
+				return err
+			}
+		}
+	}
+	if wishedRelease != nil {
+		for _, secrets := range wishedRelease.DestSecrets {
+			// Prepare a dir for secrets
+			secretsPath := fmt.Sprintf("%s/%s", workdirPath, "secrets")
+			secretsFilePath := fmt.Sprintf("%s/%s", secretsDirPath, secrets.DestPath)
+			logrus.Infof("trying to create secrets file: %s", secretsFilePath)
+			if err := os.MkdirAll(secretsPath, os.ModePerm); err != nil {
+				return err
+			}
+			var secretsFile *os.File
+			if _, err := os.Stat(secretsFilePath); err == nil {
+				secretsFile, err = os.Open(secretsFilePath)
+				if err != nil {
+					return err
+				}
+				defer secretsFile.Close()
+			} else if errors.Is(err, os.ErrNotExist) {
+				secretsFile, err = os.Create(secretsFilePath)
+				if err != nil {
+					return nil
+				}
+				defer secretsFile.Close()
+			} else {
+				return err
+			}
+
+			k8sSecretObj := corev1.Secret{
+				TypeMeta: v1.TypeMeta{
+					Kind:       "Secret",
+					APIVersion: "v1",
+				},
+				ObjectMeta: v1.ObjectMeta{
+					Name:      secrets.DestPath,
+					Namespace: "flux-system",
+					Labels: map[string]string{
+						"shoebill-release": wishedRelease.Release,
+						"shoebill-chart":   wishedRelease.Chart,
+					},
+				},
+				Data: map[string][]byte{
+					secrets.DestPath: secrets.Data,
+				},
+			}
+
+			secretsFileData, err := yaml.Marshal(k8sSecretObj)
+			if err != nil {
+				return err
+			}
+
+			if err := os.WriteFile(secretsFilePath, secretsFileData, os.ModeAppend); err != nil {
+				return nil
+			}
+
+			// I have to use the sops binary here, because they do not provide a go package that can be used for encryption :(
+			sopsConfPath := fmt.Sprintf("%s/.sops.yaml", workdirPath)
+			cmd := exec.Command(sopsBin, "--encrypt", "--in-place", "--config", sopsConfPath, secretsFilePath)
+			stderr, err := cmd.StderrPipe()
+			if err != nil {
+				return err
+			}
+
+			if err := cmd.Start(); err != nil {
+				return err
+			}
+
+			errMsg, _ := io.ReadAll(stderr)
+			if err := cmd.Wait(); err != nil {
+				err := fmt.Errorf("%s - %s", err, errMsg)
+				return err
+			}
 		}
 	}
 	return nil

internal/providers/types.go

@@ -8,7 +8,7 @@ import (
 )
 
 type Provider interface {
-	SyncState(diff diff.Diff) error
+	SyncState(diff.ReleasesDiffs, diff.RepositoriesDiffs) error
 }
 
 func NewProvider(provider, path, sopsBin string, gh githelper.Githelper) (Provider, error) {