From c67a9c84e4152ffc1af17d8cddd32c69f4e1aec6 Mon Sep 17 00:00:00 2001 From: Nikolai Rodionov Date: Thu, 12 Oct 2023 13:23:50 +0200 Subject: [PATCH] WIP: Some refactoring --- internal/controller/controller.go | 77 ++++++++++++++++++------------- internal/providers/flux.go | 43 +++++++++-------- internal/utils/diff/diff.go | 9 +++- internal/utils/workdir/workdir.go | 2 - pkg/lockfile/lockfile.go | 43 +++++++++-------- pkg/release/release.go | 28 +++++++++-- pkg/repository/repository.go | 4 ++ 7 files changed, 127 insertions(+), 79 deletions(-) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index a6c49a4..858ea4a 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -24,45 +24,50 @@ func ReadTheConfig(path string) (*config.Config, error) { return conf, nil } -func Reconcile(workdirPath, sshKeyPath string, conf *config.Config, dry bool) error { - dir, err := workdir.CreateWorkdir(workdirPath) +func Reconcile(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool) error { + // Start by creating a directory where everything should be happening + configPath := filepath.Dir(conf.ConfigPath) + workdirPath, err := workdir.CreateWorkdir(definedWorkdirPath) if err != nil { return err } - // Prepare repositories + + // Prepare helm repositories for _, repository := range conf.Repositories { - if err := repository.ValidateURL(); err != nil { - return err - } if err := repository.KindFromUrl(); err != nil { return err } } + // Configure a git client gh := githelper.NewGit(sshKeyPath) + // The main logic starts here for _, cluster := range conf.Clusters { - fullPath := fmt.Sprintf("%s/%s", dir, cluster.Name) - provider, err := providers.NewProvider(cluster.Provider, fullPath, conf.SopsBin, gh) + // Create a dir for the cluster git repo + clusterWorkdirPath := fmt.Sprintf("%s/%s", workdirPath, cluster.Name) + + // Init a gitops provider (Currently onle flux is supported) + provider, err := providers.NewProvider(cluster.Provider, clusterWorkdirPath, conf.SopsBin, gh) if err != nil { return err } - if err := cluster.CloneRepo(gh, fullPath, dry); err != nil { + if err := cluster.CloneRepo(gh, clusterWorkdirPath, dry); err != nil { return err } - err = cluster.BootstrapRepo(gh, fullPath, dry) + if err := cluster.BootstrapRepo(gh, clusterWorkdirPath, dry); err != nil { + return err + } + + // Read the lockfile generated by the shoebill + lockfileData, err := lockfile.NewFromFile(clusterWorkdirPath) if err != nil { return err } - lockfileData, err := lockfile.NewFromFile(fullPath) - if err != nil { - return err - } - - reposExisting, err := lockfileData.ReposFromLockfile() + currentRepositories, err := lockfileData.ReposFromLockfile() if err != nil { return err } @@ -71,61 +76,69 @@ func Reconcile(workdirPath, sshKeyPath string, conf *config.Config, dry bool) er return err } + // Init the helm client hh := helmhelper.NewHelm() + // Init the sops client sops := sopshelper.NewSops() for _, release := range conf.Releases { release.InitRelease() - err := release.VersionHandler(dir, hh) + + err := release.VersionHandler(workdirPath, hh) if err != nil { return err } - release.ValuesHandler(filepath.Dir(conf.ConfigPath)) - if err := release.SecretsHandler(filepath.Dir(conf.ConfigPath), sops); err != nil { + + release.ValuesHandler(configPath) + + if err := release.SecretsHandler(configPath, sops); err != nil { return err } } - rsObj := release.FindReleaseByNames(cluster.Releases, conf.Releases) - cluster.PopulateReleases(rsObj) + releaseObj := release.FindReleaseByNames(cluster.Releases, conf.Releases) + cluster.PopulateReleases(releaseObj) + releasesCurrent, err := release.ReleasesFromLockfile(lockfileData, conf.Repositories) if err != nil { return err } - diffRls, err := diff.DiffReleases(releasesCurrent, cluster.ReleasesObj) - if err != nil { - return err - } - lockfile, err := diffRls.Resolve(reposExisting, fullPath) + // Compare releases from the lockfile to ones from the current cluster config + diffReleases, err := diff.DiffReleases(releasesCurrent, cluster.ReleasesObj) if err != nil { return err } - if err := provider.SyncState(*diffRls); err != nil { + lockfile, err := diffReleases.Resolve(currentRepositories, clusterWorkdirPath) + if err != nil { return err } - if err := kustomize.Generate(fullPath, gh); err != nil { + if err := provider.SyncState(*diffReleases); err != nil { return err } - if err := lockfile.WriteToFile(fullPath); err != nil { + if err := kustomize.Generate(clusterWorkdirPath, gh); err != nil { return err } - if err := gh.AddAllAndCommit(fullPath, "Update the lockfile"); err != nil { + + if err := lockfile.WriteToFile(clusterWorkdirPath); err != nil { + return err + } + if err := gh.AddAllAndCommit(clusterWorkdirPath, "Update the lockfile"); err != nil { return err } if !dry { - if err := gh.Push(fullPath); err != nil { + if err := gh.Push(clusterWorkdirPath); err != nil { return err } } } if !dry { - if err := workdir.RemoveWorkdir(dir); err != nil { + if err := workdir.RemoveWorkdir(workdirPath); err != nil { return err } } diff --git a/internal/providers/flux.go b/internal/providers/flux.go index 8f182b1..06dde24 100644 --- a/internal/providers/flux.go +++ b/internal/providers/flux.go @@ -33,17 +33,19 @@ func FluxProvider(path, sopsBin string, gh githelper.Githelper) Provider { } } +// TODO: This function is ugly as hell, I need to do something about it func (f *Flux) SyncState(diff diff.Diff) error { entity := "repository" - srcPath := fmt.Sprintf("%s/src", f.path) - filePath := fmt.Sprintf("%s/%s-", srcPath, entity) + srcDirPath := fmt.Sprintf("%s/src", f.path) + // It should containe either release or repository as a prefix, because it's how files are called + entiryFilePath := fmt.Sprintf("%s/%s-", srcDirPath, entity) + for _, repo := range diff.DeletedRepositories { - if err := os.Remove(filePath + repo.Name + ".yaml"); err != nil { + if err := os.Remove(entiryFilePath + repo.Name + ".yaml"); err != nil { return err } message := `chore(repository): Removed a repo: %s - A repo has been removed from the cluster: Name: %s URL: %s @@ -58,7 +60,7 @@ func (f *Flux) SyncState(diff diff.Diff) error { if err != nil { return err } - if err := os.WriteFile(filePath+repo.Name+".yaml", manifest, os.ModeExclusive); err != nil { + if err := os.WriteFile(entiryFilePath+repo.Name+".yaml", manifest, os.ModeExclusive); err != nil { return err } message := `chore(repository): Update a repo: %s @@ -76,7 +78,7 @@ func (f *Flux) SyncState(diff diff.Diff) error { if err != nil { return err } - file, err := os.Create(filePath + repo.Name + ".yaml") + file, err := os.Create(entiryFilePath + repo.Name + ".yaml") if err != nil { return err } @@ -92,24 +94,24 @@ func (f *Flux) SyncState(diff diff.Diff) error { if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repo.Name, repo.Name, repo.URL)); err != nil { return err } - } entity = "release" - filePath = fmt.Sprintf("%s/%s-", srcPath, entity) + entiryFilePath = fmt.Sprintf("%s/%s-", srcDirPath, entity) + // Added are simply copying all the values for _, release := range diff.AddedReleases { - if err := SyncValues(release, srcPath); err != nil { + if err := SyncValues(release, srcDirPath); err != nil { return err } - if err := SyncSecrets(release, srcPath, f.path, f.sopsBin); err != nil { + if err := SyncSecrets(release, srcDirPath, f.path, f.sopsBin); err != nil { return err } manifest, err := GenerateRelease(release) if err != nil { return err } - file, err := os.Create(filePath + release.Release + ".yaml") + file, err := os.Create(entiryFilePath + release.Release + ".yaml") if err != nil { return err @@ -128,13 +130,12 @@ func (f *Flux) SyncState(diff diff.Diff) error { if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Release, release.Release, release.Namespace, release.Version, release.Repository, release.Release)); err != nil { return err } - } for _, release := range diff.UpdatedReleases { - SyncValues(release, srcPath) + SyncValues(release, srcDirPath) - if err := SyncSecrets(release, srcPath, f.path, f.sopsBin); err != nil { + if err := SyncSecrets(release, srcDirPath, f.path, f.sopsBin); err != nil { return err } @@ -143,13 +144,13 @@ func (f *Flux) SyncState(diff diff.Diff) error { return err } - if err := os.WriteFile(filePath+release.Release+".yaml", manifest, os.ModeExclusive); err != nil { + if err := os.WriteFile(entiryFilePath+release.Release+".yaml", manifest, os.ModeExclusive); err != nil { return err } message := `chore(release): Update a release: %s - A release has been updated: + A release has been updated: Name: %s Namespace: %s Version: %s @@ -158,14 +159,13 @@ func (f *Flux) SyncState(diff diff.Diff) error { if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Release, release.Release, release.Namespace, release.Version, release.Repository, release.Release)); err != nil { return err } - } for _, release := range diff.DeletedReleases { - if err := os.Remove(filePath + release.Release + ".yaml"); err != nil { + if err := os.Remove(entiryFilePath + release.Release + ".yaml"); err != nil { return err } - files, err := filepath.Glob(fmt.Sprintf("%s/values/%s*", srcPath, release.Release)) + files, err := filepath.Glob(fmt.Sprintf("%s/values/%s*", srcDirPath, release.Release)) if err != nil { return err } @@ -175,7 +175,7 @@ func (f *Flux) SyncState(diff diff.Diff) error { } } - files, err = filepath.Glob(fmt.Sprintf("%s/secrets/%s*", srcPath, release.Release)) + files, err = filepath.Glob(fmt.Sprintf("%s/secrets/%s*", srcDirPath, release.Release)) if err != nil { return err } @@ -274,6 +274,9 @@ func GenerateRelease(release *release.Release) ([]byte, error) { } func SyncValues(release *release.Release, path string) error { + for values := range release.DestValues { + + } for _, valueFile := range release.Values { // Prepare a dir for values valuesPath := fmt.Sprintf("%s/%s", path, "values") diff --git a/internal/utils/diff/diff.go b/internal/utils/diff/diff.go index 45f5654..444ba8a 100644 --- a/internal/utils/diff/diff.go +++ b/internal/utils/diff/diff.go @@ -81,10 +81,13 @@ func DiffReleases(src, dest release.Releases) (*Diff, error) { func (diff *Diff) Resolve(repositories repository.Repositories, path string) (lockfile.LockFile, error) { lockfile := lockfile.LockFile{} reposWished := repository.Repositories{} + for _, p := range diff.PreservedReleases { + logrus.Infof("preserving %s", p.Release) lockfile = append(lockfile, p.LockEntry()) reposWished = append(reposWished, p.RepositoryObj) } + for _, a := range diff.AddedReleases { logrus.Infof("adding %s", a.Release) lockfile = append(lockfile, a.LockEntry()) @@ -97,7 +100,11 @@ func (diff *Diff) Resolve(repositories repository.Repositories, path string) (lo reposWished = append(reposWished, u.RepositoryObj) } - // Repo Wished is the list of all repos that are required by the current setup, we need to + for _, d := range diff.DeletedReleases { + logrus.Infof("removing %s", d.Release) + } + + // Repo Wished is the list of all repos that are required by the current setup // Existing repos are all the repos in the lockfile for _, repoExisting := range repositories { diff --git a/internal/utils/workdir/workdir.go b/internal/utils/workdir/workdir.go index 1373f5b..55ab37b 100644 --- a/internal/utils/workdir/workdir.go +++ b/internal/utils/workdir/workdir.go @@ -5,7 +5,6 @@ import "os" func CreateWorkdir(path string) (workdir string, err error) { if len(path) > 0 { // Create a dir using the path - // It should not be removed after the execution if err := os.Mkdir(path, 0777); err != nil { return path, err } @@ -13,7 +12,6 @@ func CreateWorkdir(path string) (workdir string, err error) { workdir = path } else { // Create a temporary dir - // It should be removed after the execution workdir, err = os.MkdirTemp("", "shoebill") if err != nil { return workdir, err diff --git a/pkg/lockfile/lockfile.go b/pkg/lockfile/lockfile.go index 2ce9246..ddfbb2e 100644 --- a/pkg/lockfile/lockfile.go +++ b/pkg/lockfile/lockfile.go @@ -18,6 +18,8 @@ type LockEntry struct { Namespace string RepoUrl string RepoName string + Values []string + Secrets []string } type LockRepository struct { @@ -27,51 +29,52 @@ type LockRepository struct { type LockFile []*LockEntry -func NewFromFile(dir string) (LockFile, error) { +// Init the LockFile object by reading the yaml file +func NewFromFile(lockfileDirPath string) (LockFile, error) { var lockEntries LockFile - lockfilePath := fmt.Sprintf("%s/%s", dir, LOCKFILE_NAME) + lockfilePath := fmt.Sprintf("%s/%s", lockfileDirPath, LOCKFILE_NAME) + logrus.Infof("reading the lockfile file: %s", lockfilePath) - lockFile, err := os.ReadFile(lockfilePath) + + lockFileData, err := os.ReadFile(lockfilePath) if err != nil { return nil, err } - if err := yaml.Unmarshal(lockFile, &lockEntries); err != nil { + + if err := yaml.Unmarshal(lockFileData, &lockEntries); err != nil { return nil, err } + return lockEntries, nil } func (lockfile LockFile) ReposFromLockfile() (repository.Repositories, error) { - reposEntries := []LockRepository{} + repositories := repository.Repositories{} for _, lockentry := range lockfile { - newRepoEntry := &LockRepository{ + newRepoEntry := &repository.Repository{ URL: lockentry.RepoUrl, Name: lockentry.RepoName, } - reposEntries = append(reposEntries, *newRepoEntry) + repositories = append(repositories, newRepoEntry) } - allKeys := make(map[string]bool) - dedupedEntries := []LockRepository{} - for _, repo := range reposEntries { + // Lockfile contains an entry per a release, so one repo might be met several times + allKeys := make(map[string]bool) + dedupedRepositories := repository.Repositories{} + + for _, repo := range repositories { if _, value := allKeys[repo.Name]; !value { allKeys[repo.Name] = true - dedupedEntries = append(dedupedEntries, repo) + dedupedRepositories = append(dedupedRepositories, repo) } } - repos := repository.Repositories{} - for _, repoEntry := range dedupedEntries { - repo := &repository.Repository{ - Name: repoEntry.Name, - URL: repoEntry.URL, - } - if err := repo.KindFromUrl(); err != nil { + for _, repoEntry := range dedupedRepositories { + if err := repoEntry.KindFromUrl(); err != nil { return nil, err } - repos = append(repos, repo) } - return repos, nil + return dedupedRepositories, nil } func (lf LockFile) WriteToFile(dir string) error { diff --git a/pkg/release/release.go b/pkg/release/release.go index d47f013..6b31ec6 100644 --- a/pkg/release/release.go +++ b/pkg/release/release.go @@ -2,6 +2,7 @@ package release import ( "fmt" + "path/filepath" "reflect" "strings" @@ -30,6 +31,14 @@ type Release struct { // Private fields that should be pupulated during the run-time RepositoryObj *repository.Repository `yaml:"-"` UnencryptedSecrets map[string][]byte `yaml:"-"` + DestValues []ValuesHolder `yaml:"-"` + DestSecrets []string `yaml:"-"` +} + +type ValuesHolder struct { + SrcPath string + DestPath string + Data []byte } type Releases []*Release @@ -74,6 +83,11 @@ func (r *Release) VersionHandler(dir string, hh helmhelper.Helmhelper) error { func (r *Release) ValuesHandler(dir string) { for i := range r.Values { r.Values[i] = fmt.Sprintf("%s/%s", dir, strings.ReplaceAll(r.Values[i], "./", "")) + destValues := fmt.Sprintf("%s-%s-%s", r.Namespace, r.Release, filepath.Base(r.Values[i])) + r.DestValues = append(r.DestValues, ValuesHolder{ + SrcPath: r.Values[i], + DestPath: destValues, + }) } } @@ -84,6 +98,8 @@ func (r *Release) SecretsHandler(dir string, sops sopshelper.SopsHelper) error { if err != nil { return err } + destSecrets := fmt.Sprintf("%s-%s-%s", r.Namespace, r.Release, filepath.Base(r.Secrets[i])) + r.DestSecrets = append(r.DestSecrets, destSecrets) r.UnencryptedSecrets[path] = res } return nil @@ -91,13 +107,15 @@ func (r *Release) SecretsHandler(dir string, sops sopshelper.SopsHelper) error { func FindReleaseByNames(releases []string, releasesObj Releases) Releases { result := Releases{} - for _, rObj := range releasesObj { - for _, r := range releases { - if rObj.Release == r { - result = append(result, rObj) + + for _, repoObj := range releasesObj { + for _, release := range releases { + if repoObj.Release == release { + result = append(result, repoObj) } } } + return result } @@ -135,6 +153,8 @@ func (r *Release) LockEntry() *lockfile.LockEntry { Namespace: r.Namespace, RepoUrl: r.RepositoryObj.URL, RepoName: r.RepositoryObj.Name, + Values: r.DestValues, + Secrets: r.DestSecrets, } } diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index 2cc580b..41bfa31 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -42,6 +42,10 @@ func (r *Repository) ValidateURL() error { func (r *Repository) KindFromUrl() error { // It panics if URL is not valid, // but invalid url should not pass the ValidateURL function + if err := r.ValidateURL(); err != nil { + return err + } + prefix := r.URL[:strings.IndexByte(r.URL, ':')] switch prefix { case "oci":