allanger/shoebill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

WIP: Some refactoring

Browse Source
This commit is contained in:
Nikolai Rodionov 2023-10-12 13:23:50 +02:00
parent 6a7e541b82
commit c67a9c84e4
No known key found for this signature in database
GPG Key ID: 906851F91B1DA3EF
7 changed files with 127 additions and 79 deletions
internal
controller
controller.go
providers
flux.go
utils
diff
diff.go
workdir
workdir.go
pkg
lockfile
lockfile.go
release
release.go
repository
repository.go

77
internal/controller/controller.go

@ -24,45 +24,50 @@ func ReadTheConfig(path string) (*config.Config, error) {
return conf, nil
}
func Reconcile(workdirPath, sshKeyPath string, conf *config.Config, dry bool) error {
dir, err := workdir.CreateWorkdir(workdirPath)
func Reconcile(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool) error {
// Start by creating a directory where everything should be happening
configPath := filepath.Dir(conf.ConfigPath)
workdirPath, err := workdir.CreateWorkdir(definedWorkdirPath)
if err != nil {
return err
}
// Prepare repositories
// Prepare helm repositories
for _, repository := range conf.Repositories {
if err := repository.ValidateURL(); err != nil {
return err
}
if err := repository.KindFromUrl(); err != nil {
return err
}
}
// Configure a git client
gh := githelper.NewGit(sshKeyPath)
// The main logic starts here
for _, cluster := range conf.Clusters {
fullPath := fmt.Sprintf("%s/%s", dir, cluster.Name)
provider, err := providers.NewProvider(cluster.Provider, fullPath, conf.SopsBin, gh)
// Create a dir for the cluster git repo
clusterWorkdirPath := fmt.Sprintf("%s/%s", workdirPath, cluster.Name)
// Init a gitops provider (Currently onle flux is supported)
provider, err := providers.NewProvider(cluster.Provider, clusterWorkdirPath, conf.SopsBin, gh)
if err != nil {
return err
}
if err := cluster.CloneRepo(gh, fullPath, dry); err != nil {
if err := cluster.CloneRepo(gh, clusterWorkdirPath, dry); err != nil {
return err
}
err = cluster.BootstrapRepo(gh, fullPath, dry)
if err := cluster.BootstrapRepo(gh, clusterWorkdirPath, dry); err != nil {
return err
}
// Read the lockfile generated by the shoebill
lockfileData, err := lockfile.NewFromFile(clusterWorkdirPath)
if err != nil {
return err
}
lockfileData, err := lockfile.NewFromFile(fullPath)
if err != nil {
return err
}
reposExisting, err := lockfileData.ReposFromLockfile()
currentRepositories, err := lockfileData.ReposFromLockfile()
if err != nil {
return err
}
@ -71,61 +76,69 @@ func Reconcile(workdirPath, sshKeyPath string, conf *config.Config, dry bool) er
return err
}
// Init the helm client
hh := helmhelper.NewHelm()
// Init the sops client
sops := sopshelper.NewSops()
for _, release := range conf.Releases {
release.InitRelease()
err := release.VersionHandler(dir, hh)
err := release.VersionHandler(workdirPath, hh)
if err != nil {
return err
}
release.ValuesHandler(filepath.Dir(conf.ConfigPath))
if err := release.SecretsHandler(filepath.Dir(conf.ConfigPath), sops); err != nil {
release.ValuesHandler(configPath)
if err := release.SecretsHandler(configPath, sops); err != nil {
return err
}
}
rsObj := release.FindReleaseByNames(cluster.Releases, conf.Releases)
cluster.PopulateReleases(rsObj)
releaseObj := release.FindReleaseByNames(cluster.Releases, conf.Releases)
cluster.PopulateReleases(releaseObj)
releasesCurrent, err := release.ReleasesFromLockfile(lockfileData, conf.Repositories)
if err != nil {
return err
}
diffRls, err := diff.DiffReleases(releasesCurrent, cluster.ReleasesObj)
if err != nil {
return err
}
lockfile, err := diffRls.Resolve(reposExisting, fullPath)
// Compare releases from the lockfile to ones from the current cluster config
diffReleases, err := diff.DiffReleases(releasesCurrent, cluster.ReleasesObj)
if err != nil {
return err
}
if err := provider.SyncState(*diffRls); err != nil {
lockfile, err := diffReleases.Resolve(currentRepositories, clusterWorkdirPath)
if err != nil {
return err
}
if err := kustomize.Generate(fullPath, gh); err != nil {
if err := provider.SyncState(*diffReleases); err != nil {
return err
}
if err := lockfile.WriteToFile(fullPath); err != nil {
if err := kustomize.Generate(clusterWorkdirPath, gh); err != nil {
return err
}
if err := gh.AddAllAndCommit(fullPath, "Update the lockfile"); err != nil {
if err := lockfile.WriteToFile(clusterWorkdirPath); err != nil {
return err
}
if err := gh.AddAllAndCommit(clusterWorkdirPath, "Update the lockfile"); err != nil {
return err
}
if !dry {
if err := gh.Push(fullPath); err != nil {
if err := gh.Push(clusterWorkdirPath); err != nil {
return err
}
}
}
if !dry {
if err := workdir.RemoveWorkdir(dir); err != nil {
if err := workdir.RemoveWorkdir(workdirPath); err != nil {
return err
}
}

43
internal/providers/flux.go

@ -33,17 +33,19 @@ func FluxProvider(path, sopsBin string, gh githelper.Githelper) Provider {
}
}
// TODO: This function is ugly as hell, I need to do something about it
func (f *Flux) SyncState(diff diff.Diff) error {
entity := "repository"
srcPath := fmt.Sprintf("%s/src", f.path)
filePath := fmt.Sprintf("%s/%s-", srcPath, entity)
srcDirPath := fmt.Sprintf("%s/src", f.path)
// It should containe either release or repository as a prefix, because it's how files are called
entiryFilePath := fmt.Sprintf("%s/%s-", srcDirPath, entity)
for _, repo := range diff.DeletedRepositories {
if err := os.Remove(filePath + repo.Name + ".yaml"); err != nil {
if err := os.Remove(entiryFilePath + repo.Name + ".yaml"); err != nil {
return err
}
message := `chore(repository): Removed a repo: %s
A repo has been removed from the cluster:
Name: %s
URL: %s
@ -58,7 +60,7 @@ func (f *Flux) SyncState(diff diff.Diff) error {
if err != nil {
return err
}
if err := os.WriteFile(filePath+repo.Name+".yaml", manifest, os.ModeExclusive); err != nil {
if err := os.WriteFile(entiryFilePath+repo.Name+".yaml", manifest, os.ModeExclusive); err != nil {
return err
}
message := `chore(repository): Update a repo: %s
@ -76,7 +78,7 @@ func (f *Flux) SyncState(diff diff.Diff) error {
if err != nil {
return err
}
file, err := os.Create(filePath + repo.Name + ".yaml")
file, err := os.Create(entiryFilePath + repo.Name + ".yaml")
if err != nil {
return err
}
@ -92,24 +94,24 @@ func (f *Flux) SyncState(diff diff.Diff) error {
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repo.Name, repo.Name, repo.URL)); err != nil {
return err
}
}
entity = "release"
filePath = fmt.Sprintf("%s/%s-", srcPath, entity)
entiryFilePath = fmt.Sprintf("%s/%s-", srcDirPath, entity)
// Added are simply copying all the values
for _, release := range diff.AddedReleases {
if err := SyncValues(release, srcPath); err != nil {
if err := SyncValues(release, srcDirPath); err != nil {
return err
}
if err := SyncSecrets(release, srcPath, f.path, f.sopsBin); err != nil {
if err := SyncSecrets(release, srcDirPath, f.path, f.sopsBin); err != nil {
return err
}
manifest, err := GenerateRelease(release)
if err != nil {
return err
}
file, err := os.Create(filePath + release.Release + ".yaml")
file, err := os.Create(entiryFilePath + release.Release + ".yaml")
if err != nil {
return err
@ -128,13 +130,12 @@ func (f *Flux) SyncState(diff diff.Diff) error {
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Release, release.Release, release.Namespace, release.Version, release.Repository, release.Release)); err != nil {
return err
}
}
for _, release := range diff.UpdatedReleases {
SyncValues(release, srcPath)
SyncValues(release, srcDirPath)
if err := SyncSecrets(release, srcPath, f.path, f.sopsBin); err != nil {
if err := SyncSecrets(release, srcDirPath, f.path, f.sopsBin); err != nil {
return err
}
@ -143,13 +144,13 @@ func (f *Flux) SyncState(diff diff.Diff) error {
return err
}
if err := os.WriteFile(filePath+release.Release+".yaml", manifest, os.ModeExclusive); err != nil {
if err := os.WriteFile(entiryFilePath+release.Release+".yaml", manifest, os.ModeExclusive); err != nil {
return err
}
message := `chore(release): Update a release: %s
A release has been updated:
A release has been updated:
Name: %s
Namespace: %s
Version: %s
@ -158,14 +159,13 @@ func (f *Flux) SyncState(diff diff.Diff) error {
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Release, release.Release, release.Namespace, release.Version, release.Repository, release.Release)); err != nil {
return err
}
}
for _, release := range diff.DeletedReleases {
if err := os.Remove(filePath + release.Release + ".yaml"); err != nil {
if err := os.Remove(entiryFilePath + release.Release + ".yaml"); err != nil {
return err
}
files, err := filepath.Glob(fmt.Sprintf("%s/values/%s*", srcPath, release.Release))
files, err := filepath.Glob(fmt.Sprintf("%s/values/%s*", srcDirPath, release.Release))
if err != nil {
return err
}
@ -175,7 +175,7 @@ func (f *Flux) SyncState(diff diff.Diff) error {
}
}
files, err = filepath.Glob(fmt.Sprintf("%s/secrets/%s*", srcPath, release.Release))
files, err = filepath.Glob(fmt.Sprintf("%s/secrets/%s*", srcDirPath, release.Release))
if err != nil {
return err
}
@ -274,6 +274,9 @@ func GenerateRelease(release *release.Release) ([]byte, error) {
}
func SyncValues(release *release.Release, path string) error {
for values := range release.DestValues {
}
for _, valueFile := range release.Values {
// Prepare a dir for values
valuesPath := fmt.Sprintf("%s/%s", path, "values")

9
internal/utils/diff/diff.go

@ -81,10 +81,13 @@ func DiffReleases(src, dest release.Releases) (*Diff, error) {
func (diff *Diff) Resolve(repositories repository.Repositories, path string) (lockfile.LockFile, error) {
lockfile := lockfile.LockFile{}
reposWished := repository.Repositories{}
for _, p := range diff.PreservedReleases {
logrus.Infof("preserving %s", p.Release)
lockfile = append(lockfile, p.LockEntry())
reposWished = append(reposWished, p.RepositoryObj)
}
for _, a := range diff.AddedReleases {
logrus.Infof("adding %s", a.Release)
lockfile = append(lockfile, a.LockEntry())
@ -97,7 +100,11 @@ func (diff *Diff) Resolve(repositories repository.Repositories, path string) (lo
reposWished = append(reposWished, u.RepositoryObj)
}
// Repo Wished is the list of all repos that are required by the current setup, we need to
for _, d := range diff.DeletedReleases {
logrus.Infof("removing %s", d.Release)
}
// Repo Wished is the list of all repos that are required by the current setup
// Existing repos are all the repos in the lockfile
for _, repoExisting := range repositories {

2
internal/utils/workdir/workdir.go

@ -5,7 +5,6 @@ import "os"
func CreateWorkdir(path string) (workdir string, err error) {
if len(path) > 0 {
// Create a dir using the path
// It should not be removed after the execution
if err := os.Mkdir(path, 0777); err != nil {
return path, err
}
@ -13,7 +12,6 @@ func CreateWorkdir(path string) (workdir string, err error) {
workdir = path
} else {
// Create a temporary dir
// It should be removed after the execution
workdir, err = os.MkdirTemp("", "shoebill")
if err != nil {
return workdir, err

43
pkg/lockfile/lockfile.go

@ -18,6 +18,8 @@ type LockEntry struct {
Namespace string
RepoUrl string
RepoName string
Values []string
Secrets []string
}
type LockRepository struct {
@ -27,51 +29,52 @@ type LockRepository struct {
type LockFile []*LockEntry
func NewFromFile(dir string) (LockFile, error) {
// Init the LockFile object by reading the yaml file
func NewFromFile(lockfileDirPath string) (LockFile, error) {
var lockEntries LockFile
lockfilePath := fmt.Sprintf("%s/%s", dir, LOCKFILE_NAME)
lockfilePath := fmt.Sprintf("%s/%s", lockfileDirPath, LOCKFILE_NAME)
logrus.Infof("reading the lockfile file: %s", lockfilePath)
lockFile, err := os.ReadFile(lockfilePath)
lockFileData, err := os.ReadFile(lockfilePath)
if err != nil {
return nil, err
}
if err := yaml.Unmarshal(lockFile, &lockEntries); err != nil {
if err := yaml.Unmarshal(lockFileData, &lockEntries); err != nil {
return nil, err
}
return lockEntries, nil
}
func (lockfile LockFile) ReposFromLockfile() (repository.Repositories, error) {
reposEntries := []LockRepository{}
repositories := repository.Repositories{}
for _, lockentry := range lockfile {
newRepoEntry := &LockRepository{
newRepoEntry := &repository.Repository{
URL: lockentry.RepoUrl,
Name: lockentry.RepoName,
}
reposEntries = append(reposEntries, *newRepoEntry)
repositories = append(repositories, newRepoEntry)
}
allKeys := make(map[string]bool)
dedupedEntries := []LockRepository{}
for _, repo := range reposEntries {
// Lockfile contains an entry per a release, so one repo might be met several times
allKeys := make(map[string]bool)
dedupedRepositories := repository.Repositories{}
for _, repo := range repositories {
if _, value := allKeys[repo.Name]; !value {
allKeys[repo.Name] = true
dedupedEntries = append(dedupedEntries, repo)
dedupedRepositories = append(dedupedRepositories, repo)
}
}
repos := repository.Repositories{}
for _, repoEntry := range dedupedEntries {
repo := &repository.Repository{
Name: repoEntry.Name,
URL: repoEntry.URL,
}
if err := repo.KindFromUrl(); err != nil {
for _, repoEntry := range dedupedRepositories {
if err := repoEntry.KindFromUrl(); err != nil {
return nil, err
}
repos = append(repos, repo)
}
return repos, nil
return dedupedRepositories, nil
}
func (lf LockFile) WriteToFile(dir string) error {

28
pkg/release/release.go

@ -2,6 +2,7 @@ package release
import (
"fmt"
"path/filepath"
"reflect"
"strings"
@ -30,6 +31,14 @@ type Release struct {
// Private fields that should be pupulated during the run-time
RepositoryObj *repository.Repository `yaml:"-"`
UnencryptedSecrets map[string][]byte `yaml:"-"`
DestValues []ValuesHolder `yaml:"-"`
DestSecrets []string `yaml:"-"`
}
type ValuesHolder struct {
SrcPath string
DestPath string
Data []byte
}
type Releases []*Release
@ -74,6 +83,11 @@ func (r *Release) VersionHandler(dir string, hh helmhelper.Helmhelper) error {
func (r *Release) ValuesHandler(dir string) {
for i := range r.Values {
r.Values[i] = fmt.Sprintf("%s/%s", dir, strings.ReplaceAll(r.Values[i], "./", ""))
destValues := fmt.Sprintf("%s-%s-%s", r.Namespace, r.Release, filepath.Base(r.Values[i]))
r.DestValues = append(r.DestValues, ValuesHolder{
SrcPath: r.Values[i],
DestPath: destValues,
})
}
}
@ -84,6 +98,8 @@ func (r *Release) SecretsHandler(dir string, sops sopshelper.SopsHelper) error {
if err != nil {
return err
}
destSecrets := fmt.Sprintf("%s-%s-%s", r.Namespace, r.Release, filepath.Base(r.Secrets[i]))
r.DestSecrets = append(r.DestSecrets, destSecrets)
r.UnencryptedSecrets[path] = res
}
return nil
@ -91,13 +107,15 @@ func (r *Release) SecretsHandler(dir string, sops sopshelper.SopsHelper) error {
func FindReleaseByNames(releases []string, releasesObj Releases) Releases {
result := Releases{}
for _, rObj := range releasesObj {
for _, r := range releases {
if rObj.Release == r {
result = append(result, rObj)
for _, repoObj := range releasesObj {
for _, release := range releases {
if repoObj.Release == release {
result = append(result, repoObj)
}
}
}
return result
}
@ -135,6 +153,8 @@ func (r *Release) LockEntry() *lockfile.LockEntry {
Namespace: r.Namespace,
RepoUrl: r.RepositoryObj.URL,
RepoName: r.RepositoryObj.Name,
Values: r.DestValues,
Secrets: r.DestSecrets,
}
}

4
pkg/repository/repository.go

@ -42,6 +42,10 @@ func (r *Repository) ValidateURL() error {
func (r *Repository) KindFromUrl() error {
// It panics if URL is not valid,
// but invalid url should not pass the ValidateURL function
if err := r.ValidateURL(); err != nil {
return err
}
prefix := r.URL[:strings.IndexByte(r.URL, ':')]
switch prefix {
case "oci":