allanger/shoebill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

WIP: Trying to implement diff

Browse Source
This commit is contained in:
Nikolai Rodionov
2023-10-20 13:31:30 +02:00
parent 93ad3389b2
commit ed3d45a7c4
15 changed files with 197 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
internal/controller/controller.go
View File

@ -24,7 +24,14 @@ func ReadTheConfig(path string) (*config.Config, error) {
return conf, nil
}
func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry, diffArg bool) error {
// func cloneSnapshoot(gh githelper.Githelper, snapshotDir, snapshotBranch string) error {
// if err := gh.CloneRepo(snapshotBranch, snapshotUrl, false); err != nil {
// return err
// }
// return nil
// }
func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool, diffArg string) error {
// Start by creating a directory where everything should be happening
configPath := filepath.Dir(conf.ConfigPath)
workdirPath, err := workdir.CreateWorkdir(definedWorkdirPath)
@ -41,6 +48,10 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry, diffA
// Configure a git client
gh := githelper.NewGit(sshKeyPath)
// if len(diffArg) > 0 {
// snapshotDir := fmt.Sprint("%s/.snapshot", workdirPath)
// cloneSnapshoot(gh, snapshotDir, diffArg)
// }
// The main logic starts here
for _, cluster := range conf.Clusters {
@ -88,12 +99,15 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry, diffA
return err
}
if diffArg {
_, err := hh.PullChart(workdirPath, *release.RepositoryObj, release.Chart, release.Version)
if len(diffArg) > 0 {
_, err := hh.PullChart(workdirPath, release.ToHelmReleaseData())
if err != nil {
return err
}
hh.RenderChart(workdirPath, *release.RepositoryObj, release.Chart, release.Version)
if err := hh.RenderChart(workdirPath, release.ToHelmReleaseData()); err != nil {
return err
}
}
if err := release.ValuesHandler(configPath); err != nil {
@ -113,9 +127,9 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry, diffA
return err
}
if diffArg {
if len(diffArg) > 0 {
for _, releaseCurrent := range releasesCurrent {
hh.PullChart(workdirPath, *releaseCurrent.RepositoryObj, releaseCurrent.Chart, releaseCurrent.Version)
hh.PullChart(workdirPath, releaseCurrent.ToHelmReleaseData())
}
}
@ -130,7 +144,8 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry, diffA
return err
}
if err := provider.SyncState(diffReleases, diffRepos); err != nil {
hashesPerRelease, err := provider.SyncState(diffReleases, diffRepos)
if err != nil {
return err
}
@ -138,10 +153,12 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry, diffA
return err
}
lockfile.AddHashes(hashesPerRelease)
if err := lockfile.WriteToFile(clusterWorkdirPath); err != nil {
return err
}
if err := gh.AddAllAndCommit(clusterWorkdirPath, "Update the lockfile"); err != nil {
if _, err := gh.AddAllAndCommit(clusterWorkdirPath, "Update the lockfile"); err != nil {
return err
}
if !dry {

78
internal/providers/flux.go
View File

@ -10,6 +10,7 @@ import (
"git.badhouseplants.net/allanger/shoebill/internal/utils/diff"
"git.badhouseplants.net/allanger/shoebill/internal/utils/githelper"
"git.badhouseplants.net/allanger/shoebill/pkg/lockfile"
"git.badhouseplants.net/allanger/shoebill/pkg/release"
"git.badhouseplants.net/allanger/shoebill/pkg/repository"
release_v2beta1 "github.com/fluxcd/helm-controller/api/v2beta1"
@ -35,7 +36,7 @@ func FluxProvider(path, sopsBin string, gh githelper.Githelper) Provider {
}
// TODO: This function is ugly as hell, I need to do something about it
func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.RepositoriesDiffs) error {
func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.RepositoriesDiffs) (lockfile.HashesPerReleases, error) {
entity := "repository"
srcDirPath := fmt.Sprintf("%s/src", f.path)
// It should containe either release or repository as a prefix, because it's how files are called
@ -46,14 +47,14 @@ func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.Reposi
case diff.ACTION_ADD:
manifest, err := GenerateRepository(repository.Wished)
if err != nil {
return err
return nil, err
}
file, err := os.Create(entiryFilePath + repository.Wished.Name + ".yaml")
if err != nil {
return err
return nil, err
}
if _, err := file.Write(manifest); err != nil {
return err
return nil, err
}
message := `chore(repository): Add a repo: %s
@ -61,18 +62,18 @@ func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.Reposi
Name: %s
URL: %s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Wished.Name, repository.Wished.Name, repository.Wished.URL)); err != nil {
return err
if _, err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Wished.Name, repository.Wished.Name, repository.Wished.URL)); err != nil {
return nil, err
}
case diff.ACTION_PRESERVE:
case diff.ACTION_UPDATE:
manifest, err := GenerateRepository(repository.Wished)
if err != nil {
return err
return nil, err
}
if err := os.WriteFile(entiryFilePath+repository.Wished.Name+".yaml", manifest, os.ModeExclusive); err != nil {
return err
return nil, err
}
message := `chore(repository): Update a repo: %s
@ -80,13 +81,13 @@ func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.Reposi
Name: %s
URL: %s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Wished.Name, repository.Wished.Name, repository.Wished.URL)); err != nil {
return err
if _, err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Wished.Name, repository.Wished.Name, repository.Wished.URL)); err != nil {
return nil, err
}
case diff.ACTION_DELETE:
if err := os.Remove(entiryFilePath + repository.Current.Name + ".yaml"); err != nil {
return err
return nil, err
}
message := `chore(repository): Removed a repo: %s
@ -94,40 +95,42 @@ func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.Reposi
Name: %s
URL: %s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Current.Name, repository.Current.Name, repository.Current.URL)); err != nil {
return err
if _, err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repository.Current.Name, repository.Current.Name, repository.Current.URL)); err != nil {
return nil, err
}
default:
return fmt.Errorf("unknown action is requests: %s", repository.Action)
return nil, fmt.Errorf("unknown action is requests: %s", repository.Action)
}
}
hashesPerReleases := lockfile.HashesPerReleases{}
entity = "release"
entiryFilePath = fmt.Sprintf("%s/%s-", srcDirPath, entity)
for _, release := range releasesDiffs {
var hash string
var err error
if err := SyncValues(release.Current, release.Wished, srcDirPath); err != nil {
return err
return nil, err
}
if err := SyncSecrets(release.Current, release.Wished, f.path, f.sopsBin); err != nil {
return err
return nil, err
}
switch release.Action {
case diff.ACTION_ADD:
manifest, err := GenerateRelease(release.Wished)
if err != nil {
return err
return nil, err
}
file, err := os.Create(entiryFilePath + release.Wished.Release + ".yaml")
if err != nil {
return err
return nil, err
}
if _, err := file.Write(manifest); err != nil {
return err
return nil, err
}
message := `chore(release): Add a new release: %s
@ -137,18 +140,19 @@ func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.Reposi
Version: %s
Chart: %s/%s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Wished.Release, release.Wished.Release, release.Wished.Namespace, release.Wished.Version, release.Wished.Repository, release.Wished.Release)); err != nil {
return err
hash, err = f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Wished.Release, release.Wished.Release, release.Wished.Namespace, release.Wished.Version, release.Wished.Repository, release.Wished.Release))
if err != nil {
return nil, err
}
case diff.ACTION_UPDATE:
manifest, err := GenerateRelease(release.Wished)
if err != nil {
return err
return nil, err
}
if err := os.WriteFile(entiryFilePath+release.Wished.Release+".yaml", manifest, os.ModeExclusive); err != nil {
return err
return nil, err
}
message := `chore(release): Update a release: %s
@ -159,13 +163,14 @@ func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.Reposi
Version: %s
Chart: %s/%s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Wished.Release, release.Wished.Release, release.Wished.Namespace, release.Wished.Version, release.Wished.Repository, release.Wished.Release)); err != nil {
return err
hash, err = f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Wished.Release, release.Wished.Release, release.Wished.Namespace, release.Wished.Version, release.Wished.Repository, release.Wished.Release))
if err != nil {
return nil, err
}
case diff.ACTION_DELETE:
if err := os.Remove(entiryFilePath + release.Current.Release + ".yaml"); err != nil {
return err
return nil, err
}
message := `chore(release): Remove a release: %s
@ -175,16 +180,24 @@ func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.Reposi
Version: %s
Chart: %s/%s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Current.Release, release.Current.Release, release.Current.Namespace, release.Current.Version, release.Current.Repository, release.Current.Release)); err != nil {
return err
hash, err = f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Current.Release, release.Current.Release, release.Current.Namespace, release.Current.Version, release.Current.Repository, release.Current.Release))
if err != nil {
return nil, err
}
default:
return fmt.Errorf("unknown action is requests: %s", release.Action)
return nil, fmt.Errorf("unknown action is requests: %s", release.Action)
}
hashPerRelease := &lockfile.HashPerRelease{
Release: release.Wished.Release,
Namespace: release.Wished.Namespace,
CommitHash: hash,
}
hashesPerReleases = append(hashesPerReleases, hashPerRelease)
}
return nil
return hashesPerReleases, nil
}
@ -341,10 +354,9 @@ func SyncSecrets(currentRelease, wishedRelease *release.Release, workdirPath, so
if wishedRelease != nil {
for _, secrets := range wishedRelease.DestSecrets {
// Prepare a dir for secrets
secretsPath := fmt.Sprintf("%s/%s", workdirPath, "secrets")
secretsFilePath := fmt.Sprintf("%s/%s", secretsDirPath, secrets.DestPath)
logrus.Infof("trying to create secrets file: %s", secretsFilePath)
if err := os.MkdirAll(secretsPath, os.ModePerm); err != nil {
if err := os.MkdirAll(secretsDirPath, os.ModePerm); err != nil {
return err
}
var secretsFile *os.File

3
internal/providers/types.go
View File

@ -5,10 +5,11 @@ import (
"git.badhouseplants.net/allanger/shoebill/internal/utils/diff"
"git.badhouseplants.net/allanger/shoebill/internal/utils/githelper"
"git.badhouseplants.net/allanger/shoebill/pkg/lockfile"
)
type Provider interface {
SyncState(diff.ReleasesDiffs, diff.RepositoriesDiffs) error
SyncState(diff.ReleasesDiffs, diff.RepositoriesDiffs) (lockfile.HashesPerReleases, error)
}
func NewProvider(provider, path, sopsBin string, gh githelper.Githelper) (Provider, error) {

16
internal/utils/githelper/git.go
View File

@ -73,24 +73,26 @@ func (g *Git) CloneRepo(workdir, gitURL string, dry bool) error {
return nil
}
func (g *Git) AddAllAndCommit(workdir, message string) error {
func (g *Git) AddAllAndCommit(workdir, message string) (string, error) {
r, err := git.PlainOpen(workdir)
if err != nil {
return err
return "", err
}
w, err := r.Worktree()
if err != nil {
return err
return "", err
}
if _, err := w.Add("."); err != nil {
return err
return "", err
}
if _, err := w.Commit(message, &git.CommitOptions{}); err != nil {
return err
sha, err := w.Commit(message, &git.CommitOptions{})
if err != nil {
return "", err
}
return nil
return sha.String(), nil
}
func (g *Git) Push(workdir string) error {

4
internal/utils/githelper/mock.go
View File

@ -10,8 +10,8 @@ func (m *Mock) CloneRepo(workdir, gitURL string, dry bool) error {
return nil
}
func (g *Mock) AddAllAndCommit(workdir, message string) error {
return nil
func (g *Mock) AddAllAndCommit(workdir, message string) (string, error) {
return "HASH", nil
}
func (g *Mock) Push(workdir string) error {
return nil

2
internal/utils/githelper/types.go
View File

@ -2,6 +2,6 @@ package githelper
type Githelper interface {
CloneRepo(workdir, gitURL string, dry bool) error
AddAllAndCommit(workdir, message string) error
AddAllAndCommit(workdir, message string) (string, error)
Push(workdir string) error
}

74
internal/utils/helmhelper/helm.go
View File

@ -4,11 +4,11 @@ import (
"fmt"
"os"
"git.badhouseplants.net/allanger/shoebill/pkg/repository"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/engine"
"helm.sh/helm/v3/pkg/getter"
@ -22,20 +22,16 @@ func NewHelm() Helmhelper {
return &Helm{}
}
type ChartData struct {
Version string
}
func getDownloadDirPath(workdirPath string) string {
return fmt.Sprintf("%s/.charts", workdirPath)
}
func getChartDirPath(downloadDirPath, repository, chart, version string) string {
return fmt.Sprintf("%s/%s-%s-%s", downloadDirPath, repository, chart, version)
func getChartDirPath(downloadDirPath string, release *ReleaseData) string {
return fmt.Sprintf("%s/%s-%s-%s", downloadDirPath, release.RepositoryName, release.Chart, release.Version)
}
func (h *Helm) PullChart(workdirPath string, repository repository.Repository, chart, version string) (path string, err error) {
func (h *Helm) PullChart(workdirPath string, release *ReleaseData) (path string, err error) {
downloadDirPath := getDownloadDirPath(workdirPath)
if err := os.MkdirAll(downloadDirPath, 0777); err != nil {
return "", err
@ -43,7 +39,7 @@ func (h *Helm) PullChart(workdirPath string, repository repository.Repository, c
config := new(action.Configuration)
cl := cli.New()
chartDir := getChartDirPath(downloadDirPath, repository.Name, chart, version)
chartDir := getChartDirPath(downloadDirPath, release)
_, err = os.Stat(chartDir)
if err != nil && !os.IsNotExist(err) {
@ -59,10 +55,10 @@ func (h *Helm) PullChart(workdirPath string, repository repository.Repository, c
var path string
// Download the chart to the workdir
if repository.Kind != "oci" {
if release.RepositoryKind != "oci" {
r, err := repo.NewChartRepository(&repo.Entry{
Name: repository.Name,
URL: repository.URL,
Name: release.RepositoryName,
URL: release.RepositoryURL,
}, getter.All(cl))
if err != nil {
return "", err
@ -70,7 +66,7 @@ func (h *Helm) PullChart(workdirPath string, repository repository.Repository, c
path = r.Config.Name
} else {
path = repository.URL
path = release.RepositoryURL
}
client := action.NewPullWithOpts(action.WithConfig(config))
@ -78,7 +74,7 @@ func (h *Helm) PullChart(workdirPath string, repository repository.Repository, c
client.DestDir = chartDir
client.Settings = cl
chartRemote := fmt.Sprintf("%s/%s", path, chart)
chartRemote := fmt.Sprintf("%s/%s", path, release.Chart)
logrus.Infof("trying to pull: %s", chartRemote)
if _, err = client.Run(chartRemote); err != nil {
return "", err
@ -92,7 +88,7 @@ func (h *Helm) PullChart(workdirPath string, repository repository.Repository, c
return path, nil
}
func (h *Helm) FindLatestVersion(workdirPath, chart string, repository repository.Repository) (version string, err error) {
func (h *Helm) FindLatestVersion(workdirPath string, release *ReleaseData) (version string, err error) {
downloadDirPath := getDownloadDirPath(workdirPath)
if err := os.MkdirAll(downloadDirPath, 0777); err != nil {
return "", err
@ -100,14 +96,13 @@ func (h *Helm) FindLatestVersion(workdirPath, chart string, repository repositor
config := new(action.Configuration)
cl := cli.New()
chartDir := getChartDirPath(downloadDirPath, repository.Name, chart, "latest")
chartPath, err := h.PullChart(workdirPath, repository, chart, "latest")
chartDir := getChartDirPath(downloadDirPath, release)
chartPath, err := h.PullChart(workdirPath, release)
if err != nil {
return "", err
}
showAction := action.NewShowWithConfig(action.ShowChart, config)
action.New
res, err := showAction.LocateChart(fmt.Sprintf("%s/%s", chartDir, chartPath), cl)
if err != nil {
@ -122,34 +117,45 @@ func (h *Helm) FindLatestVersion(workdirPath, chart string, repository repositor
if err != nil {
return "", err
}
logrus.Infof("the latest version of %s is %s", chart, chartData.Version)
versionedChartDir := getChartDirPath(downloadDirPath, repository.Name, chart, chartData.Version)
logrus.Infof("the latest version of %s is %s", release.Chart, chartData.Version)
versionedChartDir := getChartDirPath(downloadDirPath, release)
os.Rename(chartDir, versionedChartDir)
return chartData.Version, err
}
func (h *Helm) RenderChart(workdirPath string, repository repository.Repository, ch2art, version string) error {
func (h *Helm) RenderChart(workdirPath string, release *ReleaseData) error {
downloadDirPath := getDownloadDirPath(workdirPath)
chartDirPath := getChartDirPath(downloadDirPath, repository.Name, ch2art, version)
chartDirPath := getChartDirPath(downloadDirPath, release)
chartPath, err := getChartPathFromDir(chartDirPath)
if err != nil {
return nil
return err
}
logrus.Info(fmt.Sprintf("%s/%s", chartDirPath, chartPath))
chartObj, err := loader.Load(fmt.Sprintf("%s/%s", chartDirPath, chartPath))
if err != nil {
return err
}
config := new(action.Configuration)
client := action.NewInstall(config)
client.RepoURL = repository.URL
client.ReleaseName = ch2art
res, err := engine.Render(chartObj, nil)
values := chartutil.Values{}
values["Values"] = chartObj.Values
values["Release"] = map[string]string{
"Name": release.Name,
"Namespace": release.Namespace,
}
values["Capabilities"] = map[string]map[string]string{
"KubeVersion": {
"Version": "v1.27.9",
"GitVersion": "v1.27.9",
},
}
files, err := engine.Engine{Strict: false}.Render(chartObj, values)
if err != nil {
return err
}
logrus.Infof("%v", res)
logrus.Info(files)
for file, data := range files {
logrus.Infof("%s - %s", file, data)
}
logrus.Info("I'm here")
return nil
}
@ -165,10 +171,10 @@ func getChartPathFromDir(downloadDir string) (file string, err error) {
return files[0].Name(), nil
}
func chartFromString(info string) (*ChartData, error) {
chartData := new(ChartData)
if err := yaml.Unmarshal([]byte(info), &chartData); err != nil {
func chartFromString(info string) (*ReleaseData, error) {
releaseData := new(ReleaseData)
if err := yaml.Unmarshal([]byte(info), &releaseData); err != nil {
return nil, err
}
return chartData, nil
return releaseData, nil
}

10
internal/utils/helmhelper/mock.go
View File

@ -1,9 +1,5 @@
package helmhelper
import (
"git.badhouseplants.net/allanger/shoebill/pkg/repository"
)
const (
MOCK_LATEST_VERSION = "v1.12.1"
MOCK_CHART_PATH = ".charts/repo-release-latest/release-latest.gz"
@ -15,14 +11,14 @@ func NewHelmMock() Helmhelper {
return &Mock{}
}
func (h *Mock) FindLatestVersion(dir, chart string, repository repository.Repository) (version string, err error) {
func (h *Mock) FindLatestVersion(workdir string, release *ReleaseData) (version string, err error) {
return MOCK_LATEST_VERSION, nil
}
func (h *Mock) PullChart(workdirPath string, repository repository.Repository, chart, version string) (path string, err error) {
func (h *Mock) PullChart(workdirPath string, release *ReleaseData) (path string, err error) {
return MOCK_CHART_PATH, nil
}
func (h *Mock) RenderChart(workdirPath string, repository repository.Repository, ch2art, version string) error {
func (h *Mock) RenderChart(workdirPath string, release *ReleaseData) error {
return nil
}

19
internal/utils/helmhelper/types.go
View File

@ -1,9 +1,18 @@
package helmhelper
import "git.badhouseplants.net/allanger/shoebill/pkg/repository"
type Helmhelper interface {
FindLatestVersion(dir, chart string, repository repository.Repository) (string, error)
PullChart(workdirPath string, repository repository.Repository, chart, version string) (string, error)
RenderChart(workdirPath string, repository repository.Repository, ch2art, version string) error
FindLatestVersion(workdirPath string, release *ReleaseData) (string, error)
PullChart(workdirPath string, release *ReleaseData) (string, error)
RenderChart(workdirPath string, release *ReleaseData) error
}
type ReleaseData struct {
Name string
Chart string
Namespace string
Version string
RepositoryName string
RepositoryURL string
RepositoryKind string
ValuesData string
}

5
internal/utils/kustomize/kustomize.go
View File

@ -128,11 +128,10 @@ func Generate(path string, gh githelper.Githelper) error {
Name: "helm-root",
Namespace: "flux-system",
},
Resources: kustomize.Files,
Resources: append(kustomize.Files, kustomize.ConfigMaps...),
GeneratorOptions: &kustomize_types.GeneratorOptions{
DisableNameSuffixHash: true,
},
ConfigMapGenerator: kustomize.CmGeneratorFromFiles(),
}
if len(kustomize.Secrets) > 0 {
@ -172,7 +171,7 @@ func Generate(path string, gh githelper.Githelper) error {
return nil
}
if err := gh.AddAllAndCommit(path, "Update the root kustomization"); err != nil {
if _, err := gh.AddAllAndCommit(path, "Update the root kustomization"); err != nil {
return err
}