diff --git a/LICENSE b/LICENSE index 13ca539..03cc888 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Creative Commons Attribution 4.0 International +rrCreative Commons Attribution 4.0 International Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. diff --git a/go.mod b/go.mod index e2b08b2..8c4af9b 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( helm.sh/helm/v3 v3.12.2 k8s.io/api v0.29.0-alpha.0 k8s.io/apimachinery v0.29.0-alpha.0 + k8s.io/utils v0.0.0-20230505201702-9f6742963106 sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 sigs.k8s.io/yaml v1.3.0 ) @@ -216,7 +217,6 @@ require ( k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect k8s.io/kubectl v0.27.2 // indirect - k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect oras.land/oras-go v1.2.3 // indirect sigs.k8s.io/controller-runtime v0.15.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 07ac36b..9bc306a 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -14,6 +14,7 @@ import ( "git.badhouseplants.net/allanger/shoebill/pkg/config" "git.badhouseplants.net/allanger/shoebill/pkg/lockfile" "git.badhouseplants.net/allanger/shoebill/pkg/release" + "github.com/sirupsen/logrus" ) func ReadTheConfig(path string) (*config.Config, error) { @@ -45,6 +46,29 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool, return err } } + if err := conf.Charts.PopulateRepositories(conf.Repositories); err != nil { + return err + } + + if err := conf.Releases.PopulateCharts(conf.Charts); err != nil { + return err + } + + // Init the helm client + hh := helmhelper.NewHelm() + for _, ch := range conf.Charts { + err := ch.VersionHandler(workdirPath, hh) + if err != nil { + return err + } + logrus.Info("BEFORE") + if err := ch.SyncMirrors(workdirPath, conf.Mirrors, hh); err != nil { + return err + } + logrus.Info("AFTER") + } + + // Configure a git client gh := githelper.NewGit(sshKeyPath) @@ -87,28 +111,21 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool, return err } - // Init the helm client - hh := helmhelper.NewHelm() - // Init the sops client sops := sopshelper.NewSops() for _, release := range conf.Releases { - err := release.VersionHandler(workdirPath, hh) - if err != nil { - return err - } - if len(diffArg) > 0 { - _, err := hh.PullChart(workdirPath, release.ToHelmReleaseData()) - if err != nil { - return err - } - if err := hh.RenderChart(workdirPath, release.ToHelmReleaseData()); err != nil { - return err - } + // if len(diffArg) > 0 { + // _, err := hh.PullChart(workdirPath, release.ToHelmReleaseData()) + // if err != nil { + // return err + // } + // if err := hh.RenderChart(workdirPath, release.ToHelmReleaseData()); err != nil { + // return err + // } - } + // } if err := release.ValuesHandler(configPath); err != nil { return err @@ -127,12 +144,6 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool, return err } - if len(diffArg) > 0 { - for _, releaseCurrent := range releasesCurrent { - hh.PullChart(workdirPath, releaseCurrent.ToHelmReleaseData()) - } - } - // Compare releases from the lockfile to ones from the current cluster config diffReleases, err := diff.DiffReleases(releasesCurrent, cluster.ReleasesObj) if err != nil { diff --git a/internal/providers/flux.go b/internal/providers/flux.go index 7d7ad77..5c4d1b4 100644 --- a/internal/providers/flux.go +++ b/internal/providers/flux.go @@ -188,12 +188,14 @@ func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.Reposi default: return nil, fmt.Errorf("unknown action is requests: %s", release.Action) } - hashPerRelease := &lockfile.HashPerRelease{ - Release: release.Wished.Release, - Namespace: release.Wished.Namespace, - CommitHash: hash, - } - hashesPerReleases = append(hashesPerReleases, hashPerRelease) + if release.Wished != nil { + hashPerRelease := &lockfile.HashPerRelease{ + Release: release.Wished.Release, + Namespace: release.Wished.Namespace, + CommitHash: hash, + } + hashesPerReleases = append(hashesPerReleases, hashPerRelease) + } } diff --git a/internal/utils/helmhelper/helm.go b/internal/utils/helmhelper/helm.go index 1676250..cf8cc90 100644 --- a/internal/utils/helmhelper/helm.go +++ b/internal/utils/helmhelper/helm.go @@ -3,14 +3,12 @@ package helmhelper import ( "fmt" "os" + "strings" "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" "helm.sh/helm/v3/pkg/action" - "helm.sh/helm/v3/pkg/chart/loader" - "helm.sh/helm/v3/pkg/chartutil" "helm.sh/helm/v3/pkg/cli" - "helm.sh/helm/v3/pkg/engine" "helm.sh/helm/v3/pkg/getter" "helm.sh/helm/v3/pkg/registry" "helm.sh/helm/v3/pkg/repo" @@ -26,12 +24,12 @@ func getDownloadDirPath(workdirPath string) string { return fmt.Sprintf("%s/.charts", workdirPath) } -func getChartDirPath(downloadDirPath string, release *ReleaseData) string { - return fmt.Sprintf("%s/%s-%s-%s", downloadDirPath, release.RepositoryName, release.Chart, release.Version) +func getChartDirPath(downloadDirPath string, release *ChartData) string { + return fmt.Sprintf("%s/%s-%s-%s", downloadDirPath, release.RepositoryName, release.Name, release.Version) } -func (h *Helm) PullChart(workdirPath string, release *ReleaseData) (path string, err error) { +func (h *Helm) PullChart(workdirPath string, release *ChartData) (path string, err error) { downloadDirPath := getDownloadDirPath(workdirPath) if err := os.MkdirAll(downloadDirPath, 0777); err != nil { return "", err @@ -71,10 +69,11 @@ func (h *Helm) PullChart(workdirPath string, release *ReleaseData) (path string, client := action.NewPullWithOpts(action.WithConfig(config)) client.SetRegistryClient(registry) + client.Untar = true client.DestDir = chartDir client.Settings = cl - chartRemote := fmt.Sprintf("%s/%s", path, release.Chart) + chartRemote := fmt.Sprintf("%s/%s", path, release.Name) logrus.Infof("trying to pull: %s", chartRemote) if _, err = client.Run(chartRemote); err != nil { return "", err @@ -88,7 +87,7 @@ func (h *Helm) PullChart(workdirPath string, release *ReleaseData) (path string, return path, nil } -func (h *Helm) FindLatestVersion(workdirPath string, release *ReleaseData) (version string, err error) { +func (h *Helm) FindLatestVersion(workdirPath string, release *ChartData) (version string, err error) { downloadDirPath := getDownloadDirPath(workdirPath) if err := os.MkdirAll(downloadDirPath, 0777); err != nil { return "", err @@ -117,47 +116,95 @@ func (h *Helm) FindLatestVersion(workdirPath string, release *ReleaseData) (vers if err != nil { return "", err } - logrus.Infof("the latest version of %s is %s", release.Chart, chartData.Version) + logrus.Infof("the latest version of %s is %s", release.Name, chartData.Version) + release.Version = chartData.Version versionedChartDir := getChartDirPath(downloadDirPath, release) - os.Rename(chartDir, versionedChartDir) + if err := os.Rename(chartDir, versionedChartDir); err != nil { + return "", err + } return chartData.Version, err } -func (h *Helm) RenderChart(workdirPath string, release *ReleaseData) error { +func (h *Helm) PushChart(workdirPath string, server, prefix, username, password string, chartdata *ChartData) (err error) { downloadDirPath := getDownloadDirPath(workdirPath) - chartDirPath := getChartDirPath(downloadDirPath, release) - chartPath, err := getChartPathFromDir(chartDirPath) - if err != nil { + chartDir := getChartDirPath(downloadDirPath, chartdata) + _, err = os.Stat(chartDir) + if err != nil && !os.IsNotExist(err) { + return err + } else if os.IsNotExist(err) { + if err := os.Mkdir(chartDir, 0777); err != nil { + return err + } + } + regclient, err := registry.NewClient() + options := registry.LoginOptBasicAuth("allanger", "") + tls := registry.LoginOptInsecure(true) + serverClean := strings.ReplaceAll(server, "oci://", "") + + if err :=regclient.Login(serverClean, options, tls); err != nil { return err } - logrus.Info(fmt.Sprintf("%s/%s", chartDirPath, chartPath)) - chartObj, err := loader.Load(fmt.Sprintf("%s/%s", chartDirPath, chartPath)) - if err != nil { - return err - } - values := chartutil.Values{} - values["Values"] = chartObj.Values - values["Release"] = map[string]string{ - "Name": release.Name, - "Namespace": release.Namespace, - } - values["Capabilities"] = map[string]map[string]string{ - "KubeVersion": { - "Version": "v1.27.9", - "GitVersion": "v1.27.9", - }, - } - files, err := engine.Engine{Strict: false}.Render(chartObj, values) - if err != nil { - return err - } - logrus.Info(files) - for file, data := range files { - logrus.Infof("%s - %s", file, data) - } - logrus.Info("I'm here") - return nil + if err != nil { + return err + } + versionedChartDir := fmt.Sprintf("%s/%s", getChartDirPath(downloadDirPath, chartdata), chartdata.Name) + + tar := action.NewPackage() + tar.Destination = downloadDirPath + logrus.Info(versionedChartDir) + logrus.Info(chartDir) + tarname, err := tar.Run(versionedChartDir, nil) + // tarpath := fmt.Sprintf("%s/%s", versionedChartDir, tarname) + if err != nil { + return err + } + client := action.NewPushWithOpts(action.WithPushConfig(&action.Configuration{})) + logrus.Infof("trying to push: %s", tarname) + client.Settings = &cli.EnvSettings{} + smth, err := client.Run(tarname, fmt.Sprintf("%s/%s", server, prefix)) + if err != nil { + return err + } + logrus.Info(smth) + return nil } +func (h *Helm) RenderChart(workdirPath string, release *ReleaseData) error { + return nil +// downloadDirPath := getDownloadDirPath(workdirPath) +// chartDirPath := getChartDirPath(downloadDirPath, release) +// chartPath, err := getChartPathFromDir(chartDirPath) +// if err != nil { +// return err +// } +// logrus.Info(fmt.Sprintf("%s/%s", chartDirPath, chartPath)) +// chartObj, err := loader.Load(fmt.Sprintf("%s/%s", chartDirPath, chartPath)) +// if err != nil { +// return err +// } +// values := chartutil.Values{} +// values["Values"] = chartObj.Values +// values["Release"] = map[string]string{ +// "Name": release.Name, +// "Namespace": release.Namespace, +// } +// values["Capabilities"] = map[string]map[string]string{ +// "KubeVersion": { +// "Version": "v1.27.9", +// "GitVersion": "v1.27.9", +// }, +// } +// files, err := engine.Engine{Strict: false}.Render(chartObj, values) +// if err != nil { +// return err +// } +// logrus.Info(files) +// for file, data := range files { +// logrus.Infof("%s - %s", file, data) +// } +// logrus.Info("I'm here") +// return nil +} + func getChartPathFromDir(downloadDir string) (file string, err error) { files, err := os.ReadDir(downloadDir) @@ -171,8 +218,8 @@ func getChartPathFromDir(downloadDir string) (file string, err error) { return files[0].Name(), nil } -func chartFromString(info string) (*ReleaseData, error) { - releaseData := new(ReleaseData) +func chartFromString(info string) (*ChartData, error) { + releaseData := new(ChartData) if err := yaml.Unmarshal([]byte(info), &releaseData); err != nil { return nil, err } diff --git a/internal/utils/helmhelper/mock.go b/internal/utils/helmhelper/mock.go index d047f44..7295ae9 100644 --- a/internal/utils/helmhelper/mock.go +++ b/internal/utils/helmhelper/mock.go @@ -11,14 +11,18 @@ func NewHelmMock() Helmhelper { return &Mock{} } -func (h *Mock) FindLatestVersion(workdir string, release *ReleaseData) (version string, err error) { +func (h *Mock) FindLatestVersion(workdir string, release *ChartData) (version string, err error) { return MOCK_LATEST_VERSION, nil } -func (h *Mock) PullChart(workdirPath string, release *ReleaseData) (path string, err error) { +func (h *Mock) PullChart(workdirPath string, release *ChartData) (path string, err error) { return MOCK_CHART_PATH, nil } func (h *Mock) RenderChart(workdirPath string, release *ReleaseData) error { return nil } + +func (h *Mock) PushChart(chartDir string, server, prefix, username, password string, chartdata *ChartData) (err error) { + return nil +} diff --git a/internal/utils/helmhelper/types.go b/internal/utils/helmhelper/types.go index eb66be3..9db3722 100644 --- a/internal/utils/helmhelper/types.go +++ b/internal/utils/helmhelper/types.go @@ -1,9 +1,10 @@ package helmhelper type Helmhelper interface { - FindLatestVersion(workdirPath string, release *ReleaseData) (string, error) - PullChart(workdirPath string, release *ReleaseData) (string, error) + FindLatestVersion(workdirPath string, release *ChartData) (string, error) + PullChart(workdirPath string, release *ChartData) (string, error) RenderChart(workdirPath string, release *ReleaseData) error + PushChart(chartDir, server, prefix, username, password string, chart *ChartData) error } type ReleaseData struct { @@ -16,3 +17,12 @@ type ReleaseData struct { RepositoryKind string ValuesData string } + + +type ChartData struct { + Name string + Version string + RepositoryName string + RepositoryURL string + RepositoryKind string +} diff --git a/pkg/chart/chart.go b/pkg/chart/chart.go new file mode 100644 index 0000000..4050bc6 --- /dev/null +++ b/pkg/chart/chart.go @@ -0,0 +1,113 @@ +package chart + +import ( + "fmt" + + "git.badhouseplants.net/allanger/shoebill/internal/utils/helmhelper" + "git.badhouseplants.net/allanger/shoebill/pkg/mirror" + "git.badhouseplants.net/allanger/shoebill/pkg/repository" + "github.com/sirupsen/logrus" + "k8s.io/utils/strings/slices" +) +type Chart struct { + // Internal name of the chart + Name string + // Official name of the chart + Chart string + // Name of the repository to pull from + // Defined in repositories + Repository string + // Version of a chart + Version string + Mirrors []string + // Private fields that should be pupulated during the run-time + RepositoryObj *repository.Repository `yaml:"-"` + MirrorObjs mirror.Mirrors `yaml:"-"` +} + +type Charts []*Chart + +// Possible version placeholders +const ( + VERSION_LATEST = "latest" +) + +func (r *Chart) MirrorObjsFromName(mirrors mirror.Mirrors) { + var mirObj mirror.Mirrors + for _, mir := range mirrors{ + if slices.Contains(r.Mirrors, mir.Name){ + mirObj = append(mirObj, mir) + } + } + r.MirrorObjs = mirObj +} + +func (ch *Chart) SyncMirrors(workDir string, mirrors mirror.Mirrors, hh helmhelper.Helmhelper) error { + if len(ch.Mirrors) > 0 { + ch.MirrorObjsFromName(mirrors) + _, err := hh.PullChart(workDir, ch.ToHelmReleaseData()) + if err != nil { + return err + } + for _, mr := range ch.MirrorObjs { + + err := hh.PushChart(workDir, mr.OCI.URL, mr.OCI.Prefix, "", "", ch.ToHelmReleaseData()) + if err != nil { + return err + } + } + } + return nil +} +// RepositoryObjFromName gather the whole repository object by its name +func (r *Chart) RepositoryObjFromName(repos repository.Repositories) error { + for _, repo := range repos { + if repo.Name == r.Repository { + r.RepositoryObj = repo + } + } + if r.RepositoryObj == nil { + return fmt.Errorf("couldn't gather the RepositoryObj for %s", r.Repository) + } + return nil +} + +func (chs *Charts) PopulateRepositories(repos repository.Repositories) error { + for _, ch := range *chs { + if err := ch.RepositoryObjFromName(repos); err != nil { + return err + } + } + return nil +} +// Replace the version placeholder with the fixed version +func (r *Chart) VersionHandler(dir string, hh helmhelper.Helmhelper) error { + if len(r.Version) == 0 { + r.Version = VERSION_LATEST + } + switch r.Version { + case VERSION_LATEST: + version, err := hh.FindLatestVersion(dir, r.ToHelmReleaseData()) + if err != nil { + return err + } + r.Version = version + + } + return nil +} + +func (r *Chart) ToHelmReleaseData() *helmhelper.ChartData { + // valuesData = + // for _, data := range r.DestValues { + + // } + logrus.Info(r) + return &helmhelper.ChartData{ + Name: r.Chart, + Version: r.Version, + RepositoryName: r.RepositoryObj.Name, + RepositoryURL: r.RepositoryObj.URL, + RepositoryKind: r.RepositoryObj.Kind, + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 1493b14..f38d256 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -3,7 +3,9 @@ package config import ( "os" + "git.badhouseplants.net/allanger/shoebill/pkg/chart" "git.badhouseplants.net/allanger/shoebill/pkg/cluster" + "git.badhouseplants.net/allanger/shoebill/pkg/mirror" "git.badhouseplants.net/allanger/shoebill/pkg/release" "git.badhouseplants.net/allanger/shoebill/pkg/repository" "github.com/sirupsen/logrus" @@ -14,6 +16,8 @@ type Config struct { Repositories repository.Repositories Releases release.Releases Clusters cluster.Clusters + Charts chart.Charts + Mirrors mirror.Mirrors ConfigPath string `yaml:"-"` SopsBin string `yaml:"-"` } diff --git a/pkg/mirror/mirror.go b/pkg/mirror/mirror.go new file mode 100644 index 0000000..095d1f0 --- /dev/null +++ b/pkg/mirror/mirror.go @@ -0,0 +1,22 @@ +package mirror + +import ( + "git.badhouseplants.net/allanger/shoebill/internal/utils/helmhelper" +) + +type Mirror struct { + Name string + OCI *OCIMirror +} + +type OCIMirror struct { + URL string + Prefix string +} + +type Mirrors []*Mirror + +func (m *Mirror) Auth(dir string, hh helmhelper.Helmhelper) error{ + + return nil +} diff --git a/pkg/release/release.go b/pkg/release/release.go index 32e6139..30f2a59 100644 --- a/pkg/release/release.go +++ b/pkg/release/release.go @@ -9,7 +9,9 @@ import ( "git.badhouseplants.net/allanger/shoebill/internal/utils/helmhelper" "git.badhouseplants.net/allanger/shoebill/internal/utils/sopshelper" + "git.badhouseplants.net/allanger/shoebill/pkg/chart" "git.badhouseplants.net/allanger/shoebill/pkg/lockfile" + "git.badhouseplants.net/allanger/shoebill/pkg/mirror" "git.badhouseplants.net/allanger/shoebill/pkg/repository" "github.com/sirupsen/logrus" ) @@ -29,8 +31,11 @@ type Release struct { Values []string // Secrets SOPS encrypted Secrets []string + Mirror string // Private fields that should be pupulated during the run-time RepositoryObj *repository.Repository `yaml:"-"` + ChartObj *chart.Chart `yaml:"-"` + MirrorObj *mirror.Mirror `yaml:"-"` DestValues ValuesHolders `yaml:"-"` DestSecrets ValuesHolders `yaml:"-"` } @@ -82,28 +87,39 @@ func (r *Release) RepositoryObjFromName(repos repository.Repositories) error { return nil } +// RepositoryObjFromName gather the whole repository object by its name +func (r *Release) MirrorObjFromName(mirrors mirror.Mirrors) error { + for _, mir := range mirrors { + if mir.Name == r.Mirror { + r.RepositoryObj = &repository.Repository{ + Name: mir.Name, + URL: fmt.Sprintf("%s/%s", mir.OCI.URL, mir.OCI.Prefix), + } + } + } + if r.RepositoryObj == nil { + return fmt.Errorf("couldn't gather the RepositoryObj for %s", r.Repository) + } + return nil +} + +func (r *Release) ChartObjFromName(chs chart.Charts) error { + for _, ch := range chs { + if ch.Name == r.Chart { + r.ChartObj = ch + } + } + if r.ChartObj == nil { + return fmt.Errorf("couldn't gather the ChartObj for %s", r.Chart) + } + return nil +} + // Possible version placeholders const ( VERSION_LATEST = "latest" ) -// Replace the version placeholder with the fixed version -func (r *Release) VersionHandler(dir string, hh helmhelper.Helmhelper) error { - if len(r.Version) == 0 { - r.Version = VERSION_LATEST - } - switch r.Version { - case VERSION_LATEST: - version, err := hh.FindLatestVersion(dir, r.ToHelmReleaseData()) - if err != nil { - return err - } - r.Version = version - - } - return nil -} - func (r *Release) ValuesHandler(dir string) error { for i := range r.Values { r.Values[i] = fmt.Sprintf("%s/%s", dir, strings.ReplaceAll(r.Values[i], "./", "")) @@ -241,3 +257,12 @@ func (rs *Releases) PopulateRepositories(repos repository.Repositories) error { } return nil } + +func (rs *Releases) PopulateCharts(chs chart.Charts) error { + for _, r := range *rs { + if err := r.ChartObjFromName(chs); err != nil { + return err + } + } + return nil +} diff --git a/shoebill.yaml b/shoebill.yaml index 704119f..82380ae 100644 --- a/shoebill.yaml +++ b/shoebill.yaml @@ -9,10 +9,48 @@ repositories: - name: bitnami-oci url: oci://registry-1.docker.io/bitnamicharts +mirrors: + - name: badhouseplants-test + oci: + url: oci://git.badhouseplants.net + prefix: allanger/test + +charts: + - name: metrics-server + chart: metrics-server + repository: metrics-server + version: 3.11.0 + mirrors: + - badhouseplants-test + - name: istio-base + chart: base + repository: istio + version: latest + mirrors: + - badhouseplants-test + - name: istio-gateway + chart: gateway + repository: istio + version: 1.19.2 + mirrors: + - badhouseplants-test + - name: istiod + chart: istiod + repository: istio + version: 1.19.2 + mirrors: + - badhouseplants-test + - name: postgresql + chart: postgresql + repository: bitnami-oci + version: latest + mirrors: + - badhouseplants-test releases: - name: metrics-server repository: metrics-server + mirror: badhouseplants-net chart: metrics-server version: 3.11.0 installed: true @@ -21,15 +59,15 @@ releases: - name: istio-base repository: istio - chart: base + chart: istio-base installed: true namespace: istio-system createNamespace: false version: 1.19.2 - + - name: istio-ingressgateway repository: istio - chart: gateway + chart: istio-gateway version: 1.19.2 installed: true namespace: istio-system @@ -42,7 +80,7 @@ releases: installed: true namespace: istio-system createNamespace: false - + - name: postgresql-server chart: postgresql repository: bitnami-oci @@ -50,10 +88,9 @@ releases: version: latest values: - ./examples/one-config/values/postgresql.yaml - secrets: + secrets: - ./examples/one-config/secrets/postgresql.yaml - clusters: - name: cluster-shoebill-test git: git@git.badhouseplants.net:allanger/shoebill-test.git @@ -70,5 +107,3 @@ clusters: - istio-ingressgateway - istiod - postgresql-server - -