From 8e7a8998d34641bc4539a45da742c664a061cf2e Mon Sep 17 00:00:00 2001 From: Nikolai Rodionov Date: Wed, 20 Dec 2023 22:19:08 +0100 Subject: [PATCH] A big progress A big progress --- .../vaultwarden/virtual-service.yaml | 30 ++++++++ .../patches/vaultwarden/values-test.yaml | 13 ++++ .../patches/vaultwarden/values-vs.yaml | 8 +++ .../patches/vaultwarden/values.yaml | 17 +++++ internal/controller/controller.go | 3 +- internal/utils/helmhelper/helm.go | 25 +++---- pkg/chart/chart.go | 64 ++++++++++++++++- pkg/chart/patches/patches.go | 72 +++++++++++++++++++ pkg/release/release.go | 11 ++- shoebill.yaml | 22 +++++- 10 files changed, 243 insertions(+), 22 deletions(-) create mode 100644 examples/one-config/extensions/vaultwarden/virtual-service.yaml create mode 100644 examples/one-config/patches/vaultwarden/values-test.yaml create mode 100644 examples/one-config/patches/vaultwarden/values-vs.yaml create mode 100644 examples/one-config/patches/vaultwarden/values.yaml create mode 100644 pkg/chart/patches/patches.go diff --git a/examples/one-config/extensions/vaultwarden/virtual-service.yaml b/examples/one-config/extensions/vaultwarden/virtual-service.yaml new file mode 100644 index 0000000..28cb268 --- /dev/null +++ b/examples/one-config/extensions/vaultwarden/virtual-service.yaml @@ -0,0 +1,30 @@ +{{- if .Values.virtualservice.enabled -}} +{{- $fullName := include "vaultwarden.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if $.Capabilities.APIVersions.Has "networking.istio.io/v1beta1" }} +apiVersion: networking.istio.io/v1beta1 +kind: VirtalService +metadata: + name: {{ $fullName }} + labels: + {{- include "vaultwarden.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + gateways: + - {{ .Values.virtaulservice.gatewayRef }} + hosts: + - ci.badhouseplants.ne + http: + - match: + - uri: + prefix: / + route: + - destination: + host: woodpecker-ci-server + port: + number: 80 +{{- end }} +{{- end }} diff --git a/examples/one-config/patches/vaultwarden/values-test.yaml b/examples/one-config/patches/vaultwarden/values-test.yaml new file mode 100644 index 0000000..4c205de --- /dev/null +++ b/examples/one-config/patches/vaultwarden/values-test.yaml @@ -0,0 +1,13 @@ +--- +name: Replace image in default values +targets: + - ./templates/deployment.yaml + - ./templates/pvc.yaml + - ./templates/secret.yaml + - ./templates/service.yaml +before: |- + ..labels: +after: |- + labels: + "giantswarm.io/team": honeybudger + diff --git a/examples/one-config/patches/vaultwarden/values-vs.yaml b/examples/one-config/patches/vaultwarden/values-vs.yaml new file mode 100644 index 0000000..c75bdc0 --- /dev/null +++ b/examples/one-config/patches/vaultwarden/values-vs.yaml @@ -0,0 +1,8 @@ +--- +name: Replace image in default values +targets: + - values.yaml +after: |- + virtualservice: + enables: false + diff --git a/examples/one-config/patches/vaultwarden/values.yaml b/examples/one-config/patches/vaultwarden/values.yaml new file mode 100644 index 0000000..0781207 --- /dev/null +++ b/examples/one-config/patches/vaultwarden/values.yaml @@ -0,0 +1,17 @@ +--- +name: Replace image in default values +targets: + - values.yaml +before: |- + image: + repository: registry.hub.docker.com/vaultwarden/server + pullPolicy: IfNotPresent + # Overrides the image tag whose default is the chart appVersion. + tag: "" +after: |- + image: + repository: registry.hub.docker.com/vaultwarden/server + pullPolicy: Always + # Overrides the image tag whose default is the chart appVersion. + tag: "" + diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 9bc306a..20fb840 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -57,6 +57,7 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool, // Init the helm client hh := helmhelper.NewHelm() for _, ch := range conf.Charts { + ch.ExtensionsHandler(configPath) err := ch.VersionHandler(workdirPath, hh) if err != nil { return err @@ -107,7 +108,7 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool, return err } - if err := conf.Releases.PopulateRepositories(conf.Repositories); err != nil { + if err := conf.Releases.PopulateRepositories(conf.Repositories, conf.Mirrors); err != nil { return err } diff --git a/internal/utils/helmhelper/helm.go b/internal/utils/helmhelper/helm.go index cf8cc90..5b746c7 100644 --- a/internal/utils/helmhelper/helm.go +++ b/internal/utils/helmhelper/helm.go @@ -83,8 +83,11 @@ func (h *Helm) PullChart(workdirPath string, release *ChartData) (path string, e if err != nil { return "", err } - - return path, nil + logrus.Info(path) + logrus.Info(chartDir) + chartPath := fmt.Sprintf("%s/%s", chartDir, path) + logrus.Info(chartPath) + return chartPath, nil } func (h *Helm) FindLatestVersion(workdirPath string, release *ChartData) (version string, err error) { @@ -103,7 +106,7 @@ func (h *Helm) FindLatestVersion(workdirPath string, release *ChartData) (versio showAction := action.NewShowWithConfig(action.ShowChart, config) - res, err := showAction.LocateChart(fmt.Sprintf("%s/%s", chartDir, chartPath), cl) + res, err := showAction.LocateChart(chartPath, cl) if err != nil { return "", err } @@ -125,14 +128,12 @@ func (h *Helm) FindLatestVersion(workdirPath string, release *ChartData) (versio return chartData.Version, err } -func (h *Helm) PushChart(workdirPath string, server, prefix, username, password string, chartdata *ChartData) (err error) { - downloadDirPath := getDownloadDirPath(workdirPath) - chartDir := getChartDirPath(downloadDirPath, chartdata) - _, err = os.Stat(chartDir) +func (h *Helm) PushChart(chartPath string, server, prefix, username, password string, chartdata *ChartData) (err error) { + _, err = os.Stat(chartPath) if err != nil && !os.IsNotExist(err) { return err } else if os.IsNotExist(err) { - if err := os.Mkdir(chartDir, 0777); err != nil { + if err := os.Mkdir(chartPath, 0777); err != nil { return err } } @@ -147,13 +148,9 @@ func (h *Helm) PushChart(workdirPath string, server, prefix, username, password if err != nil { return err } - versionedChartDir := fmt.Sprintf("%s/%s", getChartDirPath(downloadDirPath, chartdata), chartdata.Name) - tar := action.NewPackage() - tar.Destination = downloadDirPath - logrus.Info(versionedChartDir) - logrus.Info(chartDir) - tarname, err := tar.Run(versionedChartDir, nil) + tar.Destination = chartPath + tarname, err := tar.Run(chartPath, nil) // tarpath := fmt.Sprintf("%s/%s", versionedChartDir, tarname) if err != nil { return err diff --git a/pkg/chart/chart.go b/pkg/chart/chart.go index 4050bc6..75d1f69 100644 --- a/pkg/chart/chart.go +++ b/pkg/chart/chart.go @@ -2,8 +2,11 @@ package chart import ( "fmt" + "os" + "strings" "git.badhouseplants.net/allanger/shoebill/internal/utils/helmhelper" + "git.badhouseplants.net/allanger/shoebill/pkg/chart/patches" "git.badhouseplants.net/allanger/shoebill/pkg/mirror" "git.badhouseplants.net/allanger/shoebill/pkg/repository" "github.com/sirupsen/logrus" @@ -20,6 +23,8 @@ type Chart struct { // Version of a chart Version string Mirrors []string + Extensions []string + Patches []string // Private fields that should be pupulated during the run-time RepositoryObj *repository.Repository `yaml:"-"` MirrorObjs mirror.Mirrors `yaml:"-"` @@ -45,13 +50,52 @@ func (r *Chart) MirrorObjsFromName(mirrors mirror.Mirrors) { func (ch *Chart) SyncMirrors(workDir string, mirrors mirror.Mirrors, hh helmhelper.Helmhelper) error { if len(ch.Mirrors) > 0 { ch.MirrorObjsFromName(mirrors) - _, err := hh.PullChart(workDir, ch.ToHelmReleaseData()) + path, err := hh.PullChart(workDir, ch.ToHelmReleaseData()) if err != nil { return err } - for _, mr := range ch.MirrorObjs { + for _, ext := range ch.Extensions { - err := hh.PushChart(workDir, mr.OCI.URL, mr.OCI.Prefix, "", "", ch.ToHelmReleaseData()) + files, err := os.ReadDir(ext) + if err != nil { + return err + } + for _, filePath := range files { + extensionFilePath := fmt.Sprintf("%s/%s", ext, filePath.Name()) + + file, err := os.ReadFile(extensionFilePath) + if err != nil { + return err + } + logrus.Info(path) + extenrsionTargetDir := fmt.Sprintf("%s/templates/extensions", path) + if err := os.MkdirAll(extenrsionTargetDir, os.ModePerm); err != nil { + return err + } + extensionTargetPath := fmt.Sprintf("%s/%s", extenrsionTargetDir, filePath.Name()) + if err := os.WriteFile(extensionTargetPath, file, os.ModePerm); err != nil { + return err + } + } + } + for _, patch := range ch.Patches { + files, err := os.ReadDir(patch) + if err != nil { + return err + } + for _, filePath := range files { + fullPatchPath := fmt.Sprintf("%s/%s", patch, filePath.Name()) + ptch, err := patches.NewPatchFromFile(fullPatchPath) + if err != nil { + return err + } + if err := ptch.Apply(path); err != nil { + return err + } + } + } + for _, mr := range ch.MirrorObjs { + err := hh.PushChart(path, mr.OCI.URL, mr.OCI.Prefix, "", "", ch.ToHelmReleaseData()) if err != nil { return err } @@ -59,6 +103,8 @@ func (ch *Chart) SyncMirrors(workDir string, mirrors mirror.Mirrors, hh helmhelp } return nil } + + // RepositoryObjFromName gather the whole repository object by its name func (r *Chart) RepositoryObjFromName(repos repository.Repositories) error { for _, repo := range repos { @@ -97,6 +143,18 @@ func (r *Chart) VersionHandler(dir string, hh helmhelper.Helmhelper) error { return nil } +func (r *Chart) ExtensionsHandler(dir string) { + for i := range r.Extensions { + r.Extensions[i] = fmt.Sprintf("%s/%s", dir, strings.ReplaceAll(r.Extensions[i], "./", "")) + } +} + +func (r *Chart) PatchesHandler(dir string) { + for i := range r.Patches { + r.Patches[i] = fmt.Sprintf("%s/%s", dir, strings.ReplaceAll(r.Patches[i], "./", "")) + } +} + func (r *Chart) ToHelmReleaseData() *helmhelper.ChartData { // valuesData = // for _, data := range r.DestValues { diff --git a/pkg/chart/patches/patches.go b/pkg/chart/patches/patches.go new file mode 100644 index 0000000..8f29308 --- /dev/null +++ b/pkg/chart/patches/patches.go @@ -0,0 +1,72 @@ +package patches + +import ( + "fmt" + "os" + "regexp" + + "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +type Patch struct { + Name string + Targets []string + Before string + After string +} + +type Patches []*Patch + +func NewPatchFromFile(filePath string) (*Patch, error){ + var patch Patch + logrus.Infof("reading a new patch file: %s", filePath) + patchFile, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + + if err := yaml.Unmarshal(patchFile, &patch); err != nil { + return nil, err + } + return &patch, nil +} + +func(p *Patch) Apply(chartDir string) error { + logrus.Infof("Applying patch: %s", p.Name) + if len(p.Before) > 0 { + beforeCompiled, err := regexp.Compile(p.Before) + if err != nil { + return err + } + + for _, target := range p.Targets { + fullTarget := fmt.Sprintf("%s/%s", chartDir, target) + file, err := os.ReadFile(fullTarget) + if err != nil { + return err + } + newfile := beforeCompiled.ReplaceAll(file, []byte(p.After)) + err = os.WriteFile(fullTarget, newfile, os.ModePerm) + if err != nil { + return err + } + } + } else { + for _, target := range p.Targets { + fullTarget := fmt.Sprintf("%s/%s", chartDir, target) + f, err := os.OpenFile(fullTarget, os.O_APPEND|os.O_CREATE|os.O_WRONLY, os.ModePerm) + if err != nil { + return err + } + defer f.Close() + if _, err := f.Write([]byte(p.After + "\n")); err != nil { + return err + } + if err := f.Close(); err != nil { + return err + } + } + } + return nil +} diff --git a/pkg/release/release.go b/pkg/release/release.go index 30f2a59..cbf08f5 100644 --- a/pkg/release/release.go +++ b/pkg/release/release.go @@ -35,7 +35,6 @@ type Release struct { // Private fields that should be pupulated during the run-time RepositoryObj *repository.Repository `yaml:"-"` ChartObj *chart.Chart `yaml:"-"` - MirrorObj *mirror.Mirror `yaml:"-"` DestValues ValuesHolders `yaml:"-"` DestSecrets ValuesHolders `yaml:"-"` } @@ -98,7 +97,7 @@ func (r *Release) MirrorObjFromName(mirrors mirror.Mirrors) error { } } if r.RepositoryObj == nil { - return fmt.Errorf("couldn't gather the RepositoryObj for %s", r.Repository) + return fmt.Errorf("couldn't gather the RepositoryObj from mirror for %s", r.Repository) } return nil } @@ -249,11 +248,17 @@ func (src Releases) Diff(dest Releases) Diff { return diff } -func (rs *Releases) PopulateRepositories(repos repository.Repositories) error { +func (rs *Releases) PopulateRepositories(repos repository.Repositories, mirrors mirror.Mirrors) error { for _, r := range *rs { + if len(r.Mirror) > 0 { + if err := r.MirrorObjFromName(mirrors); err != nil { + return err + } + } else { if err := r.RepositoryObjFromName(repos); err != nil { return err } + } } return nil } diff --git a/shoebill.yaml b/shoebill.yaml index 82380ae..4acd5ad 100644 --- a/shoebill.yaml +++ b/shoebill.yaml @@ -8,6 +8,10 @@ repositories: url: https://istio-release.storage.googleapis.com/charts - name: bitnami-oci url: oci://registry-1.docker.io/bitnamicharts + - name: badhouseplants-test + url: oci://git.badhouseplants.net/allanger/test + - name: badhouseplants + url: https://badhouseplants.github.io/helm-charts/ mirrors: - name: badhouseplants-test @@ -16,6 +20,17 @@ mirrors: prefix: allanger/test charts: + - name: vaultwarden + chart: vaultwarden + repository: badhouseplants + version: 1.0.0 + mirrors: + - badhouseplants-test + extensions: + - ./examples/one-config/extensions/vaultwarden/ + patches: + - ./examples/one-config/patches/vaultwarden + - name: metrics-server chart: metrics-server repository: metrics-server @@ -40,6 +55,7 @@ charts: version: 1.19.2 mirrors: - badhouseplants-test + - name: postgresql chart: postgresql repository: bitnami-oci @@ -50,7 +66,7 @@ charts: releases: - name: metrics-server repository: metrics-server - mirror: badhouseplants-net + mirror: badhouseplants-test chart: metrics-server version: 3.11.0 installed: true @@ -59,6 +75,7 @@ releases: - name: istio-base repository: istio + mirror: badhouseplants-test chart: istio-base installed: true namespace: istio-system @@ -67,6 +84,7 @@ releases: - name: istio-ingressgateway repository: istio + mirror: badhouseplants-test chart: istio-gateway version: 1.19.2 installed: true @@ -75,6 +93,7 @@ releases: - name: istiod repository: istio + mirror: badhouseplants-test version: latest chart: istiod installed: true @@ -84,6 +103,7 @@ releases: - name: postgresql-server chart: postgresql repository: bitnami-oci + mirror: badhouseplants-test namespace: postgresql-server version: latest values: