allanger/shoebill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Activity

Add mirrors

Browse Source
This commit is contained in:
Nikolai Rodionov 2023-12-20 17:36:14 +01:00
parent ed3d45a7c4
commit 398ffeb963
Signed by: allanger
GPG Key ID: 0AA46A90E25592AD
12 changed files with 375 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
LICENSE
View File

@ -1,4 +1,4 @@
Creative Commons Attribution 4.0 International
rrCreative Commons Attribution 4.0 International
Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible.

2
go.mod
View File

@ -20,6 +20,7 @@ require (
helm.sh/helm/v3 v3.12.2
k8s.io/api v0.29.0-alpha.0
k8s.io/apimachinery v0.29.0-alpha.0
k8s.io/utils v0.0.0-20230505201702-9f6742963106
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3
sigs.k8s.io/yaml v1.3.0
)
@ -216,7 +217,6 @@ require (
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
k8s.io/kubectl v0.27.2 // indirect
k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect
oras.land/oras-go v1.2.3 // indirect
sigs.k8s.io/controller-runtime v0.15.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

55
internal/controller/controller.go
View File

@ -14,6 +14,7 @@ import (
"git.badhouseplants.net/allanger/shoebill/pkg/config"
"git.badhouseplants.net/allanger/shoebill/pkg/lockfile"
"git.badhouseplants.net/allanger/shoebill/pkg/release"
"github.com/sirupsen/logrus"
)
func ReadTheConfig(path string) (*config.Config, error) {
@ -45,6 +46,29 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool,
return err
}
}
if err := conf.Charts.PopulateRepositories(conf.Repositories); err != nil {
return err
}
if err := conf.Releases.PopulateCharts(conf.Charts); err != nil {
return err
}
// Init the helm client
hh := helmhelper.NewHelm()
for _, ch := range conf.Charts {
err := ch.VersionHandler(workdirPath, hh)
if err != nil {
return err
}
logrus.Info("BEFORE")
if err := ch.SyncMirrors(workdirPath, conf.Mirrors, hh); err != nil {
return err
}
logrus.Info("AFTER")
}
// Configure a git client
gh := githelper.NewGit(sshKeyPath)
@ -87,28 +111,21 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool,
return err
}
// Init the helm client
hh := helmhelper.NewHelm()
// Init the sops client
sops := sopshelper.NewSops()
for _, release := range conf.Releases {
err := release.VersionHandler(workdirPath, hh)
if err != nil {
return err
}
if len(diffArg) > 0 {
_, err := hh.PullChart(workdirPath, release.ToHelmReleaseData())
if err != nil {
return err
}
if err := hh.RenderChart(workdirPath, release.ToHelmReleaseData()); err != nil {
return err
}
// if len(diffArg) > 0 {
// _, err := hh.PullChart(workdirPath, release.ToHelmReleaseData())
// if err != nil {
// return err
// }
// if err := hh.RenderChart(workdirPath, release.ToHelmReleaseData()); err != nil {
// return err
// }
}
// }
if err := release.ValuesHandler(configPath); err != nil {
return err
@ -127,12 +144,6 @@ func Sync(definedWorkdirPath, sshKeyPath string, conf *config.Config, dry bool,
return err
}
if len(diffArg) > 0 {
for _, releaseCurrent := range releasesCurrent {
hh.PullChart(workdirPath, releaseCurrent.ToHelmReleaseData())
}
}
// Compare releases from the lockfile to ones from the current cluster config
diffReleases, err := diff.DiffReleases(releasesCurrent, cluster.ReleasesObj)
if err != nil {

14
internal/providers/flux.go
View File

@ -188,12 +188,14 @@ func (f *Flux) SyncState(releasesDiffs diff.ReleasesDiffs, repoDiffs diff.Reposi
default:
return nil, fmt.Errorf("unknown action is requests: %s", release.Action)
}
hashPerRelease := &lockfile.HashPerRelease{
Release: release.Wished.Release,
Namespace: release.Wished.Namespace,
CommitHash: hash,
}
hashesPerReleases = append(hashesPerReleases, hashPerRelease)
if release.Wished != nil {
hashPerRelease := &lockfile.HashPerRelease{
Release: release.Wished.Release,
Namespace: release.Wished.Namespace,
CommitHash: hash,
}
hashesPerReleases = append(hashesPerReleases, hashPerRelease)
}
}

133
internal/utils/helmhelper/helm.go
View File

@ -3,14 +3,12 @@ package helmhelper
import (
"fmt"
"os"
"strings"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/engine"
"helm.sh/helm/v3/pkg/getter"
"helm.sh/helm/v3/pkg/registry"
"helm.sh/helm/v3/pkg/repo"
@ -26,12 +24,12 @@ func getDownloadDirPath(workdirPath string) string {
return fmt.Sprintf("%s/.charts", workdirPath)
}
func getChartDirPath(downloadDirPath string, release *ReleaseData) string {
return fmt.Sprintf("%s/%s-%s-%s", downloadDirPath, release.RepositoryName, release.Chart, release.Version)
func getChartDirPath(downloadDirPath string, release *ChartData) string {
return fmt.Sprintf("%s/%s-%s-%s", downloadDirPath, release.RepositoryName, release.Name, release.Version)
}
func (h *Helm) PullChart(workdirPath string, release *ReleaseData) (path string, err error) {
func (h *Helm) PullChart(workdirPath string, release *ChartData) (path string, err error) {
downloadDirPath := getDownloadDirPath(workdirPath)
if err := os.MkdirAll(downloadDirPath, 0777); err != nil {
return "", err
@ -71,10 +69,11 @@ func (h *Helm) PullChart(workdirPath string, release *ReleaseData) (path string,
client := action.NewPullWithOpts(action.WithConfig(config))
client.SetRegistryClient(registry)
client.Untar = true
client.DestDir = chartDir
client.Settings = cl
chartRemote := fmt.Sprintf("%s/%s", path, release.Chart)
chartRemote := fmt.Sprintf("%s/%s", path, release.Name)
logrus.Infof("trying to pull: %s", chartRemote)
if _, err = client.Run(chartRemote); err != nil {
return "", err
@ -88,7 +87,7 @@ func (h *Helm) PullChart(workdirPath string, release *ReleaseData) (path string,
return path, nil
}
func (h *Helm) FindLatestVersion(workdirPath string, release *ReleaseData) (version string, err error) {
func (h *Helm) FindLatestVersion(workdirPath string, release *ChartData) (version string, err error) {
downloadDirPath := getDownloadDirPath(workdirPath)
if err := os.MkdirAll(downloadDirPath, 0777); err != nil {
return "", err
@ -117,47 +116,95 @@ func (h *Helm) FindLatestVersion(workdirPath string, release *ReleaseData) (vers
if err != nil {
return "", err
}
logrus.Infof("the latest version of %s is %s", release.Chart, chartData.Version)
logrus.Infof("the latest version of %s is %s", release.Name, chartData.Version)
release.Version = chartData.Version
versionedChartDir := getChartDirPath(downloadDirPath, release)
os.Rename(chartDir, versionedChartDir)
if err := os.Rename(chartDir, versionedChartDir); err != nil {
return "", err
}
return chartData.Version, err
}
func (h *Helm) RenderChart(workdirPath string, release *ReleaseData) error {
func (h *Helm) PushChart(workdirPath string, server, prefix, username, password string, chartdata *ChartData) (err error) {
downloadDirPath := getDownloadDirPath(workdirPath)
chartDirPath := getChartDirPath(downloadDirPath, release)
chartPath, err := getChartPathFromDir(chartDirPath)
if err != nil {
chartDir := getChartDirPath(downloadDirPath, chartdata)
_, err = os.Stat(chartDir)
if err != nil && !os.IsNotExist(err) {
return err
} else if os.IsNotExist(err) {
if err := os.Mkdir(chartDir, 0777); err != nil {
return err
}
}
regclient, err := registry.NewClient()
options := registry.LoginOptBasicAuth("allanger", "")
tls := registry.LoginOptInsecure(true)
serverClean := strings.ReplaceAll(server, "oci://", "")
if err :=regclient.Login(serverClean, options, tls); err != nil {
return err
}
logrus.Info(fmt.Sprintf("%s/%s", chartDirPath, chartPath))
chartObj, err := loader.Load(fmt.Sprintf("%s/%s", chartDirPath, chartPath))
if err != nil {
return err
}
values := chartutil.Values{}
values["Values"] = chartObj.Values
values["Release"] = map[string]string{
"Name": release.Name,
"Namespace": release.Namespace,
}
values["Capabilities"] = map[string]map[string]string{
"KubeVersion": {
"Version": "v1.27.9",
"GitVersion": "v1.27.9",
},
}
files, err := engine.Engine{Strict: false}.Render(chartObj, values)
if err != nil {
return err
}
logrus.Info(files)
for file, data := range files {
logrus.Infof("%s - %s", file, data)
}
logrus.Info("I'm here")
return nil
if err != nil {
return err
}
versionedChartDir := fmt.Sprintf("%s/%s", getChartDirPath(downloadDirPath, chartdata), chartdata.Name)
tar := action.NewPackage()
tar.Destination = downloadDirPath
logrus.Info(versionedChartDir)
logrus.Info(chartDir)
tarname, err := tar.Run(versionedChartDir, nil)
// tarpath := fmt.Sprintf("%s/%s", versionedChartDir, tarname)
if err != nil {
return err
}
client := action.NewPushWithOpts(action.WithPushConfig(&action.Configuration{}))
logrus.Infof("trying to push: %s", tarname)
client.Settings = &cli.EnvSettings{}
smth, err := client.Run(tarname, fmt.Sprintf("%s/%s", server, prefix))
if err != nil {
return err
}
logrus.Info(smth)
return nil
}
func (h *Helm) RenderChart(workdirPath string, release *ReleaseData) error {
return nil
// downloadDirPath := getDownloadDirPath(workdirPath)
// chartDirPath := getChartDirPath(downloadDirPath, release)
// chartPath, err := getChartPathFromDir(chartDirPath)
// if err != nil {
// return err
// }
// logrus.Info(fmt.Sprintf("%s/%s", chartDirPath, chartPath))
// chartObj, err := loader.Load(fmt.Sprintf("%s/%s", chartDirPath, chartPath))
// if err != nil {
// return err
// }
// values := chartutil.Values{}
// values["Values"] = chartObj.Values
// values["Release"] = map[string]string{
// "Name": release.Name,
// "Namespace": release.Namespace,
// }
// values["Capabilities"] = map[string]map[string]string{
// "KubeVersion": {
// "Version": "v1.27.9",
// "GitVersion": "v1.27.9",
// },
// }
// files, err := engine.Engine{Strict: false}.Render(chartObj, values)
// if err != nil {
// return err
// }
// logrus.Info(files)
// for file, data := range files {
// logrus.Infof("%s - %s", file, data)
// }
// logrus.Info("I'm here")
// return nil
}
func getChartPathFromDir(downloadDir string) (file string, err error) {
files, err := os.ReadDir(downloadDir)
@ -171,8 +218,8 @@ func getChartPathFromDir(downloadDir string) (file string, err error) {
return files[0].Name(), nil
}
func chartFromString(info string) (*ReleaseData, error) {
releaseData := new(ReleaseData)
func chartFromString(info string) (*ChartData, error) {
releaseData := new(ChartData)
if err := yaml.Unmarshal([]byte(info), &releaseData); err != nil {
return nil, err
}

8
internal/utils/helmhelper/mock.go
View File

@ -11,14 +11,18 @@ func NewHelmMock() Helmhelper {
return &Mock{}
}
func (h *Mock) FindLatestVersion(workdir string, release *ReleaseData) (version string, err error) {
func (h *Mock) FindLatestVersion(workdir string, release *ChartData) (version string, err error) {
return MOCK_LATEST_VERSION, nil
}
func (h *Mock) PullChart(workdirPath string, release *ReleaseData) (path string, err error) {
func (h *Mock) PullChart(workdirPath string, release *ChartData) (path string, err error) {
return MOCK_CHART_PATH, nil
}
func (h *Mock) RenderChart(workdirPath string, release *ReleaseData) error {
return nil
}
func (h *Mock) PushChart(chartDir string, server, prefix, username, password string, chartdata *ChartData) (err error) {
return nil
}

14
internal/utils/helmhelper/types.go
View File

@ -1,9 +1,10 @@
package helmhelper
type Helmhelper interface {
FindLatestVersion(workdirPath string, release *ReleaseData) (string, error)
PullChart(workdirPath string, release *ReleaseData) (string, error)
FindLatestVersion(workdirPath string, release *ChartData) (string, error)
PullChart(workdirPath string, release *ChartData) (string, error)
RenderChart(workdirPath string, release *ReleaseData) error
PushChart(chartDir, server, prefix, username, password string, chart *ChartData) error
}
type ReleaseData struct {
@ -16,3 +17,12 @@ type ReleaseData struct {
RepositoryKind string
ValuesData string
}
type ChartData struct {
Name string
Version string
RepositoryName string
RepositoryURL string
RepositoryKind string
}

113
pkg/chart/chart.go Normal file
View File

@ -0,0 +1,113 @@
package chart
import (
"fmt"
"git.badhouseplants.net/allanger/shoebill/internal/utils/helmhelper"
"git.badhouseplants.net/allanger/shoebill/pkg/mirror"
"git.badhouseplants.net/allanger/shoebill/pkg/repository"
"github.com/sirupsen/logrus"
"k8s.io/utils/strings/slices"
)
type Chart struct {
// Internal name of the chart
Name string
// Official name of the chart
Chart string
// Name of the repository to pull from
// Defined in repositories
Repository string
// Version of a chart
Version string
Mirrors []string
// Private fields that should be pupulated during the run-time
RepositoryObj *repository.Repository `yaml:"-"`
MirrorObjs mirror.Mirrors `yaml:"-"`
}
type Charts []*Chart
// Possible version placeholders
const (
VERSION_LATEST = "latest"
)
func (r *Chart) MirrorObjsFromName(mirrors mirror.Mirrors) {
var mirObj mirror.Mirrors
for _, mir := range mirrors{
if slices.Contains(r.Mirrors, mir.Name){
mirObj = append(mirObj, mir)
}
}
r.MirrorObjs = mirObj
}
func (ch *Chart) SyncMirrors(workDir string, mirrors mirror.Mirrors, hh helmhelper.Helmhelper) error {
if len(ch.Mirrors) > 0 {
ch.MirrorObjsFromName(mirrors)
_, err := hh.PullChart(workDir, ch.ToHelmReleaseData())
if err != nil {
return err
}
for _, mr := range ch.MirrorObjs {
err := hh.PushChart(workDir, mr.OCI.URL, mr.OCI.Prefix, "", "", ch.ToHelmReleaseData())
if err != nil {
return err
}
}
}
return nil
}
// RepositoryObjFromName gather the whole repository object by its name
func (r *Chart) RepositoryObjFromName(repos repository.Repositories) error {
for _, repo := range repos {
if repo.Name == r.Repository {
r.RepositoryObj = repo
}
}
if r.RepositoryObj == nil {
return fmt.Errorf("couldn't gather the RepositoryObj for %s", r.Repository)
}
return nil
}
func (chs *Charts) PopulateRepositories(repos repository.Repositories) error {
for _, ch := range *chs {
if err := ch.RepositoryObjFromName(repos); err != nil {
return err
}
}
return nil
}
// Replace the version placeholder with the fixed version
func (r *Chart) VersionHandler(dir string, hh helmhelper.Helmhelper) error {
if len(r.Version) == 0 {
r.Version = VERSION_LATEST
}
switch r.Version {
case VERSION_LATEST:
version, err := hh.FindLatestVersion(dir, r.ToHelmReleaseData())
if err != nil {
return err
}
r.Version = version
}
return nil
}
func (r *Chart) ToHelmReleaseData() *helmhelper.ChartData {
// valuesData =
// for _, data := range r.DestValues {
// }
logrus.Info(r)
return &helmhelper.ChartData{
Name: r.Chart,
Version: r.Version,
RepositoryName: r.RepositoryObj.Name,
RepositoryURL: r.RepositoryObj.URL,
RepositoryKind: r.RepositoryObj.Kind,
}
}

4
pkg/config/config.go
View File

@ -3,7 +3,9 @@ package config
import (
"os"
"git.badhouseplants.net/allanger/shoebill/pkg/chart"
"git.badhouseplants.net/allanger/shoebill/pkg/cluster"
"git.badhouseplants.net/allanger/shoebill/pkg/mirror"
"git.badhouseplants.net/allanger/shoebill/pkg/release"
"git.badhouseplants.net/allanger/shoebill/pkg/repository"
"github.com/sirupsen/logrus"
@ -14,6 +16,8 @@ type Config struct {
Repositories repository.Repositories
Releases release.Releases
Clusters cluster.Clusters
Charts chart.Charts
Mirrors mirror.Mirrors
ConfigPath string `yaml:"-"`
SopsBin string `yaml:"-"`
}

22
pkg/mirror/mirror.go Normal file
View File

@ -0,0 +1,22 @@
package mirror
import (
"git.badhouseplants.net/allanger/shoebill/internal/utils/helmhelper"
)
type Mirror struct {
Name string
OCI *OCIMirror
}
type OCIMirror struct {
URL string
Prefix string
}
type Mirrors []*Mirror
func (m *Mirror) Auth(dir string, hh helmhelper.Helmhelper) error{
return nil
}

59
pkg/release/release.go
View File

@ -9,7 +9,9 @@ import (
"git.badhouseplants.net/allanger/shoebill/internal/utils/helmhelper"
"git.badhouseplants.net/allanger/shoebill/internal/utils/sopshelper"
"git.badhouseplants.net/allanger/shoebill/pkg/chart"
"git.badhouseplants.net/allanger/shoebill/pkg/lockfile"
"git.badhouseplants.net/allanger/shoebill/pkg/mirror"
"git.badhouseplants.net/allanger/shoebill/pkg/repository"
"github.com/sirupsen/logrus"
)
@ -29,8 +31,11 @@ type Release struct {
Values []string
// Secrets SOPS encrypted
Secrets []string
Mirror string
// Private fields that should be pupulated during the run-time
RepositoryObj *repository.Repository `yaml:"-"`
ChartObj *chart.Chart `yaml:"-"`
MirrorObj *mirror.Mirror `yaml:"-"`
DestValues ValuesHolders `yaml:"-"`
DestSecrets ValuesHolders `yaml:"-"`
}
@ -82,28 +87,39 @@ func (r *Release) RepositoryObjFromName(repos repository.Repositories) error {
return nil
}
// RepositoryObjFromName gather the whole repository object by its name
func (r *Release) MirrorObjFromName(mirrors mirror.Mirrors) error {
for _, mir := range mirrors {
if mir.Name == r.Mirror {
r.RepositoryObj = &repository.Repository{
Name: mir.Name,
URL: fmt.Sprintf("%s/%s", mir.OCI.URL, mir.OCI.Prefix),
}
}
}
if r.RepositoryObj == nil {
return fmt.Errorf("couldn't gather the RepositoryObj for %s", r.Repository)
}
return nil
}
func (r *Release) ChartObjFromName(chs chart.Charts) error {
for _, ch := range chs {
if ch.Name == r.Chart {
r.ChartObj = ch
}
}
if r.ChartObj == nil {
return fmt.Errorf("couldn't gather the ChartObj for %s", r.Chart)
}
return nil
}
// Possible version placeholders
const (
VERSION_LATEST = "latest"
)
// Replace the version placeholder with the fixed version
func (r *Release) VersionHandler(dir string, hh helmhelper.Helmhelper) error {
if len(r.Version) == 0 {
r.Version = VERSION_LATEST
}
switch r.Version {
case VERSION_LATEST:
version, err := hh.FindLatestVersion(dir, r.ToHelmReleaseData())
if err != nil {
return err
}
r.Version = version
}
return nil
}
func (r *Release) ValuesHandler(dir string) error {
for i := range r.Values {
r.Values[i] = fmt.Sprintf("%s/%s", dir, strings.ReplaceAll(r.Values[i], "./", ""))
@ -241,3 +257,12 @@ func (rs *Releases) PopulateRepositories(repos repository.Repositories) error {
}
return nil
}
func (rs *Releases) PopulateCharts(chs chart.Charts) error {
for _, r := range *rs {
if err := r.ChartObjFromName(chs); err != nil {
return err
}
}
return nil
}

51
shoebill.yaml
View File

@ -9,10 +9,48 @@ repositories:
- name: bitnami-oci
url: oci://registry-1.docker.io/bitnamicharts
mirrors:
- name: badhouseplants-test
oci:
url: oci://git.badhouseplants.net
prefix: allanger/test
charts:
- name: metrics-server
chart: metrics-server
repository: metrics-server
version: 3.11.0
mirrors:
- badhouseplants-test
- name: istio-base
chart: base
repository: istio
version: latest
mirrors:
- badhouseplants-test
- name: istio-gateway
chart: gateway
repository: istio
version: 1.19.2
mirrors:
- badhouseplants-test
- name: istiod
chart: istiod
repository: istio
version: 1.19.2
mirrors:
- badhouseplants-test
- name: postgresql
chart: postgresql
repository: bitnami-oci
version: latest
mirrors:
- badhouseplants-test
releases:
- name: metrics-server
repository: metrics-server
mirror: badhouseplants-net
chart: metrics-server
version: 3.11.0
installed: true
@ -21,15 +59,15 @@ releases:
- name: istio-base
repository: istio
chart: base
chart: istio-base
installed: true
namespace: istio-system
createNamespace: false
version: 1.19.2
- name: istio-ingressgateway
repository: istio
chart: gateway
chart: istio-gateway
version: 1.19.2
installed: true
namespace: istio-system
@ -42,7 +80,7 @@ releases:
installed: true
namespace: istio-system
createNamespace: false
- name: postgresql-server
chart: postgresql
repository: bitnami-oci
@ -50,10 +88,9 @@ releases:
version: latest
values:
- ./examples/one-config/values/postgresql.yaml
secrets:
secrets:
- ./examples/one-config/secrets/postgresql.yaml
clusters:
- name: cluster-shoebill-test
git: git@git.badhouseplants.net:allanger/shoebill-test.git
@ -70,5 +107,3 @@ clusters:
- istio-ingressgateway
- istiod
- postgresql-server