Initial logic is implemented

This commit is contained in:
Nikolai Rodionov
2023-07-20 11:26:25 +02:00
committed by Nikolai Rodionov
parent 619a86b7f8
commit 625450ca25
51 changed files with 4533 additions and 5 deletions

179
internal/utils/diff/diff.go Normal file
View File

@ -0,0 +1,179 @@
package diff
import (
"fmt"
"reflect"
"git.badhouseplants.net/allanger/shoebill/pkg/lockfile"
"git.badhouseplants.net/allanger/shoebill/pkg/release"
"git.badhouseplants.net/allanger/shoebill/pkg/repository"
"github.com/sirupsen/logrus"
)
type ReleasesDiff struct {
Action string
Current *release.Release
Wished *release.Release
}
type ReleasesDiffs []*ReleasesDiff
type RepositoriesDiff struct {
Action string
Current *repository.Repository
Wished *repository.Repository
}
type RepositoriesDiffs []*RepositoriesDiff
const (
ACTION_PRESERVE = "preserve"
ACTION_ADD = "add"
ACTION_UPDATE = "update"
ACTION_DELETE = "delete"
)
// TODO(@allanger): Naming should be better
func DiffReleases(currentReleases, wishedReleases release.Releases) (ReleasesDiffs, error) {
newDiff := ReleasesDiffs{}
for _, currentRelease := range currentReleases {
found := false
for _, wishedRelease := range wishedReleases {
if currentRelease.Release == wishedRelease.Release {
found = true
if reflect.DeepEqual(currentRelease, wishedRelease) {
newDiff = append(newDiff, &ReleasesDiff{
Action: ACTION_PRESERVE,
Current: currentRelease,
Wished: wishedRelease,
})
continue
} else {
if err := wishedRelease.RepositoryObj.KindFromUrl(); err != nil {
return nil, err
}
newDiff = append(newDiff, &ReleasesDiff{
Action: ACTION_UPDATE,
Current: currentRelease,
Wished: wishedRelease,
})
}
}
}
if !found {
newDiff = append(newDiff, &ReleasesDiff{
Action: ACTION_DELETE,
Current: currentRelease,
Wished: nil,
})
}
}
for _, wishedRelease := range wishedReleases {
found := false
for _, rSrc := range currentReleases {
if rSrc.Release == wishedRelease.Release {
found = true
continue
}
}
if !found {
if err := wishedRelease.RepositoryObj.KindFromUrl(); err != nil {
return nil, err
}
newDiff = append(newDiff, &ReleasesDiff{
Action: ACTION_ADD,
Current: nil,
Wished: wishedRelease,
})
}
}
return newDiff, nil
}
func (diff ReleasesDiffs) Resolve(currentRepositories repository.Repositories, path string) (lockfile.LockFile, RepositoriesDiffs, error) {
lockfile := lockfile.LockFile{}
wishedRepos := repository.Repositories{}
repoDiffs := RepositoriesDiffs{}
for _, diff := range diff {
switch diff.Action {
case ACTION_ADD:
logrus.Infof("adding %s", diff.Wished.Release)
lockfile = append(lockfile, diff.Wished.LockEntry())
wishedRepos = append(wishedRepos, diff.Wished.RepositoryObj)
case ACTION_PRESERVE:
logrus.Infof("preserving %s", diff.Wished.Release)
lockfile = append(lockfile, diff.Wished.LockEntry())
wishedRepos = append(wishedRepos, diff.Wished.RepositoryObj)
case ACTION_UPDATE:
logrus.Infof("updating %s", diff.Wished.Release)
lockfile = append(lockfile, diff.Wished.LockEntry())
wishedRepos = append(wishedRepos, diff.Wished.RepositoryObj)
case ACTION_DELETE:
logrus.Infof("removing %s", diff.Current.Release)
default:
return nil, nil, fmt.Errorf("unknown action is requests: %s", diff.Action)
}
}
// Repo Wished is the list of all repos that are required by the current setup
// Existing repos are all the repos in the lockfile
for _, currentRepo := range currentRepositories {
found := false
i := 0
for _, wishedRepo := range wishedRepos {
// If there is the same repo in the wished repos and in the lockfile
// We need either to udpate, or preserve. If it can't be found, just remove
// from the reposWished slice
if wishedRepo.Name == currentRepo.Name {
// If !found, should be gone from the repo
found = true
if err := wishedRepo.ValidateURL(); err != nil {
return nil, nil, err
}
if err := wishedRepo.KindFromUrl(); err != nil {
return nil, nil, err
}
if !reflect.DeepEqual(wishedRepos, currentRepo) {
repoDiffs = append(repoDiffs, &RepositoriesDiff{
Action: ACTION_UPDATE,
Current: currentRepo,
Wished: wishedRepo,
})
} else {
repoDiffs = append(repoDiffs, &RepositoriesDiff{
Action: ACTION_PRESERVE,
Current: currentRepo,
Wished: wishedRepo,
})
}
} else {
wishedRepos[i] = wishedRepo
i++
}
}
wishedRepos = wishedRepos[:i]
if !found {
repoDiffs = append(repoDiffs, &RepositoriesDiff{
Action: ACTION_DELETE,
Current: currentRepo,
Wished: nil,
})
}
}
for _, addedRepo := range wishedRepos {
repoDiffs = append(repoDiffs, &RepositoriesDiff{
Action: ACTION_ADD,
Current: nil,
Wished: addedRepo,
})
}
return lockfile, repoDiffs, nil
}

View File

@ -0,0 +1,115 @@
package githelper
import (
"errors"
"os"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/sirupsen/logrus"
)
type Git struct {
SshPrivateKeyPath string
}
func NewGit(sshPrivateKeyPath string) Githelper {
return &Git{
SshPrivateKeyPath: sshPrivateKeyPath,
}
}
func (g *Git) CloneRepo(workdir, gitURL string, dry bool) error {
// TODO(@allanger): Support ssh keys with passwords
publicKeys, err := ssh.NewPublicKeysFromFile("git", g.SshPrivateKeyPath, "")
if err != nil {
return err
}
_, err = git.PlainClone(workdir, false, &git.CloneOptions{URL: gitURL, Auth: publicKeys})
if err != nil && !errors.Is(err, git.ErrEmptyUrls) {
logrus.Info("the repo seems to be empty, I'll try to bootsrap it")
// Initialize the repo
err := os.Mkdir(workdir, 0077700)
if err != nil {
return err
}
r, err := git.PlainInit(workdir, false)
if err != nil {
return err
}
logrus.Infof("adding an origin remote: %s", gitURL)
if _, err := r.CreateRemote(&config.RemoteConfig{Name: "origin", URLs: []string{gitURL}}); err != nil {
return err
}
logrus.Info("getting the worktree")
w, err := r.Worktree()
if err != nil {
return err
}
if err := r.Storer.SetReference(plumbing.NewHashReference(plumbing.Main, plumbing.ZeroHash)); err != nil {
return err
}
logrus.Info("creating an empty 'Init Commit'")
if _, err := w.Commit("Init Commit", &git.CommitOptions{
AllowEmptyCommits: true,
}); err != nil {
return err
}
if !dry {
if err := r.Push(&git.PushOptions{RemoteName: "origin"}); err != nil {
return err
}
}
} else if err != nil {
return err
}
return nil
}
func (g *Git) AddAllAndCommit(workdir, message string) (string, error) {
r, err := git.PlainOpen(workdir)
if err != nil {
return "", err
}
w, err := r.Worktree()
if err != nil {
return "", err
}
if _, err := w.Add("."); err != nil {
return "", err
}
sha, err := w.Commit(message, &git.CommitOptions{})
if err != nil {
return "", err
}
return sha.String(), nil
}
func (g *Git) Push(workdir string) error {
r, err := git.PlainOpen(workdir)
if err != nil {
return err
}
publicKeys, err := ssh.NewPublicKeysFromFile("git", g.SshPrivateKeyPath, "")
if err != nil {
return err
}
if err := r.Push(&git.PushOptions{
RemoteName: "origin",
Auth: publicKeys,
}); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,18 @@
package githelper
type Mock struct{}
func NewGitMock() Githelper {
return &Mock{}
}
func (m *Mock) CloneRepo(workdir, gitURL string, dry bool) error {
return nil
}
func (g *Mock) AddAllAndCommit(workdir, message string) (string, error) {
return "HASH", nil
}
func (g *Mock) Push(workdir string) error {
return nil
}

View File

@ -0,0 +1,7 @@
package githelper
type Githelper interface {
CloneRepo(workdir, gitURL string, dry bool) error
AddAllAndCommit(workdir, message string) (string, error)
Push(workdir string) error
}

View File

@ -0,0 +1,180 @@
package helmhelper
import (
"fmt"
"os"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/chartutil"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/engine"
"helm.sh/helm/v3/pkg/getter"
"helm.sh/helm/v3/pkg/registry"
"helm.sh/helm/v3/pkg/repo"
)
type Helm struct{}
func NewHelm() Helmhelper {
return &Helm{}
}
func getDownloadDirPath(workdirPath string) string {
return fmt.Sprintf("%s/.charts", workdirPath)
}
func getChartDirPath(downloadDirPath string, release *ReleaseData) string {
return fmt.Sprintf("%s/%s-%s-%s", downloadDirPath, release.RepositoryName, release.Chart, release.Version)
}
func (h *Helm) PullChart(workdirPath string, release *ReleaseData) (path string, err error) {
downloadDirPath := getDownloadDirPath(workdirPath)
if err := os.MkdirAll(downloadDirPath, 0777); err != nil {
return "", err
}
config := new(action.Configuration)
cl := cli.New()
chartDir := getChartDirPath(downloadDirPath, release)
_, err = os.Stat(chartDir)
if err != nil && !os.IsNotExist(err) {
return "", nil
} else if os.IsNotExist(err) {
if err := os.Mkdir(chartDir, 0777); err != nil {
return "", err
}
registry, err := registry.NewClient()
if err != nil {
return "", err
}
var path string
// Download the chart to the workdir
if release.RepositoryKind != "oci" {
r, err := repo.NewChartRepository(&repo.Entry{
Name: release.RepositoryName,
URL: release.RepositoryURL,
}, getter.All(cl))
if err != nil {
return "", err
}
path = r.Config.Name
} else {
path = release.RepositoryURL
}
client := action.NewPullWithOpts(action.WithConfig(config))
client.SetRegistryClient(registry)
client.DestDir = chartDir
client.Settings = cl
chartRemote := fmt.Sprintf("%s/%s", path, release.Chart)
logrus.Infof("trying to pull: %s", chartRemote)
if _, err = client.Run(chartRemote); err != nil {
return "", err
}
}
path, err = getChartPathFromDir(chartDir)
if err != nil {
return "", err
}
return path, nil
}
func (h *Helm) FindLatestVersion(workdirPath string, release *ReleaseData) (version string, err error) {
downloadDirPath := getDownloadDirPath(workdirPath)
if err := os.MkdirAll(downloadDirPath, 0777); err != nil {
return "", err
}
config := new(action.Configuration)
cl := cli.New()
chartDir := getChartDirPath(downloadDirPath, release)
chartPath, err := h.PullChart(workdirPath, release)
if err != nil {
return "", err
}
showAction := action.NewShowWithConfig(action.ShowChart, config)
res, err := showAction.LocateChart(fmt.Sprintf("%s/%s", chartDir, chartPath), cl)
if err != nil {
return "", err
}
res, err = showAction.Run(res)
if err != nil {
return "", nil
}
chartData, err := chartFromString(res)
if err != nil {
return "", err
}
logrus.Infof("the latest version of %s is %s", release.Chart, chartData.Version)
versionedChartDir := getChartDirPath(downloadDirPath, release)
os.Rename(chartDir, versionedChartDir)
return chartData.Version, err
}
func (h *Helm) RenderChart(workdirPath string, release *ReleaseData) error {
downloadDirPath := getDownloadDirPath(workdirPath)
chartDirPath := getChartDirPath(downloadDirPath, release)
chartPath, err := getChartPathFromDir(chartDirPath)
if err != nil {
return err
}
logrus.Info(fmt.Sprintf("%s/%s", chartDirPath, chartPath))
chartObj, err := loader.Load(fmt.Sprintf("%s/%s", chartDirPath, chartPath))
if err != nil {
return err
}
values := chartutil.Values{}
values["Values"] = chartObj.Values
values["Release"] = map[string]string{
"Name": release.Name,
"Namespace": release.Namespace,
}
values["Capabilities"] = map[string]map[string]string{
"KubeVersion": {
"Version": "v1.27.9",
"GitVersion": "v1.27.9",
},
}
files, err := engine.Engine{Strict: false}.Render(chartObj, values)
if err != nil {
return err
}
logrus.Info(files)
for file, data := range files {
logrus.Infof("%s - %s", file, data)
}
logrus.Info("I'm here")
return nil
}
func getChartPathFromDir(downloadDir string) (file string, err error) {
files, err := os.ReadDir(downloadDir)
if err != nil {
return "", err
} else if len(files) == 0 {
return "", fmt.Errorf("expected to have one file, got zero in a dir %s", downloadDir)
} else if len(files) > 1 {
return "", fmt.Errorf("expected to have only one file in a dir %s", downloadDir)
}
return files[0].Name(), nil
}
func chartFromString(info string) (*ReleaseData, error) {
releaseData := new(ReleaseData)
if err := yaml.Unmarshal([]byte(info), &releaseData); err != nil {
return nil, err
}
return releaseData, nil
}

View File

@ -0,0 +1,24 @@
package helmhelper
const (
MOCK_LATEST_VERSION = "v1.12.1"
MOCK_CHART_PATH = ".charts/repo-release-latest/release-latest.gz"
)
type Mock struct{}
func NewHelmMock() Helmhelper {
return &Mock{}
}
func (h *Mock) FindLatestVersion(workdir string, release *ReleaseData) (version string, err error) {
return MOCK_LATEST_VERSION, nil
}
func (h *Mock) PullChart(workdirPath string, release *ReleaseData) (path string, err error) {
return MOCK_CHART_PATH, nil
}
func (h *Mock) RenderChart(workdirPath string, release *ReleaseData) error {
return nil
}

View File

@ -0,0 +1,18 @@
package helmhelper
type Helmhelper interface {
FindLatestVersion(workdirPath string, release *ReleaseData) (string, error)
PullChart(workdirPath string, release *ReleaseData) (string, error)
RenderChart(workdirPath string, release *ReleaseData) error
}
type ReleaseData struct {
Name string
Chart string
Namespace string
Version string
RepositoryName string
RepositoryURL string
RepositoryKind string
ValuesData string
}

View File

@ -0,0 +1,179 @@
package kustomize
import (
"bytes"
"errors"
"fmt"
"html/template"
"os"
"path/filepath"
"git.badhouseplants.net/allanger/shoebill/internal/utils/githelper"
"github.com/sirupsen/logrus"
kustomize_types "sigs.k8s.io/kustomize/api/types"
"sigs.k8s.io/yaml"
)
type Kusmtomize struct {
Files []string
ConfigMaps []string
Secrets []string
}
func (k *Kusmtomize) PopulateResources(path string) error {
// Main sources
files, err := os.ReadDir(fmt.Sprintf("%s/src", path))
if err != nil {
return err
}
for _, file := range files {
if file.Name() != ".gitkeep" && !file.IsDir() {
k.Files = append(k.Files, fmt.Sprintf("src/%s", file.Name()))
}
}
// Values
files, err = os.ReadDir(fmt.Sprintf("%s/src/values", path))
if err != nil {
return err
}
for _, file := range files {
k.ConfigMaps = append(k.ConfigMaps, fmt.Sprintf("src/values/%s", file.Name()))
}
// Secrets
files, err = os.ReadDir(fmt.Sprintf("%s/src/secrets", path))
if err != nil {
return err
}
for _, file := range files {
k.Secrets = append(k.Secrets, fmt.Sprintf("src/secrets/%s", file.Name()))
}
return nil
}
func (k *Kusmtomize) SecGeneratorCreate(path string) error {
logrus.Info("preparing the secret generator file")
genFileTmpl := `---
apiVersion: viaduct.ai/v1
kind: ksops
metadata:
name: shoebill-secret-gen
files:
{{- range $val := . }}
- {{ $val }}
{{- end }}
`
destFileName := fmt.Sprintf("%s/sec-generator.yaml", path)
t := template.Must(template.New("tmpl").Parse(genFileTmpl))
var genFileData bytes.Buffer
t.Execute(&genFileData, k.Secrets)
var genFile *os.File
if _, err := os.Stat(destFileName); err == nil {
genFile, err := os.Open(destFileName)
if err != nil {
return err
}
defer genFile.Close()
} else if errors.Is(err, os.ErrNotExist) {
genFile, err = os.Create(destFileName)
if err != nil {
return nil
}
defer genFile.Close()
} else {
return err
}
if err := os.WriteFile(destFileName, genFileData.Bytes(), os.ModeExclusive); err != nil {
return nil
}
return nil
}
func (k *Kusmtomize) CmGeneratorFromFiles() []kustomize_types.ConfigMapArgs {
cmGens := []kustomize_types.ConfigMapArgs{}
for _, cm := range k.ConfigMaps {
cmName := filepath.Base(cm)
cmGen := &kustomize_types.ConfigMapArgs{
GeneratorArgs: kustomize_types.GeneratorArgs{
Namespace: "flux-system",
Name: cmName,
KvPairSources: kustomize_types.KvPairSources{
FileSources: []string{cm},
},
},
}
cmGens = append(cmGens, *cmGen)
}
return cmGens
}
func Generate(path string, gh githelper.Githelper) error {
kustomize := &Kusmtomize{}
if err := kustomize.PopulateResources(path); err != nil {
return err
}
kustomization := kustomize_types.Kustomization{
TypeMeta: kustomize_types.TypeMeta{
Kind: kustomize_types.KustomizationKind,
APIVersion: kustomize_types.KustomizationVersion,
},
MetaData: &kustomize_types.ObjectMeta{
Name: "helm-root",
Namespace: "flux-system",
},
Resources: append(kustomize.Files, kustomize.ConfigMaps...),
GeneratorOptions: &kustomize_types.GeneratorOptions{
DisableNameSuffixHash: true,
},
}
if len(kustomize.Secrets) > 0 {
kustomization.Generators = []string{"sec-generator.yaml"}
if err := kustomize.SecGeneratorCreate(path); err != nil {
return err
}
} else {
if err := os.RemoveAll(fmt.Sprintf("%s/sec-generator.yaml", path)); err != nil {
return err
}
}
manifest, err := yaml.Marshal(kustomization)
if err != nil {
return err
}
dstFilePath := path + "/kustomization.yaml"
var dstFile *os.File
if _, err = os.Stat(dstFilePath); err == nil {
dstFile, err = os.Open(dstFilePath)
if err != nil {
return err
}
defer dstFile.Close()
} else if errors.Is(err, os.ErrNotExist) {
dstFile, err = os.Create(dstFilePath)
if err != nil {
return nil
}
defer dstFile.Close()
} else {
return err
}
if err := os.WriteFile(dstFilePath, manifest, os.ModeExclusive); err != nil {
return nil
}
if _, err := gh.AddAllAndCommit(path, "Update the root kustomization"); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,11 @@
package sopshelper
type SopsMock struct{}
func NewSopsMock() SopsHelper {
return &SopsMock{}
}
func (sops *SopsMock) Decrypt(filepath string) ([]byte, error) {
return nil, nil
}

View File

@ -0,0 +1,27 @@
package sopshelper
import (
// "go.mozilla.org/sops/v3/decrypt"
"os"
"github.com/getsops/sops/v3/decrypt"
"github.com/sirupsen/logrus"
)
type Sops struct{}
func NewSops() SopsHelper {
return &Sops{}
}
func (sops Sops) Decrypt(filepath string) ([]byte, error) {
logrus.Infof("trying to decrypt: %s", filepath)
encFile, err := os.ReadFile(filepath)
if err != nil {
return nil, err
}
res, err := decrypt.Data(encFile, "yaml")
if err != nil {
return nil, err
}
return res, nil
}

View File

@ -0,0 +1,5 @@
package sopshelper
type SopsHelper interface {
Decrypt(filepath string) ([]byte, error)
}

View File

@ -0,0 +1,26 @@
package workdir
import "os"
func CreateWorkdir(path string) (workdir string, err error) {
if len(path) > 0 {
// Create a dir using the path
if err := os.Mkdir(path, 0777); err != nil {
return path, err
}
// TODO(@allanger): I've got a feeling that it doesn't have to look that bad
workdir = path
} else {
// Create a temporary dir
workdir, err = os.MkdirTemp("", "shoebill")
if err != nil {
return workdir, err
}
}
return workdir, nil
}
func RemoveWorkdir(path string) (err error) {
return os.RemoveAll(path)
}