Initial logic is implemented

This commit is contained in:
Nikolai Rodionov 2023-07-20 11:26:25 +02:00
parent 619a86b7f8
commit 09a594ca60
No known key found for this signature in database
GPG Key ID: 19DB54039EBF8F10
41 changed files with 3404 additions and 4 deletions

65
.drone.yml Normal file
View File

@ -0,0 +1,65 @@
---
# ------------------------------------------------------------------------
# -- Unit tests should run on each commit
# ------------------------------------------------------------------------
kind: pipeline
type: docker
name: Run unit tests
trigger:
event:
- push
steps:
- name: Check formatting
image: registry.hub.docker.com/golangci/golangci-lint
commands:
- make lint
- name: Run unit tests
image: registry.hub.docker.com/library/golang
commands:
- make test
---
# ------------------------------------------------------------------------
# -- Build a container
# ------------------------------------------------------------------------
kind: pipeline
type: docker
name: Build a container
trigger:
event:
- push
steps:
- name: Build the builder image
image: alpine
privileged: true
environment:
GITEA_TOKEN:
from_secret: GITEA_TOKEN
BUILDAH_REG: git.badhouseplants.net/allanger/giops-builder
commands:
- ./build/build
- name: Cleanup the registry
image: git.badhouseplants.net/allanger/giops-builder:${DRONE_COMMIT_SHA}
privileged: true
environment:
GITEA_TOKEN:
from_secret: GITEA_TOKEN
GITEA_PACKAGE: giops-builder
commands:
- cleanup
- name: Build giops container and cleanuo the registry
image: git.badhouseplants.net/allanger/giops-builder:${DRONE_COMMIT_SHA}
privileged: true
environment:
GITEA_TOKEN:
from_secret: GITEA_TOKEN
commands:
- build-container
- cleanup

4
.gitignore vendored
View File

@ -1,13 +1,11 @@
# ---> Go
# If you prefer the allow list template instead of the deny list, see community template:
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
#
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
giops
# Test binary, built with `go test -c`
*.test

36
Containerfile Normal file
View File

@ -0,0 +1,36 @@
FROM registry.hub.docker.com/library/golang:1.20.5-alpine3.18 as builder
RUN apk update && apk upgrade && \
apk add --no-cache bash build-base
WORKDIR /opt/flux-helm-controller
COPY go.mod .
COPY go.sum .
RUN go mod download
COPY . .
ARG GOARCH
RUN GOOS=linux CGO_ENABLED=0 go build -tags build -o /usr/local/bin/flux-helm-controller main.go
FROM ghcr.io/allanger/dumb-downloader as dudo
RUN apt-get update -y && apt-get install tar -y
ARG HELM_VERSION=v3.12.1
ENV RUST_LOG=info
RUN dudo -l "https://get.helm.sh/helm-{{ version }}-{{ os }}-{{ arch }}.tar.gz" -d /tmp/helm.tar.gz -p $HELM_VERSION
RUN tar -xf /tmp/helm.tar.gz -C /tmp && rm -f /tmp/helm.tar.gz
RUN mkdir /out && for bin in `find /tmp | grep helm`; do cp $bin /out/; done
RUN chmod +x /out/helm
# Final container
FROM registry.hub.docker.com/library/alpine:3.18
LABEL org.opencontainers.image.authors="Nikolai Rodionov<allanger@zohomail.com>"
COPY --from=dudo /out/ /usr/bin
RUN apk update --no-cache && apk add openssh git yq rsync --no-cache
# # install operator binary
COPY --from=builder /usr/local/bin/flux-helm-controller /usr/local/bin/flux-helm-controller
ENTRYPOINT ["/usr/local/bin/flux-helm-controller"]

34
Makefile Normal file
View File

@ -0,0 +1,34 @@
# -----------------------------------------------
# -- Main rules
# -----------------------------------------------
build: tidy
@./scripts/build
tidy:
@go mod tidy
test: tidy
go test ./...
lint: tidy
golangci-lint run --timeout 2m
fmt:
go fmt ./...
# -----------------------------------------------
# -- Git helpers
# -----------------------------------------------
push_notes:
git push origin 'refs/notes/*'
fetch_notes:
git fetch origin 'refs/notes/*:refs/notes/*'
# -----------------------------------------------
# -- Helpers
# -----------------------------------------------
run:
go run main.go --config example.config.yaml --helm /Users/allanger/.rd/bin/helm --workdir test
cleanup:
rm -rf test

View File

@ -1,3 +1,4 @@
# giops
A templater for the gitops setup
A templater for the gitops setup

5
build/Containerfile Normal file
View File

@ -0,0 +1,5 @@
FROM registry.hub.docker.com/library/alpine
RUN apk update --no-cache&&\
apk add yq gettext openssl curl jq perl git\
buildah cni-plugins iptables ip6tables fuse-overlayfs --no-cache
COPY ./scripts/ /usr/bin/

34
build/build Executable file
View File

@ -0,0 +1,34 @@
# ------------------------------------------------------------------------
# -- Copyright 2023 Nikolai Rodionov (allanger)
# ------------------------------------------------------------------------
# -- Permission is hereby granted, without written agreement and without
# -- license or royalty fees, to use, copy, modify, and distribute this
# -- software and its documentation for any purpose, provided that the
# -- above copyright notice and the following two paragraphs appear in
# -- all copies of this software.
# --
# -- IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
# -- DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
# -- ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
# -- IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
# -- DAMAGE.
# --
# -- THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
# -- BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
# -- FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
# -- ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
# -- PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
# ---------------------------------------------------------------------------
#! /bin/sh
apk update
apk add buildah cni-plugins iptables ip6tables fuse-overlayfs
buildah login -u allanger -p $GITEA_TOKEN git.badhouseplants.net
buildah build -t $BUILDAH_REG:$DRONE_COMMIT_SHA ./build
buildah tag $BUILDAH_REG:$DRONE_COMMIT_SHA $BUILDAH_REG:latest
if [ -z ${BUILD_DEBUG+x} ]; then
buildah push $BUILDAH_REG:$DRONE_COMMIT_SHA;
buildah push $BUILDAH_REG:latest;
fi

55
build/scripts/build-container Executable file
View File

@ -0,0 +1,55 @@
#!/usr/bin/perl
# ------------------------------------------------------------------------
# -- Copyright 2023 Nikolai Rodionov (allanger)
# ------------------------------------------------------------------------
# -- Permission is hereby granted, without written agreement and without
# -- license or royalty fees, to use, copy, modify, and distribute this
# -- software and its documentation for any purpose, provided that the
# -- above copyright notice and the following two paragraphs appear in
# -- all copies of this software.
# --
# -- IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
# -- DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
# -- ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
# -- IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
# -- DAMAGE.
# --
# -- THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
# -- BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
# -- FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
# -- ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
# -- PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
# ---------------------------------------------------------------------------
use strict;
use warnings;
# ---------------------------------------------------------------------------
# -- Setup Git variables
# -- by default main branch should be "main"
# ---------------------------------------------------------------------------
my $git_branch = `git rev-parse --abbrev-ref HEAD`;
my $git_commit_sha = `git rev-parse HEAD`;
my $main_branch = $ENV{'GIT_MAIN_BRANCH'} || 'main';
chomp($git_branch);
chomp($git_commit_sha);
# ---------------------------------------------------------------------------
# -- Build the image with SHA tag
# -- my main build system is DRONE, so I'm using DRONE variables a lot
# ---------------------------------------------------------------------------
my $container_registry = $ENV{'CONTAINER_REGISTRY'} || 'git.badhouseplants.net';
my $image_name = $ENV{'DRONE_REPO'} || "badhouseplants/badhouseplants-net";
my $tag = "$container_registry/$image_name:$git_commit_sha";
my $username = $ENV{'DRONE_USERNAME'} || "allanger";
my $password = $ENV{'GITEA_TOKEN'} || "YOU NOT AUTHORIZED, PAL";
0 == system ("buildah login --username $username --password $password $container_registry") or die $!;
0 == system ("buildah build -t $tag .") or die $!;
0 == system ("buildah push $tag") or die $!;
# ---------------------------------------------------------------------------
# -- Push the latest if the branch is main
# ---------------------------------------------------------------------------
if ( $git_branch eq $main_branch) {
my $latest_tag = "$container_registry/$image_name:latest";
0 == system ("buildah tag $tag $latest_tag") or die $!;
0 == system ("buildah push $latest_tag") or die $!;
}
print "Thanks!\n";

74
build/scripts/cleanup Executable file
View File

@ -0,0 +1,74 @@
#!/usr/bin/perl
# ------------------------------------------------------------------------
# -- Copyright 2023 Nikolai Rodionov (allanger)
# ------------------------------------------------------------------------
# -- Permission is hereby granted, without written agreement and without
# -- license or royalty fees, to use, copy, modify, and distribute this
# -- software and its documentation for any purpose, provided that the
# -- above copyright notice and the following two paragraphs appear in
# -- all copies of this software.
# --
# -- IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR
# -- DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES
# -- ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN
# -- IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH
# -- DAMAGE.
# --
# -- THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING,
# -- BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
# -- FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS
# -- ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO
# -- PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS.
# ---------------------------------------------------------------------------
use strict;
use warnings;
# --------------------------------------
# -- Gitea variables
# --------------------------------------
my $gitea_url=$ENV{'GITEA_URL'} || 'https://git.badhouseplants.net/api/v1';
my $gitea_org=$ENV{'DRONE_REPO_NAMESPACE'} || 'badhouseplants';
my $gitea_package=$ENV{'GITEA_PACKAGE'} || $ENV{'DRONE_REPO_NAME'} ||'badhouseplants-net';
my $gitea_api="$gitea_url/packages/$gitea_org/container/$gitea_package";
my $gitea_list_api="$gitea_url/packages/$gitea_org?page=1&type=container&q=$gitea_package";
my $gitea_token=$ENV{'GITEA_TOKEN'};
my $gitea_user=$ENV{'GITEA_USER'} || $ENV{'DRONE_COMMIT_AUTHOR'};
# ---------------------------------------
# -- Get tags from Gitea
# ---------------------------------------
my $builds = "curl -X 'GET' \"$gitea_list_api\" -H 'accept: application/json' -H \"Authorization: token $gitea_token\" | jq -r '.[].version'";
my @builds_out = `$builds`;
chomp @builds_out;
# ---------------------------------------
# -- Get a list of all commits + 'latest'
# ---------------------------------------
my $commits = "";
if (defined $ENV{CLEANUP_ARGO}) {
$commits = "argocd app list -o yaml -l application=badhouseplants | yq '.[].metadata.labels.commit_sha'";
} else {
$commits = "git fetch && git log --format=format:%H --all";
}
my @commits_out = `$commits`;
chomp @commits_out;
push @commits_out, 'latest';
# --------------------------------------
# -- Rclone variables
# -------------------------------------
my $dirs = "rclone lsf badhouseplants-minio:/badhouseplants-net";
my @dirs_out = `$dirs`;
chomp @dirs_out;
# ---------------------------------------
# -- Compare builds to commits
# -- And remove obsolete imgages from
# -- registry
# ---------------------------------------
print "Cleaning up the container registry\n";
foreach my $line (@builds_out)
{
print "Checking if $line is in @commits_out\n\n";
if ( ! grep( /^$line$/, @commits_out ) ) {
my $cmd = "curl -X 'DELETE' -s \"$gitea_api/$line\" -H 'accept: application/json' -H \"Authorization: token $gitea_token\" || true";
print "Removing ${line}\n\n";
my $output = `$cmd`;
print "$output \n";
}
}

47
cmd/root.go Normal file
View File

@ -0,0 +1,47 @@
package cmd
import (
"context"
"fmt"
"os"
"git.badhouseplants.net/allanger/giops/internal/build"
"github.com/spf13/cobra"
)
var fullVersion = fmt.Sprintf("%s - %s", build.Version, build.CommitHash)
var longDescription = `---
Giops is just GitOps with a glottal T
It's a tool that is supposed to help engineers follow the GitOps practies
without fighting with GitOps being inapplicable to the real world.
Yeah, I quite hate this GitOps obsession, but since it's already there,
I think it makes sense to make it work.
---
Information about the build:
Version: %s (build on %s)
---
`
var (
rootCmd = &cobra.Command{
Use: "giops",
Short: "giops GitOps without pain, kinda",
Long: fmt.Sprintf(longDescription, fullVersion, build.BuildTime),
SilenceErrors: true,
SilenceUsage: true,
Version: build.Version,
}
)
func Execute(ctx context.Context) error {
rootCmd.PersistentFlags().Bool("server", false, "Set to true, if you want to start it in the deamon mode")
if err := rootCmd.ExecuteContext(ctx); err != nil {
fmt.Fprintf(os.Stderr, "Whoops. There was an error while executing your CLI '%s'", err)
os.Exit(1)
}
return nil
}

46
cmd/sync.go Normal file
View File

@ -0,0 +1,46 @@
package cmd
import (
"git.badhouseplants.net/allanger/giops/internal/controller"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
var (
sync = &cobra.Command{
Use: "sync",
Short: "sync does something",
Long: ``,
Run: syncCmd,
}
)
func init() {
sync.Flags().StringP("config", "c", "config.yaml", "A path to the configuration file")
sync.Flags().String("workdir", "", "A path to the workdir. On the moment of running, it should be an empty dir")
sync.Flags().String("ssh-key", "", "A path to the pricate ssh key")
sync.Flags().Bool("dry-run", false, "If set to false, will no push changes to git")
rootCmd.AddCommand(sync)
}
func syncCmd(cmd *cobra.Command, args []string) {
config := cmd.Flag("config").Value.String()
workdir := cmd.Flag("workdir").Value.String()
sshKey := cmd.Flag("ssh-key").Value.String()
dryRun, err := cmd.Flags().GetBool("dry-run")
if err != nil {
logrus.Fatal(err)
}
configObj, err := controller.ReadTheConfig(config)
if err != nil {
logrus.Fatal(err)
}
err = controller.Reconcile(workdir, sshKey, configObj, dryRun)
if err != nil {
logrus.Fatal(err)
}
logrus.Info("your config is synced")
}

View File

@ -0,0 +1,28 @@
---
import:
- ./repos-oci.yaml
- ./repos.yaml
repos:
- name: jetstack
url: https://charts.jetstack.io
- name: bitnami-oci
url: oci://registry-1.docker.io/bitnamicharts
releases:
- name: cert-manager
chart: jetstack
repo: jetstack
version: latest
namespace: cert-manager
- name: postgresql-server
chart: postgresql
repo: bitnami-oci
namespace: postgresql-server
version: latest
clusters:
- name: cluster-1
git: git@git.badhouseplants.net:giant-swarm-task/cluster-1.git
releases:
- cert-manager
- postgresql-server

View File

@ -0,0 +1,4 @@
---
repos:
- name: bitnami-oci
url: oci://registry-1.docker.io/bitnamicharts

View File

@ -0,0 +1,4 @@
---
repos:
- name: jetstack
url: https://charts.jetstack.io

View File

@ -0,0 +1,32 @@
---
repositories:
- name: jetstack
url: https://charts.jetstack.io
- name: bitnami-oci
url: oci://registry-1.docker.io/bitnamicharts
releases:
- name: cert-manager
chart: cert-manager
repository: jetstack
version: latest
namespace: cert-manager
- name: cert-manager-2
chart: cert-manager
repository: jetstack
version: latest
namespace: cert-manager
- name: postgresql-server
chart: postgresql
repository: bitnami-oci
namespace: postgresql-server
version: latest
clusters:
- name: cluster-giops-test
git: git@git.badhouseplants.net:allanger/giops-test.git
provider: flux
releases:
- postgresql-server
- cert-manager
# - cert-manager-2

171
go.mod Normal file
View File

@ -0,0 +1,171 @@
module git.badhouseplants.net/allanger/giops
go 1.20
// replace github.com/google/gnostic-models => github.com/google/gnostic-models v0.6.8
replace (
k8s.io/client-go => k8s.io/client-go v0.29.0-alpha.0
k8s.io/kubectl => k8s.io/kubectl v0.29.0-alpha.0
)
require (
github.com/fluxcd/helm-controller/api v0.35.0
github.com/fluxcd/source-controller/api v1.0.1
github.com/go-git/go-git/v5 v5.8.1
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.7.0
github.com/stretchr/testify v1.8.2
gopkg.in/yaml.v2 v2.4.0
helm.sh/helm/v3 v3.12.2
k8s.io/apimachinery v0.29.0-alpha.0
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3
sigs.k8s.io/yaml v1.3.0
)
require (
dario.cat/mergo v1.0.0 // indirect
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/BurntSushi/toml v1.2.1 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.1 // indirect
github.com/Masterminds/sprig/v3 v3.2.3 // indirect
github.com/Masterminds/squirrel v1.5.4 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
github.com/acomagu/bufpipe v1.0.4 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/chai2010/gettext-go v1.0.2 // indirect
github.com/cloudflare/circl v1.3.3 // indirect
github.com/containerd/containerd v1.7.0 // indirect
github.com/cyphar/filepath-securejoin v0.2.3 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/cli v23.0.1+incompatible // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v23.0.1+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-metrics v0.0.1 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.10.1 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fluxcd/pkg/apis/acl v0.1.0 // indirect
github.com/fluxcd/pkg/apis/kustomize v1.1.1 // indirect
github.com/fluxcd/pkg/apis/meta v1.1.1 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.4.1 // indirect
github.com/go-gorp/gorp/v3 v3.0.5 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/btree v1.0.1 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gosuri/uitable v0.0.4 // indirect
github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
github.com/jmoiron/sqlx v1.3.5 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.16.0 // indirect
github.com/lann/builder v0.0.0-20180802200727-47ae307949d0 // indirect
github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/locker v1.0.1 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/moby/term v0.0.0-20221205130635-1aeaba878587 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_golang v1.16.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/rubenv/sql-migrate v1.3.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.1.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/skeema/knownhosts v1.2.0 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/xlab/treeprint v1.2.0 // indirect
go.opentelemetry.io/otel v1.14.0 // indirect
go.opentelemetry.io/otel/trace v1.14.0 // indirect
go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.9.1 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect
google.golang.org/grpc v1.54.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/api v0.29.0-alpha.0 // indirect
k8s.io/apiextensions-apiserver v0.27.3 // indirect
k8s.io/apiserver v0.27.3 // indirect
k8s.io/cli-runtime v0.29.0-alpha.0 // indirect
k8s.io/client-go v0.29.0-alpha.0 // indirect
k8s.io/component-base v0.29.0-alpha.0 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
k8s.io/kubectl v0.27.2 // indirect
k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect
oras.land/oras-go v1.2.3 // indirect
sigs.k8s.io/controller-runtime v0.15.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/kustomize/kyaml v0.14.3 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
)

1145
go.sum Normal file

File diff suppressed because it is too large Load Diff

11
internal/build/build.go Normal file
View File

@ -0,0 +1,11 @@
package build
/*
* Build time variables, if you don't want to use Makefile for building,
* you still might have a look at to see how they should be configured
*/
var (
Version = "dev-0.0.0"
CommitHash = "n/a"
BuildTime = "n/a"
)

View File

@ -0,0 +1,70 @@
package cluster
import (
"errors"
"fmt"
"os"
"git.badhouseplants.net/allanger/giops/internal/config/release"
"git.badhouseplants.net/allanger/giops/internal/lockfile"
"git.badhouseplants.net/allanger/giops/internal/utils/githelper"
)
type Cluster struct {
// Public
Name string
Git string
Releases []string
Provider string
// Internal
ReleasesObj release.Releases `yaml:"-"`
}
type Clusters []*Cluster
func (c *Cluster) CloneRepo(gh githelper.Githelper, workdir string, dry bool) error {
return gh.CloneRepo(workdir, c.Git, dry)
}
func (c *Cluster) BootstrapRepo(gh githelper.Githelper, workdir string, dry bool) error {
// - Create an empty lockfile
lockfilePath := fmt.Sprintf("%s/%s", workdir, lockfile.LOCKFILE_NAME)
if _, err := os.Stat(lockfilePath); errors.Is(err, os.ErrNotExist) {
file, err := os.Create(lockfilePath)
if err != nil {
return err
}
if _, err := file.WriteString("[]"); err != nil {
return err
}
srcDir := fmt.Sprintf("%s/src", workdir)
if err := os.MkdirAll(srcDir, 0777); err != nil {
return err
}
_, err = os.Create(fmt.Sprintf("%s/.gitkeep", srcDir))
if err != nil {
return err
}
if err := gh.AddAllAndCommit(workdir, "Bootstrap the Giops repo"); err != nil {
return err
}
if !dry {
if err := gh.Push(workdir); err != nil {
return err
}
}
} else {
return err
}
return nil
}
func (c *Cluster) PopulateReleases(releases release.Releases) {
c.ReleasesObj = releases
}
func (c *Cluster) CreateNewLockfile() error {
return nil
}

View File

@ -0,0 +1 @@
package cluster_test

31
internal/config/config.go Normal file
View File

@ -0,0 +1,31 @@
package config
import (
"os"
"git.badhouseplants.net/allanger/giops/internal/config/cluster"
"git.badhouseplants.net/allanger/giops/internal/config/release"
"git.badhouseplants.net/allanger/giops/internal/config/repository"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
type Config struct {
Repositories repository.Repositories
Releases release.Releases
Clusters cluster.Clusters
}
// NewConfigFromFile populates the config struct from a configuration yaml file
func NewConfigFromFile(path string) (*Config, error) {
var config Config
logrus.Infof("reading the config file: %s", path)
configFile, err := os.ReadFile(path)
if err != nil {
return nil, err
}
if err := yaml.Unmarshal(configFile, &config); err != nil {
return nil, err
}
return &config, nil
}

View File

@ -0,0 +1,53 @@
package config_test
import (
"os"
"testing"
"git.badhouseplants.net/allanger/giops/internal/config"
"git.badhouseplants.net/allanger/giops/internal/config/repository"
"github.com/stretchr/testify/assert"
)
func helperCreateFile(t *testing.T) *os.File {
f, err := os.CreateTemp("", "sample")
if err != nil {
t.Error(err)
}
t.Logf("file is created: %s", f.Name())
return f
}
func helperFillFile(t *testing.T, f *os.File, content string) {
_, err := f.WriteString(content)
if err != nil {
t.Error(err)
}
}
func TestNewConfigFromFile(t *testing.T) {
f := helperCreateFile(t)
defer os.Remove(f.Name())
const configExample = `---
repositories:
- name: test
url: https://test.de
`
helperFillFile(t, f, configExample)
configGot, err := config.NewConfigFromFile(f.Name())
if err != nil {
t.Error(err)
}
repositoryWant := &repository.Repository{
Name: "test",
URL: "https://test.de",
}
configWant := &config.Config{
Repositories: repository.Repositories{repositoryWant},
}
assert.Equal(t, configWant.Repositories, configGot.Repositories)
}

View File

@ -0,0 +1,157 @@
package release
import (
"fmt"
"reflect"
"git.badhouseplants.net/allanger/giops/internal/config/repository"
"git.badhouseplants.net/allanger/giops/internal/lockfile"
"git.badhouseplants.net/allanger/giops/internal/utils/helmhelper"
"github.com/sirupsen/logrus"
)
type Release struct {
// Public fields, that can be set with yaml
Repository string
// Release name
Release string `yaml:"name"`
// Chart name
Chart string
// Chart version
Version string
// Namespace to install release
Namespace string
// Private fields that should be pupulated during the run-time
RepositoryObj *repository.Repository `yaml:"-"`
}
type Releases []*Release
// RepositoryObjFromName gather the whole repository object by its name
func (r *Release) RepositoryObjFromName(repos repository.Repositories) error {
for _, repo := range repos {
if repo.Name == r.Repository {
r.RepositoryObj = repo
}
}
if r.RepositoryObj == nil {
return fmt.Errorf("couldn't gather the RepositoryObj for %s", r.Repository)
}
return nil
}
// Possible version placeholders
const (
VERSION_LATEST = "latest"
)
// Replace the version placeholder with the fixed version
func (r *Release) VersionHandler(dir string, hh helmhelper.Helmhelper) error {
switch r.Version {
case VERSION_LATEST:
version, err := hh.FindLatestVersion(dir, r.Chart, *r.RepositoryObj)
if err != nil {
return err
}
r.Version = version
}
return nil
}
func FindReleaseByNames(releases []string, releasesObj Releases) Releases {
result := Releases{}
for _, rObj := range releasesObj {
for _, r := range releases {
if rObj.Release == r {
result = append(result, rObj)
}
}
}
return result
}
// Helpers
func ReleasesFromLockfile(lockfile lockfile.LockFile, repos repository.Repositories) (Releases, error) {
releases := Releases{}
for _, releaseLocked := range lockfile {
repoName, err := repos.NameByUrl(releaseLocked.RepoUrl)
if err != nil {
return releases, err
}
release := &Release{
Repository: repoName,
Release: releaseLocked.Release,
Chart: releaseLocked.Chart,
Version: releaseLocked.Version,
Namespace: releaseLocked.Namespace,
}
if err := release.RepositoryObjFromName(repos); err != nil {
return nil, err
}
releases = append(releases, release)
}
return releases, nil
}
func (r *Release) LockEntry() *lockfile.LockEntry {
return &lockfile.LockEntry{
Chart: r.Chart,
Release: r.Release,
Version: r.Version,
Namespace: r.Namespace,
RepoUrl: r.RepositoryObj.URL,
RepoName: r.RepositoryObj.Name,
}
}
type Diff struct {
Added Releases
Deleted Releases
Updated Releases
}
// TODO(@allanger): Naming should be better
func (src Releases) Diff(dest Releases) Diff {
diff := Diff{}
for _, rSrc := range src {
found := false
for _, rDest := range dest {
logrus.Infof("comparing %s to %s", rSrc.Release, rDest.Release)
if rSrc.Release == rDest.Release {
found = true
if reflect.DeepEqual(rSrc, rDest) {
continue
} else {
diff.Updated = append(diff.Updated, rDest)
}
}
}
if !found {
diff.Deleted = append(diff.Added, rSrc)
}
}
for _, rDest := range dest {
found := false
for _, rSrc := range src {
if rSrc.Release == rDest.Release {
found = true
continue
}
}
if !found {
diff.Added = append(diff.Added, rDest)
}
}
return diff
}
func (rs *Releases) PopulateRepositories(repos repository.Repositories) error {
for _, r := range *rs {
if err := r.RepositoryObjFromName(repos); err != nil {
return err
}
}
return nil
}

View File

@ -0,0 +1,126 @@
package release_test
import (
"fmt"
"testing"
"git.badhouseplants.net/allanger/giops/internal/config/release"
"git.badhouseplants.net/allanger/giops/internal/config/repository"
"git.badhouseplants.net/allanger/giops/internal/utils/helmhelper"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v2"
)
func TestRepositoryObjFromNameExisting(t *testing.T) {
repos := []*repository.Repository{
{
Name: "test0",
URL: "https://test.test",
},
{
Name: "test1",
URL: "oco://test.test",
},
}
release := &release.Release{
Repository: "test0",
}
err := release.RepositoryObjFromName(repos)
if err != nil {
t.Error(err)
}
assert.Equal(
t,
release.RepositoryObj.Name,
"test0",
fmt.Sprintf("unexpected repo name: %s", release.RepositoryObj.Name),
)
assert.Equal(
t,
release.RepositoryObj.URL,
"https://test.test",
fmt.Sprintf("unexpected repo url: %s", release.RepositoryObj.URL),
)
}
func TestRepositoryObjFromNameNonExisting(t *testing.T) {
repos := []*repository.Repository{
{
Name: "test0",
URL: "https://test.test",
},
{
Name: "test1",
URL: "oco://test.test",
},
}
release := &release.Release{
Repository: "test_notfound",
}
err := release.RepositoryObjFromName(repos)
assert.ErrorContains(t, err,
"couldn't gather the RepositoryObj for test_notfound",
fmt.Sprintf("got an unexpected error: %s", err),
)
}
func TestRepositoryObjParsing(t *testing.T) {
t.Log("Repository Object should be empty after parsing")
rls := &release.Release{}
const yamlSnippet = `---
repository: test
repositoryObj:
name: test
url: test.test
`
if err := yaml.Unmarshal([]byte(yamlSnippet), &rls); err != nil {
t.Error(err)
}
assert.Equal(t, (*repository.Repository)(nil), rls.RepositoryObj, "release object should be empty")
}
func TestRepositoryObjFillingUp(t *testing.T) {
rls := &release.Release{
Repository: "test1",
}
expectedRepo := &repository.Repository{
Name: "test1",
URL: "oci://test.test",
Kind: repository.HELM_REPO_OCI,
}
var repos repository.Repositories = repository.Repositories{
&repository.Repository{
Name: "test1",
URL: "https://test.test",
Kind: repository.HELM_REPO_DEFAULT,
},
expectedRepo,
}
if err := rls.RepositoryObjFromName(repos); err != nil {
t.Error(err)
}
assert.Equal(t, expectedRepo, rls.RepositoryObj, "release object should be empty")
}
func TestVersionHandlerLatest(t *testing.T) {
hh := helmhelper.NewHelmMock()
rls := &release.Release{
Repository: "test1",
Version: "latest",
RepositoryObj: new(repository.Repository),
}
if err := rls.VersionHandler("", hh); err != nil {
t.Error(err)
}
assert.Equal(t, helmhelper.MOCK_LATEST_VERSION, rls.Version, "unexpected latest version")
}

View File

@ -0,0 +1,64 @@
package repository
import (
"fmt"
"regexp"
"strings"
)
/*
* Helm repo kinds: default/oci
*/
const (
HELM_REPO_OCI = "oci"
HELM_REPO_DEFAULT = "default"
)
type Repository struct {
Name string
URL string
Kind string `yaml:"-"`
}
type Repositories []*Repository
// ValidateURL returns error if the repo URL doens't follow the format
func (r *Repository) ValidateURL() error {
// An regex that should check if a string is a valid repo URL
const urlRegex = "^(http|https|oci):\\/\\/.*"
valid, err := regexp.MatchString(urlRegex, r.URL)
if err != nil {
return nil
}
if !valid {
return fmt.Errorf("it's not a valid repo URL: %s", r.URL)
}
return nil
}
// KindFromUrl sets Repository.Kind according to the prefix of an URL
func (r *Repository) KindFromUrl() error {
// It panics if URL is not valid,
// but invalid url should not pass the ValidateURL function
prefix := r.URL[:strings.IndexByte(r.URL, ':')]
switch prefix {
case "oci":
r.Kind = HELM_REPO_OCI
case "https", "http":
r.Kind = HELM_REPO_DEFAULT
default:
return fmt.Errorf("unknown repo kind: %s", prefix)
}
return nil
}
func (rs Repositories) NameByUrl(repoURL string) (string, error) {
for _, r := range rs {
if repoURL == r.URL {
return r.Name, nil
}
}
return "", fmt.Errorf("repo couldn't be found in the config: %s", repoURL)
}

View File

@ -0,0 +1,107 @@
package repository_test
import (
"fmt"
"testing"
"git.badhouseplants.net/allanger/giops/internal/config/repository"
"github.com/stretchr/testify/assert"
)
func TestValidateURLHttps(t *testing.T) {
repo := &repository.Repository{
Name: "test",
URL: "https://test.test",
}
err := repo.ValidateURL()
assert.NoError(t, err, fmt.Sprintf("unexpected err occured: %s", err))
}
func TestValidateURLOci(t *testing.T) {
repo := &repository.Repository{
Name: "test",
URL: "oci://test.test",
}
err := repo.ValidateURL()
assert.NoError(t, err, fmt.Sprintf("unexpected err occured: %s", err))
}
func TestValidateURLInvalid(t *testing.T) {
repo := &repository.Repository{
Name: "test",
URL: "invalid://test.test",
}
err := repo.ValidateURL()
assert.ErrorContains(t, err,
"it's not a valid repo URL: invalid://test.test",
fmt.Sprintf("got unexpected err: %s", err),
)
}
func TestValidateURLNonURL(t *testing.T) {
repo := &repository.Repository{
Name: "test",
URL: "test",
}
err := repo.ValidateURL()
assert.ErrorContains(t, err,
"it's not a valid repo URL: test",
fmt.Sprintf("got unexpected err: %s", err),
)
}
func TestKindFromUrlDefaultHttps(t *testing.T) {
repo := &repository.Repository{
Name: "test",
URL: "https://test.test",
}
if err := repo.KindFromUrl(); err != nil {
t.Error(err)
}
assert.Equal(t, repo.Kind,
repository.HELM_REPO_DEFAULT,
fmt.Sprintf("got unexpected repo type: %s", repo.Kind),
)
}
func TestKindFromUrlDefaultHttp(t *testing.T) {
repo := &repository.Repository{
Name: "test",
URL: "http://test.test",
}
if err := repo.KindFromUrl(); err != nil {
t.Error(err)
}
assert.Equal(t, repo.Kind,
repository.HELM_REPO_DEFAULT,
fmt.Sprintf("got unexpected repo type: %s", repo.Kind),
)
}
func TestKindFromUrlDefaultOci(t *testing.T) {
repo := &repository.Repository{
Name: "test",
URL: "oci://test.test",
}
if err := repo.KindFromUrl(); err != nil {
t.Error(err)
}
assert.Equal(t, repo.Kind,
repository.HELM_REPO_OCI,
fmt.Sprintf("got unexpected repo type: %s", repo.Kind),
)
}
func TestKindFromUrlDefaultInvalid(t *testing.T) {
repo := &repository.Repository{
Name: "test",
URL: "invalid:url",
}
err := repo.KindFromUrl()
assert.ErrorContains(t, err,
"unknown repo kind: invalid",
fmt.Sprintf("got unexpected err: %s", err))
}

View File

@ -0,0 +1,132 @@
package controller
import (
"fmt"
"git.badhouseplants.net/allanger/giops/internal/config"
"git.badhouseplants.net/allanger/giops/internal/config/release"
"git.badhouseplants.net/allanger/giops/internal/lockfile"
"git.badhouseplants.net/allanger/giops/internal/providers"
"git.badhouseplants.net/allanger/giops/internal/utils/diff"
"git.badhouseplants.net/allanger/giops/internal/utils/githelper"
"git.badhouseplants.net/allanger/giops/internal/utils/helmhelper"
"git.badhouseplants.net/allanger/giops/internal/utils/kustomize"
"git.badhouseplants.net/allanger/giops/internal/utils/workdir"
)
func ReadTheConfig(path string) (*config.Config, error) {
conf, err := config.NewConfigFromFile(path)
if err != nil {
return nil, err
}
return conf, nil
}
/*
* First it must prepare the workdir
* It must create a directorry and clone all the repos that are listed in the config
*/
func Reconcile(workdirPath, sshKeyPath string, conf *config.Config, dry bool) error {
dir, err := workdir.CreateWorkdir(workdirPath)
if err != nil {
return err
}
for _, repository := range conf.Repositories {
if err := repository.ValidateURL(); err != nil {
return err
}
if err := repository.KindFromUrl(); err != nil {
return err
}
}
gh := githelper.NewGit(sshKeyPath)
for _, cluster := range conf.Clusters {
/*
* 1. Clone the cluster repo
* 2. Check if repo is already configured
* Yes -> Bootsrap the repo if it's not configured
* - Create the lockfile
* - ...
* No -> Get the current state
* 3. Turn the config file into the lockfile format and compare the actual to the desired
*/
fullPath := fmt.Sprintf("%s/%s", dir, cluster.Name)
provider, err := providers.NewProvider(cluster.Provider, fullPath, gh)
if err != nil {
return err
}
if err := cluster.CloneRepo(gh, fullPath, dry); err != nil {
return err
}
err = cluster.BootstrapRepo(gh, fullPath, dry)
if err != nil {
return err
}
lockfileData, err := lockfile.NewFromFile(fullPath)
if err != nil {
return err
}
reposExisting, err := lockfileData.ReposFromLockfile()
if err != nil {
return err
}
if err := conf.Releases.PopulateRepositories(conf.Repositories); err != nil {
return err
}
hh := helmhelper.NewHelm()
for _, release := range conf.Releases {
err := release.VersionHandler(workdirPath, hh)
if err != nil {
return err
}
}
rsObj := release.FindReleaseByNames(cluster.Releases, conf.Releases)
cluster.PopulateReleases(rsObj)
releasesCurrent, err := release.ReleasesFromLockfile(lockfileData, conf.Repositories)
if err != nil {
return err
}
diffRls, err := diff.DiffReleases(releasesCurrent, cluster.ReleasesObj)
if err != nil {
return err
}
lockfile, err := diffRls.Resolve(reposExisting, fullPath)
if err != nil {
return err
}
if err := provider.SyncState(*diffRls); err != nil {
return err
}
if err := kustomize.Generate(fullPath, gh); err != nil {
return err
}
if err := lockfile.WriteToFile(fullPath); err != nil {
return err
}
if err := gh.AddAllAndCommit(fullPath, "Update the lockfile"); err != nil {
return err
}
if !dry {
if err := gh.Push(fullPath); err != nil {
return err
}
}
}
if !dry {
if err := workdir.RemoveWorkdir(dir); err != nil {
return err
}
}
return nil
}

View File

@ -0,0 +1,87 @@
package lockfile
import (
"fmt"
"os"
"git.badhouseplants.net/allanger/giops/internal/config/repository"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
const LOCKFILE_NAME = "giops.lock.yaml"
type LockEntry struct {
Chart string
Release string
Version string
Namespace string
RepoUrl string
RepoName string
}
type LockRepository struct {
URL string
Name string
}
type LockFile []*LockEntry
func NewFromFile(dir string) (LockFile, error) {
var lockEntries LockFile
lockfilePath := fmt.Sprintf("%s/%s", dir, LOCKFILE_NAME)
logrus.Infof("reading the lockfile file: %s", lockfilePath)
lockFile, err := os.ReadFile(lockfilePath)
if err != nil {
return nil, err
}
if err := yaml.Unmarshal(lockFile, &lockEntries); err != nil {
return nil, err
}
return lockEntries, nil
}
func (lockfile LockFile) ReposFromLockfile() (repository.Repositories, error) {
reposEntries := []LockRepository{}
for _, lockentry := range lockfile {
newRepoEntry := &LockRepository{
URL: lockentry.RepoUrl,
Name: lockentry.RepoName,
}
reposEntries = append(reposEntries, *newRepoEntry)
}
allKeys := make(map[string]bool)
dedupedEntries := []LockRepository{}
for _, repo := range reposEntries {
if _, value := allKeys[repo.Name]; !value {
allKeys[repo.Name] = true
dedupedEntries = append(dedupedEntries, repo)
}
}
repos := repository.Repositories{}
for _, repoEntry := range dedupedEntries {
repo := &repository.Repository{
Name: repoEntry.Name,
URL: repoEntry.URL,
}
if err := repo.KindFromUrl(); err != nil {
return nil, err
}
repos = append(repos, repo)
}
return repos, nil
}
func (lf LockFile) WriteToFile(dir string) error {
lockfilePath := fmt.Sprintf("%s/%s", dir, LOCKFILE_NAME)
lockfileContent, err := yaml.Marshal(lf)
if err != nil {
return err
}
if err := os.WriteFile(lockfilePath, lockfileContent, os.ModeExclusive); err != nil {
return nil
}
return nil
}

212
internal/providers/flux.go Normal file
View File

@ -0,0 +1,212 @@
package providers
import (
"fmt"
"os"
"git.badhouseplants.net/allanger/giops/internal/config/release"
"git.badhouseplants.net/allanger/giops/internal/config/repository"
"git.badhouseplants.net/allanger/giops/internal/utils/diff"
"git.badhouseplants.net/allanger/giops/internal/utils/githelper"
release_v2beta1 "github.com/fluxcd/helm-controller/api/v2beta1"
helmrepo_v1beta2 "github.com/fluxcd/source-controller/api/v1beta2"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/yaml"
)
type Flux struct {
path string
gh githelper.Githelper
}
func FluxProvider(path string, gh githelper.Githelper) Provider {
return &Flux{
path: path,
gh: gh,
}
}
func (f *Flux) SyncState(diff diff.Diff) error {
entity := "repository"
srcPath := fmt.Sprintf("%s/src", f.path)
filePath := fmt.Sprintf("%s/%s-", srcPath, entity)
for _, repo := range diff.DeletedRepositories {
if err := os.Remove(filePath + repo.Name + ".yaml"); err != nil {
return err
}
message := `chore(repository): Removed a repo: %s
A repo has been removed from the cluster:
Name: %s
URL: %s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repo.Name, repo.Name, repo.URL)); err != nil {
return err
}
}
for _, repo := range diff.UpdatedRepositories {
manifest, err := GenerateRepository(repo)
if err != nil {
return err
}
if err := os.WriteFile(filePath+repo.Name+".yaml", manifest, os.ModeExclusive); err != nil {
return err
}
message := `chore(repository): Update a repo: %s
A repo has been updated:
Name: %s
URL: %s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repo.Name, repo.Name, repo.URL)); err != nil {
return err
}
}
for _, repo := range diff.AddedRepositories {
manifest, err := GenerateRepository(repo)
if err != nil {
return err
}
file, err := os.Create(filePath + repo.Name + ".yaml")
if err != nil {
return err
}
if _, err := file.Write(manifest); err != nil {
return err
}
message := `chore(repository): Add a repo: %s
A new repo added to the cluster:
Name: %s
URL: %s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, repo.Name, repo.Name, repo.URL)); err != nil {
return err
}
}
entity = "release"
filePath = fmt.Sprintf("%s/%s-", srcPath, entity)
for _, release := range diff.AddedReleases {
manifest, err := GenerateRelease(release)
if err != nil {
return err
}
file, err := os.Create(filePath + release.Release + ".yaml")
if err != nil {
return err
}
if _, err := file.Write(manifest); err != nil {
return err
}
message := `chore(release): Add a new release: %s
A new release is added to the cluster:
Name: %s
Namespace: %s
Version: %s
Chart: %s/%s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Release, release.Release, release.Namespace, release.Version, release.Repository, release.Release)); err != nil {
return err
}
}
for _, release := range diff.UpdatedReleases {
manifest, err := GenerateRelease(release)
if err != nil {
return err
}
if err := os.WriteFile(filePath+release.Release+".yaml", manifest, os.ModeExclusive); err != nil {
return err
}
message := `chore(release): Update a release: %s
A release has been updated:
Name: %s
Namespace: %s
Version: %s
Chart: %s/%s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Release, release.Release, release.Namespace, release.Version, release.Repository, release.Release)); err != nil {
return err
}
}
for _, release := range diff.DeletedReleases {
if err := os.Remove(filePath + release.Release + ".yaml"); err != nil {
return err
}
message := `chore(release): Remove a release: %s
A release has been removed from the cluster:
Name: %s
Namespace: %s
Version: %s
Chart: %s/%s
`
if err := f.gh.AddAllAndCommit(f.path, fmt.Sprintf(message, release.Release, release.Release, release.Namespace, release.Version, release.Repository, release.Release)); err != nil {
return err
}
}
return nil
}
func GenerateRepository(repo *repository.Repository) ([]byte, error) {
fluxRepo := &helmrepo_v1beta2.HelmRepository{
TypeMeta: v1.TypeMeta{
Kind: helmrepo_v1beta2.HelmRepositoryKind,
APIVersion: helmrepo_v1beta2.GroupVersion.String(),
},
ObjectMeta: v1.ObjectMeta{
Name: repo.Name,
Namespace: "flux-namespace",
},
Spec: helmrepo_v1beta2.HelmRepositorySpec{
URL: repo.URL,
Type: repo.Kind,
},
}
return yaml.Marshal(&fluxRepo)
}
// GenerateRelease and put
func GenerateRelease(release *release.Release) ([]byte, error) {
fluxRelease := &release_v2beta1.HelmRelease{
TypeMeta: v1.TypeMeta{
Kind: release_v2beta1.HelmReleaseKind,
APIVersion: release_v2beta1.GroupVersion.String(),
},
ObjectMeta: v1.ObjectMeta{
Name: release.Release,
Namespace: "flux-namespace",
},
Spec: release_v2beta1.HelmReleaseSpec{
Chart: release_v2beta1.HelmChartTemplate{
Spec: release_v2beta1.HelmChartTemplateSpec{
Chart: release.Chart,
Version: release.Version,
SourceRef: release_v2beta1.CrossNamespaceObjectReference{
Kind: helmrepo_v1beta2.HelmRepositoryKind,
Name: release.RepositoryObj.Name,
Namespace: "flux-namespace",
},
},
},
ReleaseName: release.Release,
Install: &release_v2beta1.Install{
CRDs: release_v2beta1.Create,
CreateNamespace: true,
},
TargetNamespace: "release-namespace",
},
}
return yaml.Marshal(&fluxRelease)
}

View File

@ -0,0 +1,21 @@
package providers
import (
"fmt"
"git.badhouseplants.net/allanger/giops/internal/utils/diff"
"git.badhouseplants.net/allanger/giops/internal/utils/githelper"
)
type Provider interface {
SyncState(diff diff.Diff) error
}
func NewProvider(provider, path string, gh githelper.Githelper) (Provider, error) {
switch provider {
case "flux":
return FluxProvider(path, gh), nil
default:
return nil, fmt.Errorf("provider is not supported: %s", provider)
}
}

144
internal/utils/diff/diff.go Normal file
View File

@ -0,0 +1,144 @@
package diff
import (
"reflect"
"git.badhouseplants.net/allanger/giops/internal/config/release"
"git.badhouseplants.net/allanger/giops/internal/config/repository"
"git.badhouseplants.net/allanger/giops/internal/lockfile"
"github.com/sirupsen/logrus"
)
type Diff struct {
AddedReleases release.Releases
DeletedReleases release.Releases
UpdatedReleases release.Releases
PreservedReleases release.Releases
AddedRepositories repository.Repositories
DeletedRepositories repository.Repositories
UpdatedRepositories repository.Repositories
PreservedRepositories repository.Repositories
}
const (
ACTION_ADD = "add"
ACTION_UPDATE = "update"
ACTION_DELETE = "delete"
)
// TODO(@allanger): Naming should be better
func DiffReleases(src, dest release.Releases) (*Diff, error) {
diff := &Diff{}
for _, rSrc := range src {
found := false
for _, rDest := range dest {
if rSrc.Release == rDest.Release {
found = true
if reflect.DeepEqual(rSrc, rDest) {
diff.PreservedReleases = append(diff.PreservedReleases, rSrc)
continue
} else {
if err := rDest.RepositoryObj.KindFromUrl(); err != nil {
return nil, err
}
diff.UpdatedReleases = append(diff.UpdatedReleases, rDest)
}
}
}
if !found {
// for i, entry := range lockFile {
// if entry.Release == rSrc.Release {
// lockFile[i] = lockFile[len(lockFile)-1]
// lockFile = lockFile[:len(lockFile)-1]
// }
// }
diff.DeletedReleases = append(diff.DeletedReleases, rSrc)
}
}
for _, rDest := range dest {
found := false
for _, rSrc := range src {
if rSrc.Release == rDest.Release {
found = true
continue
}
}
if !found {
if err := rDest.RepositoryObj.KindFromUrl(); err != nil {
return nil, err
}
diff.AddedReleases = append(diff.AddedReleases, rDest)
}
}
return diff, nil
}
func (diff *Diff) Resolve(repositories repository.Repositories, path string) (lockfile.LockFile, error) {
lockfile := lockfile.LockFile{}
reposWished := repository.Repositories{}
for _, p := range diff.PreservedReleases {
lockfile = append(lockfile, p.LockEntry())
reposWished = append(reposWished, p.RepositoryObj)
}
for _, a := range diff.AddedReleases {
logrus.Infof("adding %s", a.Release)
lockfile = append(lockfile, a.LockEntry())
reposWished = append(reposWished, a.RepositoryObj)
}
for _, u := range diff.UpdatedReleases {
logrus.Infof("updating %s", u.Release)
lockfile = append(lockfile, u.LockEntry())
reposWished = append(reposWished, u.RepositoryObj)
}
// Repo Wished is the list of all repos that are required by the current setup, we need to
// Existing repos are all the repos in the lockfile
for _, repoExisting := range repositories {
found := false
i := 0
for _, repoWished := range reposWished {
logrus.Infof("DEBUG: exst %s tp wished %s", repoExisting.Name, repoWished.Name)
// If there is the same repo in the wished repos and in the lockfile
// We need either to udpate, or preserve. If it can't be found, just remove
// from the reposWished slice
if repoWished.Name == repoExisting.Name {
// If !found, should be gone from the repo
found = true
if err := repoWished.ValidateURL(); err != nil {
return nil, err
}
if err := repoWished.KindFromUrl(); err != nil {
return nil, err
}
if !reflect.DeepEqual(reposWished, repoExisting) {
logrus.Info("DEBUG: Exists")
diff.UpdatedRepositories = append(diff.UpdatedRepositories, repoWished)
} else {
logrus.Info("DEBUG: Updated")
diff.PreservedRepositories = append(diff.PreservedRepositories, repoWished)
}
// Delete the
} else {
reposWished[i] = repoWished
logrus.Infof("%v -- %v", repoExisting, repoWished)
i++
}
}
reposWished = reposWished[:i]
if !found {
logrus.Infof("HERE I AM: %s", repoExisting)
diff.DeletedRepositories = append(diff.DeletedRepositories, repoExisting)
}
}
for _, repo := range reposWished {
logrus.Infof("DEBUG: Will add %s", repo.Name)
}
diff.AddedRepositories = append(diff.AddedRepositories, reposWished...)
return lockfile, nil
}

View File

@ -0,0 +1,113 @@
package githelper
import (
"errors"
"os"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/sirupsen/logrus"
)
type Git struct {
SshPrivateKeyPath string
}
func NewGit(sshPrivateKeyPath string) Githelper {
return &Git{
SshPrivateKeyPath: sshPrivateKeyPath,
}
}
func (g *Git) CloneRepo(workdir, gitURL string, dry bool) error {
// TODO(@allanger): Support ssh keys with passwords
publicKeys, err := ssh.NewPublicKeysFromFile("git", g.SshPrivateKeyPath, "")
if err != nil {
return err
}
_, err = git.PlainClone(workdir, false, &git.CloneOptions{URL: gitURL, Auth: publicKeys})
if err != nil && !errors.Is(err, git.ErrEmptyUrls) {
logrus.Info("the repo seems to be empty, I'll try to bootsrap it")
// Initialize the repo
err := os.Mkdir(workdir, 0077700)
if err != nil {
return err
}
r, err := git.PlainInit(workdir, false)
if err != nil {
return err
}
logrus.Infof("adding an origin remote: %s", gitURL)
if _, err := r.CreateRemote(&config.RemoteConfig{Name: "origin", URLs: []string{gitURL}}); err != nil {
return err
}
logrus.Info("getting the worktree")
w, err := r.Worktree()
if err != nil {
return err
}
if err := r.Storer.SetReference(plumbing.NewHashReference(plumbing.Main, plumbing.ZeroHash)); err != nil {
return err
}
logrus.Info("creating an empty 'Init Commit'")
if _, err := w.Commit("Init Commit", &git.CommitOptions{
AllowEmptyCommits: true,
}); err != nil {
return err
}
if !dry {
if err := r.Push(&git.PushOptions{RemoteName: "origin"}); err != nil {
return err
}
}
} else if err != nil {
return err
}
return nil
}
func (g *Git) AddAllAndCommit(workdir, message string) error {
r, err := git.PlainOpen(workdir)
if err != nil {
return err
}
w, err := r.Worktree()
if err != nil {
return err
}
if _, err := w.Add("."); err != nil {
return err
}
if _, err := w.Commit(message, &git.CommitOptions{}); err != nil {
return err
}
return nil
}
func (g *Git) Push(workdir string) error {
r, err := git.PlainOpen(workdir)
if err != nil {
return err
}
publicKeys, err := ssh.NewPublicKeysFromFile("git", g.SshPrivateKeyPath, "")
if err != nil {
return err
}
if err := r.Push(&git.PushOptions{
RemoteName: "origin",
Auth: publicKeys,
}); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,18 @@
package githelper
type Mock struct{}
func NewGitMock() Githelper {
return &Mock{}
}
func (m *Mock) CloneRepo(workdir, gitURL string, dry bool) error {
return nil
}
func (g *Mock) AddAllAndCommit(workdir, message string) error {
return nil
}
func (g *Mock) Push(workdir string) error {
return nil
}

View File

@ -0,0 +1,7 @@
package githelper
type Githelper interface {
CloneRepo(workdir, gitURL string, dry bool) error
AddAllAndCommit(workdir, message string) error
Push(workdir string) error
}

View File

@ -0,0 +1,120 @@
package helmhelper
import (
"fmt"
"os"
"git.badhouseplants.net/allanger/giops/internal/config/repository"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/getter"
"helm.sh/helm/v3/pkg/registry"
"helm.sh/helm/v3/pkg/repo"
)
type Helm struct{}
func NewHelm() Helmhelper {
return &Helm{}
}
type ChartData struct {
Version string
}
func (h *Helm) FindLatestVersion(dir, chart string, repository repository.Repository) (version string, err error) {
downloadDir := fmt.Sprintf("%s/.charts", dir)
if err := os.MkdirAll(downloadDir, 0777); err != nil {
return "", err
}
// If file doesn't exist
config := new(action.Configuration)
cl := cli.New()
chartDir := fmt.Sprintf("%s/%s-%s", downloadDir, repository.Name, chart)
_, err = os.Stat(chartDir)
if err != nil && !os.IsNotExist(err) {
return "", nil
} else if os.IsNotExist(err) {
if err := os.Mkdir(chartDir, 0777); err != nil {
return "", err
}
registry, err := registry.NewClient()
if err != nil {
return "", err
}
var path string
// Download the chart to the workdir
if repository.Kind != "oci" {
r, err := repo.NewChartRepository(&repo.Entry{
Name: repository.Name,
URL: repository.URL,
}, getter.All(cl))
if err != nil {
return "", err
}
path = r.Config.Name
} else {
path = repository.URL
}
client := action.NewPullWithOpts(action.WithConfig(config))
client.SetRegistryClient(registry)
client.DestDir = chartDir
client.Settings = cl
chartRemote := fmt.Sprintf("%s/%s", path, chart)
logrus.Infof("trying to pull: %s", chartRemote)
if _, err = client.Run(chartRemote); err != nil {
return "", err
}
}
showAction := action.NewShowWithConfig(action.ShowChart, config)
chartPath, err := getChartPathFromDir(chartDir)
if err != nil {
return "", err
}
res, err := showAction.LocateChart(fmt.Sprintf("%s/%s", chartDir, chartPath), cl)
if err != nil {
return "", err
}
res, err = showAction.Run(res)
if err != nil {
return "", nil
}
chartData, err := chartFromString(res)
if err != nil {
return "", err
}
logrus.Infof("the latest version of %s is %s", chart, chartData.Version)
return chartData.Version, err
}
func getChartPathFromDir(dir string) (file string, err error) {
files, err := os.ReadDir(dir)
if err != nil {
return "", err
} else if len(files) == 0 {
return "", fmt.Errorf("expected to have one file, got zero in a dir %s", dir)
} else if len(files) > 1 {
return "", fmt.Errorf("expected to have only one file in a dir %s", dir)
}
return files[0].Name(), nil
}
func chartFromString(info string) (*ChartData, error) {
chartData := new(ChartData)
if err := yaml.Unmarshal([]byte(info), &chartData); err != nil {
return nil, err
}
return chartData, nil
}

View File

@ -0,0 +1,15 @@
package helmhelper
import "git.badhouseplants.net/allanger/giops/internal/config/repository"
const MOCK_LATEST_VERSION = "v1.12.1"
type Mock struct{}
func NewHelmMock() Helmhelper {
return &Mock{}
}
func (h *Mock) FindLatestVersion(dir, chart string, repository repository.Repository) (version string, err error) {
return MOCK_LATEST_VERSION, nil
}

View File

@ -0,0 +1,7 @@
package helmhelper
import "git.badhouseplants.net/allanger/giops/internal/config/repository"
type Helmhelper interface {
FindLatestVersion(dir, chart string, repository repository.Repository) (string, error)
}

View File

@ -0,0 +1,64 @@
package kustomize
import (
"fmt"
"os"
"git.badhouseplants.net/allanger/giops/internal/utils/githelper"
kustomize_types "sigs.k8s.io/kustomize/api/types"
"sigs.k8s.io/yaml"
)
type Kusmtomize struct {
Files []string
}
func (k *Kusmtomize) PopulateResources(path string) error {
files, err := os.ReadDir(fmt.Sprintf("%s/src", path))
if err != nil {
return err
}
for _, file := range files {
if file.Name() != ".gitkeep" {
k.Files = append(k.Files, fmt.Sprintf("src/%s", file.Name()))
}
}
return nil
}
func Generate(path string, gh githelper.Githelper) error {
kustomize := &Kusmtomize{}
if err := kustomize.PopulateResources(path); err != nil {
return err
}
kustomization := kustomize_types.Kustomization{
TypeMeta: kustomize_types.TypeMeta{
Kind: kustomize_types.KustomizationKind,
APIVersion: kustomize_types.ComponentVersion,
},
MetaData: &kustomize_types.ObjectMeta{
Name: "helm-root",
Namespace: "flux-system",
},
Resources: kustomize.Files,
}
manifest, err := yaml.Marshal(kustomization)
if err != nil {
return err
}
file, err := os.Create(path + "/kustomization.yaml")
if err != nil {
return err
}
if _, err := file.Write(manifest); err != nil {
return err
}
if err := gh.AddAllAndCommit(path, "Update the root kustomization"); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,28 @@
package workdir
import "os"
func CreateWorkdir(path string) (workdir string, err error) {
if len(path) > 0 {
// Create a dir using the path
// It should not be removed after the execution
if err := os.Mkdir(path, 0777); err != nil {
return "", err
}
// TODO(@allanger): I've got a feeling that it doesn't have to look that bad
workdir = path
} else {
// Create a temporary dir
// It should be removed after the execution
workdir, err = os.MkdirTemp("", "giops")
if err != nil {
return "", err
}
}
return workdir, nil
}
func RemoveWorkdir(path string) (err error) {
return os.RemoveAll(path)
}

15
main.go Normal file
View File

@ -0,0 +1,15 @@
package main
import (
"context"
"git.badhouseplants.net/allanger/giops/cmd"
"github.com/sirupsen/logrus"
)
func main() {
ctx := context.Background()
if err := cmd.Execute(ctx); err != nil {
logrus.Fatal(err)
}
}

18
scripts/build Executable file
View File

@ -0,0 +1,18 @@
#!/usr/bin/env bash
PACKAGE="git.badhouseplants.net/allanger/giops"
VERSION="$(git describe --tags --always --abbrev=0 --match='v[0-9]*.[0-9]*.[0-9]*' 2> /dev/null | sed 's/^.//')"
COMMIT_HASH="$(git rev-parse --short HEAD)"
BUILD_TIMESTAMP=$(date '+%Y-%m-%dT%H:%M:%S')
# STEP 2: Build the ldflags
LDFLAGS=(
"-X '${PACKAGE}/internal/build.Version=${VERSION}'"
"-X '${PACKAGE}/internal/build.CommitHash=${COMMIT_HASH}'"
"-X '${PACKAGE}/internal/build.BuildTime=${BUILD_TIMESTAMP}'"
)
# STEP 3: Actual Go build process
go build -ldflags="${LDFLAGS[*]}"