Accountant: Inventar, where, Checkmk, Abrechnung
This commit is contained in:
br committed
1 parent
9f16148cde
commit
dd93277750
59 files changed
+7522
-113
No files matched your search
@@ -0,0 +1,329 @@
|
||||
package accountant
|
||||
|
||||
// Checkmk-Ausgabe des Accountants (Spec 5.3, Roadmap 6e): ein Service je URL
|
||||
// (Störfall: wohin springen?) und je Payee (Frische der Berichte), im
|
||||
// Local-Check-Format. Der Text kommt aus einer text/template-Vorlage:
|
||||
// eingebaut, überschreibbar durch <data-dir>/checkmk/checkmk.tmpl (wirkt
|
||||
// sofort). Werte aus Berichten sind Fremdeingaben (Containername,
|
||||
// Compose-Verzeichnis stammen vom Kunden-Host): Sie sind bereinigt
|
||||
// (einzeilig, ASCII, ohne | und ") und HTML-escaped; nur bereinigt stehen
|
||||
// sie unter .Unescaped – falls in Checkmk das HTML-Escaping abgeschaltet ist,
|
||||
// öffnet deren Ausgabe Script-Injektion in die Oberfläche.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"html"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"sultan/pkg/checkmk"
|
||||
"sultan/pkg/ledger"
|
||||
)
|
||||
|
||||
// CheckmkTemplateName ist der Dateiname der eigenen Vorlage in <data-dir>/checkmk.
|
||||
const CheckmkTemplateName = "checkmk.tmpl"
|
||||
|
||||
// Checkmk-Status.
|
||||
const (
|
||||
cmkOK = 0
|
||||
cmkWarn = 1
|
||||
cmkCrit = 2
|
||||
)
|
||||
|
||||
// defaultCheckmkTemplate ist die eingebaute Vorlage (sultan accountant
|
||||
// dump-checkmk exportiert sie zum Anpassen). Jede Zeile:
|
||||
// <status> "<service>" - <OK|WARN|CRIT> - <text>
|
||||
const defaultCheckmkTemplate = `{{- /*
|
||||
Checkmk-Vorlage des Sultan-Accountants (text/template).
|
||||
Eine Zeile je Service: <status> "<service>" - <OK|WARN|CRIT> - <text>
|
||||
.Payees: .Service .State .Problem .ID .Hostname .Age .LastReport
|
||||
.URLs: .Service .State .Problem .Host .URL .Tenant .Payee .Node .Router .Workload
|
||||
.Backend .BackendFQDN .BackendURL .ComposeProject .ComposeDir .Health .Product
|
||||
Werte sind bereinigt und HTML-escaped; .Unescaped.<Feld> nur bereinigt.
|
||||
Funktionen: status (0/1/2 → OK/WARN/CRIT)
|
||||
*/ -}}
|
||||
{{- range .Payees -}}
|
||||
{{.State}} "{{.Service}}" - {{status .State}} - {{with .Problem}}{{.}}; {{end}}letzter Bericht {{.Age}}{{with .Hostname}} von {{.}}{{end}}
|
||||
{{end -}}
|
||||
{{- range .URLs -}}
|
||||
{{.State}} "{{.Service}}" - {{status .State}} - {{with .Problem}}{{.}}; {{end}}{{.URL}} -> {{or .BackendURL .Backend}} ({{.Tenant}} auf {{.Node}} via {{.Payee}}{{with .ComposeDir}}, compose {{.}}{{end}}{{with .Health}}, {{.}}{{end}}{{with .Product}}, Produkt {{.}}{{end}})
|
||||
{{end -}}
|
||||
`
|
||||
|
||||
// CheckmkData ist der Inhalt, den die Vorlage sieht.
|
||||
type CheckmkData struct {
|
||||
Now time.Time
|
||||
Payees []CheckmkPayee
|
||||
URLs []CheckmkURL
|
||||
}
|
||||
|
||||
// PayeeFields sind die Textfelder eines Payees.
|
||||
type PayeeFields struct {
|
||||
ID string
|
||||
Hostname string
|
||||
Problem string
|
||||
}
|
||||
|
||||
// CheckmkPayee ist der Frische-Service eines Payees.
|
||||
type CheckmkPayee struct {
|
||||
PayeeFields // bereinigt und HTML-escaped
|
||||
Unescaped PayeeFields // nur bereinigt
|
||||
Service string // Name des Checkmk-Services
|
||||
State int // 0 OK, 1 WARN, 2 CRIT
|
||||
Age string // z. B. "vor 5 min", "nie"
|
||||
LastReport time.Time // Nullzeit = nie
|
||||
}
|
||||
|
||||
// URLFields sind die Textfelder einer URL.
|
||||
type URLFields struct {
|
||||
Host, URL string
|
||||
Tenant, Payee, Node string
|
||||
Router, Workload string
|
||||
Backend, BackendFQDN, BackendURL string
|
||||
ComposeProject, ComposeDir string
|
||||
Health, Product string
|
||||
Problem string
|
||||
}
|
||||
|
||||
// CheckmkURL ist der Service einer URL.
|
||||
type CheckmkURL struct {
|
||||
URLFields // bereinigt und HTML-escaped
|
||||
Unescaped URLFields // nur bereinigt
|
||||
Service string
|
||||
State int
|
||||
}
|
||||
|
||||
// clean macht Fremdtext einzeilig, ASCII und frei von Trennzeichen des
|
||||
// Local-Check-Formats.
|
||||
func clean(s string) string {
|
||||
return strings.NewReplacer("|", "/", `"`, "'").Replace(checkmk.ASCII(s))
|
||||
}
|
||||
|
||||
func escapeFields[T any](v T, f func(*T, func(string) string)) T {
|
||||
f(&v, html.EscapeString)
|
||||
return v
|
||||
}
|
||||
|
||||
func (u *URLFields) apply(fn func(string) string) {
|
||||
for _, p := range []*string{&u.Host, &u.URL, &u.Tenant, &u.Payee, &u.Node, &u.Router, &u.Workload,
|
||||
&u.Backend, &u.BackendFQDN, &u.BackendURL, &u.ComposeProject, &u.ComposeDir, &u.Health, &u.Product, &u.Problem} {
|
||||
*p = fn(*p)
|
||||
}
|
||||
}
|
||||
|
||||
func (p *PayeeFields) apply(fn func(string) string) {
|
||||
p.ID, p.Hostname, p.Problem = fn(p.ID), fn(p.Hostname), fn(p.Problem)
|
||||
}
|
||||
|
||||
func age(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Minute:
|
||||
return "vor weniger als 1 min"
|
||||
case d < time.Hour:
|
||||
return fmt.Sprintf("vor %d min", int(d/time.Minute))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("vor %d h", int(d/time.Hour))
|
||||
default:
|
||||
return fmt.Sprintf("vor %d Tagen", int(d/(24*time.Hour)))
|
||||
}
|
||||
}
|
||||
|
||||
// freshness bewertet das Alter eines Berichts.
|
||||
func freshness(last, now time.Time, warn, crit time.Duration) (int, string) {
|
||||
switch d := now.Sub(last); {
|
||||
case last.IsZero():
|
||||
return cmkWarn, "noch kein Bericht"
|
||||
case d > crit:
|
||||
return cmkCrit, "Bericht veraltet (" + age(d) + ")"
|
||||
case d > warn:
|
||||
return cmkWarn, "Bericht veraltet (" + age(d) + ")"
|
||||
default:
|
||||
return cmkOK, ""
|
||||
}
|
||||
}
|
||||
|
||||
// buildCheckmk bewertet Payees und URLs. reports enthält den letzten Bericht
|
||||
// je Payee-ID (fehlt bei Payees ohne Bericht).
|
||||
func buildCheckmk(payees []ledger.Payee, reports map[string]Report, now time.Time, staleWarn, staleCrit time.Duration) CheckmkData {
|
||||
d := CheckmkData{Now: now}
|
||||
for _, p := range payees {
|
||||
state, problem := freshness(p.LastReport, now, staleWarn, staleCrit)
|
||||
raw := PayeeFields{ID: clean(p.ID), Hostname: clean(p.Hostname), Problem: problem}
|
||||
cp := CheckmkPayee{
|
||||
Unescaped: raw, PayeeFields: escapeFields(raw, (*PayeeFields).apply),
|
||||
Service: "Sultan Payee " + raw.ID, State: state, Age: "nie", LastReport: p.LastReport,
|
||||
}
|
||||
if !p.LastReport.IsZero() {
|
||||
cp.Age = age(now.Sub(p.LastReport))
|
||||
}
|
||||
d.Payees = append(d.Payees, cp)
|
||||
|
||||
rep, ok := reports[p.ID]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
nodes := map[string]ReportNode{}
|
||||
for _, n := range rep.Nodes {
|
||||
nodes[n.ID] = n
|
||||
}
|
||||
for _, r := range rep.Routes {
|
||||
for _, host := range r.Hosts {
|
||||
d.URLs = append(d.URLs, urlService(p.ID, host, r, nodes[r.Node], state, problem))
|
||||
}
|
||||
}
|
||||
}
|
||||
slices.SortFunc(d.Payees, func(a, b CheckmkPayee) int { return strings.Compare(a.Service, b.Service) })
|
||||
|
||||
// Eindeutige Servicenamen: derselbe Host mehrfach → Zusatz (payee/knoten).
|
||||
slices.SortFunc(d.URLs, func(a, b CheckmkURL) int {
|
||||
return cmp.Or(strings.Compare(a.Unescaped.Host, b.Unescaped.Host), strings.Compare(a.Unescaped.Payee, b.Unescaped.Payee),
|
||||
strings.Compare(a.Unescaped.Node, b.Unescaped.Node), strings.Compare(a.Unescaped.Router, b.Unescaped.Router))
|
||||
})
|
||||
seen := map[string]bool{}
|
||||
for i := range d.URLs {
|
||||
u := &d.URLs[i]
|
||||
u.Service = "Sultan URL " + u.Unescaped.Host
|
||||
if seen[u.Service] {
|
||||
u.Service += " (" + u.Unescaped.Payee + "/" + u.Unescaped.Node + "/" + u.Unescaped.Router + ")"
|
||||
}
|
||||
seen[u.Service] = true
|
||||
}
|
||||
slices.SortFunc(d.URLs, func(a, b CheckmkURL) int { return strings.Compare(a.Service, b.Service) })
|
||||
return d
|
||||
}
|
||||
|
||||
func urlService(payeeID, host string, r ReportRoute, node ReportNode, payeeState int, payeeProblem string) CheckmkURL {
|
||||
raw := URLFields{
|
||||
Host: clean(host), Tenant: clean(r.Tenant), Payee: clean(payeeID), Node: clean(r.Node),
|
||||
Router: clean(r.Router), Workload: clean(r.Workload), Backend: clean(r.Backend), BackendFQDN: clean(r.BackendFQDN),
|
||||
ComposeProject: clean(r.ComposeProject), ComposeDir: clean(r.ComposeDir), Health: clean(r.Health), Product: clean(r.Product),
|
||||
}
|
||||
if r.Protocol == "tcp" {
|
||||
raw.URL, raw.BackendURL = raw.Host, raw.BackendFQDN
|
||||
} else {
|
||||
raw.URL = "https://" + raw.Host
|
||||
if raw.BackendFQDN != "" {
|
||||
raw.BackendURL = "http://" + raw.BackendFQDN
|
||||
}
|
||||
}
|
||||
|
||||
state := cmkOK
|
||||
var problems []string
|
||||
if !r.Active {
|
||||
problems = append(problems, "von Policy verworfen")
|
||||
}
|
||||
if r.Down {
|
||||
problems = append(problems, "gestoppt/Wartungsseite")
|
||||
}
|
||||
if r.Health == "unhealthy" {
|
||||
problems = append(problems, "unhealthy")
|
||||
}
|
||||
if !node.Connected && !node.Maintenance {
|
||||
problems = append(problems, "Knoten getrennt")
|
||||
}
|
||||
switch {
|
||||
case node.Maintenance:
|
||||
// Angekündigte Wartung: kein Alarm, aber sichtbar.
|
||||
problems = append([]string{"Wartung"}, problems...)
|
||||
case len(problems) > 0:
|
||||
state = cmkWarn
|
||||
}
|
||||
if payeeState > state {
|
||||
state = payeeState
|
||||
problems = append([]string{"Bericht von " + payeeID + ": " + payeeProblem}, problems...)
|
||||
}
|
||||
raw.Problem = clean(strings.Join(problems, ", "))
|
||||
return CheckmkURL{Unescaped: raw, URLFields: escapeFields(raw, (*URLFields).apply), State: state}
|
||||
}
|
||||
|
||||
var cmkFuncs = template.FuncMap{
|
||||
"status": func(s int) string {
|
||||
switch s {
|
||||
case cmkOK:
|
||||
return "OK"
|
||||
case cmkWarn:
|
||||
return "WARN"
|
||||
default:
|
||||
return "CRIT"
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
var builtinCheckmk = template.Must(template.New(CheckmkTemplateName).Funcs(cmkFuncs).Parse(defaultCheckmkTemplate))
|
||||
|
||||
// renderCheckmk rendert d mit der eigenen Vorlage aus dataDir (falls
|
||||
// vorhanden), sonst mit der eingebauten. Eine fehlerhafte eigene Vorlage
|
||||
// fällt auf die eingebaute zurück und erscheint als CRIT-Service.
|
||||
func renderCheckmk(dataDir string, d CheckmkData) string {
|
||||
path := filepath.Join(dataDir, "checkmk", CheckmkTemplateName)
|
||||
var problem error
|
||||
if raw, err := os.ReadFile(path); err == nil {
|
||||
t, err := template.New(CheckmkTemplateName).Funcs(cmkFuncs).Parse(string(raw))
|
||||
if err == nil {
|
||||
var buf bytes.Buffer
|
||||
if err = t.Execute(&buf, d); err == nil {
|
||||
return buf.String()
|
||||
}
|
||||
}
|
||||
problem = err
|
||||
} else if !errors.Is(err, os.ErrNotExist) {
|
||||
problem = err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if problem != nil {
|
||||
fmt.Fprintf(&buf, "%d \"Sultan Accountant Vorlage\" - CRIT - %s fehlerhaft, eingebaute Vorlage aktiv: %s\n",
|
||||
cmkCrit, CheckmkTemplateName, clean(problem.Error()))
|
||||
}
|
||||
if err := builtinCheckmk.Execute(&buf, d); err != nil {
|
||||
fmt.Fprintf(&buf, "%d \"Sultan Accountant Vorlage\" - CRIT - eingebaute Vorlage: %s\n", cmkCrit, clean(err.Error()))
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// DumpCheckmk schreibt die eingebaute Vorlage nach <dataDir>/checkmk zum
|
||||
// Anpassen; eine vorhandene Datei bleibt ohne force unangetastet.
|
||||
func DumpCheckmk(dataDir string, force bool) (string, error) {
|
||||
dir := filepath.Join(dataDir, "checkmk")
|
||||
path := filepath.Join(dir, CheckmkTemplateName)
|
||||
if _, err := os.Stat(path); err == nil && !force {
|
||||
return "", fmt.Errorf("%s existiert bereits (überschreiben mit --force): %w", path, os.ErrExist)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return path, os.WriteFile(path, []byte(defaultCheckmkTemplate), 0o644)
|
||||
}
|
||||
|
||||
func (a *Accountant) handleCheckmk(w http.ResponseWriter, r *http.Request) {
|
||||
payees, err := a.ledger.ListPayees(r.Context())
|
||||
if err != nil {
|
||||
a.log.Error("Checkmk: Payees lesen", "err", err)
|
||||
http.Error(w, "interner Fehler", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
reports := map[string]Report{}
|
||||
for _, p := range payees {
|
||||
body, _, err := a.ledger.LatestReport(r.Context(), p.ID)
|
||||
if err != nil {
|
||||
continue // noch kein Bericht
|
||||
}
|
||||
var rep Report
|
||||
if err := json.Unmarshal(body, &rep); err != nil {
|
||||
a.log.Error("Checkmk: Bericht unlesbar", "payee", p.ID, "err", err)
|
||||
continue
|
||||
}
|
||||
reports[p.ID] = rep
|
||||
}
|
||||
d := buildCheckmk(payees, reports, time.Now(), a.cfg.StaleWarn, a.cfg.StaleCrit)
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Write([]byte(renderCheckmk(a.cfg.DataDir, d)))
|
||||
}
|
||||
Reference in new issue
Block a user