software steht so weit.
jetzt build + docker build und testen
This commit is contained in:
1 parent
42f0c0accb
commit
ca6cdcca63
90 files changed
+10298
-480
No files matched your search
+34
-11
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"sultan/pkg/license"
|
||||
"sultan/pkg/policy"
|
||||
@@ -14,8 +15,8 @@ import (
|
||||
// directory liefert der Registry Mandant, Lizenz und Drain-Status eines
|
||||
// Knotens aus dem Store.
|
||||
//
|
||||
// Solange Migration v2 fehlt (Spec Abschnitt 9), wird die Lizenz aus dem
|
||||
// Mandanten-Datensatz abgeleitet: ohne Features und ohne Mahnstufe.
|
||||
// Die Lizenz wird aus dem Mandanten-Datensatz abgeleitet (inkl. Features);
|
||||
// die Mahnstufe ergibt sich aus der offenen Zahlung (DunningSchedule).
|
||||
type directory struct {
|
||||
store *store.Store
|
||||
}
|
||||
@@ -36,15 +37,37 @@ func (d *directory) NodeContext(ctx context.Context, nodeID string) (registry.No
|
||||
return registry.NodeContext{}, err
|
||||
}
|
||||
return registry.NodeContext{
|
||||
Tenant: policy.Tenant{ID: t.ID, Status: t.Status},
|
||||
License: license.License{
|
||||
TenantID: t.ID,
|
||||
Tier: license.Tier(t.Tier),
|
||||
MaxRoutes: t.MaxRoutes,
|
||||
AllowedDomains: t.AllowedDomains,
|
||||
Features: []license.Feature{},
|
||||
ValidUntil: t.ValidUntil,
|
||||
Tenant: policy.Tenant{
|
||||
ID: t.ID,
|
||||
Status: t.Status,
|
||||
DunningLevel: policy.DunningSchedule{}.Level(t.PaymentOverdueSince, time.Now()),
|
||||
},
|
||||
Drained: node.Status == store.NodeDrained,
|
||||
License: licenseOf(t),
|
||||
Drained: node.Status == store.NodeDrained,
|
||||
Maintenance: maintenanceOf(node),
|
||||
Disconnected: node.Status == store.NodeOffline,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// licenseOf leitet die (unsignierte) Lizenz aus dem Mandanten-Datensatz ab.
|
||||
func licenseOf(t store.Tenant) license.License {
|
||||
features := make([]license.Feature, 0, len(t.Features))
|
||||
for _, f := range t.Features {
|
||||
features = append(features, license.Feature(f))
|
||||
}
|
||||
return license.License{
|
||||
TenantID: t.ID,
|
||||
Tier: license.Tier(t.Tier),
|
||||
MaxRoutes: t.MaxRoutes,
|
||||
AllowedDomains: t.AllowedDomains,
|
||||
Features: features,
|
||||
ValidUntil: t.ValidUntil,
|
||||
}
|
||||
}
|
||||
|
||||
func maintenanceOf(n store.Node) *policy.Maintenance {
|
||||
if n.Maintenance == nil {
|
||||
return nil
|
||||
}
|
||||
return &policy.Maintenance{Until: n.Maintenance.Until, PublicReason: n.Maintenance.PublicReason}
|
||||
}
|
||||
@@ -3,9 +3,12 @@ package server
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"sultan/pkg/auth"
|
||||
"sultan/pkg/policy"
|
||||
@@ -28,17 +31,40 @@ type EnrollResponse struct {
|
||||
MasterPublicKey string `json:"master_public_key"` // zum Prüfen von Lizenzen/Leases
|
||||
}
|
||||
|
||||
// MaintenanceRequest ist der Body von POST /api/v1/maintenance: Der Knoten
|
||||
// meldet Wartung an oder ab (z. B. aus einem Patch-/Reboot-Skript).
|
||||
type MaintenanceRequest struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Duration string `json:"duration,omitempty"` // Go-Dauer; "" = 4h, "0" = unbefristet
|
||||
Reason string `json:"reason,omitempty"` // intern
|
||||
PublicReason string `json:"public_reason,omitempty"`
|
||||
}
|
||||
|
||||
// RotateKeyRequest ist der Body von POST /api/v1/rotate-key. Der Request ist
|
||||
// mit dem bisherigen Schlüssel signiert; Proof (auth.RotationProof) beweist
|
||||
// den Besitz des neuen.
|
||||
type RotateKeyRequest struct {
|
||||
PublicKey string `json:"public_key"`
|
||||
Proof string `json:"proof"`
|
||||
}
|
||||
|
||||
// DefaultMaintenance ist die Wartungsdauer ohne Angabe.
|
||||
const DefaultMaintenance = 4 * time.Hour
|
||||
|
||||
func (s *Server) agentRoutes() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("POST /api/v1/enroll", s.handleEnroll)
|
||||
mux.HandleFunc("GET /api/v1/stream", s.handleStream)
|
||||
mux.HandleFunc("POST /api/v1/state", s.handleState)
|
||||
mux.HandleFunc("POST /api/v1/maintenance", s.handleMaintenance)
|
||||
mux.HandleFunc("POST /api/v1/rotate-key", s.handleRotateKey)
|
||||
return mux
|
||||
}
|
||||
|
||||
func (s *Server) providerRoutes() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("GET /v1/traefik-config", s.registry.Handler())
|
||||
mux.HandleFunc("GET /metrics/checkmk", s.handleCheckmk)
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Write([]byte("ok\n"))
|
||||
})
|
||||
@@ -46,6 +72,13 @@ func (s *Server) providerRoutes() http.Handler {
|
||||
}
|
||||
|
||||
func (s *Server) handleEnroll(w http.ResponseWriter, r *http.Request) {
|
||||
// Vor der Signaturprüfung: Fluten kosten so kaum etwas.
|
||||
if ok, retry := s.enrollRL.allow(remoteIP(r)); !ok {
|
||||
w.Header().Set("Retry-After", strconv.Itoa(int(math.Ceil(retry.Seconds()))))
|
||||
s.log.Warn("Enrollment gebremst (zu viele Versuche)", "ip", remoteIP(r))
|
||||
http.Error(w, "zu viele Versuche", http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
pub, err := s.verifier.VerifyUnregistered(r)
|
||||
if err != nil {
|
||||
authError(w, err)
|
||||
@@ -144,6 +177,91 @@ func (s *Server) handleState(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (s *Server) handleMaintenance(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := s.verifier.Verify(r)
|
||||
if err != nil {
|
||||
authError(w, err)
|
||||
return
|
||||
}
|
||||
var req MaintenanceRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "ungültiger Body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if !req.Enabled {
|
||||
err = s.store.ClearNodeMaintenance(r.Context(), id.Node.ID)
|
||||
} else {
|
||||
d := DefaultMaintenance
|
||||
if req.Duration != "" {
|
||||
if d, err = time.ParseDuration(req.Duration); err != nil || d < 0 {
|
||||
http.Error(w, "duration ungültig (z. B. 30m, 4h; 0 = unbefristet)", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
}
|
||||
m := store.Maintenance{Reason: req.Reason, PublicReason: req.PublicReason}
|
||||
if d > 0 {
|
||||
m.Until = time.Now().Add(d)
|
||||
}
|
||||
err = s.store.SetNodeMaintenance(r.Context(), id.Node.ID, m)
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, store.ErrInvalid):
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
case err != nil:
|
||||
s.log.Error("Wartung setzen", "node", id.Node.ID, "err", err)
|
||||
http.Error(w, "interner Fehler", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
if req.Enabled {
|
||||
s.log.Info("Wartung angemeldet", "node", id.Node.ID, "duration", req.Duration, "reason", req.Reason,
|
||||
"public_reason", req.PublicReason)
|
||||
} else {
|
||||
s.log.Info("Wartung beendet", "node", id.Node.ID)
|
||||
}
|
||||
// Sofort wirksam, nicht erst beim nächsten Refresh.
|
||||
if err := s.registry.Refresh(r.Context()); err != nil {
|
||||
s.log.Error("Neubewertung nach Wartungsänderung", "err", err)
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// handleRotateKey tauscht den Schlüssel eines Knotens. Idempotent: Ist der
|
||||
// neue Schlüssel schon aktiv (Wiederholung nach einem Abbruch, signiert mit
|
||||
// dem neuen Schlüssel), antwortet er ebenfalls mit 204.
|
||||
func (s *Server) handleRotateKey(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := s.verifier.Verify(r)
|
||||
if err != nil {
|
||||
authError(w, err)
|
||||
return
|
||||
}
|
||||
var req RotateKeyRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "ungültiger Body", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
pub, err := auth.DecodePublicKey(req.PublicKey)
|
||||
if err != nil || !auth.VerifyRotationProof(id.Node.ID, pub, req.Proof) {
|
||||
http.Error(w, "Besitznachweis für den neuen Schlüssel ungültig", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
encoded := auth.EncodePublicKey(pub)
|
||||
if encoded != id.Node.PublicKey {
|
||||
err := s.store.UpdateNodePublicKey(r.Context(), id.Node.ID, encoded)
|
||||
switch {
|
||||
case errors.Is(err, store.ErrExists):
|
||||
http.Error(w, "Schlüssel bereits vergeben", http.StatusConflict)
|
||||
return
|
||||
case err != nil:
|
||||
s.log.Error("Schlüssel tauschen", "node", id.Node.ID, "err", err)
|
||||
http.Error(w, "interner Fehler", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
s.log.Info("Knotenschlüssel gewechselt", "node", id.Node.ID)
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func authError(w http.ResponseWriter, err error) {
|
||||
code := auth.StatusCode(err)
|
||||
msg := http.StatusText(code)
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"sultan/pkg/checkmk"
|
||||
"sultan/pkg/notify"
|
||||
"sultan/pkg/policy"
|
||||
"sultan/pkg/store"
|
||||
)
|
||||
|
||||
// retention: Aufbewahrung von Einladungen, abgelaufenen Leases und
|
||||
// zugestellten Befehlen (entspricht der Community-Lease-Laufzeit).
|
||||
const retention = 10 * 24 * time.Hour
|
||||
|
||||
func commercial(t store.Tenant) bool { return t.Tier != store.TierCommunity }
|
||||
|
||||
// runJobs läuft bei jedem Refresh: Vertragsablauf, Benachrichtigungen,
|
||||
// Aufräumen.
|
||||
func (s *Server) runJobs(ctx context.Context) {
|
||||
now := time.Now()
|
||||
tenants, err := s.store.ListTenants(ctx)
|
||||
if err != nil {
|
||||
s.log.Error("Job: Mandanten lesen", "err", err)
|
||||
return
|
||||
}
|
||||
for _, t := range tenants {
|
||||
s.expireContract(ctx, t, now)
|
||||
if s.notifier != nil && commercial(t) {
|
||||
s.notifyContract(ctx, t, now)
|
||||
s.notifyDunning(ctx, t, now)
|
||||
}
|
||||
}
|
||||
if s.notifier != nil {
|
||||
s.notifyOffline(ctx, now)
|
||||
}
|
||||
s.housekeeping(ctx)
|
||||
}
|
||||
|
||||
// expireContract setzt kommerzielle Verträge ab dem Ablaufzeitpunkt dauerhaft
|
||||
// auf expired (Community wird automatisch neu lizenziert).
|
||||
func (s *Server) expireContract(ctx context.Context, t store.Tenant, now time.Time) {
|
||||
if !commercial(t) || t.Status == store.TenantExpired || t.Status == store.TenantQuarantined || now.Before(t.ValidUntil) {
|
||||
return
|
||||
}
|
||||
if err := s.store.SetTenantStatus(ctx, t.ID, store.TenantExpired); err != nil {
|
||||
s.log.Error("Job: Vertrag auf expired setzen", "tenant", t.ID, "err", err)
|
||||
return
|
||||
}
|
||||
s.log.Warn("Vertrag abgelaufen – Mandant auf expired gesetzt", "tenant", t.ID, "valid_until", t.ValidUntil)
|
||||
}
|
||||
|
||||
func (s *Server) notifyContract(ctx context.Context, t store.Tenant, now time.Time) {
|
||||
day := t.ValidUntil.UTC().Format(time.DateOnly)
|
||||
days := int(t.ValidUntil.Sub(now).Hours() / 24)
|
||||
switch {
|
||||
case t.Status == store.TenantExpired || !now.Before(t.ValidUntil):
|
||||
s.sendOnce(ctx, "contract:"+t.ID+":"+day+":0", notify.Message{
|
||||
Severity: notify.Critical,
|
||||
Title: "Vertrag " + t.ID + " abgelaufen",
|
||||
Body: fmt.Sprintf("Der Vertrag von *%s* ist am %s abgelaufen. Die Seiten zeigen die Landingpage.", t.ID, day),
|
||||
})
|
||||
case days <= checkmk.ContractCritDays:
|
||||
s.sendOnce(ctx, "contract:"+t.ID+":"+day+":14", notify.Message{
|
||||
Severity: notify.Critical,
|
||||
Title: fmt.Sprintf("Vertrag %s läuft in %d Tagen ab", t.ID, days),
|
||||
Body: fmt.Sprintf("Vertragsende für *%s*: %s. Verlängerung erforderlich.", t.ID, day),
|
||||
})
|
||||
case days <= checkmk.ContractWarnDays:
|
||||
s.sendOnce(ctx, "contract:"+t.ID+":"+day+":30", notify.Message{
|
||||
Severity: notify.Warning,
|
||||
Title: fmt.Sprintf("Vertrag %s läuft in %d Tagen ab", t.ID, days),
|
||||
Body: fmt.Sprintf("Vertragsende für *%s*: %s.", t.ID, day),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) notifyDunning(ctx context.Context, t store.Tenant, now time.Time) {
|
||||
level := policy.DunningSchedule{}.Level(t.PaymentOverdueSince, now)
|
||||
if level == 0 {
|
||||
return
|
||||
}
|
||||
sev := notify.Warning
|
||||
if level >= 2 {
|
||||
sev = notify.Critical
|
||||
}
|
||||
text := map[int]string{1: "Hinweisseite mit 5 s Weiterleitung", 2: "Hinweisseite mit 10 s Weiterleitung", 3: "Kunde offline (Einladungsseite)"}[min(level, 3)]
|
||||
s.sendOnce(ctx, fmt.Sprintf("dunning:%s:%s:%d", t.ID, t.PaymentOverdueSince.UTC().Format(time.DateOnly), level), notify.Message{
|
||||
Severity: sev,
|
||||
Title: fmt.Sprintf("Mahnstufe %d für %s", level, t.ID),
|
||||
Body: fmt.Sprintf("Zahlung von *%s* seit %s offen. Wirkung: %s.", t.ID, t.PaymentOverdueSince.UTC().Format(time.DateOnly), text),
|
||||
})
|
||||
}
|
||||
|
||||
// notifyOffline alarmiert einmal pro Ausfall, wenn ein Knoten länger als
|
||||
// OfflineAlertAfter getrennt ist – nicht bei angemeldeter Wartung oder drain.
|
||||
func (s *Server) notifyOffline(ctx context.Context, now time.Time) {
|
||||
nodes, err := s.store.ListNodes(ctx, "")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, n := range nodes {
|
||||
if n.Status != store.NodeOffline || n.Maintenance.Active(now) || now.Sub(n.LastSeen) < s.cfg.OfflineAlertAfter {
|
||||
continue
|
||||
}
|
||||
s.sendOnce(ctx, "offline:"+n.ID+":"+n.LastSeen.UTC().Format(time.RFC3339Nano), notify.Message{
|
||||
Severity: notify.Critical,
|
||||
Title: "Knoten " + n.ID + " offline",
|
||||
Body: fmt.Sprintf("Knoten *%s* (Mandant %s, %s) ist seit %s getrennt. Besucher sehen die Wartungsseite, solange die Lease gilt.",
|
||||
n.ID, n.TenantID, n.Hostname, n.LastSeen.Local().Format("2006-01-02 15:04")),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// notifyBackOnline meldet „wieder online“ – nur für Ausfälle, zu denen ein
|
||||
// Offline-Alarm verschickt wurde, und je Ausfall genau einmal.
|
||||
func (s *Server) notifyBackOnline(nodeID string) {
|
||||
ctx := context.Background()
|
||||
prefix := "offline:" + nodeID + ":"
|
||||
keys, err := s.store.NotificationKeys(ctx, prefix)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, k := range keys {
|
||||
msg := notify.Message{Severity: notify.Info, Title: "Knoten " + nodeID + " wieder online"}
|
||||
if since, err := time.Parse(time.RFC3339Nano, k[len(prefix):]); err == nil {
|
||||
msg.Body = fmt.Sprintf("Knoten *%s* ist nach %s wieder verbunden.", nodeID, outage(time.Since(since)))
|
||||
} else {
|
||||
msg.Body = fmt.Sprintf("Knoten *%s* ist wieder verbunden.", nodeID)
|
||||
}
|
||||
s.sendOnce(ctx, "recovered:"+k, msg)
|
||||
}
|
||||
}
|
||||
|
||||
func outage(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Minute:
|
||||
return fmt.Sprintf("%d s", int(d/time.Second))
|
||||
case d < time.Hour:
|
||||
return fmt.Sprintf("%d min", int(d/time.Minute))
|
||||
default:
|
||||
return fmt.Sprintf("%d h %d min", int(d/time.Hour), int(d%time.Hour/time.Minute))
|
||||
}
|
||||
}
|
||||
|
||||
// sendOnce verschickt eine Nachricht genau einmal je Schlüssel; scheitert
|
||||
// der Versand, wird es beim nächsten Durchlauf erneut versucht.
|
||||
func (s *Server) sendOnce(ctx context.Context, key string, m notify.Message) {
|
||||
if s.store.Notified(ctx, key) {
|
||||
return
|
||||
}
|
||||
if err := s.notifier.Send(ctx, m); err != nil {
|
||||
s.log.Warn("Benachrichtigung nicht zugestellt – nächster Versuch beim nächsten Durchlauf", "key", key, "err", err)
|
||||
return
|
||||
}
|
||||
if err := s.store.RecordNotification(ctx, key); err != nil {
|
||||
s.log.Error("Benachrichtigung vermerken", "key", key, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) housekeeping(ctx context.Context) {
|
||||
for name, purge := range map[string]func(context.Context, time.Duration) (int64, error){
|
||||
"Einladungen": s.store.PurgeEnrollmentTokens,
|
||||
"Leases": s.store.PurgeLeases,
|
||||
"Befehle": s.store.PurgeNodeCommands,
|
||||
} {
|
||||
if n, err := purge(ctx, retention); err != nil {
|
||||
s.log.Error("Aufräumen", "was", name, "err", err)
|
||||
} else if n > 0 {
|
||||
s.log.Info("Aufgeräumt", "was", name, "anzahl", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// recordViolation zählt Verstöße für Checkmk (letzte 24 h, im Speicher).
|
||||
func (s *Server) recordViolation(at time.Time) {
|
||||
s.vmu.Lock()
|
||||
defer s.vmu.Unlock()
|
||||
s.violations = append(s.violations, at)
|
||||
}
|
||||
|
||||
func (s *Server) violations24h(now time.Time) int {
|
||||
s.vmu.Lock()
|
||||
defer s.vmu.Unlock()
|
||||
cutoff := now.Add(-24 * time.Hour)
|
||||
i := 0
|
||||
for i < len(s.violations) && s.violations[i].Before(cutoff) {
|
||||
i++
|
||||
}
|
||||
s.violations = s.violations[i:]
|
||||
return len(s.violations)
|
||||
}
|
||||
|
||||
// handleCheckmk liefert den Zustand im Checkmk-Local-Check-Format.
|
||||
func (s *Server) handleCheckmk(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, now := r.Context(), time.Now()
|
||||
state := checkmk.State{Violations24h: s.violations24h(now), OfflineCrit: s.cfg.OfflineAlertAfter, Location: time.Local}
|
||||
tenants, err := s.store.ListTenants(ctx)
|
||||
if err != nil {
|
||||
http.Error(w, "interner Fehler", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
for _, t := range tenants {
|
||||
state.Tenants = append(state.Tenants, checkmk.Tenant{
|
||||
ID: t.ID, Commercial: commercial(t), Status: string(t.Status), ValidUntil: t.ValidUntil,
|
||||
DunningLevel: policy.DunningSchedule{}.Level(t.PaymentOverdueSince, now), OverdueSince: t.PaymentOverdueSince,
|
||||
})
|
||||
}
|
||||
nodes, err := s.store.ListNodes(ctx, "")
|
||||
if err != nil {
|
||||
http.Error(w, "interner Fehler", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
for _, n := range nodes {
|
||||
cn := checkmk.Node{ID: n.ID, Status: string(n.Status), OfflineSince: n.LastSeen}
|
||||
if n.Maintenance.Active(now) {
|
||||
cn.MaintenanceActive, cn.MaintenanceUntil, cn.MaintenanceReason = true, n.Maintenance.Until, n.Maintenance.Reason
|
||||
}
|
||||
if l, err := s.store.GetLease(ctx, n.ID); err == nil {
|
||||
cn.LeaseUntil = l.ValidUntil
|
||||
}
|
||||
state.Nodes = append(state.Nodes, cn)
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
w.Write([]byte(checkmk.Render(state, now)))
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"sultan/pkg/license"
|
||||
"sultan/pkg/store"
|
||||
"sultan/pkg/stream"
|
||||
)
|
||||
|
||||
// ensureLease stellt die Lease eines verbundenen Knotens aus bzw. erneuert
|
||||
// sie ab renew_after und schickt sie per lease_renew. Mit always wird auch
|
||||
// eine noch gültige Lease erneut zugestellt (beim Verbinden).
|
||||
func (s *Server) ensureLease(nodeID string, always bool) {
|
||||
ctx := context.Background()
|
||||
now := time.Now()
|
||||
current, err := s.store.GetLease(ctx, nodeID)
|
||||
if err == nil && now.Before(current.RenewAfter) {
|
||||
if always {
|
||||
s.sendLease(nodeID, []byte(current.Document))
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil && !errors.Is(err, store.ErrNotFound) {
|
||||
s.log.Error("Lease lesen", "node", nodeID, "err", err)
|
||||
return
|
||||
}
|
||||
node, err := s.store.GetNode(ctx, nodeID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
tenant, err := s.store.GetTenant(ctx, node.TenantID)
|
||||
if err != nil || tenant.Status == store.TenantQuarantined {
|
||||
return
|
||||
}
|
||||
lic := licenseOf(tenant)
|
||||
terms, ok := license.TermsFor(lic.Tier, node.Maintenance.Active(now), lic.ValidUntil, now, s.cfg.LeaseDuration, nil)
|
||||
if !ok {
|
||||
s.log.Warn("Keine Lease: Vertrag abgelaufen", "node", nodeID, "tenant", tenant.ID)
|
||||
return
|
||||
}
|
||||
lease, err := (&license.Issuer{Key: s.master}).IssueLease(lic, nodeID, terms, now)
|
||||
if err != nil {
|
||||
s.log.Error("Lease ausstellen", "node", nodeID, "err", err)
|
||||
return
|
||||
}
|
||||
doc, _ := json.Marshal(lease)
|
||||
if err := s.store.PutLease(ctx, store.Lease{
|
||||
NodeID: nodeID, IssuedAt: lease.IssuedAt, RenewAfter: lease.RenewAfter, ValidUntil: lease.ValidUntil, Document: string(doc),
|
||||
}); err != nil {
|
||||
s.log.Error("Lease speichern", "node", nodeID, "err", err)
|
||||
return
|
||||
}
|
||||
s.log.Info("Lease ausgestellt", "node", nodeID, "valid_until", terms.ValidUntil, "renew_after", terms.RenewAfter)
|
||||
s.sendLease(nodeID, doc)
|
||||
}
|
||||
|
||||
func (s *Server) sendLease(nodeID string, doc []byte) {
|
||||
if err := s.hub.Send(nodeID, stream.Event{Type: stream.EventLeaseRenew, Payload: doc}); err != nil {
|
||||
s.log.Debug("Lease nicht zugestellt", "node", nodeID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// leaseValid meldet, ob der Knoten eine noch gültige Lease hat.
|
||||
func (s *Server) leaseValid(nodeID string) bool {
|
||||
l, err := s.store.GetLease(context.Background(), nodeID)
|
||||
return err == nil && time.Now().Before(l.ValidUntil)
|
||||
}
|
||||
|
||||
// dispatchCommands stellt Befehle aus der Warteschlange zu.
|
||||
func (s *Server) dispatchCommands(nodeID string) {
|
||||
ctx := context.Background()
|
||||
cmds, err := s.store.PendingNodeCommands(ctx, nodeID)
|
||||
if err != nil {
|
||||
s.log.Error("Befehle lesen", "node", nodeID, "err", err)
|
||||
return
|
||||
}
|
||||
for _, c := range cmds {
|
||||
payload, _ := json.Marshal(stream.NodeCommand{Command: c.Command, Reason: c.Reason})
|
||||
if err := s.hub.Send(nodeID, stream.Event{Type: stream.EventNodeCommand, Payload: payload}); err != nil {
|
||||
return // später erneut (nächster Refresh oder nächste Verbindung)
|
||||
}
|
||||
if err := s.store.MarkNodeCommandSent(ctx, c.ID); err != nil {
|
||||
s.log.Error("Befehl als gesendet markieren", "node", nodeID, "err", err)
|
||||
return
|
||||
}
|
||||
s.log.Info("Befehl zugestellt", "node", nodeID, "command", c.Command, "reason", c.Reason)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// rateLimiter begrenzt Versuche je Schlüssel (Client-IP) auf limit pro
|
||||
// window (gleitendes Fenster). limit < 0 = unbegrenzt.
|
||||
type rateLimiter struct {
|
||||
limit int
|
||||
window time.Duration
|
||||
now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
hits map[string][]time.Time
|
||||
lastPurge time.Time
|
||||
}
|
||||
|
||||
// allow verbucht einen Versuch; bei Ablehnung liefert es die Wartezeit.
|
||||
func (l *rateLimiter) allow(key string) (bool, time.Duration) {
|
||||
if l.limit < 0 {
|
||||
return true, 0
|
||||
}
|
||||
now := time.Now()
|
||||
if l.now != nil {
|
||||
now = l.now()
|
||||
}
|
||||
cutoff := now.Add(-l.window)
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
if l.hits == nil {
|
||||
l.hits = map[string][]time.Time{}
|
||||
}
|
||||
if now.Sub(l.lastPurge) >= l.window { // vergessen, was nicht mehr zählt
|
||||
for k, ts := range l.hits {
|
||||
if !ts[len(ts)-1].After(cutoff) {
|
||||
delete(l.hits, k)
|
||||
}
|
||||
}
|
||||
l.lastPurge = now
|
||||
}
|
||||
ts := l.hits[key]
|
||||
i := 0
|
||||
for i < len(ts) && !ts[i].After(cutoff) {
|
||||
i++
|
||||
}
|
||||
ts = ts[i:]
|
||||
if len(ts) >= l.limit {
|
||||
l.hits[key] = ts
|
||||
return false, ts[0].Add(l.window).Sub(now)
|
||||
}
|
||||
l.hits[key] = append(ts, now)
|
||||
return true, 0
|
||||
}
|
||||
|
||||
func (l *rateLimiter) size() int {
|
||||
l.mu.Lock()
|
||||
defer l.mu.Unlock()
|
||||
return len(l.hits)
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRateLimiter(t *testing.T) {
|
||||
now := time.Date(2026, 10, 2, 12, 0, 0, 0, time.UTC)
|
||||
l := &rateLimiter{limit: 3, window: time.Minute, now: func() time.Time { return now }}
|
||||
|
||||
for i := range 3 {
|
||||
if ok, _ := l.allow("1.2.3.4"); !ok {
|
||||
t.Fatalf("Versuch %d abgelehnt", i+1)
|
||||
}
|
||||
}
|
||||
ok, retry := l.allow("1.2.3.4")
|
||||
if ok || retry != time.Minute {
|
||||
t.Errorf("4. Versuch: ok=%v retry=%v, erwartet abgelehnt mit 1 min", ok, retry)
|
||||
}
|
||||
if ok, _ := l.allow("5.6.7.8"); !ok {
|
||||
t.Error("andere IP mitbetroffen")
|
||||
}
|
||||
|
||||
now = now.Add(40 * time.Second)
|
||||
if _, retry := l.allow("1.2.3.4"); retry != 20*time.Second {
|
||||
t.Errorf("Retry-After nach 40 s: %v, erwartet 20 s", retry)
|
||||
}
|
||||
now = now.Add(20 * time.Second)
|
||||
if ok, _ := l.allow("1.2.3.4"); !ok {
|
||||
t.Error("nach Ablauf des Fensters weiter gesperrt")
|
||||
}
|
||||
}
|
||||
|
||||
// Die Zählerliste darf bei vielen IPs nicht unbegrenzt wachsen.
|
||||
func TestRateLimiter_ForgetsOldEntries(t *testing.T) {
|
||||
now := time.Date(2026, 10, 2, 12, 0, 0, 0, time.UTC)
|
||||
l := &rateLimiter{limit: 1, window: time.Minute, now: func() time.Time { return now }}
|
||||
for i := range 1000 {
|
||||
l.allow(string(rune('a'+i%26)) + time.Duration(i).String())
|
||||
}
|
||||
now = now.Add(2 * time.Minute)
|
||||
l.allow("neu")
|
||||
if n := l.size(); n != 1 {
|
||||
t.Errorf("%d Einträge nach Ablauf, erwartet 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimiter_Disabled(t *testing.T) {
|
||||
l := &rateLimiter{limit: -1, window: time.Minute}
|
||||
for range 100 {
|
||||
if ok, _ := l.allow("x"); !ok {
|
||||
t.Fatal("Limit -1 muss unbegrenzt sein")
|
||||
}
|
||||
}
|
||||
}
|
||||
+227
-13
@@ -5,6 +5,8 @@
|
||||
// Zustandsmeldungen der Agents, jeder Request Ed25519-signiert.
|
||||
// - Provider-Port (Default 127.0.0.1:8080, Klartext, nur lokal): die
|
||||
// Traefik-Konfiguration für Traefiks HTTP-Provider und /healthz.
|
||||
// - Lander-Port (Default 127.0.0.1:8081, nur lokal): Hinweis-,
|
||||
// Wartungs- und Einladungsseiten, zu denen die Policy umleitet.
|
||||
//
|
||||
// Persistenter Zustand liegt im Datenverzeichnis (Default ./suldata):
|
||||
// sultan.db, server.key/server.crt (TLS) und master.key (Lizenzen).
|
||||
@@ -13,6 +15,7 @@ package server
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
@@ -25,6 +28,8 @@ import (
|
||||
|
||||
"sultan/pkg/auth"
|
||||
"sultan/pkg/crypto"
|
||||
"sultan/pkg/lander"
|
||||
"sultan/pkg/notify"
|
||||
"sultan/pkg/policy"
|
||||
"sultan/pkg/registry"
|
||||
"sultan/pkg/store"
|
||||
@@ -33,12 +38,35 @@ import (
|
||||
|
||||
// Config ist die Konfiguration des Servers; Nullwerte bekommen Defaults.
|
||||
type Config struct {
|
||||
DataDir string // Default ./suldata
|
||||
AgentAddr string // Default :8443
|
||||
ProviderAddr string // Default 127.0.0.1:8080 – Klartext, nicht öffentlich binden
|
||||
InterceptorURL string // Default http://127.0.0.1:8081
|
||||
RefreshInterval time.Duration // Default 1m: Neubewertung aller Knoten
|
||||
DataDir string // Default ./suldata
|
||||
AgentAddr string // Default :8443
|
||||
ProviderAddr string // Default 127.0.0.1:8080 – Klartext, nicht öffentlich binden
|
||||
// PublicURLs: Adressen, unter denen Agents den Server erreichen (z. B.
|
||||
// öffentlich und NetBird). Werden gemerkt; die erste steckt in
|
||||
// Einladungen, die ganze Liste bekommen Agents per config-Befehl.
|
||||
PublicURLs []string
|
||||
LanderAddr string // Default 127.0.0.1:8081; "none" = kein eingebauter Lander
|
||||
InterceptorURL string // Default: Adresse des eingebauten Landers
|
||||
HosterName string // Lander: Name des Hosting-Anbieters
|
||||
HosterURL string // Lander: Angebotsseite
|
||||
ProURL string // Lander: Pro-Hinweis auf Community-Wartungsseiten
|
||||
RefreshInterval time.Duration // Default 1m: Neubewertung aller Knoten, Lease-Erneuerung, Befehle
|
||||
LeaseDuration time.Duration // 0 = Laufzeiten je Tier (Spec 4.1a); > 0 feste Laufzeit (Tests, Betrieb)
|
||||
Logger *slog.Logger // Default: Text auf stderr
|
||||
|
||||
// Benachrichtigungen: Slack-kompatible Webhooks und ntfy-Topics (je
|
||||
// mehrfach); Notify ersetzt beides (Tests, eigene Ziele).
|
||||
SlackWebhooks []string
|
||||
NtfyURLs []string
|
||||
NtfyToken string
|
||||
Notify notify.Target
|
||||
OfflineAlertAfter time.Duration // Default 15m: Alarm für getrennte Knoten
|
||||
|
||||
// EnrollLimit: Enrollment-Versuche pro Minute und Client-IP (Default 10,
|
||||
// -1 = unbegrenzt) – gegen das Durchprobieren von Einladungen.
|
||||
EnrollLimit int
|
||||
|
||||
interceptorDerived bool // InterceptorURL folgt der gebundenen Lander-Adresse
|
||||
}
|
||||
|
||||
func (c Config) withDefaults() Config {
|
||||
@@ -51,18 +79,55 @@ func (c Config) withDefaults() Config {
|
||||
if c.ProviderAddr == "" {
|
||||
c.ProviderAddr = "127.0.0.1:8080"
|
||||
}
|
||||
if c.LanderAddr == "" {
|
||||
c.LanderAddr = "127.0.0.1:8081"
|
||||
}
|
||||
if c.InterceptorURL == "" {
|
||||
c.InterceptorURL = "http://127.0.0.1:8081"
|
||||
if c.LanderAddr != "none" {
|
||||
c.InterceptorURL = "http://" + loopbackAddr(c.LanderAddr)
|
||||
c.interceptorDerived = true
|
||||
}
|
||||
}
|
||||
if c.RefreshInterval <= 0 {
|
||||
c.RefreshInterval = time.Minute
|
||||
}
|
||||
if c.OfflineAlertAfter <= 0 {
|
||||
c.OfflineAlertAfter = 15 * time.Minute
|
||||
}
|
||||
if c.EnrollLimit == 0 {
|
||||
c.EnrollLimit = 10
|
||||
}
|
||||
if c.Logger == nil {
|
||||
c.Logger = slog.New(slog.NewTextHandler(os.Stderr, nil))
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// SettingPublicURLs ist der Store-Schlüssel der gemerkten Adressen (JSON-Liste).
|
||||
const SettingPublicURLs = "public_urls"
|
||||
|
||||
// LoadPublicURLs liest die gemerkten Adressen (leer, wenn keine gesetzt sind).
|
||||
func LoadPublicURLs(ctx context.Context, st *store.Store) ([]string, error) {
|
||||
raw, err := st.Setting(ctx, SettingPublicURLs)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var urls []string
|
||||
if err := json.Unmarshal([]byte(raw), &urls); err != nil {
|
||||
return nil, fmt.Errorf("server: %s beschädigt: %w", SettingPublicURLs, err)
|
||||
}
|
||||
return urls, nil
|
||||
}
|
||||
|
||||
// PublicURLs liefert die gemerkten Adressen.
|
||||
func (s *Server) PublicURLs(ctx context.Context) ([]string, error) {
|
||||
return LoadPublicURLs(ctx, s.store)
|
||||
}
|
||||
|
||||
// Server ist ein laufender bzw. startbereiter Sultan-Server.
|
||||
type Server struct {
|
||||
cfg Config
|
||||
@@ -73,12 +138,20 @@ type Server struct {
|
||||
hub *stream.Hub
|
||||
verifier *auth.Verifier
|
||||
registry *registry.Registry
|
||||
lander *lander.Lander
|
||||
notifier notify.Target // nil = keine Benachrichtigungen
|
||||
enrollRL *rateLimiter
|
||||
|
||||
vmu sync.Mutex
|
||||
violations []time.Time // Zeitpunkte der Policy-Verstöße (für Checkmk)
|
||||
|
||||
mu sync.Mutex
|
||||
agentAddr string
|
||||
providerAddr string
|
||||
landerAddr string
|
||||
agentSrv *http.Server
|
||||
providerSrv *http.Server
|
||||
landerSrv *http.Server
|
||||
serveErr chan error
|
||||
stopRefresh chan struct{}
|
||||
shutdownOnce sync.Once
|
||||
@@ -89,6 +162,13 @@ type Server struct {
|
||||
// TLS-Zertifikat und Master-Key. Lauschen beginnt erst mit Start.
|
||||
func New(ctx context.Context, cfg Config) (*Server, error) {
|
||||
cfg = cfg.withDefaults()
|
||||
for i, raw := range cfg.PublicURLs {
|
||||
u, err := crypto.ValidateServerURL(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("server: --public-url: %w", err)
|
||||
}
|
||||
cfg.PublicURLs[i] = u
|
||||
}
|
||||
if err := os.MkdirAll(cfg.DataDir, 0o700); err != nil {
|
||||
return nil, fmt.Errorf("server: Datenverzeichnis: %w", err)
|
||||
}
|
||||
@@ -96,6 +176,13 @@ func New(ctx context.Context, cfg Config) (*Server, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(cfg.PublicURLs) > 0 {
|
||||
raw, _ := json.Marshal(cfg.PublicURLs)
|
||||
if err := st.SetSetting(ctx, SettingPublicURLs, string(raw)); err != nil {
|
||||
st.Close()
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
cert, err := crypto.LoadOrCreateServerCert(cfg.DataDir)
|
||||
if err != nil {
|
||||
st.Close()
|
||||
@@ -114,10 +201,31 @@ func New(ctx context.Context, cfg Config) (*Server, error) {
|
||||
cert: cert,
|
||||
master: master,
|
||||
verifier: &auth.Verifier{Directory: st},
|
||||
serveErr: make(chan error, 2),
|
||||
serveErr: make(chan error, 3),
|
||||
stopRefresh: make(chan struct{}),
|
||||
}
|
||||
s.hub = &stream.Hub{OnConnect: s.nodeOnline, OnDisconnect: s.nodeOffline}
|
||||
s.hub = &stream.Hub{OnConnect: s.nodeOnline, OnDisconnect: s.nodeOffline, SigningKey: master}
|
||||
s.enrollRL = &rateLimiter{limit: cfg.EnrollLimit, window: time.Minute}
|
||||
s.notifier = cfg.Notify
|
||||
if s.notifier == nil {
|
||||
var targets notify.Multi
|
||||
for _, u := range cfg.SlackWebhooks {
|
||||
targets = append(targets, notify.Slack{URL: u})
|
||||
}
|
||||
for _, u := range cfg.NtfyURLs {
|
||||
targets = append(targets, notify.Ntfy{URL: u, Token: cfg.NtfyToken})
|
||||
}
|
||||
if len(targets) > 0 {
|
||||
s.notifier = targets
|
||||
}
|
||||
}
|
||||
s.lander = &lander.Lander{
|
||||
Dir: filepath.Join(cfg.DataDir, "landing"),
|
||||
HosterName: cfg.HosterName,
|
||||
HosterURL: cfg.HosterURL,
|
||||
ProURL: cfg.ProURL,
|
||||
Logger: cfg.Logger,
|
||||
}
|
||||
s.registry = ®istry.Registry{
|
||||
Directory: &directory{store: st},
|
||||
Policy: &policy.Engine{InterceptorURL: cfg.InterceptorURL},
|
||||
@@ -157,6 +265,22 @@ func (s *Server) AgentAddr() string { s.mu.Lock(); defer s.mu.Unlock(); return s
|
||||
// ProviderAddr ist die tatsächlich gebundene Adresse des Provider-Ports.
|
||||
func (s *Server) ProviderAddr() string { s.mu.Lock(); defer s.mu.Unlock(); return s.providerAddr }
|
||||
|
||||
// LanderAddr ist die gebundene Adresse des Landers ("" wenn abgeschaltet).
|
||||
func (s *Server) LanderAddr() string { s.mu.Lock(); defer s.mu.Unlock(); return s.landerAddr }
|
||||
|
||||
// loopbackAddr macht aus ":8081" bzw. "0.0.0.0:8081" eine lokal erreichbare
|
||||
// Adresse (127.0.0.1:8081) für die Interceptor-URL.
|
||||
func loopbackAddr(addr string) string {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return addr
|
||||
}
|
||||
if ip := net.ParseIP(host); host == "" || (ip != nil && ip.IsUnspecified()) {
|
||||
host = "127.0.0.1"
|
||||
}
|
||||
return net.JoinHostPort(host, port)
|
||||
}
|
||||
|
||||
// Start bindet beide Ports und beginnt zu bedienen; er kehrt sofort zurück.
|
||||
func (s *Server) Start() error {
|
||||
agentLn, err := net.Listen("tcp", s.cfg.AgentAddr)
|
||||
@@ -168,6 +292,18 @@ func (s *Server) Start() error {
|
||||
agentLn.Close()
|
||||
return fmt.Errorf("server: Provider-Port %s: %w", s.cfg.ProviderAddr, err)
|
||||
}
|
||||
var landerLn net.Listener
|
||||
if s.cfg.LanderAddr != "none" {
|
||||
if landerLn, err = net.Listen("tcp", s.cfg.LanderAddr); err != nil {
|
||||
agentLn.Close()
|
||||
providerLn.Close()
|
||||
return fmt.Errorf("server: Lander-Port %s: %w", s.cfg.LanderAddr, err)
|
||||
}
|
||||
if s.cfg.interceptorDerived {
|
||||
// Noch läuft nichts nebenläufig: die Policy darf angepasst werden.
|
||||
s.registry.Policy.InterceptorURL = "http://" + loopbackAddr(landerLn.Addr().String())
|
||||
}
|
||||
}
|
||||
|
||||
agentSrv := &http.Server{
|
||||
Handler: s.agentRoutes(),
|
||||
@@ -183,14 +319,22 @@ func (s *Server) Start() error {
|
||||
s.mu.Lock()
|
||||
s.agentAddr, s.providerAddr = agentLn.Addr().String(), providerLn.Addr().String()
|
||||
s.agentSrv, s.providerSrv = agentSrv, providerSrv
|
||||
if landerLn != nil {
|
||||
s.landerAddr = landerLn.Addr().String()
|
||||
s.landerSrv = &http.Server{Handler: s.lander, ReadHeaderTimeout: 10 * time.Second}
|
||||
}
|
||||
landerSrv := s.landerSrv
|
||||
s.mu.Unlock()
|
||||
|
||||
go s.serve(func() error { return agentSrv.ServeTLS(agentLn, "", "") })
|
||||
go s.serve(func() error { return providerSrv.Serve(providerLn) })
|
||||
if landerSrv != nil {
|
||||
go s.serve(func() error { return landerSrv.Serve(landerLn) })
|
||||
}
|
||||
go s.refreshLoop()
|
||||
|
||||
s.log.Info("Sultan-Server gestartet",
|
||||
"agent", s.AgentAddr(), "provider", s.ProviderAddr(),
|
||||
"agent", s.AgentAddr(), "provider", s.ProviderAddr(), "lander", s.LanderAddr(),
|
||||
"Fingerprint", string(s.cert.Fingerprint), "data_dir", s.cfg.DataDir)
|
||||
return nil
|
||||
}
|
||||
@@ -209,9 +353,15 @@ func (s *Server) refreshLoop() {
|
||||
case <-s.stopRefresh:
|
||||
return
|
||||
case <-t.C:
|
||||
s.pruneDisconnected()
|
||||
for _, id := range s.hub.Nodes() {
|
||||
s.ensureLease(id, false)
|
||||
s.dispatchCommands(id)
|
||||
}
|
||||
if err := s.registry.Refresh(context.Background()); err != nil {
|
||||
s.log.Error("Neubewertung fehlgeschlagen", "err", err)
|
||||
}
|
||||
s.runJobs(context.Background())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -241,10 +391,10 @@ func (s *Server) Shutdown(ctx context.Context) error {
|
||||
close(s.stopRefresh)
|
||||
s.hub.Close()
|
||||
s.mu.Lock()
|
||||
agentSrv, providerSrv := s.agentSrv, s.providerSrv
|
||||
agentSrv, providerSrv, landerSrv := s.agentSrv, s.providerSrv, s.landerSrv
|
||||
s.mu.Unlock()
|
||||
var errs []error
|
||||
for _, srv := range []*http.Server{agentSrv, providerSrv} {
|
||||
for _, srv := range []*http.Server{agentSrv, providerSrv, landerSrv} {
|
||||
if srv != nil {
|
||||
errs = append(errs, srv.Shutdown(ctx))
|
||||
}
|
||||
@@ -257,16 +407,62 @@ func (s *Server) Shutdown(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// nodeOnline/nodeOffline halten nodes.status aktuell; ein gedrainter
|
||||
// Knoten bleibt gedraint. Beim Trennen verschwinden seine Routen.
|
||||
// Knoten bleibt gedraint. Beim Trennen bleiben die Routen als Wartungsseite
|
||||
// stehen, solange eine Wartung angemeldet ist oder die Lease gilt; erst
|
||||
// danach (pruneDisconnected) verschwinden sie. drain entfernt sie sofort.
|
||||
func (s *Server) nodeOnline(nodeID string) {
|
||||
s.setNodeStatus(nodeID, store.NodeOnline)
|
||||
s.log.Info("Knoten verbunden", "node", nodeID)
|
||||
s.ensureLease(nodeID, true)
|
||||
s.dispatchCommands(nodeID)
|
||||
s.sendServerURLs(nodeID)
|
||||
if s.notifier != nil {
|
||||
go s.notifyBackOnline(nodeID) // nicht den Stream-Aufbau aufhalten
|
||||
}
|
||||
}
|
||||
|
||||
// sendServerURLs schickt dem Agenten die Adressliste (Ausweichweg).
|
||||
func (s *Server) sendServerURLs(nodeID string) {
|
||||
urls, err := s.PublicURLs(context.Background())
|
||||
if err != nil || len(urls) == 0 {
|
||||
return
|
||||
}
|
||||
payload, _ := json.Marshal(stream.NodeCommand{Command: stream.CmdConfig, ServerURLs: urls})
|
||||
if err := s.hub.Send(nodeID, stream.Event{Type: stream.EventNodeCommand, Payload: payload}); err != nil {
|
||||
s.log.Debug("Adressliste nicht zugestellt", "node", nodeID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) nodeOffline(nodeID string) {
|
||||
s.registry.Remove(nodeID)
|
||||
s.setNodeStatus(nodeID, store.NodeOffline)
|
||||
s.log.Info("Knoten getrennt", "node", nodeID)
|
||||
if err := s.store.MarkNodeSeen(context.Background(), nodeID); err != nil {
|
||||
s.log.Debug("offline-seit setzen", "node", nodeID, "err", err)
|
||||
}
|
||||
if s.inMaintenance(nodeID) || s.leaseValid(nodeID) {
|
||||
s.log.Info("Knoten getrennt – Routen bleiben als Wartungsseite (Wartung bzw. Lease)", "node", nodeID)
|
||||
if err := s.registry.Refresh(context.Background()); err != nil {
|
||||
s.log.Error("Neubewertung nach Trennung", "err", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
s.registry.Remove(nodeID)
|
||||
s.log.Info("Knoten getrennt – Routen entfernt", "node", nodeID)
|
||||
}
|
||||
|
||||
func (s *Server) inMaintenance(nodeID string) bool {
|
||||
n, err := s.store.GetNode(context.Background(), nodeID)
|
||||
return err == nil && n.Maintenance.Active(time.Now())
|
||||
}
|
||||
|
||||
// pruneDisconnected entfernt Routen getrennter Knoten, deren Wartung und
|
||||
// Lease inzwischen abgelaufen sind.
|
||||
func (s *Server) pruneDisconnected() {
|
||||
for _, id := range s.registry.Nodes() {
|
||||
if !s.hub.Connected(id) && !s.inMaintenance(id) && !s.leaseValid(id) {
|
||||
s.registry.Remove(id)
|
||||
s.log.Info("Getrennter Knoten: Wartung und Lease abgelaufen – Routen entfernt", "node", id)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Server) setNodeStatus(nodeID string, status store.NodeStatus) {
|
||||
@@ -284,4 +480,22 @@ func (s *Server) logViolation(v registry.Violation) {
|
||||
s.log.Warn("Policy-Verstoß",
|
||||
"node", v.NodeID, "kind", string(v.Kind), "router", v.Router,
|
||||
"workload", v.Workload, "detail", v.Detail, "observed", v.Observed)
|
||||
s.recordViolation(time.Now())
|
||||
if s.notifier == nil {
|
||||
return
|
||||
}
|
||||
// Asynchron: OnViolation läuft unter der Sperre der Registry. Die
|
||||
// Registry meldet jeden Verstoß nur beim ersten Auftreten.
|
||||
msg := notify.Message{
|
||||
Severity: notify.Warning,
|
||||
Title: "Policy-Verstoß auf Knoten " + v.NodeID,
|
||||
Body: fmt.Sprintf("Knoten *%s*, Router `%s`, Art `%s`: %s", v.NodeID, v.Router, v.Kind, v.Detail),
|
||||
}
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
if err := s.notifier.Send(ctx, msg); err != nil {
|
||||
s.log.Warn("Benachrichtigung über Policy-Verstoß nicht zugestellt", "err", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
+725
-9
@@ -3,6 +3,7 @@ package server
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -17,6 +19,8 @@ import (
|
||||
|
||||
"sultan/pkg/auth"
|
||||
"sultan/pkg/crypto"
|
||||
"sultan/pkg/license"
|
||||
"sultan/pkg/notify"
|
||||
"sultan/pkg/policy"
|
||||
"sultan/pkg/registry"
|
||||
"sultan/pkg/store"
|
||||
@@ -58,6 +62,7 @@ func startServer(t *testing.T, mutate ...func(*Config)) (*Server, *syncBuffer) {
|
||||
DataDir: filepath.Join(t.TempDir(), "suldata"),
|
||||
AgentAddr: "127.0.0.1:0",
|
||||
ProviderAddr: "127.0.0.1:0",
|
||||
LanderAddr: "127.0.0.1:0",
|
||||
RefreshInterval: time.Hour,
|
||||
Logger: slog.New(slog.NewTextHandler(logs, nil)),
|
||||
}
|
||||
@@ -92,7 +97,7 @@ func createTenant(t *testing.T, s *Server, id string, domains ...string) {
|
||||
// issueToken erzeugt einen Enrollment-Token wie `sultan token` es tun wird.
|
||||
func issueToken(t *testing.T, s *Server, tenant string) string {
|
||||
t.Helper()
|
||||
tok, err := crypto.NewEnrollmentToken(s.Fingerprint())
|
||||
tok, err := crypto.NewEnrollmentToken(s.Fingerprint(), "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -116,11 +121,10 @@ func newAgent(t *testing.T, s *Server, token string) *testAgent {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ks := &crypto.KnownServers{Path: filepath.Join(t.TempDir(), "known_servers")}
|
||||
priv, _ := auth.GenerateKey()
|
||||
return &testAgent{
|
||||
base: "https://" + s.AgentAddr(),
|
||||
hc: &http.Client{Transport: &http.Transport{TLSClientConfig: crypto.ClientTLSConfig(s.AgentAddr(), ks, tok.Fingerprint)}},
|
||||
hc: &http.Client{Transport: &http.Transport{TLSClientConfig: crypto.PinnedTLSConfig(tok.Fingerprint)}},
|
||||
signer: &auth.Signer{Key: priv},
|
||||
secret: tok.Secret,
|
||||
}
|
||||
@@ -151,13 +155,46 @@ func (a *testAgent) enroll(t *testing.T, nodeID string) *http.Response {
|
||||
|
||||
// connect hält den Stream im Hintergrund offen, bis cancel aufgerufen wird.
|
||||
func (a *testAgent) connect(t *testing.T) (cancel func()) {
|
||||
t.Helper()
|
||||
cancel, _ = a.connectEvents(t)
|
||||
return cancel
|
||||
}
|
||||
|
||||
// connectEvents hält den Stream offen und liefert die empfangenen Events.
|
||||
func (a *testAgent) connectEvents(t *testing.T) (cancel func(), events <-chan stream.Event) {
|
||||
t.Helper()
|
||||
sctx, cancel := context.WithCancel(ctx)
|
||||
ch := make(chan stream.Event, 64)
|
||||
c := &stream.Client{URL: a.base + "/api/v1/stream", HTTPClient: a.hc, Prepare: a.signer.Sign, MinBackoff: 10 * time.Millisecond}
|
||||
done := make(chan struct{})
|
||||
go func() { c.Run(sctx, func(stream.Event) error { return nil }); close(done) }()
|
||||
go func() {
|
||||
c.Run(sctx, func(ev stream.Event) error {
|
||||
select {
|
||||
case ch <- ev:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
})
|
||||
close(done)
|
||||
}()
|
||||
t.Cleanup(func() { cancel(); <-done })
|
||||
return cancel
|
||||
return cancel, ch
|
||||
}
|
||||
|
||||
// nextEvent wartet auf das nächste Event des Typs typ.
|
||||
func nextEvent(t *testing.T, events <-chan stream.Event, typ stream.EventType) stream.Event {
|
||||
t.Helper()
|
||||
deadline := time.After(3 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case ev := <-events:
|
||||
if ev.Type == typ {
|
||||
return ev
|
||||
}
|
||||
case <-deadline:
|
||||
t.Fatalf("kein Event %s empfangen", typ)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func wpState(domain string) registry.NodeState {
|
||||
@@ -187,7 +224,7 @@ func TestConfigDefaults(t *testing.T) {
|
||||
if c.DataDir != "./suldata" || c.AgentAddr != ":8443" || c.ProviderAddr != "127.0.0.1:8080" {
|
||||
t.Errorf("Defaults: %+v", c)
|
||||
}
|
||||
if c.InterceptorURL != "http://127.0.0.1:8081" || c.RefreshInterval != time.Minute || c.Logger == nil {
|
||||
if c.InterceptorURL != "http://127.0.0.1:8081" || c.LanderAddr != "127.0.0.1:8081" || c.RefreshInterval != time.Minute || c.Logger == nil {
|
||||
t.Errorf("Defaults: %+v", c)
|
||||
}
|
||||
}
|
||||
@@ -299,8 +336,11 @@ func TestEndToEnd(t *testing.T) {
|
||||
t.Errorf("Traefik-Konfiguration: %s", cfg)
|
||||
}
|
||||
|
||||
// Ungeplant getrennt: Routen bleiben als Wartungsseite, solange die Lease gilt.
|
||||
stop()
|
||||
eventually(t, "Route nach Trennung entfernt", func() bool { return providerConfig(t, s) == "{}" })
|
||||
eventually(t, "Wartungsseite nach Trennung", func() bool {
|
||||
return strings.Contains(providerConfig(t, s), policy.InterceptorService)
|
||||
})
|
||||
eventually(t, "Knoten offline", func() bool {
|
||||
n, _ := s.store.GetNode(ctx, "node-1")
|
||||
return n.Status == store.NodeOffline
|
||||
@@ -442,7 +482,7 @@ func TestDirectory(t *testing.T) {
|
||||
func TestRun_ShutdownWithOpenStream(t *testing.T) {
|
||||
cfg := Config{
|
||||
DataDir: filepath.Join(t.TempDir(), "suldata"), AgentAddr: "127.0.0.1:0", ProviderAddr: "127.0.0.1:0",
|
||||
Logger: slog.New(slog.DiscardHandler),
|
||||
LanderAddr: "127.0.0.1:0", Logger: slog.New(slog.DiscardHandler),
|
||||
}
|
||||
s, err := New(ctx, cfg)
|
||||
if err != nil {
|
||||
@@ -474,7 +514,7 @@ func TestStart_PortInUse(t *testing.T) {
|
||||
a, _ := startServer(t)
|
||||
b, err := New(ctx, Config{
|
||||
DataDir: filepath.Join(t.TempDir(), "suldata"), AgentAddr: a.AgentAddr(), ProviderAddr: "127.0.0.1:0",
|
||||
Logger: slog.New(slog.DiscardHandler),
|
||||
LanderAddr: "127.0.0.1:0", Logger: slog.New(slog.DiscardHandler),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -484,3 +524,679 @@ func TestStart_PortInUse(t *testing.T) {
|
||||
t.Error("belegter Port: erwartet Fehler")
|
||||
}
|
||||
}
|
||||
|
||||
// Wie Traefik: Konfiguration lesen, Router zum Interceptor finden, dessen
|
||||
// Header-Middleware anwenden und den Lander aufrufen.
|
||||
func TestLander_EndToEndMaintenance(t *testing.T) {
|
||||
s, _ := startServer(t, func(c *Config) { c.ProURL = "https://hoster.example/pro" })
|
||||
if s.LanderAddr() == "" || strings.HasSuffix(s.LanderAddr(), ":0") {
|
||||
t.Fatalf("Lander-Adresse: %q", s.LanderAddr())
|
||||
}
|
||||
createTenant(t, s, "kunde", "kunde.de")
|
||||
agent := newAgent(t, s, issueToken(t, s, "kunde"))
|
||||
agent.enroll(t, "n1")
|
||||
agent.connect(t)
|
||||
eventually(t, "Stream", func() bool { return s.hub.Connected("n1") })
|
||||
st := wpState("kunde.de")
|
||||
st.DownServices = []string{"wp"}
|
||||
if r := agent.do(t, http.MethodPost, "/api/v1/state", st); r.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("State: %d", r.StatusCode)
|
||||
}
|
||||
|
||||
var cfg struct {
|
||||
HTTP struct {
|
||||
Routers map[string]struct {
|
||||
Service string
|
||||
Middlewares []string
|
||||
}
|
||||
Services map[string]struct {
|
||||
LoadBalancer struct{ Servers []struct{ URL string } }
|
||||
}
|
||||
Middlewares map[string]struct {
|
||||
Headers struct{ CustomRequestHeaders map[string]string }
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := json.Unmarshal([]byte(providerConfig(t, s)), &cfg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
router := cfg.HTTP.Routers["kunde__n1__wp"]
|
||||
interceptor := cfg.HTTP.Services[router.Service].LoadBalancer.Servers[0].URL
|
||||
if interceptor != "http://"+s.LanderAddr() {
|
||||
t.Fatalf("Interceptor-URL %q, erwartet den gebundenen Lander %q", interceptor, s.LanderAddr())
|
||||
}
|
||||
req, _ := http.NewRequest(http.MethodGet, interceptor+"/shop", nil)
|
||||
for _, mw := range router.Middlewares {
|
||||
for k, v := range cfg.HTTP.Middlewares[mw].Headers.CustomRequestHeaders {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != http.StatusServiceUnavailable || resp.Header.Get("Retry-After") != "60" ||
|
||||
!strings.Contains(string(body), "Gleich wieder da") {
|
||||
t.Errorf("Lander: %d %q\n%s", resp.StatusCode, resp.Header.Get("Retry-After"), body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLander_CustomTemplatesFromDataDir(t *testing.T) {
|
||||
s, _ := startServer(t)
|
||||
dir := filepath.Join(s.cfg.DataDir, "landing")
|
||||
os.MkdirAll(dir, 0o755)
|
||||
os.WriteFile(filepath.Join(dir, "maintenance.html"), []byte("Eigene Seite {{.Tenant}}"), 0o644)
|
||||
req, _ := http.NewRequest(http.MethodGet, "http://"+s.LanderAddr()+"/", nil)
|
||||
req.Header.Set("X-Sultan-Stage", "maintenance")
|
||||
req.Header.Set("X-Sultan-Tenant", "kunde")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if body, _ := io.ReadAll(resp.Body); string(body) != "Eigene Seite kunde" {
|
||||
t.Errorf("body = %q", body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLander_Disabled(t *testing.T) {
|
||||
s, _ := startServer(t, func(c *Config) { c.LanderAddr = "none"; c.InterceptorURL = "http://lander.intern:8081" })
|
||||
if s.LanderAddr() != "" {
|
||||
t.Errorf("Lander läuft trotz none: %q", s.LanderAddr())
|
||||
}
|
||||
if s.registry.Policy.InterceptorURL != "http://lander.intern:8081" {
|
||||
t.Errorf("explizite Interceptor-URL überschrieben: %q", s.registry.Policy.InterceptorURL)
|
||||
}
|
||||
}
|
||||
|
||||
// Die öffentliche Adresse wird im Store gemerkt, damit `sultan token invite`
|
||||
// (eigener Prozess) sie in Einladungen einbetten kann.
|
||||
func TestPublicURLs(t *testing.T) {
|
||||
dir := filepath.Join(t.TempDir(), "suldata")
|
||||
discard := slog.New(slog.DiscardHandler)
|
||||
if _, err := New(ctx, Config{DataDir: dir, PublicURLs: []string{"https://ok.example:8443", "http://ingress:8443"}, Logger: discard}); err == nil {
|
||||
t.Error("http-URL akzeptiert")
|
||||
}
|
||||
s, err := New(ctx, Config{DataDir: dir, PublicURLs: []string{"https://Ingress.example:8443/", "https://100.64.0.1:8443"}, Logger: discard})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
want := []string{"https://ingress.example:8443", "https://100.64.0.1:8443"}
|
||||
if got, err := s.PublicURLs(ctx); err != nil || !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("gespeichert: %v %v", got, err)
|
||||
}
|
||||
s.Shutdown(ctx)
|
||||
|
||||
// Neustart ohne --public-url: gemerkte Adressen bleiben.
|
||||
s, err = New(ctx, Config{DataDir: dir, Logger: discard})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Shutdown(ctx)
|
||||
if got, _ := s.PublicURLs(ctx); !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("nach Neustart: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
// Beim Verbinden bekommt der Agent die vollständige Adressliste (signiert).
|
||||
func TestConfigCommandOnConnect(t *testing.T) {
|
||||
urls := []string{"https://ingress.example:8443", "https://100.64.0.1:8443"}
|
||||
s, _ := startServer(t, func(c *Config) { c.PublicURLs = urls })
|
||||
createTenant(t, s, "kunde", "kunde.de")
|
||||
agent := newAgent(t, s, issueToken(t, s, "kunde"))
|
||||
agent.enroll(t, "n1")
|
||||
_, events := agent.connectEvents(t)
|
||||
|
||||
ev := nextEvent(t, events, stream.EventNodeCommand)
|
||||
if err := stream.VerifyEvent(ev, s.MasterPublicKey()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cmd, err := stream.ParseNodeCommand(ev)
|
||||
if err != nil || cmd.Command != stream.CmdConfig || !reflect.DeepEqual(cmd.ServerURLs, urls) {
|
||||
t.Errorf("config-Befehl: %+v %v", cmd, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoConfigCommandWithoutPublicURLs(t *testing.T) {
|
||||
s, _ := startServer(t)
|
||||
createTenant(t, s, "kunde", "kunde.de")
|
||||
agent := newAgent(t, s, issueToken(t, s, "kunde"))
|
||||
agent.enroll(t, "n1")
|
||||
_, events := agent.connectEvents(t)
|
||||
nextEvent(t, events, stream.EventLeaseRenew)
|
||||
select {
|
||||
case ev := <-events:
|
||||
if ev.Type == stream.EventNodeCommand {
|
||||
t.Errorf("config ohne Adressen gesendet: %s", ev.Payload)
|
||||
}
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
|
||||
// connectedAgent: enrollt, verbindet und meldet wpState(kunde.de).
|
||||
func connectedAgent(t *testing.T, s *Server, node string) (*testAgent, func()) {
|
||||
t.Helper()
|
||||
if _, err := s.store.GetTenant(ctx, "kunde"); err != nil {
|
||||
createTenant(t, s, "kunde", "kunde.de")
|
||||
}
|
||||
agent := newAgent(t, s, issueToken(t, s, "kunde"))
|
||||
if r := agent.enroll(t, node); r.StatusCode != http.StatusCreated {
|
||||
t.Fatalf("Enrollment: %d", r.StatusCode)
|
||||
}
|
||||
stop := agent.connect(t)
|
||||
eventually(t, "Stream", func() bool { return s.hub.Connected(node) })
|
||||
if r := agent.do(t, http.MethodPost, "/api/v1/state", wpState("kunde.de")); r.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("State: %d", r.StatusCode)
|
||||
}
|
||||
return agent, stop
|
||||
}
|
||||
|
||||
func TestMaintenance_AgentEndpoint(t *testing.T) {
|
||||
s, logs := startServer(t)
|
||||
agent, _ := connectedAgent(t, s, "n1")
|
||||
|
||||
resp := agent.do(t, http.MethodPost, "/api/v1/maintenance", MaintenanceRequest{
|
||||
Enabled: true, Duration: "2h", Reason: "Kernel 6.12", PublicReason: "Sicherheitsupdate",
|
||||
})
|
||||
if resp.StatusCode != http.StatusNoContent {
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
t.Fatalf("Wartung an: %d %s", resp.StatusCode, b)
|
||||
}
|
||||
n, _ := s.store.GetNode(ctx, "n1")
|
||||
if m := n.Maintenance; m == nil || m.Reason != "Kernel 6.12" || m.PublicReason != "Sicherheitsupdate" ||
|
||||
time.Until(m.Until) < 119*time.Minute || time.Until(m.Until) > 121*time.Minute {
|
||||
t.Errorf("gespeichert: %+v", m)
|
||||
}
|
||||
// Sofort wirksam, nicht erst beim nächsten Refresh.
|
||||
if cfg := providerConfig(t, s); !strings.Contains(cfg, policy.InterceptorService) {
|
||||
t.Errorf("Wartung nicht sofort wirksam: %s", cfg)
|
||||
}
|
||||
if !strings.Contains(logs.String(), "Kernel 6.12") {
|
||||
t.Error("interner Grund nicht protokolliert")
|
||||
}
|
||||
|
||||
if r := agent.do(t, http.MethodPost, "/api/v1/maintenance", MaintenanceRequest{Enabled: false}); r.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("Wartung aus: %d", r.StatusCode)
|
||||
}
|
||||
if n, _ := s.store.GetNode(ctx, "n1"); n.Maintenance != nil {
|
||||
t.Error("Wartung nicht beendet")
|
||||
}
|
||||
if cfg := providerConfig(t, s); strings.Contains(cfg, policy.InterceptorService) {
|
||||
t.Errorf("nach Wartungsende noch umgeleitet: %s", cfg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintenance_DefaultAndUnlimited(t *testing.T) {
|
||||
s, _ := startServer(t)
|
||||
agent, _ := connectedAgent(t, s, "n1")
|
||||
|
||||
agent.do(t, http.MethodPost, "/api/v1/maintenance", MaintenanceRequest{Enabled: true})
|
||||
n, _ := s.store.GetNode(ctx, "n1")
|
||||
if d := time.Until(n.Maintenance.Until); d < 239*time.Minute || d > 241*time.Minute {
|
||||
t.Errorf("Default-Dauer %v, erwartet 4h", d)
|
||||
}
|
||||
agent.do(t, http.MethodPost, "/api/v1/maintenance", MaintenanceRequest{Enabled: true, Duration: "0"})
|
||||
if n, _ := s.store.GetNode(ctx, "n1"); n.Maintenance == nil || !n.Maintenance.Until.IsZero() {
|
||||
t.Errorf("Dauer 0 = unbefristet: %+v", n.Maintenance)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMaintenance_InvalidRequests(t *testing.T) {
|
||||
s, _ := startServer(t)
|
||||
agent, _ := connectedAgent(t, s, "n1")
|
||||
for name, req := range map[string]any{
|
||||
"Dauer unlesbar": MaintenanceRequest{Enabled: true, Duration: "bald"},
|
||||
"Dauer negativ": MaintenanceRequest{Enabled: true, Duration: "-1h"},
|
||||
"Grund zu lang": MaintenanceRequest{Enabled: true, PublicReason: strings.Repeat("x", 301)},
|
||||
"Grund mehrzeilig": MaintenanceRequest{Enabled: true, PublicReason: "Zeile 1\nZeile 2"},
|
||||
"kein JSON-Objekt": "an",
|
||||
} {
|
||||
if r := agent.do(t, http.MethodPost, "/api/v1/maintenance", req); r.StatusCode != http.StatusBadRequest {
|
||||
t.Errorf("%s: Status %d", name, r.StatusCode)
|
||||
}
|
||||
}
|
||||
plain, _ := http.NewRequest(http.MethodPost, agent.base+"/api/v1/maintenance", strings.NewReader(`{"enabled":true}`))
|
||||
if resp, err := agent.hc.Do(plain); err != nil || resp.StatusCode != http.StatusUnauthorized {
|
||||
t.Errorf("unsigniert: %v %v", resp, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Reboot während der Wartung: Routen bleiben als Wartungsseite stehen, bis
|
||||
// die Wartung endet – erst dann verschwinden sie.
|
||||
func TestMaintenance_DisconnectKeepsRoutesUntilEnd(t *testing.T) {
|
||||
// Kurze Lease, damit nach Wartungsende nicht die Lease die Routen hält.
|
||||
s, _ := startServer(t, func(c *Config) { c.RefreshInterval = 20 * time.Millisecond; c.LeaseDuration = 150 * time.Millisecond })
|
||||
agent, stop := connectedAgent(t, s, "n1")
|
||||
agent.do(t, http.MethodPost, "/api/v1/maintenance", MaintenanceRequest{Enabled: true, Duration: "400ms"})
|
||||
|
||||
stop()
|
||||
eventually(t, "Knoten getrennt", func() bool { return !s.hub.Connected("n1") })
|
||||
time.Sleep(100 * time.Millisecond) // einige Refresh-Runden
|
||||
if cfg := providerConfig(t, s); !strings.Contains(cfg, policy.InterceptorService) {
|
||||
t.Fatalf("Routen während der Wartung entfernt: %s", cfg)
|
||||
}
|
||||
eventually(t, "Routen nach Wartungsende entfernt", func() bool { return providerConfig(t, s) == "{}" })
|
||||
}
|
||||
|
||||
// Per CLI (anderer Prozess, direkt im Store) gesetzte Wartung wirkt beim nächsten Refresh.
|
||||
func TestMaintenance_FromStoreAppliedOnRefresh(t *testing.T) {
|
||||
s, _ := startServer(t, func(c *Config) { c.RefreshInterval = 20 * time.Millisecond })
|
||||
connectedAgent(t, s, "n1")
|
||||
if err := s.store.SetNodeMaintenance(ctx, "n1", store.Maintenance{Until: time.Now().Add(time.Hour)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "Umleitung", func() bool { return strings.Contains(providerConfig(t, s), policy.InterceptorService) })
|
||||
}
|
||||
|
||||
func TestDirectory_Maintenance(t *testing.T) {
|
||||
s, _ := startServer(t)
|
||||
createTenant(t, s, "kunde", "kunde.de")
|
||||
s.store.CreateNode(ctx, store.Node{ID: "n1", TenantID: "kunde", Hostname: "h", PublicKey: "k", IPAddress: "1.2.3.4"})
|
||||
until := time.Now().Add(time.Hour).UTC().Truncate(time.Second)
|
||||
s.store.SetNodeMaintenance(ctx, "n1", store.Maintenance{Until: until, Reason: "intern", PublicReason: "öffentlich"})
|
||||
nc, err := (&directory{store: s.store}).NodeContext(ctx, "n1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if nc.Maintenance == nil || !nc.Maintenance.Until.Equal(until) || nc.Maintenance.PublicReason != "öffentlich" {
|
||||
t.Errorf("Maintenance: %+v", nc.Maintenance)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLease_IssuedOnConnectAndSigned(t *testing.T) {
|
||||
s, _ := startServer(t)
|
||||
createTenant(t, s, "kunde", "kunde.de")
|
||||
agent := newAgent(t, s, issueToken(t, s, "kunde"))
|
||||
agent.enroll(t, "n1")
|
||||
_, events := agent.connectEvents(t)
|
||||
|
||||
ev := nextEvent(t, events, stream.EventLeaseRenew)
|
||||
if err := stream.VerifyEvent(ev, s.MasterPublicKey()); err != nil {
|
||||
t.Fatalf("Event nicht vom Server signiert: %v", err)
|
||||
}
|
||||
l, err := license.ParseLease(ev.Payload, s.MasterPublicKey())
|
||||
if err != nil {
|
||||
t.Fatalf("Lease ungültig: %v", err)
|
||||
}
|
||||
if l.NodeID != "n1" || l.TenantID != "kunde" {
|
||||
t.Errorf("Lease: %+v", l)
|
||||
}
|
||||
if d := time.Until(l.ValidUntil); d < 7*24*time.Hour-time.Minute || d > 7*24*time.Hour {
|
||||
t.Errorf("Pro-Lease läuft %v, erwartet 7 Tage", d)
|
||||
}
|
||||
stored, err := s.store.GetLease(ctx, "n1")
|
||||
if err != nil || !stored.ValidUntil.Equal(l.ValidUntil) {
|
||||
t.Errorf("gespeicherte Lease: %+v %v", stored, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLease_Renewal(t *testing.T) {
|
||||
s, _ := startServer(t, func(c *Config) { c.RefreshInterval = 20 * time.Millisecond; c.LeaseDuration = 300 * time.Millisecond })
|
||||
createTenant(t, s, "kunde", "kunde.de")
|
||||
agent := newAgent(t, s, issueToken(t, s, "kunde"))
|
||||
agent.enroll(t, "n1")
|
||||
_, events := agent.connectEvents(t)
|
||||
|
||||
first, _ := license.ParseLease(nextEvent(t, events, stream.EventLeaseRenew).Payload, s.MasterPublicKey())
|
||||
second, _ := license.ParseLease(nextEvent(t, events, stream.EventLeaseRenew).Payload, s.MasterPublicKey())
|
||||
if !second.ValidUntil.After(first.ValidUntil) {
|
||||
t.Errorf("nicht verlängert: %v → %v", first.ValidUntil, second.ValidUntil)
|
||||
}
|
||||
}
|
||||
|
||||
// Ungeplantes Trennen: Wartungsseite bis zum Ablauf der Lease, danach weg.
|
||||
func TestDisconnect_RoutesUntilLeaseExpiry(t *testing.T) {
|
||||
s, _ := startServer(t, func(c *Config) { c.RefreshInterval = 20 * time.Millisecond; c.LeaseDuration = 600 * time.Millisecond })
|
||||
_, stop := connectedAgent(t, s, "n1")
|
||||
stop()
|
||||
eventually(t, "Wartungsseite", func() bool { return strings.Contains(providerConfig(t, s), policy.InterceptorService) })
|
||||
eventually(t, "Routen nach Lease-Ablauf entfernt", func() bool { return providerConfig(t, s) == "{}" })
|
||||
}
|
||||
|
||||
// drain entfernt die Routen eines getrennten Knotens sofort – trotz gültiger Lease.
|
||||
func TestDisconnect_DrainRemovesImmediately(t *testing.T) {
|
||||
s, _ := startServer(t, func(c *Config) { c.RefreshInterval = 20 * time.Millisecond })
|
||||
_, stop := connectedAgent(t, s, "n1")
|
||||
stop()
|
||||
eventually(t, "Wartungsseite", func() bool { return strings.Contains(providerConfig(t, s), policy.InterceptorService) })
|
||||
if err := s.store.SetNodeStatus(ctx, "n1", store.NodeDrained); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "drain entfernt Routen", func() bool { return providerConfig(t, s) == "{}" })
|
||||
}
|
||||
|
||||
// Befehle aus der Warteschlange (die CLI ist ein eigener Prozess) werden
|
||||
// zugestellt – auch solche, die vor dem Verbinden eingereiht wurden.
|
||||
func TestNodeCommands_Delivered(t *testing.T) {
|
||||
s, _ := startServer(t, func(c *Config) { c.RefreshInterval = 20 * time.Millisecond })
|
||||
createTenant(t, s, "kunde", "kunde.de")
|
||||
agent := newAgent(t, s, issueToken(t, s, "kunde"))
|
||||
agent.enroll(t, "n1")
|
||||
s.store.EnqueueNodeCommand(ctx, "n1", stream.CmdResync, "")
|
||||
|
||||
_, events := agent.connectEvents(t)
|
||||
ev := nextEvent(t, events, stream.EventNodeCommand)
|
||||
if err := stream.VerifyEvent(ev, s.MasterPublicKey()); err != nil {
|
||||
t.Fatalf("Befehl nicht signiert: %v", err)
|
||||
}
|
||||
if cmd, err := stream.ParseNodeCommand(ev); err != nil || cmd.Command != stream.CmdResync {
|
||||
t.Errorf("Befehl: %+v %v", cmd, err)
|
||||
}
|
||||
|
||||
s.store.EnqueueNodeCommand(ctx, "n1", stream.CmdDrain, "Umzug")
|
||||
ev = nextEvent(t, events, stream.EventNodeCommand)
|
||||
if cmd, _ := stream.ParseNodeCommand(ev); cmd.Command != stream.CmdDrain || cmd.Reason != "Umzug" {
|
||||
t.Errorf("zweiter Befehl: %+v", cmd)
|
||||
}
|
||||
eventually(t, "als gesendet markiert", func() bool {
|
||||
cmds, _ := s.store.PendingNodeCommands(ctx, "n1")
|
||||
return len(cmds) == 0
|
||||
})
|
||||
}
|
||||
|
||||
func TestRotateKey(t *testing.T) {
|
||||
s, _ := startServer(t)
|
||||
agent, _ := connectedAgent(t, s, "n1")
|
||||
other, _ := connectedAgent(t, s, "n2")
|
||||
newKey, _ := auth.GenerateKey()
|
||||
newPub := auth.EncodePublicKey(newKey.Public().(ed25519.PublicKey))
|
||||
req := RotateKeyRequest{PublicKey: newPub, Proof: auth.RotationProof("n1", newKey)}
|
||||
|
||||
if r := agent.do(t, http.MethodPost, "/api/v1/rotate-key", RotateKeyRequest{PublicKey: newPub, Proof: "AAAA"}); r.StatusCode != http.StatusBadRequest {
|
||||
t.Errorf("falscher Besitznachweis: %d", r.StatusCode)
|
||||
}
|
||||
if r := agent.do(t, http.MethodPost, "/api/v1/rotate-key", req); r.StatusCode != http.StatusNoContent {
|
||||
t.Fatalf("Rotation: %d", r.StatusCode)
|
||||
}
|
||||
if n, _ := s.store.GetNode(ctx, "n1"); n.PublicKey != newPub {
|
||||
t.Error("Schlüssel nicht getauscht")
|
||||
}
|
||||
if r := agent.do(t, http.MethodPost, "/api/v1/rotate-key", req); r.StatusCode != http.StatusUnauthorized {
|
||||
t.Errorf("alter Schlüssel noch gültig: %d", r.StatusCode)
|
||||
}
|
||||
oldSigner := agent.signer
|
||||
agent.signer = &auth.Signer{Key: newKey}
|
||||
if r := agent.do(t, http.MethodPost, "/api/v1/rotate-key", req); r.StatusCode != http.StatusNoContent {
|
||||
t.Errorf("Wiederholung mit neuem Schlüssel (Wiederaufnahme nach Absturz): %d", r.StatusCode)
|
||||
}
|
||||
agent.signer = oldSigner
|
||||
|
||||
// Der Besitznachweis ist an die Knoten-ID gebunden: Ein Nachweis für n2
|
||||
// gilt nicht für n1.
|
||||
_ = other
|
||||
another, _ := auth.GenerateKey()
|
||||
anotherPub := auth.EncodePublicKey(another.Public().(ed25519.PublicKey))
|
||||
agent.signer = &auth.Signer{Key: newKey}
|
||||
foreign := RotateKeyRequest{PublicKey: anotherPub, Proof: auth.RotationProof("n2", another)}
|
||||
if r := agent.do(t, http.MethodPost, "/api/v1/rotate-key", foreign); r.StatusCode != http.StatusBadRequest {
|
||||
t.Errorf("Nachweis für anderen Knoten: %d", r.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectory_FeaturesPaymentDisconnected(t *testing.T) {
|
||||
s, _ := startServer(t)
|
||||
if _, err := s.store.CreateTenant(ctx, store.Tenant{
|
||||
ID: "kunde", Name: "k", Email: "k@x.de", Tier: store.TierPro, MaxRoutes: 5,
|
||||
AllowedDomains: []string{"kunde.de"}, ValidUntil: time.Now().AddDate(1, 0, 0),
|
||||
Features: []string{"custom_middleware"},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.store.CreateNode(ctx, store.Node{ID: "n1", TenantID: "kunde", Hostname: "h", PublicKey: "k", IPAddress: "1.2.3.4", Status: store.NodeOffline})
|
||||
s.store.SetTenantPayment(ctx, "kunde", time.Now().Add(-10*24*time.Hour))
|
||||
|
||||
nc, err := (&directory{store: s.store}).NodeContext(ctx, "n1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !nc.License.HasFeature(license.FeatureCustomMiddleware) {
|
||||
t.Error("Features nicht in der Lizenz")
|
||||
}
|
||||
if nc.Tenant.DunningLevel != 1 {
|
||||
t.Errorf("Mahnstufe %d, erwartet 1 (10 Tage überfällig)", nc.Tenant.DunningLevel)
|
||||
}
|
||||
if !nc.Disconnected {
|
||||
t.Error("getrennter Knoten nicht erkannt")
|
||||
}
|
||||
}
|
||||
|
||||
// recorder ist ein Benachrichtigungsziel für Tests; failFirst lässt die
|
||||
// ersten n Zustellungen scheitern.
|
||||
type recorder struct {
|
||||
mu sync.Mutex
|
||||
msgs []notify.Message
|
||||
failFirst int
|
||||
}
|
||||
|
||||
func (r *recorder) Send(_ context.Context, m notify.Message) error {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if r.failFirst > 0 {
|
||||
r.failFirst--
|
||||
return errors.New("Ziel nicht erreichbar")
|
||||
}
|
||||
r.msgs = append(r.msgs, m)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *recorder) matching(substr string) []notify.Message {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var out []notify.Message
|
||||
for _, m := range r.msgs {
|
||||
if strings.Contains(m.Title+" "+m.Body, substr) {
|
||||
out = append(out, m)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fastJobs(rec *recorder) func(*Config) {
|
||||
return func(c *Config) {
|
||||
c.RefreshInterval = 20 * time.Millisecond
|
||||
c.Notify = rec
|
||||
}
|
||||
}
|
||||
|
||||
func commercialTenant(t *testing.T, s *Server, id string, validFor time.Duration) {
|
||||
t.Helper()
|
||||
if _, err := s.store.CreateTenant(ctx, store.Tenant{
|
||||
ID: id, Name: id, Email: id + "@example.com", Tier: store.TierPro, MaxRoutes: 10,
|
||||
AllowedDomains: []string{"kunde.de"}, ValidUntil: time.Now().Add(validFor),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckmkEndpoint(t *testing.T) {
|
||||
s, _ := startServer(t)
|
||||
commercialTenant(t, s, "kunde", 20*24*time.Hour)
|
||||
connectedAgent(t, s, "n1")
|
||||
|
||||
resp, err := http.Get("http://" + s.ProviderAddr() + "/metrics/checkmk")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
body, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != 200 || !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/plain") {
|
||||
t.Fatalf("Status %d, Content-Type %q", resp.StatusCode, resp.Header.Get("Content-Type"))
|
||||
}
|
||||
out := string(body)
|
||||
for _, want := range []string{`1 "Sultan_Tenant_kunde" days_left=19;30;14`, `0 "Sultan_Payment_kunde"`, `0 "Sultan_Node_n1"`, `0 "Sultan_Policy_Violations"`} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("fehlt %q in:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestJob_ExpiresCommercialTenantOnly(t *testing.T) {
|
||||
rec := &recorder{}
|
||||
s, _ := startServer(t, fastJobs(rec))
|
||||
commercialTenant(t, s, "kunde", 150*time.Millisecond)
|
||||
if _, err := s.store.CreateTenant(ctx, store.Tenant{
|
||||
ID: "homelab", Name: "h", Email: "h@x.de", Tier: store.TierCommunity, MaxRoutes: 5,
|
||||
AllowedDomains: []string{"x.de"}, ValidUntil: time.Now().Add(150 * time.Millisecond),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
eventually(t, "kommerzieller Vertrag auf expired", func() bool {
|
||||
tn, _ := s.store.GetTenant(ctx, "kunde")
|
||||
return tn.Status == store.TenantExpired
|
||||
})
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if tn, _ := s.store.GetTenant(ctx, "homelab"); tn.Status != store.TenantActive {
|
||||
t.Errorf("Community abgelaufen: %s", tn.Status)
|
||||
}
|
||||
eventually(t, "Benachrichtigung Vertragsende", func() bool { return len(rec.matching("abgelaufen")) == 1 })
|
||||
if m := rec.matching("abgelaufen")[0]; m.Severity != notify.Critical || !strings.Contains(m.Title, "kunde") {
|
||||
t.Errorf("Nachricht: %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotifications_ContractOnce(t *testing.T) {
|
||||
rec := &recorder{}
|
||||
s, _ := startServer(t, fastJobs(rec))
|
||||
commercialTenant(t, s, "kunde", 10*24*time.Hour) // innerhalb der 14-Tage-Schwelle
|
||||
eventually(t, "Benachrichtigung", func() bool { return len(rec.matching("kunde")) > 0 })
|
||||
time.Sleep(150 * time.Millisecond) // mehrere Job-Runden
|
||||
if got := rec.matching("kunde"); len(got) != 1 || got[0].Severity != notify.Critical {
|
||||
t.Errorf("erwartet genau eine kritische Nachricht, bekommen %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotifications_Dunning(t *testing.T) {
|
||||
rec := &recorder{}
|
||||
s, _ := startServer(t, fastJobs(rec))
|
||||
commercialTenant(t, s, "kunde", 365*24*time.Hour)
|
||||
s.store.SetTenantPayment(ctx, "kunde", time.Now().Add(-9*24*time.Hour))
|
||||
eventually(t, "Mahnstufe 1", func() bool { return len(rec.matching("Mahnstufe 1")) == 1 })
|
||||
if m := rec.matching("Mahnstufe 1")[0]; m.Severity != notify.Warning {
|
||||
t.Errorf("Schweregrad: %v", m.Severity)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotifications_NodeOffline(t *testing.T) {
|
||||
rec := &recorder{}
|
||||
s, _ := startServer(t, fastJobs(rec), func(c *Config) { c.OfflineAlertAfter = 100 * time.Millisecond })
|
||||
_, stop := connectedAgent(t, s, "n1")
|
||||
_, stop2 := connectedAgent(t, s, "n2")
|
||||
agent2, _ := connectedAgent(t, s, "n3")
|
||||
agent2.do(t, http.MethodPost, "/api/v1/maintenance", MaintenanceRequest{Enabled: true})
|
||||
_ = stop2
|
||||
stop()
|
||||
eventually(t, "Offline-Alarm", func() bool { return len(rec.matching("n1")) == 1 })
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if got := rec.matching("n1"); len(got) != 1 || got[0].Severity != notify.Critical {
|
||||
t.Errorf("erwartet genau einen Alarm: %+v", got)
|
||||
}
|
||||
if len(rec.matching("n2")) != 0 || len(rec.matching("n3")) != 0 {
|
||||
t.Error("Alarm für verbundenen Knoten")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotifications_NoAlarmDuringMaintenance(t *testing.T) {
|
||||
rec := &recorder{}
|
||||
s, _ := startServer(t, fastJobs(rec), func(c *Config) { c.OfflineAlertAfter = 50 * time.Millisecond })
|
||||
agent, stop := connectedAgent(t, s, "n1")
|
||||
agent.do(t, http.MethodPost, "/api/v1/maintenance", MaintenanceRequest{Enabled: true, Duration: "1h"})
|
||||
stop()
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
if got := rec.matching("n1"); len(got) != 0 {
|
||||
t.Errorf("Alarm trotz angemeldeter Wartung: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotifications_Violation(t *testing.T) {
|
||||
rec := &recorder{}
|
||||
s, _ := startServer(t, fastJobs(rec))
|
||||
agent, _ := connectedAgent(t, s, "n1")
|
||||
agent.do(t, http.MethodPost, "/api/v1/state", wpState("bank.de"))
|
||||
eventually(t, "Verstoß gemeldet", func() bool { return len(rec.matching("bank.de")) == 1 })
|
||||
if m := rec.matching("bank.de")[0]; m.Severity != notify.Warning || !strings.Contains(m.Body, "n1") {
|
||||
t.Errorf("Nachricht: %+v", m)
|
||||
}
|
||||
}
|
||||
|
||||
// Scheitert die Zustellung, wird es beim nächsten Durchlauf erneut versucht.
|
||||
func TestNotifications_RetryOnFailure(t *testing.T) {
|
||||
rec := &recorder{failFirst: 2}
|
||||
s, _ := startServer(t, fastJobs(rec))
|
||||
commercialTenant(t, s, "kunde", 10*24*time.Hour)
|
||||
eventually(t, "Zustellung nach Wiederholung", func() bool { return len(rec.matching("kunde")) == 1 })
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if n := len(rec.matching("kunde")); n != 1 {
|
||||
t.Errorf("%d Nachrichten, erwartet 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Massenhaftes Durchprobieren von Einladungen wird gebremst – vor der
|
||||
// Signaturprüfung, pro Client-IP.
|
||||
func TestEnroll_RateLimited(t *testing.T) {
|
||||
s, _ := startServer(t, func(c *Config) { c.EnrollLimit = 3 })
|
||||
createTenant(t, s, "kunde", "kunde.de")
|
||||
agent := newAgent(t, s, issueToken(t, s, "kunde"))
|
||||
agent.secret = crypto.TokenPrefix + strings.Repeat("A", 43) // falsch geraten
|
||||
for i := range 3 {
|
||||
if r := agent.enroll(t, "n1"); r.StatusCode != http.StatusForbidden {
|
||||
t.Fatalf("Versuch %d: %d", i+1, r.StatusCode)
|
||||
}
|
||||
}
|
||||
r := agent.enroll(t, "n1")
|
||||
if r.StatusCode != http.StatusTooManyRequests || r.Header.Get("Retry-After") == "" {
|
||||
t.Errorf("4. Versuch: %d, Retry-After %q", r.StatusCode, r.Header.Get("Retry-After"))
|
||||
}
|
||||
// Auch unsignierte Fluten zählen.
|
||||
req, _ := http.NewRequest(http.MethodPost, agent.base+"/api/v1/enroll", strings.NewReader(`{}`))
|
||||
if resp, err := agent.hc.Do(req); err != nil || resp.StatusCode != http.StatusTooManyRequests {
|
||||
t.Errorf("unsigniert: %v %v", resp, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigDefaults_EnrollLimit(t *testing.T) {
|
||||
if c := (Config{}).withDefaults(); c.EnrollLimit != 10 {
|
||||
t.Errorf("EnrollLimit-Default %d, erwartet 10 pro Minute", c.EnrollLimit)
|
||||
}
|
||||
}
|
||||
|
||||
// „Wieder online“ nur nach einem Offline-Alarm und genau einmal pro Ausfall.
|
||||
func TestNotifications_BackOnline(t *testing.T) {
|
||||
rec := &recorder{}
|
||||
s, _ := startServer(t, fastJobs(rec), func(c *Config) { c.OfflineAlertAfter = 100 * time.Millisecond })
|
||||
agent, stop := connectedAgent(t, s, "n1")
|
||||
stop()
|
||||
eventually(t, "Offline-Alarm", func() bool { return len(rec.matching("offline")) == 1 })
|
||||
|
||||
agent.connect(t)
|
||||
eventually(t, "wieder online", func() bool { return len(rec.matching("wieder online")) == 1 })
|
||||
m := rec.matching("wieder online")[0]
|
||||
if m.Severity != notify.Info || !strings.Contains(m.Body, "n1") {
|
||||
t.Errorf("Nachricht: %+v", m)
|
||||
}
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if n := len(rec.matching("wieder online")); n != 1 {
|
||||
t.Errorf("%d Wieder-online-Nachrichten, erwartet 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotifications_NoBackOnlineWithoutAlarm(t *testing.T) {
|
||||
rec := &recorder{}
|
||||
s, _ := startServer(t, fastJobs(rec), func(c *Config) { c.OfflineAlertAfter = time.Hour })
|
||||
agent, stop := connectedAgent(t, s, "n1")
|
||||
stop()
|
||||
eventually(t, "offline", func() bool { n, _ := s.store.GetNode(ctx, "n1"); return n.Status == store.NodeOffline })
|
||||
agent.connect(t)
|
||||
eventually(t, "online", func() bool { return s.hub.Connected("n1") })
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if got := rec.matching("wieder online"); len(got) != 0 {
|
||||
t.Errorf("Meldung ohne vorherigen Alarm: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigDefaults_EnrollLimitDisabled(t *testing.T) {
|
||||
if c := (Config{EnrollLimit: -1}).withDefaults(); c.EnrollLimit != -1 {
|
||||
t.Errorf("-1 (unbegrenzt) überschrieben: %d", c.EnrollLimit)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user